Commit | Line | Data |
---|---|---|
80ff0fd3 DD |
1 | /***********************license start*************** |
2 | * Author: Cavium Networks | |
3 | * | |
4 | * Contact: support@caviumnetworks.com | |
5 | * This file is part of the OCTEON SDK | |
6 | * | |
7 | * Copyright (c) 2003-2008 Cavium Networks | |
8 | * | |
9 | * This file is free software; you can redistribute it and/or modify | |
10 | * it under the terms of the GNU General Public License, Version 2, as | |
11 | * published by the Free Software Foundation. | |
12 | * | |
13 | * This file is distributed in the hope that it will be useful, but | |
14 | * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty | |
15 | * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or | |
16 | * NONINFRINGEMENT. See the GNU General Public License for more | |
17 | * details. | |
18 | * | |
19 | * You should have received a copy of the GNU General Public License | |
20 | * along with this file; if not, write to the Free Software | |
21 | * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA | |
22 | * or visit http://www.gnu.org/licenses/. | |
23 | * | |
24 | * This file may also be available under a different license from Cavium. | |
25 | * Contact Cavium Networks for more information | |
26 | ***********************license end**************************************/ | |
27 | ||
28 | /* | |
29 | * | |
30 | * Support functions for managing command queues used for | |
31 | * various hardware blocks. | |
32 | * | |
33 | * The common command queue infrastructure abstracts out the | |
34 | * software necessary for adding to Octeon's chained queue | |
35 | * structures. These structures are used for commands to the | |
36 | * PKO, ZIP, DFA, RAID, and DMA engine blocks. Although each | |
37 | * hardware unit takes commands and CSRs of different types, | |
38 | * they all use basic linked command buffers to store the | |
39 | * pending request. In general, users of the CVMX API don't | |
40 | * call cvmx-cmd-queue functions directly. Instead the hardware | |
41 | * unit specific wrapper should be used. The wrappers perform | |
42 | * unit specific validation and CSR writes to submit the | |
43 | * commands. | |
44 | * | |
45 | * Even though most software will never directly interact with | |
46 | * cvmx-cmd-queue, knowledge of its internal working can help | |
47 | * in diagnosing performance problems and help with debugging. | |
48 | * | |
49 | * Command queue pointers are stored in a global named block | |
50 | * called "cvmx_cmd_queues". Except for the PKO queues, each | |
51 | * hardware queue is stored in its own cache line to reduce SMP | |
52 | * contention on spin locks. The PKO queues are stored such that | |
53 | * every 16th queue is next to each other in memory. This scheme | |
54 | * allows for queues being in separate cache lines when there | |
55 | * are low number of queues per port. With 16 queues per port, | |
56 | * the first queue for each port is in the same cache area. The | |
57 | * second queues for each port are in another area, etc. This | |
58 | * allows software to implement very efficient lockless PKO with | |
59 | * 16 queues per port using a minimum of cache lines per core. | |
60 | * All queues for a given core will be isolated in the same | |
61 | * cache area. | |
62 | * | |
63 | * In addition to the memory pointer layout, cvmx-cmd-queue | |
64 | * provides an optimized fair ll/sc locking mechanism for the | |
65 | * queues. The lock uses a "ticket / now serving" model to | |
66 | * maintain fair order on contended locks. In addition, it uses | |
67 | * predicted locking time to limit cache contention. When a core | |
68 | * know it must wait in line for a lock, it spins on the | |
69 | * internal cycle counter to completely eliminate any causes of | |
70 | * bus traffic. | |
71 | * | |
72 | */ | |
73 | ||
74 | #ifndef __CVMX_CMD_QUEUE_H__ | |
75 | #define __CVMX_CMD_QUEUE_H__ | |
76 | ||
77 | #include <linux/prefetch.h> | |
78 | ||
b0984c43 MR |
79 | #include <asm/compiler.h> |
80 | ||
a1ce3928 | 81 | #include <asm/octeon/cvmx-fpa.h> |
80ff0fd3 DD |
82 | /** |
83 | * By default we disable the max depth support. Most programs | |
84 | * don't use it and it slows down the command queue processing | |
85 | * significantly. | |
86 | */ | |
87 | #ifndef CVMX_CMD_QUEUE_ENABLE_MAX_DEPTH | |
88 | #define CVMX_CMD_QUEUE_ENABLE_MAX_DEPTH 0 | |
89 | #endif | |
90 | ||
91 | /** | |
92 | * Enumeration representing all hardware blocks that use command | |
93 | * queues. Each hardware block has up to 65536 sub identifiers for | |
94 | * multiple command queues. Not all chips support all hardware | |
95 | * units. | |
96 | */ | |
97 | typedef enum { | |
98 | CVMX_CMD_QUEUE_PKO_BASE = 0x00000, | |
99 | ||
100 | #define CVMX_CMD_QUEUE_PKO(queue) \ | |
101 | ((cvmx_cmd_queue_id_t)(CVMX_CMD_QUEUE_PKO_BASE + (0xffff&(queue)))) | |
102 | ||
103 | CVMX_CMD_QUEUE_ZIP = 0x10000, | |
104 | CVMX_CMD_QUEUE_DFA = 0x20000, | |
105 | CVMX_CMD_QUEUE_RAID = 0x30000, | |
106 | CVMX_CMD_QUEUE_DMA_BASE = 0x40000, | |
107 | ||
108 | #define CVMX_CMD_QUEUE_DMA(queue) \ | |
109 | ((cvmx_cmd_queue_id_t)(CVMX_CMD_QUEUE_DMA_BASE + (0xffff&(queue)))) | |
110 | ||
111 | CVMX_CMD_QUEUE_END = 0x50000, | |
112 | } cvmx_cmd_queue_id_t; | |
113 | ||
114 | /** | |
698a9dce | 115 | * Command write operations can fail if the command queue needs |
80ff0fd3 DD |
116 | * a new buffer and the associated FPA pool is empty. It can also |
117 | * fail if the number of queued command words reaches the maximum | |
118 | * set at initialization. | |
119 | */ | |
120 | typedef enum { | |
121 | CVMX_CMD_QUEUE_SUCCESS = 0, | |
122 | CVMX_CMD_QUEUE_NO_MEMORY = -1, | |
123 | CVMX_CMD_QUEUE_FULL = -2, | |
124 | CVMX_CMD_QUEUE_INVALID_PARAM = -3, | |
125 | CVMX_CMD_QUEUE_ALREADY_SETUP = -4, | |
126 | } cvmx_cmd_queue_result_t; | |
127 | ||
128 | typedef struct { | |
129 | /* You have lock when this is your ticket */ | |
130 | uint8_t now_serving; | |
131 | uint64_t unused1:24; | |
132 | /* Maximum outstanding command words */ | |
133 | uint32_t max_depth; | |
134 | /* FPA pool buffers come from */ | |
135 | uint64_t fpa_pool:3; | |
136 | /* Top of command buffer pointer shifted 7 */ | |
137 | uint64_t base_ptr_div128:29; | |
138 | uint64_t unused2:6; | |
139 | /* FPA buffer size in 64bit words minus 1 */ | |
140 | uint64_t pool_size_m1:13; | |
698a9dce | 141 | /* Number of commands already used in buffer */ |
80ff0fd3 DD |
142 | uint64_t index:13; |
143 | } __cvmx_cmd_queue_state_t; | |
144 | ||
145 | /** | |
698a9dce | 146 | * This structure contains the global state of all command queues. |
80ff0fd3 DD |
147 | * It is stored in a bootmem named block and shared by all |
148 | * applications running on Octeon. Tickets are stored in a differnet | |
149 | * cahce line that queue information to reduce the contention on the | |
150 | * ll/sc used to get a ticket. If this is not the case, the update | |
151 | * of queue state causes the ll/sc to fail quite often. | |
152 | */ | |
153 | typedef struct { | |
154 | uint64_t ticket[(CVMX_CMD_QUEUE_END >> 16) * 256]; | |
155 | __cvmx_cmd_queue_state_t state[(CVMX_CMD_QUEUE_END >> 16) * 256]; | |
156 | } __cvmx_cmd_queue_all_state_t; | |
157 | ||
158 | /** | |
159 | * Initialize a command queue for use. The initial FPA buffer is | |
160 | * allocated and the hardware unit is configured to point to the | |
161 | * new command queue. | |
162 | * | |
163 | * @queue_id: Hardware command queue to initialize. | |
164 | * @max_depth: Maximum outstanding commands that can be queued. | |
165 | * @fpa_pool: FPA pool the command queues should come from. | |
166 | * @pool_size: Size of each buffer in the FPA pool (bytes) | |
167 | * | |
168 | * Returns CVMX_CMD_QUEUE_SUCCESS or a failure code | |
169 | */ | |
170 | cvmx_cmd_queue_result_t cvmx_cmd_queue_initialize(cvmx_cmd_queue_id_t queue_id, | |
171 | int max_depth, int fpa_pool, | |
172 | int pool_size); | |
173 | ||
174 | /** | |
175 | * Shutdown a queue a free it's command buffers to the FPA. The | |
176 | * hardware connected to the queue must be stopped before this | |
177 | * function is called. | |
178 | * | |
179 | * @queue_id: Queue to shutdown | |
180 | * | |
181 | * Returns CVMX_CMD_QUEUE_SUCCESS or a failure code | |
182 | */ | |
183 | cvmx_cmd_queue_result_t cvmx_cmd_queue_shutdown(cvmx_cmd_queue_id_t queue_id); | |
184 | ||
185 | /** | |
186 | * Return the number of command words pending in the queue. This | |
187 | * function may be relatively slow for some hardware units. | |
188 | * | |
189 | * @queue_id: Hardware command queue to query | |
190 | * | |
191 | * Returns Number of outstanding commands | |
192 | */ | |
193 | int cvmx_cmd_queue_length(cvmx_cmd_queue_id_t queue_id); | |
194 | ||
195 | /** | |
196 | * Return the command buffer to be written to. The purpose of this | |
197 | * function is to allow CVMX routine access t othe low level buffer | |
198 | * for initial hardware setup. User applications should not call this | |
199 | * function directly. | |
200 | * | |
201 | * @queue_id: Command queue to query | |
202 | * | |
203 | * Returns Command buffer or NULL on failure | |
204 | */ | |
205 | void *cvmx_cmd_queue_buffer(cvmx_cmd_queue_id_t queue_id); | |
206 | ||
207 | /** | |
208 | * Get the index into the state arrays for the supplied queue id. | |
209 | * | |
210 | * @queue_id: Queue ID to get an index for | |
211 | * | |
212 | * Returns Index into the state arrays | |
213 | */ | |
214 | static inline int __cvmx_cmd_queue_get_index(cvmx_cmd_queue_id_t queue_id) | |
215 | { | |
216 | /* | |
217 | * Warning: This code currently only works with devices that | |
218 | * have 256 queues or less. Devices with more than 16 queues | |
25985edc | 219 | * are laid out in memory to allow cores quick access to |
80ff0fd3 DD |
220 | * every 16th queue. This reduces cache thrashing when you are |
221 | * running 16 queues per port to support lockless operation. | |
222 | */ | |
223 | int unit = queue_id >> 16; | |
224 | int q = (queue_id >> 4) & 0xf; | |
225 | int core = queue_id & 0xf; | |
226 | return unit * 256 + core * 16 + q; | |
227 | } | |
228 | ||
229 | /** | |
230 | * Lock the supplied queue so nobody else is updating it at the same | |
231 | * time as us. | |
232 | * | |
233 | * @queue_id: Queue ID to lock | |
234 | * @qptr: Pointer to the queue's global state | |
235 | */ | |
236 | static inline void __cvmx_cmd_queue_lock(cvmx_cmd_queue_id_t queue_id, | |
237 | __cvmx_cmd_queue_state_t *qptr) | |
238 | { | |
239 | extern __cvmx_cmd_queue_all_state_t | |
240 | *__cvmx_cmd_queue_state_ptr; | |
241 | int tmp; | |
242 | int my_ticket; | |
243 | prefetch(qptr); | |
244 | asm volatile ( | |
245 | ".set push\n" | |
246 | ".set noreorder\n" | |
247 | "1:\n" | |
248 | /* Atomic add one to ticket_ptr */ | |
70342287 | 249 | "ll %[my_ticket], %[ticket_ptr]\n" |
80ff0fd3 | 250 | /* and store the original value */ |
70342287 | 251 | "li %[ticket], 1\n" |
80ff0fd3 | 252 | /* in my_ticket */ |
70342287 RB |
253 | "baddu %[ticket], %[my_ticket]\n" |
254 | "sc %[ticket], %[ticket_ptr]\n" | |
255 | "beqz %[ticket], 1b\n" | |
80ff0fd3 DD |
256 | " nop\n" |
257 | /* Load the current now_serving ticket */ | |
70342287 | 258 | "lbu %[ticket], %[now_serving]\n" |
80ff0fd3 DD |
259 | "2:\n" |
260 | /* Jump out if now_serving == my_ticket */ | |
70342287 | 261 | "beq %[ticket], %[my_ticket], 4f\n" |
80ff0fd3 | 262 | /* Find out how many tickets are in front of me */ |
70342287 | 263 | " subu %[ticket], %[my_ticket], %[ticket]\n" |
80ff0fd3 DD |
264 | /* Use tickets in front of me minus one to delay */ |
265 | "subu %[ticket], 1\n" | |
266 | /* Delay will be ((tickets in front)-1)*32 loops */ | |
70342287 | 267 | "cins %[ticket], %[ticket], 5, 7\n" |
80ff0fd3 DD |
268 | "3:\n" |
269 | /* Loop here until our ticket might be up */ | |
70342287 RB |
270 | "bnez %[ticket], 3b\n" |
271 | " subu %[ticket], 1\n" | |
80ff0fd3 | 272 | /* Jump back up to check out ticket again */ |
70342287 | 273 | "b 2b\n" |
80ff0fd3 | 274 | /* Load the current now_serving ticket */ |
70342287 | 275 | " lbu %[ticket], %[now_serving]\n" |
80ff0fd3 DD |
276 | "4:\n" |
277 | ".set pop\n" : | |
94bfb75a | 278 | [ticket_ptr] "=" GCC_OFF_SMALL_ASM()(__cvmx_cmd_queue_state_ptr->ticket[__cvmx_cmd_queue_get_index(queue_id)]), |
80ff0fd3 DD |
279 | [now_serving] "=m"(qptr->now_serving), [ticket] "=r"(tmp), |
280 | [my_ticket] "=r"(my_ticket) | |
281 | ); | |
282 | } | |
283 | ||
284 | /** | |
285 | * Unlock the queue, flushing all writes. | |
286 | * | |
287 | * @qptr: Queue to unlock | |
288 | */ | |
289 | static inline void __cvmx_cmd_queue_unlock(__cvmx_cmd_queue_state_t *qptr) | |
290 | { | |
291 | qptr->now_serving++; | |
292 | CVMX_SYNCWS; | |
293 | } | |
294 | ||
295 | /** | |
296 | * Get the queue state structure for the given queue id | |
297 | * | |
298 | * @queue_id: Queue id to get | |
299 | * | |
300 | * Returns Queue structure or NULL on failure | |
301 | */ | |
302 | static inline __cvmx_cmd_queue_state_t | |
303 | *__cvmx_cmd_queue_get_state(cvmx_cmd_queue_id_t queue_id) | |
304 | { | |
305 | extern __cvmx_cmd_queue_all_state_t | |
306 | *__cvmx_cmd_queue_state_ptr; | |
307 | return &__cvmx_cmd_queue_state_ptr-> | |
308 | state[__cvmx_cmd_queue_get_index(queue_id)]; | |
309 | } | |
310 | ||
311 | /** | |
312 | * Write an arbitrary number of command words to a command queue. | |
698a9dce | 313 | * This is a generic function; the fixed number of command word |
80ff0fd3 DD |
314 | * functions yield higher performance. |
315 | * | |
316 | * @queue_id: Hardware command queue to write to | |
317 | * @use_locking: | |
70342287 RB |
318 | * Use internal locking to ensure exclusive access for queue |
319 | * updates. If you don't use this locking you must ensure | |
320 | * exclusivity some other way. Locking is strongly recommended. | |
80ff0fd3 | 321 | * @cmd_count: Number of command words to write |
698a9dce | 322 | * @cmds: Array of commands to write |
80ff0fd3 DD |
323 | * |
324 | * Returns CVMX_CMD_QUEUE_SUCCESS or a failure code | |
325 | */ | |
326 | static inline cvmx_cmd_queue_result_t cvmx_cmd_queue_write(cvmx_cmd_queue_id_t | |
327 | queue_id, | |
328 | int use_locking, | |
329 | int cmd_count, | |
330 | uint64_t *cmds) | |
331 | { | |
332 | __cvmx_cmd_queue_state_t *qptr = __cvmx_cmd_queue_get_state(queue_id); | |
333 | ||
334 | /* Make sure nobody else is updating the same queue */ | |
335 | if (likely(use_locking)) | |
336 | __cvmx_cmd_queue_lock(queue_id, qptr); | |
337 | ||
338 | /* | |
339 | * If a max queue length was specified then make sure we don't | |
340 | * exceed it. If any part of the command would be below the | |
341 | * limit we allow it. | |
342 | */ | |
343 | if (CVMX_CMD_QUEUE_ENABLE_MAX_DEPTH && unlikely(qptr->max_depth)) { | |
344 | if (unlikely | |
345 | (cvmx_cmd_queue_length(queue_id) > (int)qptr->max_depth)) { | |
346 | if (likely(use_locking)) | |
347 | __cvmx_cmd_queue_unlock(qptr); | |
348 | return CVMX_CMD_QUEUE_FULL; | |
349 | } | |
350 | } | |
351 | ||
352 | /* | |
353 | * Normally there is plenty of room in the current buffer for | |
354 | * the command. | |
355 | */ | |
356 | if (likely(qptr->index + cmd_count < qptr->pool_size_m1)) { | |
357 | uint64_t *ptr = | |
358 | (uint64_t *) cvmx_phys_to_ptr((uint64_t) qptr-> | |
359 | base_ptr_div128 << 7); | |
360 | ptr += qptr->index; | |
361 | qptr->index += cmd_count; | |
362 | while (cmd_count--) | |
363 | *ptr++ = *cmds++; | |
364 | } else { | |
365 | uint64_t *ptr; | |
366 | int count; | |
367 | /* | |
698a9dce | 368 | * We need a new command buffer. Fail if there isn't |
80ff0fd3 DD |
369 | * one available. |
370 | */ | |
371 | uint64_t *new_buffer = | |
372 | (uint64_t *) cvmx_fpa_alloc(qptr->fpa_pool); | |
373 | if (unlikely(new_buffer == NULL)) { | |
374 | if (likely(use_locking)) | |
375 | __cvmx_cmd_queue_unlock(qptr); | |
376 | return CVMX_CMD_QUEUE_NO_MEMORY; | |
377 | } | |
378 | ptr = | |
379 | (uint64_t *) cvmx_phys_to_ptr((uint64_t) qptr-> | |
380 | base_ptr_div128 << 7); | |
381 | /* | |
382 | * Figure out how many command words will fit in this | |
383 | * buffer. One location will be needed for the next | |
384 | * buffer pointer. | |
385 | */ | |
386 | count = qptr->pool_size_m1 - qptr->index; | |
387 | ptr += qptr->index; | |
388 | cmd_count -= count; | |
389 | while (count--) | |
390 | *ptr++ = *cmds++; | |
391 | *ptr = cvmx_ptr_to_phys(new_buffer); | |
392 | /* | |
393 | * The current buffer is full and has a link to the | |
394 | * next buffer. Time to write the rest of the commands | |
395 | * into the new buffer. | |
396 | */ | |
397 | qptr->base_ptr_div128 = *ptr >> 7; | |
398 | qptr->index = cmd_count; | |
399 | ptr = new_buffer; | |
400 | while (cmd_count--) | |
401 | *ptr++ = *cmds++; | |
402 | } | |
403 | ||
404 | /* All updates are complete. Release the lock and return */ | |
405 | if (likely(use_locking)) | |
406 | __cvmx_cmd_queue_unlock(qptr); | |
407 | return CVMX_CMD_QUEUE_SUCCESS; | |
408 | } | |
409 | ||
410 | /** | |
411 | * Simple function to write two command words to a command | |
412 | * queue. | |
413 | * | |
414 | * @queue_id: Hardware command queue to write to | |
415 | * @use_locking: | |
70342287 RB |
416 | * Use internal locking to ensure exclusive access for queue |
417 | * updates. If you don't use this locking you must ensure | |
418 | * exclusivity some other way. Locking is strongly recommended. | |
80ff0fd3 DD |
419 | * @cmd1: Command |
420 | * @cmd2: Command | |
421 | * | |
422 | * Returns CVMX_CMD_QUEUE_SUCCESS or a failure code | |
423 | */ | |
424 | static inline cvmx_cmd_queue_result_t cvmx_cmd_queue_write2(cvmx_cmd_queue_id_t | |
425 | queue_id, | |
426 | int use_locking, | |
427 | uint64_t cmd1, | |
428 | uint64_t cmd2) | |
429 | { | |
430 | __cvmx_cmd_queue_state_t *qptr = __cvmx_cmd_queue_get_state(queue_id); | |
431 | ||
432 | /* Make sure nobody else is updating the same queue */ | |
433 | if (likely(use_locking)) | |
434 | __cvmx_cmd_queue_lock(queue_id, qptr); | |
435 | ||
436 | /* | |
437 | * If a max queue length was specified then make sure we don't | |
438 | * exceed it. If any part of the command would be below the | |
439 | * limit we allow it. | |
440 | */ | |
441 | if (CVMX_CMD_QUEUE_ENABLE_MAX_DEPTH && unlikely(qptr->max_depth)) { | |
442 | if (unlikely | |
443 | (cvmx_cmd_queue_length(queue_id) > (int)qptr->max_depth)) { | |
444 | if (likely(use_locking)) | |
445 | __cvmx_cmd_queue_unlock(qptr); | |
446 | return CVMX_CMD_QUEUE_FULL; | |
447 | } | |
448 | } | |
449 | ||
450 | /* | |
451 | * Normally there is plenty of room in the current buffer for | |
452 | * the command. | |
453 | */ | |
454 | if (likely(qptr->index + 2 < qptr->pool_size_m1)) { | |
455 | uint64_t *ptr = | |
456 | (uint64_t *) cvmx_phys_to_ptr((uint64_t) qptr-> | |
457 | base_ptr_div128 << 7); | |
458 | ptr += qptr->index; | |
459 | qptr->index += 2; | |
460 | ptr[0] = cmd1; | |
461 | ptr[1] = cmd2; | |
462 | } else { | |
463 | uint64_t *ptr; | |
464 | /* | |
465 | * Figure out how many command words will fit in this | |
466 | * buffer. One location will be needed for the next | |
467 | * buffer pointer. | |
468 | */ | |
469 | int count = qptr->pool_size_m1 - qptr->index; | |
470 | /* | |
698a9dce | 471 | * We need a new command buffer. Fail if there isn't |
80ff0fd3 DD |
472 | * one available. |
473 | */ | |
474 | uint64_t *new_buffer = | |
475 | (uint64_t *) cvmx_fpa_alloc(qptr->fpa_pool); | |
476 | if (unlikely(new_buffer == NULL)) { | |
477 | if (likely(use_locking)) | |
478 | __cvmx_cmd_queue_unlock(qptr); | |
479 | return CVMX_CMD_QUEUE_NO_MEMORY; | |
480 | } | |
481 | count--; | |
482 | ptr = | |
483 | (uint64_t *) cvmx_phys_to_ptr((uint64_t) qptr-> | |
484 | base_ptr_div128 << 7); | |
485 | ptr += qptr->index; | |
486 | *ptr++ = cmd1; | |
487 | if (likely(count)) | |
488 | *ptr++ = cmd2; | |
489 | *ptr = cvmx_ptr_to_phys(new_buffer); | |
490 | /* | |
491 | * The current buffer is full and has a link to the | |
492 | * next buffer. Time to write the rest of the commands | |
493 | * into the new buffer. | |
494 | */ | |
495 | qptr->base_ptr_div128 = *ptr >> 7; | |
496 | qptr->index = 0; | |
497 | if (unlikely(count == 0)) { | |
498 | qptr->index = 1; | |
499 | new_buffer[0] = cmd2; | |
500 | } | |
501 | } | |
502 | ||
503 | /* All updates are complete. Release the lock and return */ | |
504 | if (likely(use_locking)) | |
505 | __cvmx_cmd_queue_unlock(qptr); | |
506 | return CVMX_CMD_QUEUE_SUCCESS; | |
507 | } | |
508 | ||
509 | /** | |
510 | * Simple function to write three command words to a command | |
511 | * queue. | |
512 | * | |
513 | * @queue_id: Hardware command queue to write to | |
514 | * @use_locking: | |
70342287 RB |
515 | * Use internal locking to ensure exclusive access for queue |
516 | * updates. If you don't use this locking you must ensure | |
517 | * exclusivity some other way. Locking is strongly recommended. | |
80ff0fd3 DD |
518 | * @cmd1: Command |
519 | * @cmd2: Command | |
520 | * @cmd3: Command | |
521 | * | |
522 | * Returns CVMX_CMD_QUEUE_SUCCESS or a failure code | |
523 | */ | |
524 | static inline cvmx_cmd_queue_result_t cvmx_cmd_queue_write3(cvmx_cmd_queue_id_t | |
525 | queue_id, | |
526 | int use_locking, | |
527 | uint64_t cmd1, | |
528 | uint64_t cmd2, | |
529 | uint64_t cmd3) | |
530 | { | |
531 | __cvmx_cmd_queue_state_t *qptr = __cvmx_cmd_queue_get_state(queue_id); | |
532 | ||
533 | /* Make sure nobody else is updating the same queue */ | |
534 | if (likely(use_locking)) | |
535 | __cvmx_cmd_queue_lock(queue_id, qptr); | |
536 | ||
537 | /* | |
538 | * If a max queue length was specified then make sure we don't | |
539 | * exceed it. If any part of the command would be below the | |
540 | * limit we allow it. | |
541 | */ | |
542 | if (CVMX_CMD_QUEUE_ENABLE_MAX_DEPTH && unlikely(qptr->max_depth)) { | |
543 | if (unlikely | |
544 | (cvmx_cmd_queue_length(queue_id) > (int)qptr->max_depth)) { | |
545 | if (likely(use_locking)) | |
546 | __cvmx_cmd_queue_unlock(qptr); | |
547 | return CVMX_CMD_QUEUE_FULL; | |
548 | } | |
549 | } | |
550 | ||
551 | /* | |
552 | * Normally there is plenty of room in the current buffer for | |
553 | * the command. | |
554 | */ | |
555 | if (likely(qptr->index + 3 < qptr->pool_size_m1)) { | |
556 | uint64_t *ptr = | |
557 | (uint64_t *) cvmx_phys_to_ptr((uint64_t) qptr-> | |
558 | base_ptr_div128 << 7); | |
559 | ptr += qptr->index; | |
560 | qptr->index += 3; | |
561 | ptr[0] = cmd1; | |
562 | ptr[1] = cmd2; | |
563 | ptr[2] = cmd3; | |
564 | } else { | |
565 | uint64_t *ptr; | |
566 | /* | |
567 | * Figure out how many command words will fit in this | |
568 | * buffer. One location will be needed for the next | |
569 | * buffer pointer | |
570 | */ | |
571 | int count = qptr->pool_size_m1 - qptr->index; | |
572 | /* | |
698a9dce | 573 | * We need a new command buffer. Fail if there isn't |
80ff0fd3 DD |
574 | * one available |
575 | */ | |
576 | uint64_t *new_buffer = | |
577 | (uint64_t *) cvmx_fpa_alloc(qptr->fpa_pool); | |
578 | if (unlikely(new_buffer == NULL)) { | |
579 | if (likely(use_locking)) | |
580 | __cvmx_cmd_queue_unlock(qptr); | |
581 | return CVMX_CMD_QUEUE_NO_MEMORY; | |
582 | } | |
583 | count--; | |
584 | ptr = | |
585 | (uint64_t *) cvmx_phys_to_ptr((uint64_t) qptr-> | |
586 | base_ptr_div128 << 7); | |
587 | ptr += qptr->index; | |
588 | *ptr++ = cmd1; | |
589 | if (count) { | |
590 | *ptr++ = cmd2; | |
591 | if (count > 1) | |
592 | *ptr++ = cmd3; | |
593 | } | |
594 | *ptr = cvmx_ptr_to_phys(new_buffer); | |
595 | /* | |
596 | * The current buffer is full and has a link to the | |
597 | * next buffer. Time to write the rest of the commands | |
598 | * into the new buffer. | |
599 | */ | |
600 | qptr->base_ptr_div128 = *ptr >> 7; | |
601 | qptr->index = 0; | |
602 | ptr = new_buffer; | |
603 | if (count == 0) { | |
604 | *ptr++ = cmd2; | |
605 | qptr->index++; | |
606 | } | |
607 | if (count < 2) { | |
608 | *ptr++ = cmd3; | |
609 | qptr->index++; | |
610 | } | |
611 | } | |
612 | ||
613 | /* All updates are complete. Release the lock and return */ | |
614 | if (likely(use_locking)) | |
615 | __cvmx_cmd_queue_unlock(qptr); | |
616 | return CVMX_CMD_QUEUE_SUCCESS; | |
617 | } | |
618 | ||
619 | #endif /* __CVMX_CMD_QUEUE_H__ */ |