1 /**********************************************************************
4 * Contact: support@cavium.com
5 * Please include "LiquidIO" in the subject.
7 * Copyright (c) 2003-2015 Cavium, Inc.
9 * This file is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License, Version 2, as
11 * published by the Free Software Foundation.
13 * This file is distributed in the hope that it will be useful, but
14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
16 * NONINFRINGEMENT. See the GNU General Public License for more
19 * This file may also be available under a different license from Cavium.
20 * Contact Cavium, Inc. for more information
21 **********************************************************************/
22 #include <linux/version.h>
23 #include <linux/types.h>
24 #include <linux/list.h>
25 #include <linux/interrupt.h>
26 #include <linux/pci.h>
27 #include <linux/kthread.h>
28 #include <linux/netdevice.h>
29 #include <linux/vmalloc.h>
30 #include "octeon_config.h"
31 #include "liquidio_common.h"
32 #include "octeon_droq.h"
33 #include "octeon_iq.h"
34 #include "response_manager.h"
35 #include "octeon_device.h"
36 #include "octeon_nic.h"
37 #include "octeon_main.h"
38 #include "octeon_network.h"
39 #include "cn66xx_regs.h"
40 #include "cn66xx_device.h"
41 #include "cn68xx_regs.h"
42 #include "cn68xx_device.h"
43 #include "liquidio_image.h"
45 #define INCR_INSTRQUEUE_PKT_COUNT(octeon_dev_ptr, iq_no, field, count) \
46 (octeon_dev_ptr->instr_queue[iq_no]->stats.field += count)
48 struct iq_post_status
{
53 static void check_db_timeout(struct work_struct
*work
);
54 static void __check_db_timeout(struct octeon_device
*oct
, u64 iq_no
);
56 static void (*reqtype_free_fn
[MAX_OCTEON_DEVICES
][REQTYPE_LAST
+ 1]) (void *);
58 static inline int IQ_INSTR_MODE_64B(struct octeon_device
*oct
, int iq_no
)
60 struct octeon_instr_queue
*iq
=
61 (struct octeon_instr_queue
*)oct
->instr_queue
[iq_no
];
65 #define IQ_INSTR_MODE_32B(oct, iq_no) (!IQ_INSTR_MODE_64B(oct, iq_no))
67 /* Define this to return the request status comaptible to old code */
68 /*#define OCTEON_USE_OLD_REQ_STATUS*/
70 /* Return 0 on success, 1 on failure */
71 int octeon_init_instr_queue(struct octeon_device
*oct
,
72 union oct_txpciq txpciq
,
75 struct octeon_instr_queue
*iq
;
76 struct octeon_iq_config
*conf
= NULL
;
77 u32 iq_no
= (u32
)txpciq
.s
.q_no
;
79 struct cavium_wq
*db_wq
;
80 int orig_node
= dev_to_node(&oct
->pci_dev
->dev
);
81 int numa_node
= cpu_to_node(iq_no
% num_online_cpus());
83 if (OCTEON_CN6XXX(oct
))
84 conf
= &(CFG_GET_IQ_CFG(CHIP_FIELD(oct
, cn6xxx
, conf
)));
87 dev_err(&oct
->pci_dev
->dev
, "Unsupported Chip %x\n",
92 if (num_descs
& (num_descs
- 1)) {
93 dev_err(&oct
->pci_dev
->dev
,
94 "Number of descriptors for instr queue %d not in power of 2.\n",
99 q_size
= (u32
)conf
->instr_type
* num_descs
;
101 iq
= oct
->instr_queue
[iq_no
];
104 set_dev_node(&oct
->pci_dev
->dev
, numa_node
);
105 iq
->base_addr
= lio_dma_alloc(oct
, q_size
,
106 (dma_addr_t
*)&iq
->base_addr_dma
);
107 set_dev_node(&oct
->pci_dev
->dev
, orig_node
);
109 iq
->base_addr
= lio_dma_alloc(oct
, q_size
,
110 (dma_addr_t
*)&iq
->base_addr_dma
);
111 if (!iq
->base_addr
) {
112 dev_err(&oct
->pci_dev
->dev
, "Cannot allocate memory for instr queue %d\n",
117 iq
->max_count
= num_descs
;
119 /* Initialize a list to holds requests that have been posted to Octeon
120 * but has yet to be fetched by octeon
122 iq
->request_list
= vmalloc_node((sizeof(*iq
->request_list
) * num_descs
),
124 if (!iq
->request_list
)
125 iq
->request_list
= vmalloc(sizeof(*iq
->request_list
) *
127 if (!iq
->request_list
) {
128 lio_dma_free(oct
, q_size
, iq
->base_addr
, iq
->base_addr_dma
);
129 dev_err(&oct
->pci_dev
->dev
, "Alloc failed for IQ[%d] nr free list\n",
134 memset(iq
->request_list
, 0, sizeof(*iq
->request_list
) * num_descs
);
136 dev_dbg(&oct
->pci_dev
->dev
, "IQ[%d]: base: %p basedma: %llx count: %d\n",
137 iq_no
, iq
->base_addr
, iq
->base_addr_dma
, iq
->max_count
);
139 iq
->txpciq
.u64
= txpciq
.u64
;
140 iq
->fill_threshold
= (u32
)conf
->db_min
;
142 iq
->host_write_index
= 0;
143 iq
->octeon_read_index
= 0;
145 iq
->last_db_time
= 0;
146 iq
->do_auto_flush
= 1;
147 iq
->db_timeout
= (u32
)conf
->db_timeout
;
148 atomic_set(&iq
->instr_pending
, 0);
150 /* Initialize the spinlock for this instruction queue */
151 spin_lock_init(&iq
->lock
);
152 spin_lock_init(&iq
->post_lock
);
154 spin_lock_init(&iq
->iq_flush_running_lock
);
156 oct
->io_qmask
.iq
|= (1ULL << iq_no
);
158 /* Set the 32B/64B mode for each input queue */
159 oct
->io_qmask
.iq64B
|= ((conf
->instr_type
== 64) << iq_no
);
160 iq
->iqcmd_64B
= (conf
->instr_type
== 64);
162 oct
->fn_list
.setup_iq_regs(oct
, iq_no
);
164 oct
->check_db_wq
[iq_no
].wq
= alloc_workqueue("check_iq_db",
167 if (!oct
->check_db_wq
[iq_no
].wq
) {
168 lio_dma_free(oct
, q_size
, iq
->base_addr
, iq
->base_addr_dma
);
169 dev_err(&oct
->pci_dev
->dev
, "check db wq create failed for iq %d\n",
174 db_wq
= &oct
->check_db_wq
[iq_no
];
176 INIT_DELAYED_WORK(&db_wq
->wk
.work
, check_db_timeout
);
177 db_wq
->wk
.ctxptr
= oct
;
178 db_wq
->wk
.ctxul
= iq_no
;
179 queue_delayed_work(db_wq
->wq
, &db_wq
->wk
.work
, msecs_to_jiffies(1));
184 int octeon_delete_instr_queue(struct octeon_device
*oct
, u32 iq_no
)
186 u64 desc_size
= 0, q_size
;
187 struct octeon_instr_queue
*iq
= oct
->instr_queue
[iq_no
];
189 cancel_delayed_work_sync(&oct
->check_db_wq
[iq_no
].wk
.work
);
190 destroy_workqueue(oct
->check_db_wq
[iq_no
].wq
);
192 if (OCTEON_CN6XXX(oct
))
194 CFG_GET_IQ_INSTR_TYPE(CHIP_FIELD(oct
, cn6xxx
, conf
));
196 vfree(iq
->request_list
);
199 q_size
= iq
->max_count
* desc_size
;
200 lio_dma_free(oct
, (u32
)q_size
, iq
->base_addr
,
207 /* Return 0 on success, 1 on failure */
208 int octeon_setup_iq(struct octeon_device
*oct
,
211 union oct_txpciq txpciq
,
215 u32 iq_no
= (u32
)txpciq
.s
.q_no
;
216 int numa_node
= cpu_to_node(iq_no
% num_online_cpus());
218 if (oct
->instr_queue
[iq_no
]) {
219 dev_dbg(&oct
->pci_dev
->dev
, "IQ is in use. Cannot create the IQ: %d again\n",
221 oct
->instr_queue
[iq_no
]->txpciq
.u64
= txpciq
.u64
;
222 oct
->instr_queue
[iq_no
]->app_ctx
= app_ctx
;
225 oct
->instr_queue
[iq_no
] =
226 vmalloc_node(sizeof(struct octeon_instr_queue
), numa_node
);
227 if (!oct
->instr_queue
[iq_no
])
228 oct
->instr_queue
[iq_no
] =
229 vmalloc(sizeof(struct octeon_instr_queue
));
230 if (!oct
->instr_queue
[iq_no
])
233 memset(oct
->instr_queue
[iq_no
], 0,
234 sizeof(struct octeon_instr_queue
));
236 oct
->instr_queue
[iq_no
]->q_index
= q_index
;
237 oct
->instr_queue
[iq_no
]->app_ctx
= app_ctx
;
238 oct
->instr_queue
[iq_no
]->ifidx
= ifidx
;
240 if (octeon_init_instr_queue(oct
, txpciq
, num_descs
)) {
241 vfree(oct
->instr_queue
[iq_no
]);
242 oct
->instr_queue
[iq_no
] = NULL
;
247 oct
->fn_list
.enable_io_queues(oct
);
251 int lio_wait_for_instr_fetch(struct octeon_device
*oct
)
253 int i
, retry
= 1000, pending
, instr_cnt
= 0;
258 /*for (i = 0; i < oct->num_iqs; i++) {*/
259 for (i
= 0; i
< MAX_OCTEON_INSTR_QUEUES(oct
); i
++) {
260 if (!(oct
->io_qmask
.iq
& (1ULL << i
)))
264 instr_queue
[i
]->instr_pending
);
266 __check_db_timeout(oct
, i
);
267 instr_cnt
+= pending
;
273 schedule_timeout_uninterruptible(1);
275 } while (retry
-- && instr_cnt
);
281 ring_doorbell(struct octeon_device
*oct
, struct octeon_instr_queue
*iq
)
283 if (atomic_read(&oct
->status
) == OCT_DEV_RUNNING
) {
284 writel(iq
->fill_cnt
, iq
->doorbell_reg
);
285 /* make sure doorbell write goes through */
288 iq
->last_db_time
= jiffies
;
293 static inline void __copy_cmd_into_iq(struct octeon_instr_queue
*iq
,
298 cmdsize
= ((iq
->iqcmd_64B
) ? 64 : 32);
299 iqptr
= iq
->base_addr
+ (cmdsize
* iq
->host_write_index
);
301 memcpy(iqptr
, cmd
, cmdsize
);
305 __post_command(struct octeon_device
*octeon_dev
__attribute__((unused
)),
306 struct octeon_instr_queue
*iq
,
307 u32 force_db
__attribute__((unused
)), u8
*cmd
)
311 /* This ensures that the read index does not wrap around to the same
312 * position if queue gets full before Octeon could fetch any instr.
314 if (atomic_read(&iq
->instr_pending
) >= (s32
)(iq
->max_count
- 1))
317 __copy_cmd_into_iq(iq
, cmd
);
319 /* "index" is returned, host_write_index is modified. */
320 index
= iq
->host_write_index
;
321 INCR_INDEX_BY1(iq
->host_write_index
, iq
->max_count
);
324 /* Flush the command into memory. We need to be sure the data is in
325 * memory before indicating that the instruction is pending.
329 atomic_inc(&iq
->instr_pending
);
334 static inline struct iq_post_status
335 __post_command2(struct octeon_device
*octeon_dev
__attribute__((unused
)),
336 struct octeon_instr_queue
*iq
,
337 u32 force_db
__attribute__((unused
)), u8
*cmd
)
339 struct iq_post_status st
;
341 st
.status
= IQ_SEND_OK
;
343 /* This ensures that the read index does not wrap around to the same
344 * position if queue gets full before Octeon could fetch any instr.
346 if (atomic_read(&iq
->instr_pending
) >= (s32
)(iq
->max_count
- 1)) {
347 st
.status
= IQ_SEND_FAILED
;
352 if (atomic_read(&iq
->instr_pending
) >= (s32
)(iq
->max_count
- 2))
353 st
.status
= IQ_SEND_STOP
;
355 __copy_cmd_into_iq(iq
, cmd
);
357 /* "index" is returned, host_write_index is modified. */
358 st
.index
= iq
->host_write_index
;
359 INCR_INDEX_BY1(iq
->host_write_index
, iq
->max_count
);
362 /* Flush the command into memory. We need to be sure the data is in
363 * memory before indicating that the instruction is pending.
367 atomic_inc(&iq
->instr_pending
);
373 octeon_register_reqtype_free_fn(struct octeon_device
*oct
, int reqtype
,
376 if (reqtype
> REQTYPE_LAST
) {
377 dev_err(&oct
->pci_dev
->dev
, "%s: Invalid reqtype: %d\n",
382 reqtype_free_fn
[oct
->octeon_id
][reqtype
] = fn
;
388 __add_to_request_list(struct octeon_instr_queue
*iq
,
389 int idx
, void *buf
, int reqtype
)
391 iq
->request_list
[idx
].buf
= buf
;
392 iq
->request_list
[idx
].reqtype
= reqtype
;
396 lio_process_iq_request_list(struct octeon_device
*oct
,
397 struct octeon_instr_queue
*iq
, u32 napi_budget
)
401 u32 old
= iq
->flush_index
;
403 unsigned int pkts_compl
= 0, bytes_compl
= 0;
404 struct octeon_soft_command
*sc
;
405 struct octeon_instr_irh
*irh
;
407 while (old
!= iq
->octeon_read_index
) {
408 reqtype
= iq
->request_list
[old
].reqtype
;
409 buf
= iq
->request_list
[old
].buf
;
411 if (reqtype
== REQTYPE_NONE
)
414 octeon_update_tx_completion_counters(buf
, reqtype
, &pkts_compl
,
418 case REQTYPE_NORESP_NET
:
419 case REQTYPE_NORESP_NET_SG
:
420 case REQTYPE_RESP_NET_SG
:
421 reqtype_free_fn
[oct
->octeon_id
][reqtype
](buf
);
423 case REQTYPE_RESP_NET
:
424 case REQTYPE_SOFT_COMMAND
:
427 irh
= (struct octeon_instr_irh
*)&sc
->cmd
.cmd2
.irh
;
429 /* We're expecting a response from Octeon.
430 * It's up to lio_process_ordered_list() to
431 * process sc. Add sc to the ordered soft
432 * command response list because we expect
433 * a response from Octeon.
435 spin_lock_bh(&oct
->response_list
436 [OCTEON_ORDERED_SC_LIST
].lock
);
437 atomic_inc(&oct
->response_list
438 [OCTEON_ORDERED_SC_LIST
].
440 list_add_tail(&sc
->node
, &oct
->response_list
441 [OCTEON_ORDERED_SC_LIST
].head
);
442 spin_unlock_bh(&oct
->response_list
443 [OCTEON_ORDERED_SC_LIST
].lock
);
446 sc
->callback(oct
, OCTEON_REQUEST_DONE
,
452 dev_err(&oct
->pci_dev
->dev
,
453 "%s Unknown reqtype: %d buf: %p at idx %d\n",
454 __func__
, reqtype
, buf
, old
);
457 iq
->request_list
[old
].buf
= NULL
;
458 iq
->request_list
[old
].reqtype
= 0;
462 INCR_INDEX_BY1(old
, iq
->max_count
);
464 if ((napi_budget
) && (inst_count
>= napi_budget
))
468 octeon_report_tx_completion_to_bql(iq
->app_ctx
, pkts_compl
,
470 iq
->flush_index
= old
;
475 /* Can only be called from process context */
477 octeon_flush_iq(struct octeon_device
*oct
, struct octeon_instr_queue
*iq
,
478 u32 pending_thresh
, u32 napi_budget
)
480 u32 inst_processed
= 0;
481 u32 tot_inst_processed
= 0;
484 if (!spin_trylock(&iq
->iq_flush_running_lock
))
487 spin_lock_bh(&iq
->lock
);
489 iq
->octeon_read_index
= oct
->fn_list
.update_iq_read_idx(iq
);
491 if (atomic_read(&iq
->instr_pending
) >= (s32
)pending_thresh
) {
493 /* Process any outstanding IQ packets. */
494 if (iq
->flush_index
== iq
->octeon_read_index
)
498 inst_processed
= lio_process_iq_request_list
500 napi_budget
- tot_inst_processed
);
503 lio_process_iq_request_list(oct
, iq
, 0);
505 if (inst_processed
) {
506 atomic_sub(inst_processed
, &iq
->instr_pending
);
507 iq
->stats
.instr_processed
+= inst_processed
;
510 tot_inst_processed
+= inst_processed
;
513 } while (tot_inst_processed
< napi_budget
);
515 if (napi_budget
&& (tot_inst_processed
>= napi_budget
))
519 iq
->last_db_time
= jiffies
;
521 spin_unlock_bh(&iq
->lock
);
523 spin_unlock(&iq
->iq_flush_running_lock
);
528 /* Process instruction queue after timeout.
529 * This routine gets called from a workqueue or when removing the module.
531 static void __check_db_timeout(struct octeon_device
*oct
, u64 iq_no
)
533 struct octeon_instr_queue
*iq
;
538 iq
= oct
->instr_queue
[iq_no
];
542 /* return immediately, if no work pending */
543 if (!atomic_read(&iq
->instr_pending
))
545 /* If jiffies - last_db_time < db_timeout do nothing */
546 next_time
= iq
->last_db_time
+ iq
->db_timeout
;
547 if (!time_after(jiffies
, (unsigned long)next_time
))
549 iq
->last_db_time
= jiffies
;
551 /* Flush the instruction queue */
552 octeon_flush_iq(oct
, iq
, 1, 0);
555 /* Called by the Poll thread at regular intervals to check the instruction
556 * queue for commands to be posted and for commands that were fetched by Octeon.
558 static void check_db_timeout(struct work_struct
*work
)
560 struct cavium_wk
*wk
= (struct cavium_wk
*)work
;
561 struct octeon_device
*oct
= (struct octeon_device
*)wk
->ctxptr
;
562 unsigned long iq_no
= wk
->ctxul
;
563 struct cavium_wq
*db_wq
= &oct
->check_db_wq
[iq_no
];
565 __check_db_timeout(oct
, iq_no
);
566 queue_delayed_work(db_wq
->wq
, &db_wq
->wk
.work
, msecs_to_jiffies(1));
570 octeon_send_command(struct octeon_device
*oct
, u32 iq_no
,
571 u32 force_db
, void *cmd
, void *buf
,
572 u32 datasize
, u32 reqtype
)
574 struct iq_post_status st
;
575 struct octeon_instr_queue
*iq
= oct
->instr_queue
[iq_no
];
577 /* Get the lock and prevent other tasks and tx interrupt handler from
580 spin_lock_bh(&iq
->post_lock
);
582 st
= __post_command2(oct
, iq
, force_db
, cmd
);
584 if (st
.status
!= IQ_SEND_FAILED
) {
585 octeon_report_sent_bytes_to_bql(buf
, reqtype
);
586 __add_to_request_list(iq
, st
.index
, buf
, reqtype
);
587 INCR_INSTRQUEUE_PKT_COUNT(oct
, iq_no
, bytes_sent
, datasize
);
588 INCR_INSTRQUEUE_PKT_COUNT(oct
, iq_no
, instr_posted
, 1);
590 if (iq
->fill_cnt
>= iq
->fill_threshold
|| force_db
)
591 ring_doorbell(oct
, iq
);
593 INCR_INSTRQUEUE_PKT_COUNT(oct
, iq_no
, instr_dropped
, 1);
596 spin_unlock_bh(&iq
->post_lock
);
598 /* This is only done here to expedite packets being flushed
599 * for cases where there are no IQ completion interrupts.
601 /*if (iq->do_auto_flush)*/
602 /* octeon_flush_iq(oct, iq, 2, 0);*/
608 octeon_prepare_soft_command(struct octeon_device
*oct
,
609 struct octeon_soft_command
*sc
,
616 struct octeon_config
*oct_cfg
;
617 struct octeon_instr_ih2
*ih2
;
618 struct octeon_instr_irh
*irh
;
619 struct octeon_instr_rdp
*rdp
;
622 BUG_ON(subcode
> 127);
624 oct_cfg
= octeon_get_conf(oct
);
626 ih2
= (struct octeon_instr_ih2
*)&sc
->cmd
.cmd2
.ih2
;
627 ih2
->tagtype
= ATOMIC_TAG
;
628 ih2
->tag
= LIO_CONTROL
;
630 ih2
->grp
= CFG_GET_CTRL_Q_GRP(oct_cfg
);
633 ih2
->dlengsz
= sc
->datasize
;
637 irh
= (struct octeon_instr_irh
*)&sc
->cmd
.cmd2
.irh
;
638 irh
->opcode
= opcode
;
639 irh
->subcode
= subcode
;
641 /* opcode/subcode specific parameters (ossp) */
642 irh
->ossp
= irh_ossp
;
643 sc
->cmd
.cmd2
.ossp
[0] = ossp0
;
644 sc
->cmd
.cmd2
.ossp
[1] = ossp1
;
647 rdp
= (struct octeon_instr_rdp
*)&sc
->cmd
.cmd2
.rdp
;
648 rdp
->pcie_port
= oct
->pcie_port
;
649 rdp
->rlen
= sc
->rdatasize
;
652 ih2
->fsz
= 40; /* irh+ossp[0]+ossp[1]+rdp+rptr = 40 bytes */
655 ih2
->fsz
= 24; /* irh + ossp[0] + ossp[1] = 24 bytes */
659 int octeon_send_soft_command(struct octeon_device
*oct
,
660 struct octeon_soft_command
*sc
)
662 struct octeon_instr_ih2
*ih2
;
663 struct octeon_instr_irh
*irh
;
664 struct octeon_instr_rdp
*rdp
;
667 ih2
= (struct octeon_instr_ih2
*)&sc
->cmd
.cmd2
.ih2
;
669 WARN_ON(!sc
->dmadptr
);
670 sc
->cmd
.cmd2
.dptr
= sc
->dmadptr
;
672 irh
= (struct octeon_instr_irh
*)&sc
->cmd
.cmd2
.irh
;
674 BUG_ON(!sc
->dmarptr
);
675 BUG_ON(!sc
->status_word
);
676 *sc
->status_word
= COMPLETION_WORD_INIT
;
678 rdp
= (struct octeon_instr_rdp
*)&sc
->cmd
.cmd2
.rdp
;
680 sc
->cmd
.cmd2
.rptr
= sc
->dmarptr
;
682 len
= (u32
)ih2
->dlengsz
;
685 sc
->timeout
= jiffies
+ sc
->wait_time
;
687 return (octeon_send_command(oct
, sc
->iq_no
, 1, &sc
->cmd
, sc
,
688 len
, REQTYPE_SOFT_COMMAND
));
691 int octeon_setup_sc_buffer_pool(struct octeon_device
*oct
)
695 struct octeon_soft_command
*sc
;
697 INIT_LIST_HEAD(&oct
->sc_buf_pool
.head
);
698 spin_lock_init(&oct
->sc_buf_pool
.lock
);
699 atomic_set(&oct
->sc_buf_pool
.alloc_buf_count
, 0);
701 for (i
= 0; i
< MAX_SOFT_COMMAND_BUFFERS
; i
++) {
702 sc
= (struct octeon_soft_command
*)
704 SOFT_COMMAND_BUFFER_SIZE
,
705 (dma_addr_t
*)&dma_addr
);
709 sc
->dma_addr
= dma_addr
;
710 sc
->size
= SOFT_COMMAND_BUFFER_SIZE
;
712 list_add_tail(&sc
->node
, &oct
->sc_buf_pool
.head
);
718 int octeon_free_sc_buffer_pool(struct octeon_device
*oct
)
720 struct list_head
*tmp
, *tmp2
;
721 struct octeon_soft_command
*sc
;
723 spin_lock(&oct
->sc_buf_pool
.lock
);
725 list_for_each_safe(tmp
, tmp2
, &oct
->sc_buf_pool
.head
) {
728 sc
= (struct octeon_soft_command
*)tmp
;
730 lio_dma_free(oct
, sc
->size
, sc
, sc
->dma_addr
);
733 INIT_LIST_HEAD(&oct
->sc_buf_pool
.head
);
735 spin_unlock(&oct
->sc_buf_pool
.lock
);
740 struct octeon_soft_command
*octeon_alloc_soft_command(struct octeon_device
*oct
,
747 u32 offset
= sizeof(struct octeon_soft_command
);
748 struct octeon_soft_command
*sc
= NULL
;
749 struct list_head
*tmp
;
751 BUG_ON((offset
+ datasize
+ rdatasize
+ ctxsize
) >
752 SOFT_COMMAND_BUFFER_SIZE
);
754 spin_lock(&oct
->sc_buf_pool
.lock
);
756 if (list_empty(&oct
->sc_buf_pool
.head
)) {
757 spin_unlock(&oct
->sc_buf_pool
.lock
);
761 list_for_each(tmp
, &oct
->sc_buf_pool
.head
)
766 atomic_inc(&oct
->sc_buf_pool
.alloc_buf_count
);
768 spin_unlock(&oct
->sc_buf_pool
.lock
);
770 sc
= (struct octeon_soft_command
*)tmp
;
772 dma_addr
= sc
->dma_addr
;
775 memset(sc
, 0, sc
->size
);
777 sc
->dma_addr
= dma_addr
;
781 sc
->ctxptr
= (u8
*)sc
+ offset
;
782 sc
->ctxsize
= ctxsize
;
785 /* Start data at 128 byte boundary */
786 offset
= (offset
+ ctxsize
+ 127) & 0xffffff80;
789 sc
->virtdptr
= (u8
*)sc
+ offset
;
790 sc
->dmadptr
= dma_addr
+ offset
;
791 sc
->datasize
= datasize
;
794 /* Start rdata at 128 byte boundary */
795 offset
= (offset
+ datasize
+ 127) & 0xffffff80;
798 BUG_ON(rdatasize
< 16);
799 sc
->virtrptr
= (u8
*)sc
+ offset
;
800 sc
->dmarptr
= dma_addr
+ offset
;
801 sc
->rdatasize
= rdatasize
;
802 sc
->status_word
= (u64
*)((u8
*)(sc
->virtrptr
) + rdatasize
- 8);
808 void octeon_free_soft_command(struct octeon_device
*oct
,
809 struct octeon_soft_command
*sc
)
811 spin_lock(&oct
->sc_buf_pool
.lock
);
813 list_add_tail(&sc
->node
, &oct
->sc_buf_pool
.head
);
815 atomic_dec(&oct
->sc_buf_pool
.alloc_buf_count
);
817 spin_unlock(&oct
->sc_buf_pool
.lock
);