2 * This file is provided under a dual BSD/GPLv2 license. When using or
3 * redistributing this file, you may do so under either license.
7 * Copyright(c) 2012 Intel Corporation. All rights reserved.
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of version 2 of the GNU General Public License as
11 * published by the Free Software Foundation.
15 * Copyright(c) 2012 Intel Corporation. All rights reserved.
17 * Redistribution and use in source and binary forms, with or without
18 * modification, are permitted provided that the following conditions
21 * * Redistributions of source code must retain the above copyright
22 * notice, this list of conditions and the following disclaimer.
23 * * Redistributions in binary form must reproduce the above copy
24 * notice, this list of conditions and the following disclaimer in
25 * the documentation and/or other materials provided with the
27 * * Neither the name of Intel Corporation nor the names of its
28 * contributors may be used to endorse or promote products derived
29 * from this software without specific prior written permission.
31 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
32 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
33 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
34 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
35 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
36 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
37 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
38 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
39 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
40 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
41 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
43 * Intel PCIe NTB Linux driver
45 * Contact Information:
46 * Jon Mason <jon.mason@intel.com>
48 #include <linux/debugfs.h>
49 #include <linux/delay.h>
50 #include <linux/dma-mapping.h>
51 #include <linux/errno.h>
52 #include <linux/export.h>
53 #include <linux/interrupt.h>
54 #include <linux/module.h>
55 #include <linux/pci.h>
56 #include <linux/slab.h>
57 #include <linux/types.h>
58 #include <linux/ntb.h>
61 #define NTB_TRANSPORT_VERSION 1
63 static unsigned int transport_mtu
= 0x401E;
64 module_param(transport_mtu
, uint
, 0644);
65 MODULE_PARM_DESC(transport_mtu
, "Maximum size of NTB transport packets");
67 static unsigned char max_num_clients
= 2;
68 module_param(max_num_clients
, byte
, 0644);
69 MODULE_PARM_DESC(max_num_clients
, "Maximum number of NTB transport clients");
71 struct ntb_queue_entry
{
72 /* ntb_queue list reference */
73 struct list_head entry
;
74 /* pointers to data to be transfered */
81 struct ntb_transport_qp
{
82 struct ntb_transport
*transport
;
83 struct ntb_device
*ndev
;
88 u8 qp_num
; /* Only 64 QP's are allowed. 0-63 */
90 void (*tx_handler
) (struct ntb_transport_qp
*qp
, void *qp_data
,
92 struct list_head tx_free_q
;
93 spinlock_t ntb_tx_free_q_lock
;
97 unsigned int tx_max_frame
;
99 void (*rx_handler
) (struct ntb_transport_qp
*qp
, void *qp_data
,
100 void *data
, int len
);
101 struct tasklet_struct rx_work
;
102 struct list_head rx_pend_q
;
103 struct list_head rx_free_q
;
104 spinlock_t ntb_rx_pend_q_lock
;
105 spinlock_t ntb_rx_free_q_lock
;
109 unsigned int rx_max_frame
;
111 void (*event_handler
) (void *data
, int status
);
112 struct delayed_work link_work
;
113 struct work_struct link_cleanup
;
115 struct dentry
*debugfs_dir
;
116 struct dentry
*debugfs_stats
;
130 struct ntb_transport_mw
{
136 struct ntb_transport_client_dev
{
137 struct list_head entry
;
141 struct ntb_transport
{
142 struct list_head entry
;
143 struct list_head client_devs
;
145 struct ntb_device
*ndev
;
146 struct ntb_transport_mw mw
[NTB_NUM_MW
];
147 struct ntb_transport_qp
*qps
;
148 unsigned int max_qps
;
149 unsigned long qp_bitmap
;
151 struct delayed_work link_work
;
152 struct work_struct link_cleanup
;
153 struct dentry
*debugfs_dir
;
157 DESC_DONE_FLAG
= 1 << 0,
158 LINK_DOWN_FLAG
= 1 << 1,
161 struct ntb_payload_header
{
176 #define QP_TO_MW(qp) ((qp) % NTB_NUM_MW)
177 #define NTB_QP_DEF_NUM_ENTRIES 100
178 #define NTB_LINK_DOWN_TIMEOUT 10
180 static int ntb_match_bus(struct device
*dev
, struct device_driver
*drv
)
182 return !strncmp(dev_name(dev
), drv
->name
, strlen(drv
->name
));
185 static int ntb_client_probe(struct device
*dev
)
187 const struct ntb_client
*drv
= container_of(dev
->driver
,
188 struct ntb_client
, driver
);
189 struct pci_dev
*pdev
= container_of(dev
->parent
, struct pci_dev
, dev
);
193 if (drv
&& drv
->probe
)
194 rc
= drv
->probe(pdev
);
201 static int ntb_client_remove(struct device
*dev
)
203 const struct ntb_client
*drv
= container_of(dev
->driver
,
204 struct ntb_client
, driver
);
205 struct pci_dev
*pdev
= container_of(dev
->parent
, struct pci_dev
, dev
);
207 if (drv
&& drv
->remove
)
215 static struct bus_type ntb_bus_type
= {
217 .match
= ntb_match_bus
,
218 .probe
= ntb_client_probe
,
219 .remove
= ntb_client_remove
,
222 static LIST_HEAD(ntb_transport_list
);
224 static int ntb_bus_init(struct ntb_transport
*nt
)
226 if (list_empty(&ntb_transport_list
)) {
227 int rc
= bus_register(&ntb_bus_type
);
232 list_add(&nt
->entry
, &ntb_transport_list
);
237 static void ntb_bus_remove(struct ntb_transport
*nt
)
239 struct ntb_transport_client_dev
*client_dev
, *cd
;
241 list_for_each_entry_safe(client_dev
, cd
, &nt
->client_devs
, entry
) {
242 dev_err(client_dev
->dev
.parent
, "%s still attached to bus, removing\n",
243 dev_name(&client_dev
->dev
));
244 list_del(&client_dev
->entry
);
245 device_unregister(&client_dev
->dev
);
248 list_del(&nt
->entry
);
250 if (list_empty(&ntb_transport_list
))
251 bus_unregister(&ntb_bus_type
);
254 static void ntb_client_release(struct device
*dev
)
256 struct ntb_transport_client_dev
*client_dev
;
257 client_dev
= container_of(dev
, struct ntb_transport_client_dev
, dev
);
263 * ntb_unregister_client_dev - Unregister NTB client device
264 * @device_name: Name of NTB client device
266 * Unregister an NTB client device with the NTB transport layer
268 void ntb_unregister_client_dev(char *device_name
)
270 struct ntb_transport_client_dev
*client
, *cd
;
271 struct ntb_transport
*nt
;
273 list_for_each_entry(nt
, &ntb_transport_list
, entry
)
274 list_for_each_entry_safe(client
, cd
, &nt
->client_devs
, entry
)
275 if (!strncmp(dev_name(&client
->dev
), device_name
,
276 strlen(device_name
))) {
277 list_del(&client
->entry
);
278 device_unregister(&client
->dev
);
281 EXPORT_SYMBOL_GPL(ntb_unregister_client_dev
);
284 * ntb_register_client_dev - Register NTB client device
285 * @device_name: Name of NTB client device
287 * Register an NTB client device with the NTB transport layer
289 int ntb_register_client_dev(char *device_name
)
291 struct ntb_transport_client_dev
*client_dev
;
292 struct ntb_transport
*nt
;
295 if (list_empty(&ntb_transport_list
))
298 list_for_each_entry(nt
, &ntb_transport_list
, entry
) {
301 client_dev
= kzalloc(sizeof(struct ntb_transport_client_dev
),
308 dev
= &client_dev
->dev
;
310 /* setup and register client devices */
311 dev_set_name(dev
, "%s", device_name
);
312 dev
->bus
= &ntb_bus_type
;
313 dev
->release
= ntb_client_release
;
314 dev
->parent
= &ntb_query_pdev(nt
->ndev
)->dev
;
316 rc
= device_register(dev
);
322 list_add_tail(&client_dev
->entry
, &nt
->client_devs
);
328 ntb_unregister_client_dev(device_name
);
332 EXPORT_SYMBOL_GPL(ntb_register_client_dev
);
335 * ntb_register_client - Register NTB client driver
336 * @drv: NTB client driver to be registered
338 * Register an NTB client driver with the NTB transport layer
340 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
342 int ntb_register_client(struct ntb_client
*drv
)
344 drv
->driver
.bus
= &ntb_bus_type
;
346 if (list_empty(&ntb_transport_list
))
349 return driver_register(&drv
->driver
);
351 EXPORT_SYMBOL_GPL(ntb_register_client
);
354 * ntb_unregister_client - Unregister NTB client driver
355 * @drv: NTB client driver to be unregistered
357 * Unregister an NTB client driver with the NTB transport layer
359 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
361 void ntb_unregister_client(struct ntb_client
*drv
)
363 driver_unregister(&drv
->driver
);
365 EXPORT_SYMBOL_GPL(ntb_unregister_client
);
367 static ssize_t
debugfs_read(struct file
*filp
, char __user
*ubuf
, size_t count
,
370 struct ntb_transport_qp
*qp
;
372 ssize_t ret
, out_offset
, out_count
;
376 buf
= kmalloc(out_count
, GFP_KERNEL
);
380 qp
= filp
->private_data
;
382 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
384 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
385 "rx_bytes - \t%llu\n", qp
->rx_bytes
);
386 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
387 "rx_pkts - \t%llu\n", qp
->rx_pkts
);
388 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
389 "rx_ring_empty - %llu\n", qp
->rx_ring_empty
);
390 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
391 "rx_err_no_buf - %llu\n", qp
->rx_err_no_buf
);
392 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
393 "rx_err_oflow - \t%llu\n", qp
->rx_err_oflow
);
394 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
395 "rx_err_ver - \t%llu\n", qp
->rx_err_ver
);
396 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
397 "rx_buff_begin - %p\n", qp
->rx_buff_begin
);
398 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
399 "rx_offset - \t%p\n", qp
->rx_offset
);
400 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
401 "rx_buff_end - \t%p\n", qp
->rx_buff_end
);
403 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
404 "tx_bytes - \t%llu\n", qp
->tx_bytes
);
405 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
406 "tx_pkts - \t%llu\n", qp
->tx_pkts
);
407 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
408 "tx_ring_full - \t%llu\n", qp
->tx_ring_full
);
409 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
410 "tx_mw_begin - \t%p\n", qp
->tx_mw_begin
);
411 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
412 "tx_offset - \t%p\n", qp
->tx_offset
);
413 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
414 "tx_mw_end - \t%p\n", qp
->tx_mw_end
);
416 out_offset
+= snprintf(buf
+ out_offset
, out_count
- out_offset
,
417 "\nQP Link %s\n", (qp
->qp_link
== NTB_LINK_UP
) ?
419 if (out_offset
> out_count
)
420 out_offset
= out_count
;
422 ret
= simple_read_from_buffer(ubuf
, count
, offp
, buf
, out_offset
);
427 static const struct file_operations ntb_qp_debugfs_stats
= {
428 .owner
= THIS_MODULE
,
430 .read
= debugfs_read
,
433 static void ntb_list_add(spinlock_t
*lock
, struct list_head
*entry
,
434 struct list_head
*list
)
438 spin_lock_irqsave(lock
, flags
);
439 list_add_tail(entry
, list
);
440 spin_unlock_irqrestore(lock
, flags
);
443 static struct ntb_queue_entry
*ntb_list_rm(spinlock_t
*lock
,
444 struct list_head
*list
)
446 struct ntb_queue_entry
*entry
;
449 spin_lock_irqsave(lock
, flags
);
450 if (list_empty(list
)) {
454 entry
= list_first_entry(list
, struct ntb_queue_entry
, entry
);
455 list_del(&entry
->entry
);
457 spin_unlock_irqrestore(lock
, flags
);
462 static void ntb_transport_setup_qp_mw(struct ntb_transport
*nt
,
465 struct ntb_transport_qp
*qp
= &nt
->qps
[qp_num
];
466 unsigned int rx_size
, num_qps_mw
;
467 u8 mw_num
= QP_TO_MW(qp_num
);
470 WARN_ON(nt
->mw
[mw_num
].virt_addr
== 0);
472 if (nt
->max_qps
% NTB_NUM_MW
&& mw_num
< nt
->max_qps
% NTB_NUM_MW
)
473 num_qps_mw
= nt
->max_qps
/ NTB_NUM_MW
+ 1;
475 num_qps_mw
= nt
->max_qps
/ NTB_NUM_MW
;
477 rx_size
= nt
->mw
[mw_num
].size
/ num_qps_mw
;
478 qp
->rx_buff_begin
= nt
->mw
[mw_num
].virt_addr
+
479 (qp_num
/ NTB_NUM_MW
* rx_size
);
480 qp
->rx_buff_end
= qp
->rx_buff_begin
+ rx_size
;
481 qp
->rx_offset
= qp
->rx_buff_begin
;
482 qp
->rx_max_frame
= min(transport_mtu
, rx_size
);
484 /* setup the hdr offsets with 0's */
485 for (offset
= qp
->rx_buff_begin
+ qp
->rx_max_frame
-
486 sizeof(struct ntb_payload_header
);
487 offset
< qp
->rx_buff_end
; offset
+= qp
->rx_max_frame
)
488 memset(offset
, 0, sizeof(struct ntb_payload_header
));
494 static int ntb_set_mw(struct ntb_transport
*nt
, int num_mw
, unsigned int size
)
496 struct ntb_transport_mw
*mw
= &nt
->mw
[num_mw
];
497 struct pci_dev
*pdev
= ntb_query_pdev(nt
->ndev
);
499 /* Alloc memory for receiving data. Must be 4k aligned */
500 mw
->size
= ALIGN(size
, 4096);
502 mw
->virt_addr
= dma_alloc_coherent(&pdev
->dev
, mw
->size
, &mw
->dma_addr
,
504 if (!mw
->virt_addr
) {
505 dev_err(&pdev
->dev
, "Unable to allocate MW buffer of size %d\n",
510 /* Notify HW the memory location of the receive buffer */
511 ntb_set_mw_addr(nt
->ndev
, num_mw
, mw
->dma_addr
);
516 static void ntb_qp_link_cleanup(struct work_struct
*work
)
518 struct ntb_transport_qp
*qp
= container_of(work
,
519 struct ntb_transport_qp
,
521 struct ntb_transport
*nt
= qp
->transport
;
522 struct pci_dev
*pdev
= ntb_query_pdev(nt
->ndev
);
524 if (qp
->qp_link
== NTB_LINK_DOWN
) {
525 cancel_delayed_work_sync(&qp
->link_work
);
529 if (qp
->event_handler
)
530 qp
->event_handler(qp
->cb_data
, NTB_LINK_DOWN
);
532 dev_info(&pdev
->dev
, "qp %d: Link Down\n", qp
->qp_num
);
533 qp
->qp_link
= NTB_LINK_DOWN
;
535 if (nt
->transport_link
== NTB_LINK_UP
)
536 schedule_delayed_work(&qp
->link_work
,
537 msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT
));
540 static void ntb_qp_link_down(struct ntb_transport_qp
*qp
)
542 schedule_work(&qp
->link_cleanup
);
545 static void ntb_transport_link_cleanup(struct work_struct
*work
)
547 struct ntb_transport
*nt
= container_of(work
, struct ntb_transport
,
551 if (nt
->transport_link
== NTB_LINK_DOWN
)
552 cancel_delayed_work_sync(&nt
->link_work
);
554 nt
->transport_link
= NTB_LINK_DOWN
;
556 /* Pass along the info to any clients */
557 for (i
= 0; i
< nt
->max_qps
; i
++)
558 if (!test_bit(i
, &nt
->qp_bitmap
))
559 ntb_qp_link_down(&nt
->qps
[i
]);
561 /* The scratchpad registers keep the values if the remote side
562 * goes down, blast them now to give them a sane value the next
563 * time they are accessed
565 for (i
= 0; i
< MAX_SPAD
; i
++)
566 ntb_write_local_spad(nt
->ndev
, i
, 0);
569 static void ntb_transport_event_callback(void *data
, enum ntb_hw_event event
)
571 struct ntb_transport
*nt
= data
;
574 case NTB_EVENT_HW_LINK_UP
:
575 schedule_delayed_work(&nt
->link_work
, 0);
577 case NTB_EVENT_HW_LINK_DOWN
:
578 schedule_work(&nt
->link_cleanup
);
585 static void ntb_transport_link_work(struct work_struct
*work
)
587 struct ntb_transport
*nt
= container_of(work
, struct ntb_transport
,
589 struct ntb_device
*ndev
= nt
->ndev
;
590 struct pci_dev
*pdev
= ntb_query_pdev(ndev
);
594 /* send the local info */
595 rc
= ntb_write_remote_spad(ndev
, VERSION
, NTB_TRANSPORT_VERSION
);
597 dev_err(&pdev
->dev
, "Error writing %x to remote spad %d\n",
602 rc
= ntb_write_remote_spad(ndev
, MW0_SZ
, ntb_get_mw_size(ndev
, 0));
604 dev_err(&pdev
->dev
, "Error writing %x to remote spad %d\n",
605 (u32
) ntb_get_mw_size(ndev
, 0), MW0_SZ
);
609 rc
= ntb_write_remote_spad(ndev
, MW1_SZ
, ntb_get_mw_size(ndev
, 1));
611 dev_err(&pdev
->dev
, "Error writing %x to remote spad %d\n",
612 (u32
) ntb_get_mw_size(ndev
, 1), MW1_SZ
);
616 rc
= ntb_write_remote_spad(ndev
, NUM_QPS
, nt
->max_qps
);
618 dev_err(&pdev
->dev
, "Error writing %x to remote spad %d\n",
619 nt
->max_qps
, NUM_QPS
);
623 rc
= ntb_read_local_spad(nt
->ndev
, QP_LINKS
, &val
);
625 dev_err(&pdev
->dev
, "Error reading spad %d\n", QP_LINKS
);
629 rc
= ntb_write_remote_spad(ndev
, QP_LINKS
, val
);
631 dev_err(&pdev
->dev
, "Error writing %x to remote spad %d\n",
636 /* Query the remote side for its info */
637 rc
= ntb_read_remote_spad(ndev
, VERSION
, &val
);
639 dev_err(&pdev
->dev
, "Error reading remote spad %d\n", VERSION
);
643 if (val
!= NTB_TRANSPORT_VERSION
)
645 dev_dbg(&pdev
->dev
, "Remote version = %d\n", val
);
647 rc
= ntb_read_remote_spad(ndev
, NUM_QPS
, &val
);
649 dev_err(&pdev
->dev
, "Error reading remote spad %d\n", NUM_QPS
);
653 if (val
!= nt
->max_qps
)
655 dev_dbg(&pdev
->dev
, "Remote max number of qps = %d\n", val
);
657 rc
= ntb_read_remote_spad(ndev
, MW0_SZ
, &val
);
659 dev_err(&pdev
->dev
, "Error reading remote spad %d\n", MW0_SZ
);
665 dev_dbg(&pdev
->dev
, "Remote MW0 size = %d\n", val
);
667 rc
= ntb_set_mw(nt
, 0, val
);
671 rc
= ntb_read_remote_spad(ndev
, MW1_SZ
, &val
);
673 dev_err(&pdev
->dev
, "Error reading remote spad %d\n", MW1_SZ
);
679 dev_dbg(&pdev
->dev
, "Remote MW1 size = %d\n", val
);
681 rc
= ntb_set_mw(nt
, 1, val
);
685 nt
->transport_link
= NTB_LINK_UP
;
687 for (i
= 0; i
< nt
->max_qps
; i
++) {
688 struct ntb_transport_qp
*qp
= &nt
->qps
[i
];
690 ntb_transport_setup_qp_mw(nt
, i
);
692 if (qp
->client_ready
== NTB_LINK_UP
)
693 schedule_delayed_work(&qp
->link_work
, 0);
699 if (ntb_hw_link_status(ndev
))
700 schedule_delayed_work(&nt
->link_work
,
701 msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT
));
704 static void ntb_qp_link_work(struct work_struct
*work
)
706 struct ntb_transport_qp
*qp
= container_of(work
,
707 struct ntb_transport_qp
,
709 struct pci_dev
*pdev
= ntb_query_pdev(qp
->ndev
);
710 struct ntb_transport
*nt
= qp
->transport
;
713 WARN_ON(nt
->transport_link
!= NTB_LINK_UP
);
715 rc
= ntb_read_local_spad(nt
->ndev
, QP_LINKS
, &val
);
717 dev_err(&pdev
->dev
, "Error reading spad %d\n", QP_LINKS
);
721 rc
= ntb_write_remote_spad(nt
->ndev
, QP_LINKS
, val
| 1 << qp
->qp_num
);
723 dev_err(&pdev
->dev
, "Error writing %x to remote spad %d\n",
724 val
| 1 << qp
->qp_num
, QP_LINKS
);
726 /* query remote spad for qp ready bits */
727 rc
= ntb_read_remote_spad(nt
->ndev
, QP_LINKS
, &val
);
729 dev_err(&pdev
->dev
, "Error reading remote spad %d\n", QP_LINKS
);
731 dev_dbg(&pdev
->dev
, "Remote QP link status = %x\n", val
);
733 /* See if the remote side is up */
734 if (1 << qp
->qp_num
& val
) {
735 qp
->qp_link
= NTB_LINK_UP
;
737 dev_info(&pdev
->dev
, "qp %d: Link Up\n", qp
->qp_num
);
738 if (qp
->event_handler
)
739 qp
->event_handler(qp
->cb_data
, NTB_LINK_UP
);
740 } else if (nt
->transport_link
== NTB_LINK_UP
)
741 schedule_delayed_work(&qp
->link_work
,
742 msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT
));
745 static void ntb_transport_init_queue(struct ntb_transport
*nt
,
748 struct ntb_transport_qp
*qp
;
749 unsigned int num_qps_mw
, tx_size
;
750 u8 mw_num
= QP_TO_MW(qp_num
);
752 qp
= &nt
->qps
[qp_num
];
756 qp
->qp_link
= NTB_LINK_DOWN
;
757 qp
->client_ready
= NTB_LINK_DOWN
;
758 qp
->event_handler
= NULL
;
760 if (nt
->max_qps
% NTB_NUM_MW
&& mw_num
< nt
->max_qps
% NTB_NUM_MW
)
761 num_qps_mw
= nt
->max_qps
/ NTB_NUM_MW
+ 1;
763 num_qps_mw
= nt
->max_qps
/ NTB_NUM_MW
;
765 tx_size
= ntb_get_mw_size(qp
->ndev
, mw_num
) / num_qps_mw
;
766 qp
->tx_mw_begin
= ntb_get_mw_vbase(nt
->ndev
, mw_num
) +
767 (qp_num
/ NTB_NUM_MW
* tx_size
);
768 qp
->tx_mw_end
= qp
->tx_mw_begin
+ tx_size
;
769 qp
->tx_offset
= qp
->tx_mw_begin
;
770 qp
->tx_max_frame
= min(transport_mtu
, tx_size
);
772 if (nt
->debugfs_dir
) {
773 char debugfs_name
[4];
775 snprintf(debugfs_name
, 4, "qp%d", qp_num
);
776 qp
->debugfs_dir
= debugfs_create_dir(debugfs_name
,
779 qp
->debugfs_stats
= debugfs_create_file("stats", S_IRUSR
,
781 &ntb_qp_debugfs_stats
);
784 INIT_DELAYED_WORK(&qp
->link_work
, ntb_qp_link_work
);
785 INIT_WORK(&qp
->link_cleanup
, ntb_qp_link_cleanup
);
787 spin_lock_init(&qp
->ntb_rx_pend_q_lock
);
788 spin_lock_init(&qp
->ntb_rx_free_q_lock
);
789 spin_lock_init(&qp
->ntb_tx_free_q_lock
);
791 INIT_LIST_HEAD(&qp
->rx_pend_q
);
792 INIT_LIST_HEAD(&qp
->rx_free_q
);
793 INIT_LIST_HEAD(&qp
->tx_free_q
);
796 int ntb_transport_init(struct pci_dev
*pdev
)
798 struct ntb_transport
*nt
;
801 nt
= kzalloc(sizeof(struct ntb_transport
), GFP_KERNEL
);
805 if (debugfs_initialized())
806 nt
->debugfs_dir
= debugfs_create_dir(KBUILD_MODNAME
, NULL
);
808 nt
->debugfs_dir
= NULL
;
810 nt
->ndev
= ntb_register_transport(pdev
, nt
);
816 nt
->max_qps
= min(nt
->ndev
->max_cbs
, max_num_clients
);
818 nt
->qps
= kcalloc(nt
->max_qps
, sizeof(struct ntb_transport_qp
),
825 nt
->qp_bitmap
= ((u64
) 1 << nt
->max_qps
) - 1;
827 for (i
= 0; i
< nt
->max_qps
; i
++)
828 ntb_transport_init_queue(nt
, i
);
830 INIT_DELAYED_WORK(&nt
->link_work
, ntb_transport_link_work
);
831 INIT_WORK(&nt
->link_cleanup
, ntb_transport_link_cleanup
);
833 rc
= ntb_register_event_callback(nt
->ndev
,
834 ntb_transport_event_callback
);
838 INIT_LIST_HEAD(&nt
->client_devs
);
839 rc
= ntb_bus_init(nt
);
843 if (ntb_hw_link_status(nt
->ndev
))
844 schedule_delayed_work(&nt
->link_work
, 0);
849 ntb_unregister_event_callback(nt
->ndev
);
853 ntb_unregister_transport(nt
->ndev
);
855 debugfs_remove_recursive(nt
->debugfs_dir
);
860 void ntb_transport_free(void *transport
)
862 struct ntb_transport
*nt
= transport
;
863 struct pci_dev
*pdev
;
866 nt
->transport_link
= NTB_LINK_DOWN
;
868 /* verify that all the qp's are freed */
869 for (i
= 0; i
< nt
->max_qps
; i
++)
870 if (!test_bit(i
, &nt
->qp_bitmap
))
871 ntb_transport_free_queue(&nt
->qps
[i
]);
875 cancel_delayed_work_sync(&nt
->link_work
);
877 debugfs_remove_recursive(nt
->debugfs_dir
);
879 ntb_unregister_event_callback(nt
->ndev
);
881 pdev
= ntb_query_pdev(nt
->ndev
);
883 for (i
= 0; i
< NTB_NUM_MW
; i
++)
884 if (nt
->mw
[i
].virt_addr
)
885 dma_free_coherent(&pdev
->dev
, nt
->mw
[i
].size
,
890 ntb_unregister_transport(nt
->ndev
);
894 static void ntb_rx_copy_task(struct ntb_transport_qp
*qp
,
895 struct ntb_queue_entry
*entry
, void *offset
)
898 struct ntb_payload_header
*hdr
;
900 BUG_ON(offset
< qp
->rx_buff_begin
||
901 offset
+ qp
->rx_max_frame
>= qp
->rx_buff_end
);
903 hdr
= offset
+ qp
->rx_max_frame
- sizeof(struct ntb_payload_header
);
904 entry
->len
= hdr
->len
;
906 memcpy(entry
->buf
, offset
, entry
->len
);
908 /* Ensure that the data is fully copied out before clearing the flag */
912 if (qp
->rx_handler
&& qp
->client_ready
== NTB_LINK_UP
)
913 qp
->rx_handler(qp
, qp
->cb_data
, entry
->cb_data
, entry
->len
);
915 ntb_list_add(&qp
->ntb_rx_free_q_lock
, &entry
->entry
, &qp
->rx_free_q
);
918 static int ntb_process_rxc(struct ntb_transport_qp
*qp
)
920 struct ntb_payload_header
*hdr
;
921 struct ntb_queue_entry
*entry
;
924 entry
= ntb_list_rm(&qp
->ntb_rx_pend_q_lock
, &qp
->rx_pend_q
);
926 hdr
= offset
+ qp
->rx_max_frame
-
927 sizeof(struct ntb_payload_header
);
928 dev_dbg(&ntb_query_pdev(qp
->ndev
)->dev
,
929 "no buffer - HDR ver %llu, len %d, flags %x\n",
930 hdr
->ver
, hdr
->len
, hdr
->flags
);
935 offset
= qp
->rx_offset
;
936 hdr
= offset
+ qp
->rx_max_frame
- sizeof(struct ntb_payload_header
);
938 if (!(hdr
->flags
& DESC_DONE_FLAG
)) {
939 ntb_list_add(&qp
->ntb_rx_pend_q_lock
, &entry
->entry
,
945 if (hdr
->ver
!= qp
->rx_pkts
) {
946 dev_dbg(&ntb_query_pdev(qp
->ndev
)->dev
,
947 "qp %d: version mismatch, expected %llu - got %llu\n",
948 qp
->qp_num
, qp
->rx_pkts
, hdr
->ver
);
949 ntb_list_add(&qp
->ntb_rx_pend_q_lock
, &entry
->entry
,
955 if (hdr
->flags
& LINK_DOWN_FLAG
) {
956 ntb_qp_link_down(qp
);
958 ntb_list_add(&qp
->ntb_rx_pend_q_lock
, &entry
->entry
,
961 /* Ensure that the data is fully copied out before clearing the
969 dev_dbg(&ntb_query_pdev(qp
->ndev
)->dev
,
970 "rx offset %p, ver %llu - %d payload received, buf size %d\n",
971 qp
->rx_offset
, hdr
->ver
, hdr
->len
, entry
->len
);
973 if (hdr
->len
<= entry
->len
)
974 ntb_rx_copy_task(qp
, entry
, offset
);
976 ntb_list_add(&qp
->ntb_rx_pend_q_lock
, &entry
->entry
,
979 /* Ensure that the data is fully copied out before clearing the
985 dev_dbg(&ntb_query_pdev(qp
->ndev
)->dev
,
986 "RX overflow! Wanted %d got %d\n",
987 hdr
->len
, entry
->len
);
990 qp
->rx_bytes
+= hdr
->len
;
994 qp
->rx_offset
+= qp
->rx_max_frame
;
995 if (qp
->rx_offset
+ qp
->rx_max_frame
>= qp
->rx_buff_end
)
996 qp
->rx_offset
= qp
->rx_buff_begin
;
1001 static void ntb_transport_rx(unsigned long data
)
1003 struct ntb_transport_qp
*qp
= (struct ntb_transport_qp
*)data
;
1007 rc
= ntb_process_rxc(qp
);
1011 static void ntb_transport_rxc_db(void *data
, int db_num
)
1013 struct ntb_transport_qp
*qp
= data
;
1015 dev_dbg(&ntb_query_pdev(qp
->ndev
)->dev
, "%s: doorbell %d received\n",
1018 tasklet_schedule(&qp
->rx_work
);
1021 static void ntb_tx_copy_task(struct ntb_transport_qp
*qp
,
1022 struct ntb_queue_entry
*entry
,
1025 struct ntb_payload_header
*hdr
;
1027 BUG_ON(offset
< qp
->tx_mw_begin
||
1028 offset
+ qp
->tx_max_frame
>= qp
->tx_mw_end
);
1030 memcpy_toio(offset
, entry
->buf
, entry
->len
);
1032 hdr
= offset
+ qp
->tx_max_frame
- sizeof(struct ntb_payload_header
);
1033 hdr
->len
= entry
->len
;
1034 hdr
->ver
= qp
->tx_pkts
;
1036 /* Ensure that the data is fully copied out before setting the flag */
1038 hdr
->flags
= entry
->flags
| DESC_DONE_FLAG
;
1040 ntb_ring_sdb(qp
->ndev
, qp
->qp_num
);
1042 /* The entry length can only be zero if the packet is intended to be a
1043 * "link down" or similar. Since no payload is being sent in these
1044 * cases, there is nothing to add to the completion queue.
1046 if (entry
->len
> 0) {
1047 qp
->tx_bytes
+= entry
->len
;
1050 qp
->tx_handler(qp
, qp
->cb_data
, entry
->cb_data
,
1054 ntb_list_add(&qp
->ntb_tx_free_q_lock
, &entry
->entry
, &qp
->tx_free_q
);
1057 static int ntb_process_tx(struct ntb_transport_qp
*qp
,
1058 struct ntb_queue_entry
*entry
)
1060 struct ntb_payload_header
*hdr
;
1063 offset
= qp
->tx_offset
;
1064 hdr
= offset
+ qp
->tx_max_frame
- sizeof(struct ntb_payload_header
);
1066 dev_dbg(&ntb_query_pdev(qp
->ndev
)->dev
, "%lld - offset %p, tx %p, entry len %d flags %x buff %p\n",
1067 qp
->tx_pkts
, offset
, qp
->tx_offset
, entry
->len
, entry
->flags
,
1074 if (entry
->len
> qp
->tx_max_frame
- sizeof(struct ntb_payload_header
)) {
1076 qp
->tx_handler(qp
->cb_data
, qp
, NULL
, -EIO
);
1078 ntb_list_add(&qp
->ntb_tx_free_q_lock
, &entry
->entry
,
1083 ntb_tx_copy_task(qp
, entry
, offset
);
1085 qp
->tx_offset
+= qp
->tx_max_frame
;
1086 if (qp
->tx_offset
+ qp
->tx_max_frame
>= qp
->tx_mw_end
)
1087 qp
->tx_offset
= qp
->tx_mw_begin
;
1094 static void ntb_send_link_down(struct ntb_transport_qp
*qp
)
1096 struct pci_dev
*pdev
= ntb_query_pdev(qp
->ndev
);
1097 struct ntb_queue_entry
*entry
;
1100 if (qp
->qp_link
== NTB_LINK_DOWN
)
1103 qp
->qp_link
= NTB_LINK_DOWN
;
1104 dev_info(&pdev
->dev
, "qp %d: Link Down\n", qp
->qp_num
);
1106 for (i
= 0; i
< NTB_LINK_DOWN_TIMEOUT
; i
++) {
1107 entry
= ntb_list_rm(&qp
->ntb_tx_free_q_lock
, &qp
->tx_free_q
);
1116 entry
->cb_data
= NULL
;
1119 entry
->flags
= LINK_DOWN_FLAG
;
1121 rc
= ntb_process_tx(qp
, entry
);
1123 dev_err(&pdev
->dev
, "ntb: QP%d unable to send linkdown msg\n",
1128 * ntb_transport_create_queue - Create a new NTB transport layer queue
1129 * @rx_handler: receive callback function
1130 * @tx_handler: transmit callback function
1131 * @event_handler: event callback function
1133 * Create a new NTB transport layer queue and provide the queue with a callback
1134 * routine for both transmit and receive. The receive callback routine will be
1135 * used to pass up data when the transport has received it on the queue. The
1136 * transmit callback routine will be called when the transport has completed the
1137 * transmission of the data on the queue and the data is ready to be freed.
1139 * RETURNS: pointer to newly created ntb_queue, NULL on error.
1141 struct ntb_transport_qp
*
1142 ntb_transport_create_queue(void *data
, struct pci_dev
*pdev
,
1143 const struct ntb_queue_handlers
*handlers
)
1145 struct ntb_queue_entry
*entry
;
1146 struct ntb_transport_qp
*qp
;
1147 struct ntb_transport
*nt
;
1148 unsigned int free_queue
;
1151 nt
= ntb_find_transport(pdev
);
1155 free_queue
= ffs(nt
->qp_bitmap
);
1159 /* decrement free_queue to make it zero based */
1162 clear_bit(free_queue
, &nt
->qp_bitmap
);
1164 qp
= &nt
->qps
[free_queue
];
1166 qp
->rx_handler
= handlers
->rx_handler
;
1167 qp
->tx_handler
= handlers
->tx_handler
;
1168 qp
->event_handler
= handlers
->event_handler
;
1170 for (i
= 0; i
< NTB_QP_DEF_NUM_ENTRIES
; i
++) {
1171 entry
= kzalloc(sizeof(struct ntb_queue_entry
), GFP_ATOMIC
);
1175 ntb_list_add(&qp
->ntb_rx_free_q_lock
, &entry
->entry
,
1179 for (i
= 0; i
< NTB_QP_DEF_NUM_ENTRIES
; i
++) {
1180 entry
= kzalloc(sizeof(struct ntb_queue_entry
), GFP_ATOMIC
);
1184 ntb_list_add(&qp
->ntb_tx_free_q_lock
, &entry
->entry
,
1188 tasklet_init(&qp
->rx_work
, ntb_transport_rx
, (unsigned long) qp
);
1190 rc
= ntb_register_db_callback(qp
->ndev
, free_queue
, qp
,
1191 ntb_transport_rxc_db
);
1195 dev_info(&pdev
->dev
, "NTB Transport QP %d created\n", qp
->qp_num
);
1200 tasklet_disable(&qp
->rx_work
);
1202 while ((entry
= ntb_list_rm(&qp
->ntb_tx_free_q_lock
, &qp
->tx_free_q
)))
1205 while ((entry
= ntb_list_rm(&qp
->ntb_rx_free_q_lock
, &qp
->rx_free_q
)))
1207 set_bit(free_queue
, &nt
->qp_bitmap
);
1211 EXPORT_SYMBOL_GPL(ntb_transport_create_queue
);
1214 * ntb_transport_free_queue - Frees NTB transport queue
1215 * @qp: NTB queue to be freed
1217 * Frees NTB transport queue
1219 void ntb_transport_free_queue(struct ntb_transport_qp
*qp
)
1221 struct pci_dev
*pdev
= ntb_query_pdev(qp
->ndev
);
1222 struct ntb_queue_entry
*entry
;
1227 cancel_delayed_work_sync(&qp
->link_work
);
1229 ntb_unregister_db_callback(qp
->ndev
, qp
->qp_num
);
1230 tasklet_disable(&qp
->rx_work
);
1232 while ((entry
= ntb_list_rm(&qp
->ntb_rx_free_q_lock
, &qp
->rx_free_q
)))
1235 while ((entry
= ntb_list_rm(&qp
->ntb_rx_pend_q_lock
, &qp
->rx_pend_q
))) {
1236 dev_warn(&pdev
->dev
, "Freeing item from a non-empty queue\n");
1240 while ((entry
= ntb_list_rm(&qp
->ntb_tx_free_q_lock
, &qp
->tx_free_q
)))
1243 set_bit(qp
->qp_num
, &qp
->transport
->qp_bitmap
);
1245 dev_info(&pdev
->dev
, "NTB Transport QP %d freed\n", qp
->qp_num
);
1247 EXPORT_SYMBOL_GPL(ntb_transport_free_queue
);
1250 * ntb_transport_rx_remove - Dequeues enqueued rx packet
1251 * @qp: NTB queue to be freed
1252 * @len: pointer to variable to write enqueued buffers length
1254 * Dequeues unused buffers from receive queue. Should only be used during
1257 * RETURNS: NULL error value on error, or void* for success.
1259 void *ntb_transport_rx_remove(struct ntb_transport_qp
*qp
, unsigned int *len
)
1261 struct ntb_queue_entry
*entry
;
1264 if (!qp
|| qp
->client_ready
== NTB_LINK_UP
)
1267 entry
= ntb_list_rm(&qp
->ntb_rx_pend_q_lock
, &qp
->rx_pend_q
);
1271 buf
= entry
->cb_data
;
1274 ntb_list_add(&qp
->ntb_rx_free_q_lock
, &entry
->entry
, &qp
->rx_free_q
);
1278 EXPORT_SYMBOL_GPL(ntb_transport_rx_remove
);
1281 * ntb_transport_rx_enqueue - Enqueue a new NTB queue entry
1282 * @qp: NTB transport layer queue the entry is to be enqueued on
1283 * @cb: per buffer pointer for callback function to use
1284 * @data: pointer to data buffer that incoming packets will be copied into
1285 * @len: length of the data buffer
1287 * Enqueue a new receive buffer onto the transport queue into which a NTB
1288 * payload can be received into.
1290 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
1292 int ntb_transport_rx_enqueue(struct ntb_transport_qp
*qp
, void *cb
, void *data
,
1295 struct ntb_queue_entry
*entry
;
1300 entry
= ntb_list_rm(&qp
->ntb_rx_free_q_lock
, &qp
->rx_free_q
);
1304 entry
->cb_data
= cb
;
1308 ntb_list_add(&qp
->ntb_rx_pend_q_lock
, &entry
->entry
, &qp
->rx_pend_q
);
1312 EXPORT_SYMBOL_GPL(ntb_transport_rx_enqueue
);
1315 * ntb_transport_tx_enqueue - Enqueue a new NTB queue entry
1316 * @qp: NTB transport layer queue the entry is to be enqueued on
1317 * @cb: per buffer pointer for callback function to use
1318 * @data: pointer to data buffer that will be sent
1319 * @len: length of the data buffer
1321 * Enqueue a new transmit buffer onto the transport queue from which a NTB
1322 * payload will be transmitted. This assumes that a lock is behing held to
1323 * serialize access to the qp.
1325 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
1327 int ntb_transport_tx_enqueue(struct ntb_transport_qp
*qp
, void *cb
, void *data
,
1330 struct ntb_queue_entry
*entry
;
1333 if (!qp
|| qp
->qp_link
!= NTB_LINK_UP
|| !len
)
1336 entry
= ntb_list_rm(&qp
->ntb_tx_free_q_lock
, &qp
->tx_free_q
);
1340 entry
->cb_data
= cb
;
1345 rc
= ntb_process_tx(qp
, entry
);
1347 ntb_list_add(&qp
->ntb_tx_free_q_lock
, &entry
->entry
,
1352 EXPORT_SYMBOL_GPL(ntb_transport_tx_enqueue
);
1355 * ntb_transport_link_up - Notify NTB transport of client readiness to use queue
1356 * @qp: NTB transport layer queue to be enabled
1358 * Notify NTB transport layer of client readiness to use queue
1360 void ntb_transport_link_up(struct ntb_transport_qp
*qp
)
1365 qp
->client_ready
= NTB_LINK_UP
;
1367 if (qp
->transport
->transport_link
== NTB_LINK_UP
)
1368 schedule_delayed_work(&qp
->link_work
, 0);
1370 EXPORT_SYMBOL_GPL(ntb_transport_link_up
);
1373 * ntb_transport_link_down - Notify NTB transport to no longer enqueue data
1374 * @qp: NTB transport layer queue to be disabled
1376 * Notify NTB transport layer of client's desire to no longer receive data on
1377 * transport queue specified. It is the client's responsibility to ensure all
1378 * entries on queue are purged or otherwise handled appropraitely.
1380 void ntb_transport_link_down(struct ntb_transport_qp
*qp
)
1382 struct pci_dev
*pdev
= ntb_query_pdev(qp
->ndev
);
1388 qp
->client_ready
= NTB_LINK_DOWN
;
1390 rc
= ntb_read_local_spad(qp
->ndev
, QP_LINKS
, &val
);
1392 dev_err(&pdev
->dev
, "Error reading spad %d\n", QP_LINKS
);
1396 rc
= ntb_write_remote_spad(qp
->ndev
, QP_LINKS
,
1397 val
& ~(1 << qp
->qp_num
));
1399 dev_err(&pdev
->dev
, "Error writing %x to remote spad %d\n",
1400 val
& ~(1 << qp
->qp_num
), QP_LINKS
);
1402 if (qp
->qp_link
== NTB_LINK_UP
)
1403 ntb_send_link_down(qp
);
1405 cancel_delayed_work_sync(&qp
->link_work
);
1407 EXPORT_SYMBOL_GPL(ntb_transport_link_down
);
1410 * ntb_transport_link_query - Query transport link state
1411 * @qp: NTB transport layer queue to be queried
1413 * Query connectivity to the remote system of the NTB transport queue
1415 * RETURNS: true for link up or false for link down
1417 bool ntb_transport_link_query(struct ntb_transport_qp
*qp
)
1419 return qp
->qp_link
== NTB_LINK_UP
;
1421 EXPORT_SYMBOL_GPL(ntb_transport_link_query
);
1424 * ntb_transport_qp_num - Query the qp number
1425 * @qp: NTB transport layer queue to be queried
1427 * Query qp number of the NTB transport queue
1429 * RETURNS: a zero based number specifying the qp number
1431 unsigned char ntb_transport_qp_num(struct ntb_transport_qp
*qp
)
1435 EXPORT_SYMBOL_GPL(ntb_transport_qp_num
);
1438 * ntb_transport_max_size - Query the max payload size of a qp
1439 * @qp: NTB transport layer queue to be queried
1441 * Query the maximum payload size permissible on the given qp
1443 * RETURNS: the max payload size of a qp
1445 unsigned int ntb_transport_max_size(struct ntb_transport_qp
*qp
)
1447 return qp
->tx_max_frame
- sizeof(struct ntb_payload_header
);
1449 EXPORT_SYMBOL_GPL(ntb_transport_max_size
);