2 * rio_cm - RapidIO Channelized Messaging Driver
4 * Copyright 2013-2016 Integrated Device Technology, Inc.
5 * Copyright (c) 2015, Prodrive Technologies
6 * Copyright (c) 2015, RapidIO Trade Association
8 * This program is free software; you can redistribute it and/or modify it
9 * under the terms of the GNU General Public License as published by the
10 * Free Software Foundation; either version 2 of the License, or (at your
11 * option) any later version.
13 * THIS PROGRAM IS DISTRIBUTED IN THE HOPE THAT IT WILL BE USEFUL,
14 * BUT WITHOUT ANY WARRANTY; WITHOUT EVEN THE IMPLIED WARRANTY OF
15 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. SEE THE
16 * GNU GENERAL PUBLIC LICENSE FOR MORE DETAILS.
19 #include <linux/module.h>
20 #include <linux/kernel.h>
21 #include <linux/dma-mapping.h>
22 #include <linux/delay.h>
23 #include <linux/sched.h>
24 #include <linux/rio.h>
25 #include <linux/rio_drv.h>
26 #include <linux/slab.h>
27 #include <linux/idr.h>
28 #include <linux/interrupt.h>
29 #include <linux/cdev.h>
31 #include <linux/poll.h>
32 #include <linux/reboot.h>
33 #include <linux/bitops.h>
34 #include <linux/printk.h>
35 #include <linux/rio_cm_cdev.h>
37 #define DRV_NAME "rio_cm"
38 #define DRV_VERSION "1.0.0"
39 #define DRV_AUTHOR "Alexandre Bounine <alexandre.bounine@idt.com>"
40 #define DRV_DESC "RapidIO Channelized Messaging Driver"
41 #define DEV_NAME "rio_cm"
43 /* Debug output filtering masks */
46 DBG_INIT
= BIT(0), /* driver init */
47 DBG_EXIT
= BIT(1), /* driver exit */
48 DBG_MPORT
= BIT(2), /* mport add/remove */
49 DBG_RDEV
= BIT(3), /* RapidIO device add/remove */
50 DBG_CHOP
= BIT(4), /* channel operations */
51 DBG_WAIT
= BIT(5), /* waiting for events */
52 DBG_TX
= BIT(6), /* message TX */
53 DBG_TX_EVENT
= BIT(7), /* message TX event */
54 DBG_RX_DATA
= BIT(8), /* inbound data messages */
55 DBG_RX_CMD
= BIT(9), /* inbound REQ/ACK/NACK messages */
60 #define riocm_debug(level, fmt, arg...) \
62 if (DBG_##level & dbg_level) \
63 pr_debug(DRV_NAME ": %s " fmt "\n", \
67 #define riocm_debug(level, fmt, arg...) \
68 no_printk(KERN_DEBUG pr_fmt(DRV_NAME fmt "\n"), ##arg)
71 #define riocm_warn(fmt, arg...) \
72 pr_warn(DRV_NAME ": %s WARNING " fmt "\n", __func__, ##arg)
74 #define riocm_error(fmt, arg...) \
75 pr_err(DRV_NAME ": %s ERROR " fmt "\n", __func__, ##arg)
79 module_param(cmbox
, int, S_IRUGO
);
80 MODULE_PARM_DESC(cmbox
, "RapidIO Mailbox number (default 1)");
82 static int chstart
= 256;
83 module_param(chstart
, int, S_IRUGO
);
84 MODULE_PARM_DESC(chstart
,
85 "Start channel number for dynamic allocation (default 256)");
88 static u32 dbg_level
= DBG_NONE
;
89 module_param(dbg_level
, uint
, S_IWUSR
| S_IRUGO
);
90 MODULE_PARM_DESC(dbg_level
, "Debugging output level (default 0 = none)");
93 MODULE_AUTHOR(DRV_AUTHOR
);
94 MODULE_DESCRIPTION(DRV_DESC
);
95 MODULE_LICENSE("GPL");
96 MODULE_VERSION(DRV_VERSION
);
98 #define RIOCM_TX_RING_SIZE 128
99 #define RIOCM_RX_RING_SIZE 128
100 #define RIOCM_CONNECT_TO 3 /* connect response TO (in sec) */
102 #define RIOCM_MAX_CHNUM 0xffff /* Use full range of u16 field */
103 #define RIOCM_CHNUM_AUTO 0
104 #define RIOCM_MAX_EP_COUNT 0x10000 /* Max number of endpoints */
116 enum rio_cm_pkt_type
{
128 struct rio_ch_base_bhdr
{
131 #define RIO_HDR_LETTER_MASK 0xffff0000
132 #define RIO_HDR_MBOX_MASK 0x0000ffff
136 } __attribute__((__packed__
));
138 struct rio_ch_chan_hdr
{
139 struct rio_ch_base_bhdr bhdr
;
145 } __attribute__((__packed__
));
148 struct list_head node
;
149 struct rio_dev
*rdev
;
155 struct list_head list
;
156 struct rio_mport
*mport
;
157 void *rx_buf
[RIOCM_RX_RING_SIZE
];
159 struct mutex rx_lock
;
161 void *tx_buf
[RIOCM_TX_RING_SIZE
];
165 struct list_head tx_reqs
;
168 struct list_head peers
;
170 struct workqueue_struct
*rx_wq
;
171 struct work_struct rx_work
;
174 struct chan_rx_ring
{
175 void *buf
[RIOCM_RX_RING_SIZE
];
180 /* Tracking RX buffers reported to upper level */
181 void *inuse
[RIOCM_RX_RING_SIZE
];
186 u16 id
; /* local channel ID */
187 struct kref ref
; /* channel refcount */
189 struct cm_dev
*cmdev
; /* associated CM device object */
190 struct rio_dev
*rdev
; /* remote RapidIO device */
191 enum rio_cm_state state
;
195 u32 loc_destid
; /* local destID */
196 u32 rem_destid
; /* remote destID */
197 u16 rem_channel
; /* remote channel ID */
198 struct list_head accept_queue
;
199 struct list_head ch_node
;
200 struct completion comp
;
201 struct completion comp_close
;
202 struct chan_rx_ring rx_ring
;
206 struct list_head node
;
207 struct rio_dev
*rdev
;
211 struct work_struct work
;
217 struct list_head node
;
218 u32 destid
; /* requester destID */
219 u16 chan
; /* requester channel ID */
220 struct cm_dev
*cmdev
;
224 * A channel_dev structure represents a CM_CDEV
225 * @cdev Character device
226 * @dev Associated device object
233 static struct rio_channel
*riocm_ch_alloc(u16 ch_num
);
234 static void riocm_ch_free(struct kref
*ref
);
235 static int riocm_post_send(struct cm_dev
*cm
, struct rio_dev
*rdev
,
236 void *buffer
, size_t len
);
237 static int riocm_ch_close(struct rio_channel
*ch
);
239 static DEFINE_SPINLOCK(idr_lock
);
240 static DEFINE_IDR(ch_idr
);
242 static LIST_HEAD(cm_dev_list
);
243 static DECLARE_RWSEM(rdev_sem
);
245 static struct class *dev_class
;
246 static unsigned int dev_major
;
247 static unsigned int dev_minor_base
;
248 static dev_t dev_number
;
249 static struct channel_dev riocm_cdev
;
251 #define is_msg_capable(src_ops, dst_ops) \
252 ((src_ops & RIO_SRC_OPS_DATA_MSG) && \
253 (dst_ops & RIO_DST_OPS_DATA_MSG))
254 #define dev_cm_capable(dev) \
255 is_msg_capable(dev->src_ops, dev->dst_ops)
257 static int riocm_cmp(struct rio_channel
*ch
, enum rio_cm_state cmp
)
261 spin_lock_bh(&ch
->lock
);
262 ret
= (ch
->state
== cmp
);
263 spin_unlock_bh(&ch
->lock
);
267 static int riocm_cmp_exch(struct rio_channel
*ch
,
268 enum rio_cm_state cmp
, enum rio_cm_state exch
)
272 spin_lock_bh(&ch
->lock
);
273 ret
= (ch
->state
== cmp
);
276 spin_unlock_bh(&ch
->lock
);
280 static enum rio_cm_state
riocm_exch(struct rio_channel
*ch
,
281 enum rio_cm_state exch
)
283 enum rio_cm_state old
;
285 spin_lock_bh(&ch
->lock
);
288 spin_unlock_bh(&ch
->lock
);
292 static struct rio_channel
*riocm_get_channel(u16 nr
)
294 struct rio_channel
*ch
;
296 spin_lock_bh(&idr_lock
);
297 ch
= idr_find(&ch_idr
, nr
);
300 spin_unlock_bh(&idr_lock
);
304 static void riocm_put_channel(struct rio_channel
*ch
)
306 kref_put(&ch
->ref
, riocm_ch_free
);
309 static void *riocm_rx_get_msg(struct cm_dev
*cm
)
314 msg
= rio_get_inb_message(cm
->mport
, cmbox
);
316 for (i
= 0; i
< RIOCM_RX_RING_SIZE
; i
++) {
317 if (cm
->rx_buf
[i
] == msg
) {
318 cm
->rx_buf
[i
] = NULL
;
324 if (i
== RIOCM_RX_RING_SIZE
)
325 riocm_warn("no record for buffer 0x%p", msg
);
332 * riocm_rx_fill - fills a ring of receive buffers for given cm device
334 * @nent: max number of entries to fill
338 static void riocm_rx_fill(struct cm_dev
*cm
, int nent
)
342 if (cm
->rx_slots
== 0)
345 for (i
= 0; i
< RIOCM_RX_RING_SIZE
&& cm
->rx_slots
&& nent
; i
++) {
346 if (cm
->rx_buf
[i
] == NULL
) {
347 cm
->rx_buf
[i
] = kmalloc(RIO_MAX_MSG_SIZE
, GFP_KERNEL
);
348 if (cm
->rx_buf
[i
] == NULL
)
350 rio_add_inb_buffer(cm
->mport
, cmbox
, cm
->rx_buf
[i
]);
358 * riocm_rx_free - frees all receive buffers associated with given cm device
363 static void riocm_rx_free(struct cm_dev
*cm
)
367 for (i
= 0; i
< RIOCM_RX_RING_SIZE
; i
++) {
368 if (cm
->rx_buf
[i
] != NULL
) {
369 kfree(cm
->rx_buf
[i
]);
370 cm
->rx_buf
[i
] = NULL
;
376 * riocm_req_handler - connection request handler
378 * @req_data: pointer to the request packet
380 * Returns: 0 if success, or
381 * -EINVAL if channel is not in correct state,
382 * -ENODEV if cannot find a channel with specified ID,
383 * -ENOMEM if unable to allocate memory to store the request
385 static int riocm_req_handler(struct cm_dev
*cm
, void *req_data
)
387 struct rio_channel
*ch
;
388 struct conn_req
*req
;
389 struct rio_ch_chan_hdr
*hh
= req_data
;
392 chnum
= ntohs(hh
->dst_ch
);
394 ch
= riocm_get_channel(chnum
);
399 if (ch
->state
!= RIO_CM_LISTEN
) {
400 riocm_debug(RX_CMD
, "channel %d is not in listen state", chnum
);
401 riocm_put_channel(ch
);
405 req
= kzalloc(sizeof(*req
), GFP_KERNEL
);
407 riocm_put_channel(ch
);
411 req
->destid
= ntohl(hh
->bhdr
.src_id
);
412 req
->chan
= ntohs(hh
->src_ch
);
415 spin_lock_bh(&ch
->lock
);
416 list_add_tail(&req
->node
, &ch
->accept_queue
);
417 spin_unlock_bh(&ch
->lock
);
419 riocm_put_channel(ch
);
425 * riocm_resp_handler - response to connection request handler
426 * @resp_data: pointer to the response packet
428 * Returns: 0 if success, or
429 * -EINVAL if channel is not in correct state,
430 * -ENODEV if cannot find a channel with specified ID,
432 static int riocm_resp_handler(void *resp_data
)
434 struct rio_channel
*ch
;
435 struct rio_ch_chan_hdr
*hh
= resp_data
;
438 chnum
= ntohs(hh
->dst_ch
);
439 ch
= riocm_get_channel(chnum
);
443 if (ch
->state
!= RIO_CM_CONNECT
) {
444 riocm_put_channel(ch
);
448 riocm_exch(ch
, RIO_CM_CONNECTED
);
449 ch
->rem_channel
= ntohs(hh
->src_ch
);
451 riocm_put_channel(ch
);
457 * riocm_close_handler - channel close request handler
458 * @req_data: pointer to the request packet
460 * Returns: 0 if success, or
461 * -ENODEV if cannot find a channel with specified ID,
462 * + error codes returned by riocm_ch_close.
464 static int riocm_close_handler(void *data
)
466 struct rio_channel
*ch
;
467 struct rio_ch_chan_hdr
*hh
= data
;
470 riocm_debug(RX_CMD
, "for ch=%d", ntohs(hh
->dst_ch
));
472 spin_lock_bh(&idr_lock
);
473 ch
= idr_find(&ch_idr
, ntohs(hh
->dst_ch
));
475 spin_unlock_bh(&idr_lock
);
478 idr_remove(&ch_idr
, ch
->id
);
479 spin_unlock_bh(&idr_lock
);
481 riocm_exch(ch
, RIO_CM_DISCONNECT
);
483 ret
= riocm_ch_close(ch
);
485 riocm_debug(RX_CMD
, "riocm_ch_close() returned %d", ret
);
491 * rio_cm_handler - function that services request (non-data) packets
493 * @data: pointer to the packet
495 static void rio_cm_handler(struct cm_dev
*cm
, void *data
)
497 struct rio_ch_chan_hdr
*hdr
;
499 if (!rio_mport_is_running(cm
->mport
))
504 riocm_debug(RX_CMD
, "OP=%x for ch=%d from %d",
505 hdr
->ch_op
, ntohs(hdr
->dst_ch
), ntohs(hdr
->src_ch
));
507 switch (hdr
->ch_op
) {
509 riocm_req_handler(cm
, data
);
512 riocm_resp_handler(data
);
515 riocm_close_handler(data
);
518 riocm_error("Invalid packet header");
526 * rio_rx_data_handler - received data packet handler
530 * Returns: 0 if success, or
531 * -ENODEV if cannot find a channel with specified ID,
532 * -EIO if channel is not in CONNECTED state,
533 * -ENOMEM if channel RX queue is full (packet discarded)
535 static int rio_rx_data_handler(struct cm_dev
*cm
, void *buf
)
537 struct rio_ch_chan_hdr
*hdr
;
538 struct rio_channel
*ch
;
542 riocm_debug(RX_DATA
, "for ch=%d", ntohs(hdr
->dst_ch
));
544 ch
= riocm_get_channel(ntohs(hdr
->dst_ch
));
546 /* Discard data message for non-existing channel */
551 /* Place pointer to the buffer into channel's RX queue */
552 spin_lock(&ch
->lock
);
554 if (ch
->state
!= RIO_CM_CONNECTED
) {
555 /* Channel is not ready to receive data, discard a packet */
556 riocm_debug(RX_DATA
, "ch=%d is in wrong state=%d",
558 spin_unlock(&ch
->lock
);
560 riocm_put_channel(ch
);
564 if (ch
->rx_ring
.count
== RIOCM_RX_RING_SIZE
) {
565 /* If RX ring is full, discard a packet */
566 riocm_debug(RX_DATA
, "ch=%d is full", ch
->id
);
567 spin_unlock(&ch
->lock
);
569 riocm_put_channel(ch
);
573 ch
->rx_ring
.buf
[ch
->rx_ring
.head
] = buf
;
576 ch
->rx_ring
.head
%= RIOCM_RX_RING_SIZE
;
580 spin_unlock(&ch
->lock
);
581 riocm_put_channel(ch
);
587 * rio_ibmsg_handler - inbound message packet handler
589 static void rio_ibmsg_handler(struct work_struct
*work
)
591 struct cm_dev
*cm
= container_of(work
, struct cm_dev
, rx_work
);
593 struct rio_ch_chan_hdr
*hdr
;
595 if (!rio_mport_is_running(cm
->mport
))
599 mutex_lock(&cm
->rx_lock
);
600 data
= riocm_rx_get_msg(cm
);
602 riocm_rx_fill(cm
, 1);
603 mutex_unlock(&cm
->rx_lock
);
610 if (hdr
->bhdr
.type
!= RIO_CM_CHAN
) {
611 /* For now simply discard packets other than channel */
612 riocm_error("Unsupported TYPE code (0x%x). Msg dropped",
618 /* Process a channel message */
619 if (hdr
->ch_op
== CM_DATA_MSG
)
620 rio_rx_data_handler(cm
, data
);
622 rio_cm_handler(cm
, data
);
626 static void riocm_inb_msg_event(struct rio_mport
*mport
, void *dev_id
,
629 struct cm_dev
*cm
= dev_id
;
631 if (rio_mport_is_running(cm
->mport
) && !work_pending(&cm
->rx_work
))
632 queue_work(cm
->rx_wq
, &cm
->rx_work
);
636 * rio_txcq_handler - TX completion handler
638 * @slot: TX queue slot
640 * TX completion handler also ensures that pending request packets are placed
641 * into transmit queue as soon as a free slot becomes available. This is done
642 * to give higher priority to request packets during high intensity data flow.
644 static void rio_txcq_handler(struct cm_dev
*cm
, int slot
)
648 /* ATTN: Add TX completion notification if/when direct buffer
649 * transfer is implemented. At this moment only correct tracking
650 * of tx_count is important.
652 riocm_debug(TX_EVENT
, "for mport_%d slot %d tx_cnt %d",
653 cm
->mport
->id
, slot
, cm
->tx_cnt
);
655 spin_lock(&cm
->tx_lock
);
656 ack_slot
= cm
->tx_ack_slot
;
658 if (ack_slot
== slot
)
659 riocm_debug(TX_EVENT
, "slot == ack_slot");
661 while (cm
->tx_cnt
&& ((ack_slot
!= slot
) ||
662 (cm
->tx_cnt
== RIOCM_TX_RING_SIZE
))) {
664 cm
->tx_buf
[ack_slot
] = NULL
;
666 ack_slot
&= (RIOCM_TX_RING_SIZE
- 1);
670 if (cm
->tx_cnt
< 0 || cm
->tx_cnt
> RIOCM_TX_RING_SIZE
)
671 riocm_error("tx_cnt %d out of sync", cm
->tx_cnt
);
673 WARN_ON((cm
->tx_cnt
< 0) || (cm
->tx_cnt
> RIOCM_TX_RING_SIZE
));
675 cm
->tx_ack_slot
= ack_slot
;
678 * If there are pending requests, insert them into transmit queue
680 if (!list_empty(&cm
->tx_reqs
) && (cm
->tx_cnt
< RIOCM_TX_RING_SIZE
)) {
681 struct tx_req
*req
, *_req
;
684 list_for_each_entry_safe(req
, _req
, &cm
->tx_reqs
, node
) {
685 list_del(&req
->node
);
686 cm
->tx_buf
[cm
->tx_slot
] = req
->buffer
;
687 rc
= rio_add_outb_message(cm
->mport
, req
->rdev
, cmbox
,
688 req
->buffer
, req
->len
);
694 cm
->tx_slot
&= (RIOCM_TX_RING_SIZE
- 1);
695 if (cm
->tx_cnt
== RIOCM_TX_RING_SIZE
)
700 spin_unlock(&cm
->tx_lock
);
703 static void riocm_outb_msg_event(struct rio_mport
*mport
, void *dev_id
,
706 struct cm_dev
*cm
= dev_id
;
708 if (cm
&& rio_mport_is_running(cm
->mport
))
709 rio_txcq_handler(cm
, slot
);
712 static int riocm_queue_req(struct cm_dev
*cm
, struct rio_dev
*rdev
,
713 void *buffer
, size_t len
)
718 treq
= kzalloc(sizeof(*treq
), GFP_KERNEL
);
723 treq
->buffer
= buffer
;
726 spin_lock_irqsave(&cm
->tx_lock
, flags
);
727 list_add_tail(&treq
->node
, &cm
->tx_reqs
);
728 spin_unlock_irqrestore(&cm
->tx_lock
, flags
);
733 * riocm_post_send - helper function that places packet into msg TX queue
735 * @rdev: target RapidIO device object (required by outbound msg interface)
736 * @buffer: pointer to a packet buffer to send
737 * @len: length of data to transfer
738 * @req: request priority flag
740 * Returns: 0 if success, or error code otherwise.
742 static int riocm_post_send(struct cm_dev
*cm
, struct rio_dev
*rdev
,
743 void *buffer
, size_t len
)
748 spin_lock_irqsave(&cm
->tx_lock
, flags
);
750 if (cm
->mport
== NULL
) {
755 if (cm
->tx_cnt
== RIOCM_TX_RING_SIZE
) {
756 riocm_debug(TX
, "Tx Queue is full");
761 cm
->tx_buf
[cm
->tx_slot
] = buffer
;
762 rc
= rio_add_outb_message(cm
->mport
, rdev
, cmbox
, buffer
, len
);
764 riocm_debug(TX
, "Add buf@%p destid=%x tx_slot=%d tx_cnt=%d",
765 buffer
, rdev
->destid
, cm
->tx_slot
, cm
->tx_cnt
);
769 cm
->tx_slot
&= (RIOCM_TX_RING_SIZE
- 1);
772 spin_unlock_irqrestore(&cm
->tx_lock
, flags
);
777 * riocm_ch_send - sends a data packet to a remote device
778 * @ch_id: local channel ID
779 * @buf: pointer to a data buffer to send (including CM header)
780 * @len: length of data to transfer (including CM header)
782 * ATTN: ASSUMES THAT THE HEADER SPACE IS RESERVED PART OF THE DATA PACKET
784 * Returns: 0 if success, or
785 * -EINVAL if one or more input parameters is/are not valid,
786 * -ENODEV if cannot find a channel with specified ID,
787 * -EAGAIN if a channel is not in CONNECTED state,
788 * + error codes returned by HW send routine.
790 static int riocm_ch_send(u16 ch_id
, void *buf
, int len
)
792 struct rio_channel
*ch
;
793 struct rio_ch_chan_hdr
*hdr
;
796 if (buf
== NULL
|| ch_id
== 0 || len
== 0 || len
> RIO_MAX_MSG_SIZE
)
799 ch
= riocm_get_channel(ch_id
);
801 riocm_error("%s(%d) ch_%d not found", current
->comm
,
802 task_pid_nr(current
), ch_id
);
806 if (!riocm_cmp(ch
, RIO_CM_CONNECTED
)) {
812 * Fill buffer header section with corresponding channel data
816 hdr
->bhdr
.src_id
= htonl(ch
->loc_destid
);
817 hdr
->bhdr
.dst_id
= htonl(ch
->rem_destid
);
818 hdr
->bhdr
.src_mbox
= cmbox
;
819 hdr
->bhdr
.dst_mbox
= cmbox
;
820 hdr
->bhdr
.type
= RIO_CM_CHAN
;
821 hdr
->ch_op
= CM_DATA_MSG
;
822 hdr
->dst_ch
= htons(ch
->rem_channel
);
823 hdr
->src_ch
= htons(ch
->id
);
824 hdr
->msg_len
= htons((u16
)len
);
826 /* ATTN: the function call below relies on the fact that underlying
827 * HW-specific add_outb_message() routine copies TX data into its own
828 * internal transfer buffer (true for all RIONET compatible mport
829 * drivers). Must be reviewed if mport driver uses the buffer directly.
832 ret
= riocm_post_send(ch
->cmdev
, ch
->rdev
, buf
, len
);
834 riocm_debug(TX
, "ch %d send_err=%d", ch
->id
, ret
);
836 riocm_put_channel(ch
);
840 static int riocm_ch_free_rxbuf(struct rio_channel
*ch
, void *buf
)
842 int i
, ret
= -EINVAL
;
844 spin_lock_bh(&ch
->lock
);
846 for (i
= 0; i
< RIOCM_RX_RING_SIZE
; i
++) {
847 if (ch
->rx_ring
.inuse
[i
] == buf
) {
848 ch
->rx_ring
.inuse
[i
] = NULL
;
849 ch
->rx_ring
.inuse_cnt
--;
855 spin_unlock_bh(&ch
->lock
);
864 * riocm_ch_receive - fetch a data packet received for the specified channel
865 * @ch: local channel ID
866 * @buf: pointer to a packet buffer
867 * @timeout: timeout to wait for incoming packet (in jiffies)
869 * Returns: 0 and valid buffer pointer if success, or NULL pointer and one of:
870 * -EAGAIN if a channel is not in CONNECTED state,
871 * -ENOMEM if in-use tracking queue is full,
872 * -ETIME if wait timeout expired,
873 * -EINTR if wait was interrupted.
875 static int riocm_ch_receive(struct rio_channel
*ch
, void **buf
, long timeout
)
881 if (!riocm_cmp(ch
, RIO_CM_CONNECTED
)) {
886 if (ch
->rx_ring
.inuse_cnt
== RIOCM_RX_RING_SIZE
) {
887 /* If we do not have entries to track buffers given to upper
888 * layer, reject request.
894 wret
= wait_for_completion_interruptible_timeout(&ch
->comp
, timeout
);
896 riocm_debug(WAIT
, "wait on %d returned %ld", ch
->id
, wret
);
900 else if (wret
== -ERESTARTSYS
)
903 ret
= riocm_cmp(ch
, RIO_CM_CONNECTED
) ? 0 : -ECONNRESET
;
908 spin_lock_bh(&ch
->lock
);
910 rxmsg
= ch
->rx_ring
.buf
[ch
->rx_ring
.tail
];
911 ch
->rx_ring
.buf
[ch
->rx_ring
.tail
] = NULL
;
914 ch
->rx_ring
.tail
%= RIOCM_RX_RING_SIZE
;
917 for (i
= 0; i
< RIOCM_RX_RING_SIZE
; i
++) {
918 if (ch
->rx_ring
.inuse
[i
] == NULL
) {
919 ch
->rx_ring
.inuse
[i
] = rxmsg
;
920 ch
->rx_ring
.inuse_cnt
++;
927 /* We have no entry to store pending message: drop it */
932 spin_unlock_bh(&ch
->lock
);
939 * riocm_ch_connect - sends a connect request to a remote device
940 * @loc_ch: local channel ID
941 * @cm: CM device to send connect request
942 * @peer: target RapidIO device
943 * @rem_ch: remote channel ID
945 * Returns: 0 if success, or
946 * -EINVAL if the channel is not in IDLE state,
947 * -EAGAIN if no connection request available immediately,
948 * -ETIME if ACK response timeout expired,
949 * -EINTR if wait for response was interrupted.
951 static int riocm_ch_connect(u16 loc_ch
, struct cm_dev
*cm
,
952 struct cm_peer
*peer
, u16 rem_ch
)
954 struct rio_channel
*ch
= NULL
;
955 struct rio_ch_chan_hdr
*hdr
;
959 ch
= riocm_get_channel(loc_ch
);
963 if (!riocm_cmp_exch(ch
, RIO_CM_IDLE
, RIO_CM_CONNECT
)) {
969 ch
->rdev
= peer
->rdev
;
971 ch
->loc_destid
= cm
->mport
->host_deviceid
;
972 ch
->rem_channel
= rem_ch
;
975 * Send connect request to the remote RapidIO device
978 hdr
= kzalloc(sizeof(*hdr
), GFP_KERNEL
);
984 hdr
->bhdr
.src_id
= htonl(ch
->loc_destid
);
985 hdr
->bhdr
.dst_id
= htonl(peer
->rdev
->destid
);
986 hdr
->bhdr
.src_mbox
= cmbox
;
987 hdr
->bhdr
.dst_mbox
= cmbox
;
988 hdr
->bhdr
.type
= RIO_CM_CHAN
;
989 hdr
->ch_op
= CM_CONN_REQ
;
990 hdr
->dst_ch
= htons(rem_ch
);
991 hdr
->src_ch
= htons(loc_ch
);
993 /* ATTN: the function call below relies on the fact that underlying
994 * HW-specific add_outb_message() routine copies TX data into its
995 * internal transfer buffer. Must be reviewed if mport driver uses
996 * this buffer directly.
998 ret
= riocm_post_send(cm
, peer
->rdev
, hdr
, sizeof(*hdr
));
1000 if (ret
!= -EBUSY
) {
1003 ret
= riocm_queue_req(cm
, peer
->rdev
, hdr
, sizeof(*hdr
));
1009 riocm_cmp_exch(ch
, RIO_CM_CONNECT
, RIO_CM_IDLE
);
1013 /* Wait for connect response from the remote device */
1014 wret
= wait_for_completion_interruptible_timeout(&ch
->comp
,
1015 RIOCM_CONNECT_TO
* HZ
);
1016 riocm_debug(WAIT
, "wait on %d returns %ld", ch
->id
, wret
);
1020 else if (wret
== -ERESTARTSYS
)
1023 ret
= riocm_cmp(ch
, RIO_CM_CONNECTED
) ? 0 : -1;
1026 riocm_put_channel(ch
);
1030 static int riocm_send_ack(struct rio_channel
*ch
)
1032 struct rio_ch_chan_hdr
*hdr
;
1035 hdr
= kzalloc(sizeof(*hdr
), GFP_KERNEL
);
1039 hdr
->bhdr
.src_id
= htonl(ch
->loc_destid
);
1040 hdr
->bhdr
.dst_id
= htonl(ch
->rem_destid
);
1041 hdr
->dst_ch
= htons(ch
->rem_channel
);
1042 hdr
->src_ch
= htons(ch
->id
);
1043 hdr
->bhdr
.src_mbox
= cmbox
;
1044 hdr
->bhdr
.dst_mbox
= cmbox
;
1045 hdr
->bhdr
.type
= RIO_CM_CHAN
;
1046 hdr
->ch_op
= CM_CONN_ACK
;
1048 /* ATTN: the function call below relies on the fact that underlying
1049 * add_outb_message() routine copies TX data into its internal transfer
1050 * buffer. Review if switching to direct buffer version.
1052 ret
= riocm_post_send(ch
->cmdev
, ch
->rdev
, hdr
, sizeof(*hdr
));
1054 if (ret
== -EBUSY
&& !riocm_queue_req(ch
->cmdev
,
1055 ch
->rdev
, hdr
, sizeof(*hdr
)))
1060 riocm_error("send ACK to ch_%d on %s failed (ret=%d)",
1061 ch
->id
, rio_name(ch
->rdev
), ret
);
1066 * riocm_ch_accept - accept incoming connection request
1067 * @ch_id: channel ID
1068 * @new_ch_id: local mport device
1069 * @timeout: wait timeout (if 0 non-blocking call, do not wait if connection
1070 * request is not available).
1072 * Returns: pointer to new channel struct if success, or error-valued pointer:
1073 * -ENODEV - cannot find specified channel or mport,
1074 * -EINVAL - the channel is not in IDLE state,
1075 * -EAGAIN - no connection request available immediately (timeout=0),
1076 * -ENOMEM - unable to allocate new channel,
1077 * -ETIME - wait timeout expired,
1078 * -EINTR - wait was interrupted.
1080 static struct rio_channel
*riocm_ch_accept(u16 ch_id
, u16
*new_ch_id
,
1083 struct rio_channel
*ch
= NULL
;
1084 struct rio_channel
*new_ch
= NULL
;
1085 struct conn_req
*req
;
1086 struct cm_peer
*peer
;
1091 ch
= riocm_get_channel(ch_id
);
1093 return ERR_PTR(-EINVAL
);
1095 if (!riocm_cmp(ch
, RIO_CM_LISTEN
)) {
1100 /* Don't sleep if this is a non blocking call */
1102 if (!try_wait_for_completion(&ch
->comp
)) {
1107 riocm_debug(WAIT
, "on %d", ch
->id
);
1109 wret
= wait_for_completion_interruptible_timeout(&ch
->comp
,
1114 } else if (wret
== -ERESTARTSYS
) {
1120 spin_lock_bh(&ch
->lock
);
1122 if (ch
->state
!= RIO_CM_LISTEN
) {
1124 } else if (list_empty(&ch
->accept_queue
)) {
1125 riocm_debug(WAIT
, "on %d accept_queue is empty on completion",
1130 spin_unlock_bh(&ch
->lock
);
1133 riocm_debug(WAIT
, "on %d returns %d", ch
->id
, err
);
1137 /* Create new channel for this connection */
1138 new_ch
= riocm_ch_alloc(RIOCM_CHNUM_AUTO
);
1140 if (IS_ERR(new_ch
)) {
1141 riocm_error("failed to get channel for new req (%ld)",
1147 spin_lock_bh(&ch
->lock
);
1149 req
= list_first_entry(&ch
->accept_queue
, struct conn_req
, node
);
1150 list_del(&req
->node
);
1151 new_ch
->cmdev
= ch
->cmdev
;
1152 new_ch
->loc_destid
= ch
->loc_destid
;
1153 new_ch
->rem_destid
= req
->destid
;
1154 new_ch
->rem_channel
= req
->chan
;
1156 spin_unlock_bh(&ch
->lock
);
1157 riocm_put_channel(ch
);
1160 down_read(&rdev_sem
);
1161 /* Find requester's device object */
1162 list_for_each_entry(peer
, &new_ch
->cmdev
->peers
, node
) {
1163 if (peer
->rdev
->destid
== new_ch
->rem_destid
) {
1164 riocm_debug(RX_CMD
, "found matching device(%s)",
1165 rio_name(peer
->rdev
));
1173 /* If peer device object not found, simply ignore the request */
1178 new_ch
->rdev
= peer
->rdev
;
1179 new_ch
->state
= RIO_CM_CONNECTED
;
1180 spin_lock_init(&new_ch
->lock
);
1182 /* Acknowledge the connection request. */
1183 riocm_send_ack(new_ch
);
1185 *new_ch_id
= new_ch
->id
;
1188 riocm_put_channel(ch
);
1191 spin_lock_bh(&idr_lock
);
1192 idr_remove(&ch_idr
, new_ch
->id
);
1193 spin_unlock_bh(&idr_lock
);
1194 riocm_put_channel(new_ch
);
1197 return ERR_PTR(err
);
1201 * riocm_ch_listen - puts a channel into LISTEN state
1202 * @ch_id: channel ID
1204 * Returns: 0 if success, or
1205 * -EINVAL if the specified channel does not exists or
1206 * is not in CHAN_BOUND state.
1208 static int riocm_ch_listen(u16 ch_id
)
1210 struct rio_channel
*ch
= NULL
;
1213 riocm_debug(CHOP
, "(ch_%d)", ch_id
);
1215 ch
= riocm_get_channel(ch_id
);
1216 if (!ch
|| !riocm_cmp_exch(ch
, RIO_CM_CHAN_BOUND
, RIO_CM_LISTEN
))
1218 riocm_put_channel(ch
);
1223 * riocm_ch_bind - associate a channel object and an mport device
1224 * @ch_id: channel ID
1225 * @mport_id: local mport device ID
1226 * @context: pointer to the additional caller's context
1228 * Returns: 0 if success, or
1229 * -ENODEV if cannot find specified mport,
1230 * -EINVAL if the specified channel does not exist or
1231 * is not in IDLE state.
1233 static int riocm_ch_bind(u16 ch_id
, u8 mport_id
, void *context
)
1235 struct rio_channel
*ch
= NULL
;
1239 riocm_debug(CHOP
, "ch_%d to mport_%d", ch_id
, mport_id
);
1241 /* Find matching cm_dev object */
1242 down_read(&rdev_sem
);
1243 list_for_each_entry(cm
, &cm_dev_list
, list
) {
1244 if ((cm
->mport
->id
== mport_id
) &&
1245 rio_mport_is_running(cm
->mport
)) {
1254 ch
= riocm_get_channel(ch_id
);
1260 spin_lock_bh(&ch
->lock
);
1261 if (ch
->state
!= RIO_CM_IDLE
) {
1262 spin_unlock_bh(&ch
->lock
);
1268 ch
->loc_destid
= cm
->mport
->host_deviceid
;
1269 ch
->context
= context
;
1270 ch
->state
= RIO_CM_CHAN_BOUND
;
1271 spin_unlock_bh(&ch
->lock
);
1273 riocm_put_channel(ch
);
1280 * riocm_ch_alloc - channel object allocation helper routine
1281 * @ch_num: channel ID (1 ... RIOCM_MAX_CHNUM, 0 = automatic)
1283 * Return value: pointer to newly created channel object,
1284 * or error-valued pointer
1286 static struct rio_channel
*riocm_ch_alloc(u16 ch_num
)
1290 struct rio_channel
*ch
;
1292 ch
= kzalloc(sizeof(*ch
), GFP_KERNEL
);
1294 return ERR_PTR(-ENOMEM
);
1297 /* If requested, try to obtain the specified channel ID */
1301 /* Obtain channel ID from the dynamic allocation range */
1303 end
= RIOCM_MAX_CHNUM
+ 1;
1306 idr_preload(GFP_KERNEL
);
1307 spin_lock_bh(&idr_lock
);
1308 id
= idr_alloc_cyclic(&ch_idr
, ch
, start
, end
, GFP_NOWAIT
);
1309 spin_unlock_bh(&idr_lock
);
1314 return ERR_PTR(id
== -ENOSPC
? -EBUSY
: id
);
1318 ch
->state
= RIO_CM_IDLE
;
1319 spin_lock_init(&ch
->lock
);
1320 INIT_LIST_HEAD(&ch
->accept_queue
);
1321 INIT_LIST_HEAD(&ch
->ch_node
);
1322 init_completion(&ch
->comp
);
1323 init_completion(&ch
->comp_close
);
1324 kref_init(&ch
->ref
);
1325 ch
->rx_ring
.head
= 0;
1326 ch
->rx_ring
.tail
= 0;
1327 ch
->rx_ring
.count
= 0;
1328 ch
->rx_ring
.inuse_cnt
= 0;
1334 * riocm_ch_create - creates a new channel object and allocates ID for it
1335 * @ch_num: channel ID (1 ... RIOCM_MAX_CHNUM, 0 = automatic)
1337 * Allocates and initializes a new channel object. If the parameter ch_num > 0
1338 * and is within the valid range, riocm_ch_create tries to allocate the
1339 * specified ID for the new channel. If ch_num = 0, channel ID will be assigned
1340 * automatically from the range (chstart ... RIOCM_MAX_CHNUM).
1341 * Module parameter 'chstart' defines start of an ID range available for dynamic
1342 * allocation. Range below 'chstart' is reserved for pre-defined ID numbers.
1343 * Available channel numbers are limited by 16-bit size of channel numbers used
1344 * in the packet header.
1346 * Return value: PTR to rio_channel structure if successful (with channel number
1347 * updated via pointer) or error-valued pointer if error.
1349 static struct rio_channel
*riocm_ch_create(u16
*ch_num
)
1351 struct rio_channel
*ch
= NULL
;
1353 ch
= riocm_ch_alloc(*ch_num
);
1356 riocm_debug(CHOP
, "Failed to allocate channel %d (err=%ld)",
1357 *ch_num
, PTR_ERR(ch
));
1365 * riocm_ch_free - channel object release routine
1366 * @ref: pointer to a channel's kref structure
1368 static void riocm_ch_free(struct kref
*ref
)
1370 struct rio_channel
*ch
= container_of(ref
, struct rio_channel
, ref
);
1373 riocm_debug(CHOP
, "(ch_%d)", ch
->id
);
1375 if (ch
->rx_ring
.inuse_cnt
) {
1377 i
< RIOCM_RX_RING_SIZE
&& ch
->rx_ring
.inuse_cnt
; i
++) {
1378 if (ch
->rx_ring
.inuse
[i
] != NULL
) {
1379 kfree(ch
->rx_ring
.inuse
[i
]);
1380 ch
->rx_ring
.inuse_cnt
--;
1385 if (ch
->rx_ring
.count
)
1386 for (i
= 0; i
< RIOCM_RX_RING_SIZE
&& ch
->rx_ring
.count
; i
++) {
1387 if (ch
->rx_ring
.buf
[i
] != NULL
) {
1388 kfree(ch
->rx_ring
.buf
[i
]);
1389 ch
->rx_ring
.count
--;
1393 complete(&ch
->comp_close
);
1396 static int riocm_send_close(struct rio_channel
*ch
)
1398 struct rio_ch_chan_hdr
*hdr
;
1402 * Send CH_CLOSE notification to the remote RapidIO device
1405 hdr
= kzalloc(sizeof(*hdr
), GFP_KERNEL
);
1409 hdr
->bhdr
.src_id
= htonl(ch
->loc_destid
);
1410 hdr
->bhdr
.dst_id
= htonl(ch
->rem_destid
);
1411 hdr
->bhdr
.src_mbox
= cmbox
;
1412 hdr
->bhdr
.dst_mbox
= cmbox
;
1413 hdr
->bhdr
.type
= RIO_CM_CHAN
;
1414 hdr
->ch_op
= CM_CONN_CLOSE
;
1415 hdr
->dst_ch
= htons(ch
->rem_channel
);
1416 hdr
->src_ch
= htons(ch
->id
);
1418 /* ATTN: the function call below relies on the fact that underlying
1419 * add_outb_message() routine copies TX data into its internal transfer
1420 * buffer. Needs to be reviewed if switched to direct buffer mode.
1422 ret
= riocm_post_send(ch
->cmdev
, ch
->rdev
, hdr
, sizeof(*hdr
));
1424 if (ret
== -EBUSY
&& !riocm_queue_req(ch
->cmdev
, ch
->rdev
,
1430 riocm_error("ch(%d) send CLOSE failed (ret=%d)", ch
->id
, ret
);
1436 * riocm_ch_close - closes a channel object with specified ID (by local request)
1437 * @ch: channel to be closed
1439 static int riocm_ch_close(struct rio_channel
*ch
)
1441 unsigned long tmo
= msecs_to_jiffies(3000);
1442 enum rio_cm_state state
;
1446 riocm_debug(CHOP
, "ch_%d by %s(%d)",
1447 ch
->id
, current
->comm
, task_pid_nr(current
));
1449 state
= riocm_exch(ch
, RIO_CM_DESTROYING
);
1450 if (state
== RIO_CM_CONNECTED
)
1451 riocm_send_close(ch
);
1453 complete_all(&ch
->comp
);
1455 riocm_put_channel(ch
);
1456 wret
= wait_for_completion_interruptible_timeout(&ch
->comp_close
, tmo
);
1458 riocm_debug(WAIT
, "wait on %d returns %ld", ch
->id
, wret
);
1461 /* Timeout on wait occurred */
1462 riocm_debug(CHOP
, "%s(%d) timed out waiting for ch %d",
1463 current
->comm
, task_pid_nr(current
), ch
->id
);
1465 } else if (wret
== -ERESTARTSYS
) {
1466 /* Wait_for_completion was interrupted by a signal */
1467 riocm_debug(CHOP
, "%s(%d) wait for ch %d was interrupted",
1468 current
->comm
, task_pid_nr(current
), ch
->id
);
1473 riocm_debug(CHOP
, "ch_%d resources released", ch
->id
);
1476 riocm_debug(CHOP
, "failed to release ch_%d resources", ch
->id
);
1483 * riocm_cdev_open() - Open character device
1485 static int riocm_cdev_open(struct inode
*inode
, struct file
*filp
)
1487 riocm_debug(INIT
, "by %s(%d) filp=%p ",
1488 current
->comm
, task_pid_nr(current
), filp
);
1490 if (list_empty(&cm_dev_list
))
1497 * riocm_cdev_release() - Release character device
1499 static int riocm_cdev_release(struct inode
*inode
, struct file
*filp
)
1501 struct rio_channel
*ch
, *_c
;
1505 riocm_debug(EXIT
, "by %s(%d) filp=%p",
1506 current
->comm
, task_pid_nr(current
), filp
);
1508 /* Check if there are channels associated with this file descriptor */
1509 spin_lock_bh(&idr_lock
);
1510 idr_for_each_entry(&ch_idr
, ch
, i
) {
1511 if (ch
&& ch
->filp
== filp
) {
1512 riocm_debug(EXIT
, "ch_%d not released by %s(%d)",
1513 ch
->id
, current
->comm
,
1514 task_pid_nr(current
));
1515 idr_remove(&ch_idr
, ch
->id
);
1516 list_add(&ch
->ch_node
, &list
);
1519 spin_unlock_bh(&idr_lock
);
1521 if (!list_empty(&list
)) {
1522 list_for_each_entry_safe(ch
, _c
, &list
, ch_node
) {
1523 list_del(&ch
->ch_node
);
1532 * cm_ep_get_list_size() - Reports number of endpoints in the network
1534 static int cm_ep_get_list_size(void __user
*arg
)
1536 u32 __user
*p
= arg
;
1541 if (get_user(mport_id
, p
))
1543 if (mport_id
>= RIO_MAX_MPORTS
)
1546 /* Find a matching cm_dev object */
1547 down_read(&rdev_sem
);
1548 list_for_each_entry(cm
, &cm_dev_list
, list
) {
1549 if (cm
->mport
->id
== mport_id
) {
1552 if (copy_to_user(arg
, &count
, sizeof(u32
)))
1563 * cm_ep_get_list() - Returns list of attached endpoints
1565 static int cm_ep_get_list(void __user
*arg
)
1568 struct cm_peer
*peer
;
1576 if (copy_from_user(&info
, arg
, sizeof(info
)))
1579 if (info
[1] >= RIO_MAX_MPORTS
|| info
[0] > RIOCM_MAX_EP_COUNT
)
1582 /* Find a matching cm_dev object */
1583 down_read(&rdev_sem
);
1584 list_for_each_entry(cm
, &cm_dev_list
, list
)
1585 if (cm
->mport
->id
== (u8
)info
[1])
1592 nent
= min(info
[0], cm
->npeers
);
1593 buf
= kcalloc(nent
+ 2, sizeof(u32
), GFP_KERNEL
);
1599 entry_ptr
= (u32
*)((uintptr_t)buf
+ 2*sizeof(u32
));
1601 list_for_each_entry(peer
, &cm
->peers
, node
) {
1602 *entry_ptr
= (u32
)peer
->rdev
->destid
;
1609 ((u32
*)buf
)[0] = i
; /* report an updated number of entries */
1610 ((u32
*)buf
)[1] = info
[1]; /* put back an mport ID */
1611 if (copy_to_user(arg
, buf
, sizeof(u32
) * (info
[0] + 2)))
1619 * cm_mport_get_list() - Returns list of available local mport devices
1621 static int cm_mport_get_list(void __user
*arg
)
1630 if (copy_from_user(&entries
, arg
, sizeof(entries
)))
1632 if (entries
== 0 || entries
> RIO_MAX_MPORTS
)
1634 buf
= kcalloc(entries
+ 1, sizeof(u32
), GFP_KERNEL
);
1638 /* Scan all registered cm_dev objects */
1639 entry_ptr
= (u32
*)((uintptr_t)buf
+ sizeof(u32
));
1640 down_read(&rdev_sem
);
1641 list_for_each_entry(cm
, &cm_dev_list
, list
) {
1642 if (count
++ < entries
) {
1643 *entry_ptr
= (cm
->mport
->id
<< 16) |
1644 cm
->mport
->host_deviceid
;
1650 *((u32
*)buf
) = count
; /* report a real number of entries */
1651 if (copy_to_user(arg
, buf
, sizeof(u32
) * (count
+ 1)))
1659 * cm_chan_create() - Create a message exchange channel
1661 static int cm_chan_create(struct file
*filp
, void __user
*arg
)
1663 u16 __user
*p
= arg
;
1665 struct rio_channel
*ch
;
1667 if (get_user(ch_num
, p
))
1670 riocm_debug(CHOP
, "ch_%d requested by %s(%d)",
1671 ch_num
, current
->comm
, task_pid_nr(current
));
1672 ch
= riocm_ch_create(&ch_num
);
1677 riocm_debug(CHOP
, "ch_%d created by %s(%d)",
1678 ch_num
, current
->comm
, task_pid_nr(current
));
1679 return put_user(ch_num
, p
);
1683 * cm_chan_close() - Close channel
1684 * @filp: Pointer to file object
1685 * @arg: Channel to close
1687 static int cm_chan_close(struct file
*filp
, void __user
*arg
)
1689 u16 __user
*p
= arg
;
1691 struct rio_channel
*ch
;
1693 if (get_user(ch_num
, p
))
1696 riocm_debug(CHOP
, "ch_%d by %s(%d)",
1697 ch_num
, current
->comm
, task_pid_nr(current
));
1699 spin_lock_bh(&idr_lock
);
1700 ch
= idr_find(&ch_idr
, ch_num
);
1702 spin_unlock_bh(&idr_lock
);
1705 if (ch
->filp
!= filp
) {
1706 spin_unlock_bh(&idr_lock
);
1709 idr_remove(&ch_idr
, ch
->id
);
1710 spin_unlock_bh(&idr_lock
);
1712 return riocm_ch_close(ch
);
1716 * cm_chan_bind() - Bind channel
1717 * @arg: Channel number
1719 static int cm_chan_bind(void __user
*arg
)
1721 struct rio_cm_channel chan
;
1723 if (copy_from_user(&chan
, arg
, sizeof(chan
)))
1725 if (chan
.mport_id
>= RIO_MAX_MPORTS
)
1728 return riocm_ch_bind(chan
.id
, chan
.mport_id
, NULL
);
1732 * cm_chan_listen() - Listen on channel
1733 * @arg: Channel number
1735 static int cm_chan_listen(void __user
*arg
)
1737 u16 __user
*p
= arg
;
1740 if (get_user(ch_num
, p
))
1743 return riocm_ch_listen(ch_num
);
1747 * cm_chan_accept() - Accept incoming connection
1748 * @filp: Pointer to file object
1749 * @arg: Channel number
1751 static int cm_chan_accept(struct file
*filp
, void __user
*arg
)
1753 struct rio_cm_accept param
;
1755 struct rio_channel
*ch
;
1757 if (copy_from_user(¶m
, arg
, sizeof(param
)))
1760 riocm_debug(CHOP
, "on ch_%d by %s(%d)",
1761 param
.ch_num
, current
->comm
, task_pid_nr(current
));
1763 accept_to
= param
.wait_to
?
1764 msecs_to_jiffies(param
.wait_to
) : 0;
1766 ch
= riocm_ch_accept(param
.ch_num
, ¶m
.ch_num
, accept_to
);
1771 riocm_debug(CHOP
, "new ch_%d for %s(%d)",
1772 ch
->id
, current
->comm
, task_pid_nr(current
));
1774 if (copy_to_user(arg
, ¶m
, sizeof(param
)))
1780 * cm_chan_connect() - Connect on channel
1781 * @arg: Channel information
1783 static int cm_chan_connect(void __user
*arg
)
1785 struct rio_cm_channel chan
;
1787 struct cm_peer
*peer
;
1790 if (copy_from_user(&chan
, arg
, sizeof(chan
)))
1792 if (chan
.mport_id
>= RIO_MAX_MPORTS
)
1795 down_read(&rdev_sem
);
1797 /* Find matching cm_dev object */
1798 list_for_each_entry(cm
, &cm_dev_list
, list
) {
1799 if (cm
->mport
->id
== chan
.mport_id
) {
1808 if (chan
.remote_destid
>= RIO_ANY_DESTID(cm
->mport
->sys_size
)) {
1813 /* Find corresponding RapidIO endpoint device object */
1816 list_for_each_entry(peer
, &cm
->peers
, node
) {
1817 if (peer
->rdev
->destid
== chan
.remote_destid
) {
1828 return riocm_ch_connect(chan
.id
, cm
, peer
, chan
.remote_channel
);
1835 * cm_chan_msg_send() - Send a message through channel
1836 * @arg: Outbound message information
1838 static int cm_chan_msg_send(void __user
*arg
)
1840 struct rio_cm_msg msg
;
1844 if (copy_from_user(&msg
, arg
, sizeof(msg
)))
1846 if (msg
.size
> RIO_MAX_MSG_SIZE
)
1849 buf
= kmalloc(msg
.size
, GFP_KERNEL
);
1853 if (copy_from_user(buf
, (void __user
*)(uintptr_t)msg
.msg
, msg
.size
)) {
1858 ret
= riocm_ch_send(msg
.ch_num
, buf
, msg
.size
);
1865 * cm_chan_msg_rcv() - Receive a message through channel
1866 * @arg: Inbound message information
1868 static int cm_chan_msg_rcv(void __user
*arg
)
1870 struct rio_cm_msg msg
;
1871 struct rio_channel
*ch
;
1874 int ret
= 0, msg_size
;
1876 if (copy_from_user(&msg
, arg
, sizeof(msg
)))
1879 if (msg
.ch_num
== 0 || msg
.size
== 0)
1882 ch
= riocm_get_channel(msg
.ch_num
);
1886 rxto
= msg
.rxto
? msecs_to_jiffies(msg
.rxto
) : MAX_SCHEDULE_TIMEOUT
;
1888 ret
= riocm_ch_receive(ch
, &buf
, rxto
);
1892 msg_size
= min(msg
.size
, (u16
)(RIO_MAX_MSG_SIZE
));
1894 if (copy_to_user((void __user
*)(uintptr_t)msg
.msg
, buf
, msg_size
))
1897 riocm_ch_free_rxbuf(ch
, buf
);
1899 riocm_put_channel(ch
);
1904 * riocm_cdev_ioctl() - IOCTL requests handler
1907 riocm_cdev_ioctl(struct file
*filp
, unsigned int cmd
, unsigned long arg
)
1910 case RIO_CM_EP_GET_LIST_SIZE
:
1911 return cm_ep_get_list_size((void __user
*)arg
);
1912 case RIO_CM_EP_GET_LIST
:
1913 return cm_ep_get_list((void __user
*)arg
);
1914 case RIO_CM_CHAN_CREATE
:
1915 return cm_chan_create(filp
, (void __user
*)arg
);
1916 case RIO_CM_CHAN_CLOSE
:
1917 return cm_chan_close(filp
, (void __user
*)arg
);
1918 case RIO_CM_CHAN_BIND
:
1919 return cm_chan_bind((void __user
*)arg
);
1920 case RIO_CM_CHAN_LISTEN
:
1921 return cm_chan_listen((void __user
*)arg
);
1922 case RIO_CM_CHAN_ACCEPT
:
1923 return cm_chan_accept(filp
, (void __user
*)arg
);
1924 case RIO_CM_CHAN_CONNECT
:
1925 return cm_chan_connect((void __user
*)arg
);
1926 case RIO_CM_CHAN_SEND
:
1927 return cm_chan_msg_send((void __user
*)arg
);
1928 case RIO_CM_CHAN_RECEIVE
:
1929 return cm_chan_msg_rcv((void __user
*)arg
);
1930 case RIO_CM_MPORT_GET_LIST
:
1931 return cm_mport_get_list((void __user
*)arg
);
1939 static const struct file_operations riocm_cdev_fops
= {
1940 .owner
= THIS_MODULE
,
1941 .open
= riocm_cdev_open
,
1942 .release
= riocm_cdev_release
,
1943 .unlocked_ioctl
= riocm_cdev_ioctl
,
1947 * riocm_add_dev - add new remote RapidIO device into channel management core
1948 * @dev: device object associated with RapidIO device
1949 * @sif: subsystem interface
1951 * Adds the specified RapidIO device (if applicable) into peers list of
1952 * the corresponding channel management device (cm_dev).
1954 static int riocm_add_dev(struct device
*dev
, struct subsys_interface
*sif
)
1956 struct cm_peer
*peer
;
1957 struct rio_dev
*rdev
= to_rio_dev(dev
);
1960 /* Check if the remote device has capabilities required to support CM */
1961 if (!dev_cm_capable(rdev
))
1964 riocm_debug(RDEV
, "(%s)", rio_name(rdev
));
1966 peer
= kmalloc(sizeof(*peer
), GFP_KERNEL
);
1970 /* Find a corresponding cm_dev object */
1971 down_write(&rdev_sem
);
1972 list_for_each_entry(cm
, &cm_dev_list
, list
) {
1973 if (cm
->mport
== rdev
->net
->hport
)
1977 up_write(&rdev_sem
);
1983 list_add_tail(&peer
->node
, &cm
->peers
);
1986 up_write(&rdev_sem
);
1991 * riocm_remove_dev - remove remote RapidIO device from channel management core
1992 * @dev: device object associated with RapidIO device
1993 * @sif: subsystem interface
1995 * Removes the specified RapidIO device (if applicable) from peers list of
1996 * the corresponding channel management device (cm_dev).
1998 static void riocm_remove_dev(struct device
*dev
, struct subsys_interface
*sif
)
2000 struct rio_dev
*rdev
= to_rio_dev(dev
);
2002 struct cm_peer
*peer
;
2003 struct rio_channel
*ch
, *_c
;
2008 /* Check if the remote device has capabilities required to support CM */
2009 if (!dev_cm_capable(rdev
))
2012 riocm_debug(RDEV
, "(%s)", rio_name(rdev
));
2014 /* Find matching cm_dev object */
2015 down_write(&rdev_sem
);
2016 list_for_each_entry(cm
, &cm_dev_list
, list
) {
2017 if (cm
->mport
== rdev
->net
->hport
) {
2024 up_write(&rdev_sem
);
2028 /* Remove remote device from the list of peers */
2030 list_for_each_entry(peer
, &cm
->peers
, node
) {
2031 if (peer
->rdev
== rdev
) {
2032 riocm_debug(RDEV
, "removing peer %s", rio_name(rdev
));
2034 list_del(&peer
->node
);
2041 up_write(&rdev_sem
);
2047 * Release channels associated with this peer
2050 spin_lock_bh(&idr_lock
);
2051 idr_for_each_entry(&ch_idr
, ch
, i
) {
2052 if (ch
&& ch
->rdev
== rdev
) {
2053 if (atomic_read(&rdev
->state
) != RIO_DEVICE_SHUTDOWN
)
2054 riocm_exch(ch
, RIO_CM_DISCONNECT
);
2055 idr_remove(&ch_idr
, ch
->id
);
2056 list_add(&ch
->ch_node
, &list
);
2059 spin_unlock_bh(&idr_lock
);
2061 if (!list_empty(&list
)) {
2062 list_for_each_entry_safe(ch
, _c
, &list
, ch_node
) {
2063 list_del(&ch
->ch_node
);
2070 * riocm_cdev_add() - Create rio_cm char device
2071 * @devno: device number assigned to device (MAJ + MIN)
2073 static int riocm_cdev_add(dev_t devno
)
2077 cdev_init(&riocm_cdev
.cdev
, &riocm_cdev_fops
);
2078 riocm_cdev
.cdev
.owner
= THIS_MODULE
;
2079 ret
= cdev_add(&riocm_cdev
.cdev
, devno
, 1);
2081 riocm_error("Cannot register a device with error %d", ret
);
2085 riocm_cdev
.dev
= device_create(dev_class
, NULL
, devno
, NULL
, DEV_NAME
);
2086 if (IS_ERR(riocm_cdev
.dev
)) {
2087 cdev_del(&riocm_cdev
.cdev
);
2088 return PTR_ERR(riocm_cdev
.dev
);
2091 riocm_debug(MPORT
, "Added %s cdev(%d:%d)",
2092 DEV_NAME
, MAJOR(devno
), MINOR(devno
));
2098 * riocm_add_mport - add new local mport device into channel management core
2099 * @dev: device object associated with mport
2100 * @class_intf: class interface
2102 * When a new mport device is added, CM immediately reserves inbound and
2103 * outbound RapidIO mailboxes that will be used.
2105 static int riocm_add_mport(struct device
*dev
,
2106 struct class_interface
*class_intf
)
2111 struct rio_mport
*mport
= to_rio_mport(dev
);
2113 riocm_debug(MPORT
, "add mport %s", mport
->name
);
2115 cm
= kzalloc(sizeof(*cm
), GFP_KERNEL
);
2121 rc
= rio_request_outb_mbox(mport
, cm
, cmbox
,
2122 RIOCM_TX_RING_SIZE
, riocm_outb_msg_event
);
2124 riocm_error("failed to allocate OBMBOX_%d on %s",
2125 cmbox
, mport
->name
);
2130 rc
= rio_request_inb_mbox(mport
, cm
, cmbox
,
2131 RIOCM_RX_RING_SIZE
, riocm_inb_msg_event
);
2133 riocm_error("failed to allocate IBMBOX_%d on %s",
2134 cmbox
, mport
->name
);
2135 rio_release_outb_mbox(mport
, cmbox
);
2141 * Allocate and register inbound messaging buffers to be ready
2142 * to receive channel and system management requests
2144 for (i
= 0; i
< RIOCM_RX_RING_SIZE
; i
++)
2145 cm
->rx_buf
[i
] = NULL
;
2147 cm
->rx_slots
= RIOCM_RX_RING_SIZE
;
2148 mutex_init(&cm
->rx_lock
);
2149 riocm_rx_fill(cm
, RIOCM_RX_RING_SIZE
);
2150 cm
->rx_wq
= create_workqueue(DRV_NAME
"/rxq");
2151 INIT_WORK(&cm
->rx_work
, rio_ibmsg_handler
);
2155 cm
->tx_ack_slot
= 0;
2156 spin_lock_init(&cm
->tx_lock
);
2158 INIT_LIST_HEAD(&cm
->peers
);
2160 INIT_LIST_HEAD(&cm
->tx_reqs
);
2162 down_write(&rdev_sem
);
2163 list_add_tail(&cm
->list
, &cm_dev_list
);
2164 up_write(&rdev_sem
);
2170 * riocm_remove_mport - remove local mport device from channel management core
2171 * @dev: device object associated with mport
2172 * @class_intf: class interface
2174 * Removes a local mport device from the list of registered devices that provide
2175 * channel management services. Returns an error if the specified mport is not
2176 * registered with the CM core.
2178 static void riocm_remove_mport(struct device
*dev
,
2179 struct class_interface
*class_intf
)
2181 struct rio_mport
*mport
= to_rio_mport(dev
);
2183 struct cm_peer
*peer
, *temp
;
2184 struct rio_channel
*ch
, *_c
;
2189 riocm_debug(MPORT
, "%s", mport
->name
);
2191 /* Find a matching cm_dev object */
2192 down_write(&rdev_sem
);
2193 list_for_each_entry(cm
, &cm_dev_list
, list
) {
2194 if (cm
->mport
== mport
) {
2195 list_del(&cm
->list
);
2200 up_write(&rdev_sem
);
2204 flush_workqueue(cm
->rx_wq
);
2205 destroy_workqueue(cm
->rx_wq
);
2207 /* Release channels bound to this mport */
2208 spin_lock_bh(&idr_lock
);
2209 idr_for_each_entry(&ch_idr
, ch
, i
) {
2210 if (ch
->cmdev
== cm
) {
2211 riocm_debug(RDEV
, "%s drop ch_%d",
2212 mport
->name
, ch
->id
);
2213 idr_remove(&ch_idr
, ch
->id
);
2214 list_add(&ch
->ch_node
, &list
);
2217 spin_unlock_bh(&idr_lock
);
2219 if (!list_empty(&list
)) {
2220 list_for_each_entry_safe(ch
, _c
, &list
, ch_node
) {
2221 list_del(&ch
->ch_node
);
2226 rio_release_inb_mbox(mport
, cmbox
);
2227 rio_release_outb_mbox(mport
, cmbox
);
2229 /* Remove and free peer entries */
2230 if (!list_empty(&cm
->peers
))
2231 riocm_debug(RDEV
, "ATTN: peer list not empty");
2232 list_for_each_entry_safe(peer
, temp
, &cm
->peers
, node
) {
2233 riocm_debug(RDEV
, "removing peer %s", rio_name(peer
->rdev
));
2234 list_del(&peer
->node
);
2240 riocm_debug(MPORT
, "%s done", mport
->name
);
2243 static int rio_cm_shutdown(struct notifier_block
*nb
, unsigned long code
,
2246 struct rio_channel
*ch
;
2249 riocm_debug(EXIT
, ".");
2251 spin_lock_bh(&idr_lock
);
2252 idr_for_each_entry(&ch_idr
, ch
, i
) {
2253 riocm_debug(EXIT
, "close ch %d", ch
->id
);
2254 if (ch
->state
== RIO_CM_CONNECTED
)
2255 riocm_send_close(ch
);
2257 spin_unlock_bh(&idr_lock
);
2263 * riocm_interface handles addition/removal of remote RapidIO devices
2265 static struct subsys_interface riocm_interface
= {
2267 .subsys
= &rio_bus_type
,
2268 .add_dev
= riocm_add_dev
,
2269 .remove_dev
= riocm_remove_dev
,
2273 * rio_mport_interface handles addition/removal local mport devices
2275 static struct class_interface rio_mport_interface __refdata
= {
2276 .class = &rio_mport_class
,
2277 .add_dev
= riocm_add_mport
,
2278 .remove_dev
= riocm_remove_mport
,
2281 static struct notifier_block rio_cm_notifier
= {
2282 .notifier_call
= rio_cm_shutdown
,
2285 static int __init
riocm_init(void)
2289 /* Create device class needed by udev */
2290 dev_class
= class_create(THIS_MODULE
, DRV_NAME
);
2291 if (IS_ERR(dev_class
)) {
2292 riocm_error("Cannot create " DRV_NAME
" class");
2293 return PTR_ERR(dev_class
);
2296 ret
= alloc_chrdev_region(&dev_number
, 0, 1, DRV_NAME
);
2298 class_destroy(dev_class
);
2302 dev_major
= MAJOR(dev_number
);
2303 dev_minor_base
= MINOR(dev_number
);
2304 riocm_debug(INIT
, "Registered class with %d major", dev_major
);
2307 * Register as rapidio_port class interface to get notifications about
2308 * mport additions and removals.
2310 ret
= class_interface_register(&rio_mport_interface
);
2312 riocm_error("class_interface_register error: %d", ret
);
2317 * Register as RapidIO bus interface to get notifications about
2318 * addition/removal of remote RapidIO devices.
2320 ret
= subsys_interface_register(&riocm_interface
);
2322 riocm_error("subsys_interface_register error: %d", ret
);
2326 ret
= register_reboot_notifier(&rio_cm_notifier
);
2328 riocm_error("failed to register reboot notifier (err=%d)", ret
);
2332 ret
= riocm_cdev_add(dev_number
);
2334 unregister_reboot_notifier(&rio_cm_notifier
);
2341 subsys_interface_unregister(&riocm_interface
);
2343 class_interface_unregister(&rio_mport_interface
);
2345 unregister_chrdev_region(dev_number
, 1);
2346 class_destroy(dev_class
);
2350 static void __exit
riocm_exit(void)
2352 riocm_debug(EXIT
, "enter");
2353 unregister_reboot_notifier(&rio_cm_notifier
);
2354 subsys_interface_unregister(&riocm_interface
);
2355 class_interface_unregister(&rio_mport_interface
);
2356 idr_destroy(&ch_idr
);
2358 device_unregister(riocm_cdev
.dev
);
2359 cdev_del(&(riocm_cdev
.cdev
));
2361 class_destroy(dev_class
);
2362 unregister_chrdev_region(dev_number
, 1);
2365 late_initcall(riocm_init
);
2366 module_exit(riocm_exit
);