projects / deliverable / linux.git / blob
? search:


96168b8190449ccc1bee6e24ef0758af1bb3c69c
[deliverable/linux.git] / drivers / rapidio / devices / rio_mport_cdev.c
1 /*
2 * RapidIO mport character device
3 *
4 * Copyright 2014-2015 Integrated Device Technology, Inc.
5 * Alexandre Bounine <alexandre.bounine@idt.com>
6 * Copyright 2014-2015 Prodrive Technologies
7 * Andre van Herk <andre.van.herk@prodrive-technologies.com>
8 * Jerry Jacobs <jerry.jacobs@prodrive-technologies.com>
9 * Copyright (C) 2014 Texas Instruments Incorporated
10 * Aurelien Jacquiot <a-jacquiot@ti.com>
11 *
12 * This program is free software; you can redistribute it and/or modify it
13 * under the terms of the GNU General Public License as published by the
14 * Free Software Foundation; either version 2 of the License, or (at your
15 * option) any later version.
16 */
17 #include <linux/module.h>
18 #include <linux/kernel.h>
19 #include <linux/cdev.h>
20 #include <linux/ioctl.h>
21 #include <linux/uaccess.h>
22 #include <linux/list.h>
23 #include <linux/fs.h>
24 #include <linux/err.h>
25 #include <linux/net.h>
26 #include <linux/poll.h>
27 #include <linux/spinlock.h>
28 #include <linux/sched.h>
29 #include <linux/kfifo.h>
30
31 #include <linux/mm.h>
32 #include <linux/slab.h>
33 #include <linux/vmalloc.h>
34 #include <linux/mman.h>
35
36 #include <linux/dma-mapping.h>
37 #ifdef CONFIG_RAPIDIO_DMA_ENGINE
38 #include <linux/dmaengine.h>
39 #endif
40
41 #include <linux/rio.h>
42 #include <linux/rio_ids.h>
43 #include <linux/rio_drv.h>
44 #include <linux/rio_mport_cdev.h>
45
46 #include "../rio.h"
47
48 #define DRV_NAME "rio_mport"
49 #define DRV_PREFIX DRV_NAME ": "
50 #define DEV_NAME "rio_mport"
51 #define DRV_VERSION "1.0.0"
52
53 /* Debug output filtering masks */
54 enum {
55 DBG_NONE = 0,
56 DBG_INIT = BIT(0), /* driver init */
57 DBG_EXIT = BIT(1), /* driver exit */
58 DBG_MPORT = BIT(2), /* mport add/remove */
59 DBG_RDEV = BIT(3), /* RapidIO device add/remove */
60 DBG_DMA = BIT(4), /* DMA transfer messages */
61 DBG_MMAP = BIT(5), /* mapping messages */
62 DBG_IBW = BIT(6), /* inbound window */
63 DBG_EVENT = BIT(7), /* event handling messages */
64 DBG_OBW = BIT(8), /* outbound window messages */
65 DBG_DBELL = BIT(9), /* doorbell messages */
66 DBG_ALL = ~0,
67 };
68
69 #ifdef DEBUG
70 #define rmcd_debug(level, fmt, arg...) \
71 do { \
72 if (DBG_##level & dbg_level) \
73 pr_debug(DRV_PREFIX "%s: " fmt "\n", __func__, ##arg); \
74 } while (0)
75 #else
76 #define rmcd_debug(level, fmt, arg...) \
77 no_printk(KERN_DEBUG pr_fmt(DRV_PREFIX fmt "\n"), ##arg)
78 #endif
79
80 #define rmcd_warn(fmt, arg...) \
81 pr_warn(DRV_PREFIX "%s WARNING " fmt "\n", __func__, ##arg)
82
83 #define rmcd_error(fmt, arg...) \
84 pr_err(DRV_PREFIX "%s ERROR " fmt "\n", __func__, ##arg)
85
86 MODULE_AUTHOR("Jerry Jacobs <jerry.jacobs@prodrive-technologies.com>");
87 MODULE_AUTHOR("Aurelien Jacquiot <a-jacquiot@ti.com>");
88 MODULE_AUTHOR("Alexandre Bounine <alexandre.bounine@idt.com>");
89 MODULE_AUTHOR("Andre van Herk <andre.van.herk@prodrive-technologies.com>");
90 MODULE_DESCRIPTION("RapidIO mport character device driver");
91 MODULE_LICENSE("GPL");
92 MODULE_VERSION(DRV_VERSION);
93
94 static int dma_timeout = 3000; /* DMA transfer timeout in msec */
95 module_param(dma_timeout, int, S_IRUGO);
96 MODULE_PARM_DESC(dma_timeout, "DMA Transfer Timeout in msec (default: 3000)");
97
98 #ifdef DEBUG
99 static u32 dbg_level = DBG_NONE;
100 module_param(dbg_level, uint, S_IWUSR | S_IWGRP | S_IRUGO);
101 MODULE_PARM_DESC(dbg_level, "Debugging output level (default 0 = none)");
102 #endif
103
104 /*
105 * An internal DMA coherent buffer
106 */
107 struct mport_dma_buf {
108 void *ib_base;
109 dma_addr_t ib_phys;
110 u32 ib_size;
111 u64 ib_rio_base;
112 bool ib_map;
113 struct file *filp;
114 };
115
116 /*
117 * Internal memory mapping structure
118 */
119 enum rio_mport_map_dir {
120 MAP_INBOUND,
121 MAP_OUTBOUND,
122 MAP_DMA,
123 };
124
125 struct rio_mport_mapping {
126 struct list_head node;
127 struct mport_dev *md;
128 enum rio_mport_map_dir dir;
129 u32 rioid;
130 u64 rio_addr;
131 dma_addr_t phys_addr; /* for mmap */
132 void *virt_addr; /* kernel address, for dma_free_coherent */
133 u64 size;
134 struct kref ref; /* refcount of vmas sharing the mapping */
135 struct file *filp;
136 };
137
138 struct rio_mport_dma_map {
139 int valid;
140 uint64_t length;
141 void *vaddr;
142 dma_addr_t paddr;
143 };
144
145 #define MPORT_MAX_DMA_BUFS 16
146 #define MPORT_EVENT_DEPTH 10
147
148 /*
149 * mport_dev driver-specific structure that represents mport device
150 * @active mport device status flag
151 * @node list node to maintain list of registered mports
152 * @cdev character device
153 * @dev associated device object
154 * @mport associated subsystem's master port device object
155 * @buf_mutex lock for buffer handling
156 * @file_mutex - lock for open files list
157 * @file_list - list of open files on given mport
158 * @properties properties of this mport
159 * @portwrites queue of inbound portwrites
160 * @pw_lock lock for port write queue
161 * @mappings queue for memory mappings
162 * @dma_chan DMA channels associated with this device
163 * @dma_ref:
164 * @comp:
165 */
166 struct mport_dev {
167 atomic_t active;
168 struct list_head node;
169 struct cdev cdev;
170 struct device dev;
171 struct rio_mport *mport;
172 struct mutex buf_mutex;
173 struct mutex file_mutex;
174 struct list_head file_list;
175 struct rio_mport_properties properties;
176 struct list_head doorbells;
177 spinlock_t db_lock;
178 struct list_head portwrites;
179 spinlock_t pw_lock;
180 struct list_head mappings;
181 #ifdef CONFIG_RAPIDIO_DMA_ENGINE
182 struct dma_chan *dma_chan;
183 struct kref dma_ref;
184 struct completion comp;
185 #endif
186 };
187
188 /*
189 * mport_cdev_priv - data structure specific to individual file object
190 * associated with an open device
191 * @md master port character device object
192 * @async_queue - asynchronous notification queue
193 * @list - file objects tracking list
194 * @db_filters inbound doorbell filters for this descriptor
195 * @pw_filters portwrite filters for this descriptor
196 * @event_fifo event fifo for this descriptor
197 * @event_rx_wait wait queue for this descriptor
198 * @fifo_lock lock for event_fifo
199 * @event_mask event mask for this descriptor
200 * @dmach DMA engine channel allocated for specific file object
201 */
202 struct mport_cdev_priv {
203 struct mport_dev *md;
204 struct fasync_struct *async_queue;
205 struct list_head list;
206 struct list_head db_filters;
207 struct list_head pw_filters;
208 struct kfifo event_fifo;
209 wait_queue_head_t event_rx_wait;
210 spinlock_t fifo_lock;
211 unsigned int event_mask; /* RIO_DOORBELL, RIO_PORTWRITE */
212 #ifdef CONFIG_RAPIDIO_DMA_ENGINE
213 struct dma_chan *dmach;
214 struct list_head async_list;
215 struct list_head pend_list;
216 spinlock_t req_lock;
217 struct mutex dma_lock;
218 struct kref dma_ref;
219 struct completion comp;
220 #endif
221 };
222
223 /*
224 * rio_mport_pw_filter - structure to describe a portwrite filter
225 * md_node node in mport device's list
226 * priv_node node in private file object's list
227 * priv reference to private data
228 * filter actual portwrite filter
229 */
230 struct rio_mport_pw_filter {
231 struct list_head md_node;
232 struct list_head priv_node;
233 struct mport_cdev_priv *priv;
234 struct rio_pw_filter filter;
235 };
236
237 /*
238 * rio_mport_db_filter - structure to describe a doorbell filter
239 * @data_node reference to device node
240 * @priv_node node in private data
241 * @priv reference to private data
242 * @filter actual doorbell filter
243 */
244 struct rio_mport_db_filter {
245 struct list_head data_node;
246 struct list_head priv_node;
247 struct mport_cdev_priv *priv;
248 struct rio_doorbell_filter filter;
249 };
250
251 static LIST_HEAD(mport_devs);
252 static DEFINE_MUTEX(mport_devs_lock);
253
254 #if (0) /* used by commented out portion of poll function : FIXME */
255 static DECLARE_WAIT_QUEUE_HEAD(mport_cdev_wait);
256 #endif
257
258 static struct class *dev_class;
259 static dev_t dev_number;
260
261 static struct workqueue_struct *dma_wq;
262
263 static void mport_release_mapping(struct kref *ref);
264
265 static int rio_mport_maint_rd(struct mport_cdev_priv *priv, void __user *arg,
266 int local)
267 {
268 struct rio_mport *mport = priv->md->mport;
269 struct rio_mport_maint_io maint_io;
270 u32 *buffer;
271 u32 offset;
272 size_t length;
273 int ret, i;
274
275 if (unlikely(copy_from_user(&maint_io, arg, sizeof(maint_io))))
276 return -EFAULT;
277
278 if ((maint_io.offset % 4) ||
279 (maint_io.length == 0) || (maint_io.length % 4))
280 return -EINVAL;
281
282 buffer = vmalloc(maint_io.length);
283 if (buffer == NULL)
284 return -ENOMEM;
285 length = maint_io.length/sizeof(u32);
286 offset = maint_io.offset;
287
288 for (i = 0; i < length; i++) {
289 if (local)
290 ret = __rio_local_read_config_32(mport,
291 offset, &buffer[i]);
292 else
293 ret = rio_mport_read_config_32(mport, maint_io.rioid,
294 maint_io.hopcount, offset, &buffer[i]);
295 if (ret)
296 goto out;
297
298 offset += 4;
299 }
300
301 if (unlikely(copy_to_user(maint_io.buffer, buffer, maint_io.length)))
302 ret = -EFAULT;
303 out:
304 vfree(buffer);
305 return ret;
306 }
307
308 static int rio_mport_maint_wr(struct mport_cdev_priv *priv, void __user *arg,
309 int local)
310 {
311 struct rio_mport *mport = priv->md->mport;
312 struct rio_mport_maint_io maint_io;
313 u32 *buffer;
314 u32 offset;
315 size_t length;
316 int ret = -EINVAL, i;
317
318 if (unlikely(copy_from_user(&maint_io, arg, sizeof(maint_io))))
319 return -EFAULT;
320
321 if ((maint_io.offset % 4) ||
322 (maint_io.length == 0) || (maint_io.length % 4))
323 return -EINVAL;
324
325 buffer = vmalloc(maint_io.length);
326 if (buffer == NULL)
327 return -ENOMEM;
328 length = maint_io.length;
329
330 if (unlikely(copy_from_user(buffer, maint_io.buffer, length))) {
331 ret = -EFAULT;
332 goto out;
333 }
334
335 offset = maint_io.offset;
336 length /= sizeof(u32);
337
338 for (i = 0; i < length; i++) {
339 if (local)
340 ret = __rio_local_write_config_32(mport,
341 offset, buffer[i]);
342 else
343 ret = rio_mport_write_config_32(mport, maint_io.rioid,
344 maint_io.hopcount,
345 offset, buffer[i]);
346 if (ret)
347 goto out;
348
349 offset += 4;
350 }
351
352 out:
353 vfree(buffer);
354 return ret;
355 }
356
357
358 /*
359 * Inbound/outbound memory mapping functions
360 */
361 static int
362 rio_mport_create_outbound_mapping(struct mport_dev *md, struct file *filp,
363 u32 rioid, u64 raddr, u32 size,
364 dma_addr_t *paddr)
365 {
366 struct rio_mport *mport = md->mport;
367 struct rio_mport_mapping *map;
368 int ret;
369
370 rmcd_debug(OBW, "did=%d ra=0x%llx sz=0x%x", rioid, raddr, size);
371
372 map = kzalloc(sizeof(struct rio_mport_mapping), GFP_KERNEL);
373 if (map == NULL)
374 return -ENOMEM;
375
376 ret = rio_map_outb_region(mport, rioid, raddr, size, 0, paddr);
377 if (ret < 0)
378 goto err_map_outb;
379
380 map->dir = MAP_OUTBOUND;
381 map->rioid = rioid;
382 map->rio_addr = raddr;
383 map->size = size;
384 map->phys_addr = *paddr;
385 map->filp = filp;
386 map->md = md;
387 kref_init(&map->ref);
388 list_add_tail(&map->node, &md->mappings);
389 return 0;
390 err_map_outb:
391 kfree(map);
392 return ret;
393 }
394
395 static int
396 rio_mport_get_outbound_mapping(struct mport_dev *md, struct file *filp,
397 u32 rioid, u64 raddr, u32 size,
398 dma_addr_t *paddr)
399 {
400 struct rio_mport_mapping *map;
401 int err = -ENOMEM;
402
403 mutex_lock(&md->buf_mutex);
404 list_for_each_entry(map, &md->mappings, node) {
405 if (map->dir != MAP_OUTBOUND)
406 continue;
407 if (rioid == map->rioid &&
408 raddr == map->rio_addr && size == map->size) {
409 *paddr = map->phys_addr;
410 err = 0;
411 break;
412 } else if (rioid == map->rioid &&
413 raddr < (map->rio_addr + map->size - 1) &&
414 (raddr + size) > map->rio_addr) {
415 err = -EBUSY;
416 break;
417 }
418 }
419
420 /* If not found, create new */
421 if (err == -ENOMEM)
422 err = rio_mport_create_outbound_mapping(md, filp, rioid, raddr,
423 size, paddr);
424 mutex_unlock(&md->buf_mutex);
425 return err;
426 }
427
428 static int rio_mport_obw_map(struct file *filp, void __user *arg)
429 {
430 struct mport_cdev_priv *priv = filp->private_data;
431 struct mport_dev *data = priv->md;
432 struct rio_mmap map;
433 dma_addr_t paddr;
434 int ret;
435
436 if (unlikely(copy_from_user(&map, arg, sizeof(struct rio_mmap))))
437 return -EFAULT;
438
439 rmcd_debug(OBW, "did=%d ra=0x%llx sz=0x%llx",
440 map.rioid, map.rio_addr, map.length);
441
442 ret = rio_mport_get_outbound_mapping(data, filp, map.rioid,
443 map.rio_addr, map.length, &paddr);
444 if (ret < 0) {
445 rmcd_error("Failed to set OBW err= %d", ret);
446 return ret;
447 }
448
449 map.handle = paddr;
450
451 if (unlikely(copy_to_user(arg, &map, sizeof(struct rio_mmap))))
452 return -EFAULT;
453 return 0;
454 }
455
456 /*
457 * rio_mport_obw_free() - unmap an OutBound Window from RapidIO address space
458 *
459 * @priv: driver private data
460 * @arg: buffer handle returned by allocation routine
461 */
462 static int rio_mport_obw_free(struct file *filp, void __user *arg)
463 {
464 struct mport_cdev_priv *priv = filp->private_data;
465 struct mport_dev *md = priv->md;
466 u64 handle;
467 struct rio_mport_mapping *map, *_map;
468
469 if (!md->mport->ops->unmap_outb)
470 return -EPROTONOSUPPORT;
471
472 if (copy_from_user(&handle, arg, sizeof(u64)))
473 return -EFAULT;
474
475 rmcd_debug(OBW, "h=0x%llx", handle);
476
477 mutex_lock(&md->buf_mutex);
478 list_for_each_entry_safe(map, _map, &md->mappings, node) {
479 if (map->dir == MAP_OUTBOUND && map->phys_addr == handle) {
480 if (map->filp == filp) {
481 rmcd_debug(OBW, "kref_put h=0x%llx", handle);
482 map->filp = NULL;
483 kref_put(&map->ref, mport_release_mapping);
484 }
485 break;
486 }
487 }
488 mutex_unlock(&md->buf_mutex);
489
490 return 0;
491 }
492
493 /*
494 * maint_hdid_set() - Set the host Device ID
495 * @priv: driver private data
496 * @arg: Device Id
497 */
498 static int maint_hdid_set(struct mport_cdev_priv *priv, void __user *arg)
499 {
500 struct mport_dev *md = priv->md;
501 uint16_t hdid;
502
503 if (copy_from_user(&hdid, arg, sizeof(uint16_t)))
504 return -EFAULT;
505
506 md->mport->host_deviceid = hdid;
507 md->properties.hdid = hdid;
508 rio_local_set_device_id(md->mport, hdid);
509
510 rmcd_debug(MPORT, "Set host device Id to %d", hdid);
511
512 return 0;
513 }
514
515 /*
516 * maint_comptag_set() - Set the host Component Tag
517 * @priv: driver private data
518 * @arg: Component Tag
519 */
520 static int maint_comptag_set(struct mport_cdev_priv *priv, void __user *arg)
521 {
522 struct mport_dev *md = priv->md;
523 uint32_t comptag;
524
525 if (copy_from_user(&comptag, arg, sizeof(uint32_t)))
526 return -EFAULT;
527
528 rio_local_write_config_32(md->mport, RIO_COMPONENT_TAG_CSR, comptag);
529
530 rmcd_debug(MPORT, "Set host Component Tag to %d", comptag);
531
532 return 0;
533 }
534
535 #ifdef CONFIG_RAPIDIO_DMA_ENGINE
536
537 struct mport_dma_req {
538 struct list_head node;
539 struct file *filp;
540 struct mport_cdev_priv *priv;
541 enum rio_transfer_sync sync;
542 struct sg_table sgt;
543 struct page **page_list;
544 unsigned int nr_pages;
545 struct rio_mport_mapping *map;
546 struct dma_chan *dmach;
547 enum dma_data_direction dir;
548 dma_cookie_t cookie;
549 enum dma_status status;
550 struct completion req_comp;
551 };
552
553 struct mport_faf_work {
554 struct work_struct work;
555 struct mport_dma_req *req;
556 };
557
558 static void mport_release_def_dma(struct kref *dma_ref)
559 {
560 struct mport_dev *md =
561 container_of(dma_ref, struct mport_dev, dma_ref);
562
563 rmcd_debug(EXIT, "DMA_%d", md->dma_chan->chan_id);
564 rio_release_dma(md->dma_chan);
565 md->dma_chan = NULL;
566 }
567
568 static void mport_release_dma(struct kref *dma_ref)
569 {
570 struct mport_cdev_priv *priv =
571 container_of(dma_ref, struct mport_cdev_priv, dma_ref);
572
573 rmcd_debug(EXIT, "DMA_%d", priv->dmach->chan_id);
574 complete(&priv->comp);
575 }
576
577 static void dma_req_free(struct mport_dma_req *req)
578 {
579 struct mport_cdev_priv *priv = req->priv;
580 unsigned int i;
581
582 dma_unmap_sg(req->dmach->device->dev,
583 req->sgt.sgl, req->sgt.nents, req->dir);
584 sg_free_table(&req->sgt);
585 if (req->page_list) {
586 for (i = 0; i < req->nr_pages; i++)
587 put_page(req->page_list[i]);
588 kfree(req->page_list);
589 }
590
591 if (req->map) {
592 mutex_lock(&req->map->md->buf_mutex);
593 kref_put(&req->map->ref, mport_release_mapping);
594 mutex_unlock(&req->map->md->buf_mutex);
595 }
596
597 kref_put(&priv->dma_ref, mport_release_dma);
598
599 kfree(req);
600 }
601
602 static void dma_xfer_callback(void *param)
603 {
604 struct mport_dma_req *req = (struct mport_dma_req *)param;
605 struct mport_cdev_priv *priv = req->priv;
606
607 req->status = dma_async_is_tx_complete(priv->dmach, req->cookie,
608 NULL, NULL);
609 complete(&req->req_comp);
610 }
611
612 static void dma_faf_cleanup(struct work_struct *_work)
613 {
614 struct mport_faf_work *work = container_of(_work,
615 struct mport_faf_work, work);
616 struct mport_dma_req *req = work->req;
617
618 dma_req_free(req);
619 kfree(work);
620 }
621
622 static void dma_faf_callback(void *param)
623 {
624 struct mport_dma_req *req = (struct mport_dma_req *)param;
625 struct mport_faf_work *work;
626
627 work = kmalloc(sizeof(*work), GFP_ATOMIC);
628 if (!work)
629 return;
630
631 INIT_WORK(&work->work, dma_faf_cleanup);
632 work->req = req;
633 queue_work(dma_wq, &work->work);
634 }
635
636 /*
637 * prep_dma_xfer() - Configure and send request to DMAengine to prepare DMA
638 * transfer object.
639 * Returns pointer to DMA transaction descriptor allocated by DMA driver on
640 * success or ERR_PTR (and/or NULL) if failed. Caller must check returned
641 * non-NULL pointer using IS_ERR macro.
642 */
643 static struct dma_async_tx_descriptor
644 *prep_dma_xfer(struct dma_chan *chan, struct rio_transfer_io *transfer,
645 struct sg_table *sgt, int nents, enum dma_transfer_direction dir,
646 enum dma_ctrl_flags flags)
647 {
648 struct rio_dma_data tx_data;
649
650 tx_data.sg = sgt->sgl;
651 tx_data.sg_len = nents;
652 tx_data.rio_addr_u = 0;
653 tx_data.rio_addr = transfer->rio_addr;
654 if (dir == DMA_MEM_TO_DEV) {
655 switch (transfer->method) {
656 case RIO_EXCHANGE_NWRITE:
657 tx_data.wr_type = RDW_ALL_NWRITE;
658 break;
659 case RIO_EXCHANGE_NWRITE_R_ALL:
660 tx_data.wr_type = RDW_ALL_NWRITE_R;
661 break;
662 case RIO_EXCHANGE_NWRITE_R:
663 tx_data.wr_type = RDW_LAST_NWRITE_R;
664 break;
665 case RIO_EXCHANGE_DEFAULT:
666 tx_data.wr_type = RDW_DEFAULT;
667 break;
668 default:
669 return ERR_PTR(-EINVAL);
670 }
671 }
672
673 return rio_dma_prep_xfer(chan, transfer->rioid, &tx_data, dir, flags);
674 }
675
676 /* Request DMA channel associated with this mport device.
677 * Try to request DMA channel for every new process that opened given
678 * mport. If a new DMA channel is not available use default channel
679 * which is the first DMA channel opened on mport device.
680 */
681 static int get_dma_channel(struct mport_cdev_priv *priv)
682 {
683 mutex_lock(&priv->dma_lock);
684 if (!priv->dmach) {
685 priv->dmach = rio_request_mport_dma(priv->md->mport);
686 if (!priv->dmach) {
687 /* Use default DMA channel if available */
688 if (priv->md->dma_chan) {
689 priv->dmach = priv->md->dma_chan;
690 kref_get(&priv->md->dma_ref);
691 } else {
692 rmcd_error("Failed to get DMA channel");
693 mutex_unlock(&priv->dma_lock);
694 return -ENODEV;
695 }
696 } else if (!priv->md->dma_chan) {
697 /* Register default DMA channel if we do not have one */
698 priv->md->dma_chan = priv->dmach;
699 kref_init(&priv->md->dma_ref);
700 rmcd_debug(DMA, "Register DMA_chan %d as default",
701 priv->dmach->chan_id);
702 }
703
704 kref_init(&priv->dma_ref);
705 init_completion(&priv->comp);
706 }
707
708 kref_get(&priv->dma_ref);
709 mutex_unlock(&priv->dma_lock);
710 return 0;
711 }
712
713 static void put_dma_channel(struct mport_cdev_priv *priv)
714 {
715 kref_put(&priv->dma_ref, mport_release_dma);
716 }
717
718 /*
719 * DMA transfer functions
720 */
721 static int do_dma_request(struct mport_dma_req *req,
722 struct rio_transfer_io *xfer,
723 enum rio_transfer_sync sync, int nents)
724 {
725 struct mport_cdev_priv *priv;
726 struct sg_table *sgt;
727 struct dma_chan *chan;
728 struct dma_async_tx_descriptor *tx;
729 dma_cookie_t cookie;
730 unsigned long tmo = msecs_to_jiffies(dma_timeout);
731 enum dma_transfer_direction dir;
732 long wret;
733 int ret = 0;
734
735 priv = req->priv;
736 sgt = &req->sgt;
737
738 chan = priv->dmach;
739 dir = (req->dir == DMA_FROM_DEVICE) ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV;
740
741 rmcd_debug(DMA, "%s(%d) uses %s for DMA_%s",
742 current->comm, task_pid_nr(current),
743 dev_name(&chan->dev->device),
744 (dir == DMA_DEV_TO_MEM)?"READ":"WRITE");
745
746 /* Initialize DMA transaction request */
747 tx = prep_dma_xfer(chan, xfer, sgt, nents, dir,
748 DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
749
750 if (!tx) {
751 rmcd_debug(DMA, "prep error for %s A:0x%llx L:0x%llx",
752 (dir == DMA_DEV_TO_MEM)?"READ":"WRITE",
753 xfer->rio_addr, xfer->length);
754 ret = -EIO;
755 goto err_out;
756 } else if (IS_ERR(tx)) {
757 ret = PTR_ERR(tx);
758 rmcd_debug(DMA, "prep error %d for %s A:0x%llx L:0x%llx", ret,
759 (dir == DMA_DEV_TO_MEM)?"READ":"WRITE",
760 xfer->rio_addr, xfer->length);
761 goto err_out;
762 }
763
764 if (sync == RIO_TRANSFER_FAF)
765 tx->callback = dma_faf_callback;
766 else
767 tx->callback = dma_xfer_callback;
768 tx->callback_param = req;
769
770 req->dmach = chan;
771 req->sync = sync;
772 req->status = DMA_IN_PROGRESS;
773 init_completion(&req->req_comp);
774
775 cookie = dmaengine_submit(tx);
776 req->cookie = cookie;
777
778 rmcd_debug(DMA, "pid=%d DMA_%s tx_cookie = %d", task_pid_nr(current),
779 (dir == DMA_DEV_TO_MEM)?"READ":"WRITE", cookie);
780
781 if (dma_submit_error(cookie)) {
782 rmcd_error("submit err=%d (addr:0x%llx len:0x%llx)",
783 cookie, xfer->rio_addr, xfer->length);
784 ret = -EIO;
785 goto err_out;
786 }
787
788 dma_async_issue_pending(chan);
789
790 if (sync == RIO_TRANSFER_ASYNC) {
791 spin_lock(&priv->req_lock);
792 list_add_tail(&req->node, &priv->async_list);
793 spin_unlock(&priv->req_lock);
794 return cookie;
795 } else if (sync == RIO_TRANSFER_FAF)
796 return 0;
797
798 wret = wait_for_completion_interruptible_timeout(&req->req_comp, tmo);
799
800 if (wret == 0) {
801 /* Timeout on wait occurred */
802 rmcd_error("%s(%d) timed out waiting for DMA_%s %d",
803 current->comm, task_pid_nr(current),
804 (dir == DMA_DEV_TO_MEM)?"READ":"WRITE", cookie);
805 return -ETIMEDOUT;
806 } else if (wret == -ERESTARTSYS) {
807 /* Wait_for_completion was interrupted by a signal but DMA may
808 * be in progress
809 */
810 rmcd_error("%s(%d) wait for DMA_%s %d was interrupted",
811 current->comm, task_pid_nr(current),
812 (dir == DMA_DEV_TO_MEM)?"READ":"WRITE", cookie);
813 return -EINTR;
814 }
815
816 if (req->status != DMA_COMPLETE) {
817 /* DMA transaction completion was signaled with error */
818 rmcd_error("%s(%d) DMA_%s %d completed with status %d (ret=%d)",
819 current->comm, task_pid_nr(current),
820 (dir == DMA_DEV_TO_MEM)?"READ":"WRITE",
821 cookie, req->status, ret);
822 ret = -EIO;
823 }
824
825 err_out:
826 return ret;
827 }
828
829 /*
830 * rio_dma_transfer() - Perform RapidIO DMA data transfer to/from
831 * the remote RapidIO device
832 * @filp: file pointer associated with the call
833 * @transfer_mode: DMA transfer mode
834 * @sync: synchronization mode
835 * @dir: DMA transfer direction (DMA_MEM_TO_DEV = write OR
836 * DMA_DEV_TO_MEM = read)
837 * @xfer: data transfer descriptor structure
838 */
839 static int
840 rio_dma_transfer(struct file *filp, uint32_t transfer_mode,
841 enum rio_transfer_sync sync, enum dma_data_direction dir,
842 struct rio_transfer_io *xfer)
843 {
844 struct mport_cdev_priv *priv = filp->private_data;
845 unsigned long nr_pages = 0;
846 struct page **page_list = NULL;
847 struct mport_dma_req *req;
848 struct mport_dev *md = priv->md;
849 struct dma_chan *chan;
850 int i, ret;
851 int nents;
852
853 if (xfer->length == 0)
854 return -EINVAL;
855 req = kzalloc(sizeof(*req), GFP_KERNEL);
856 if (!req)
857 return -ENOMEM;
858
859 ret = get_dma_channel(priv);
860 if (ret) {
861 kfree(req);
862 return ret;
863 }
864
865 /*
866 * If parameter loc_addr != NULL, we are transferring data from/to
867 * data buffer allocated in user-space: lock in memory user-space
868 * buffer pages and build an SG table for DMA transfer request
869 *
870 * Otherwise (loc_addr == NULL) contiguous kernel-space buffer is
871 * used for DMA data transfers: build single entry SG table using
872 * offset within the internal buffer specified by handle parameter.
873 */
874 if (xfer->loc_addr) {
875 unsigned long offset;
876 long pinned;
877
878 offset = (unsigned long)xfer->loc_addr & ~PAGE_MASK;
879 nr_pages = PAGE_ALIGN(xfer->length + offset) >> PAGE_SHIFT;
880
881 page_list = kmalloc_array(nr_pages,
882 sizeof(*page_list), GFP_KERNEL);
883 if (page_list == NULL) {
884 ret = -ENOMEM;
885 goto err_req;
886 }
887
888 down_read(&current->mm->mmap_sem);
889 pinned = get_user_pages(
890 (unsigned long)xfer->loc_addr & PAGE_MASK,
891 nr_pages, dir == DMA_FROM_DEVICE, 0,
892 page_list, NULL);
893 up_read(&current->mm->mmap_sem);
894
895 if (pinned != nr_pages) {
896 if (pinned < 0) {
897 rmcd_error("get_user_pages err=%ld", pinned);
898 nr_pages = 0;
899 } else
900 rmcd_error("pinned %ld out of %ld pages",
901 pinned, nr_pages);
902 ret = -EFAULT;
903 goto err_pg;
904 }
905
906 ret = sg_alloc_table_from_pages(&req->sgt, page_list, nr_pages,
907 offset, xfer->length, GFP_KERNEL);
908 if (ret) {
909 rmcd_error("sg_alloc_table failed with err=%d", ret);
910 goto err_pg;
911 }
912
913 req->page_list = page_list;
914 req->nr_pages = nr_pages;
915 } else {
916 dma_addr_t baddr;
917 struct rio_mport_mapping *map;
918
919 baddr = (dma_addr_t)xfer->handle;
920
921 mutex_lock(&md->buf_mutex);
922 list_for_each_entry(map, &md->mappings, node) {
923 if (baddr >= map->phys_addr &&
924 baddr < (map->phys_addr + map->size)) {
925 kref_get(&map->ref);
926 req->map = map;
927 break;
928 }
929 }
930 mutex_unlock(&md->buf_mutex);
931
932 if (req->map == NULL) {
933 ret = -ENOMEM;
934 goto err_req;
935 }
936
937 if (xfer->length + xfer->offset > map->size) {
938 ret = -EINVAL;
939 goto err_req;
940 }
941
942 ret = sg_alloc_table(&req->sgt, 1, GFP_KERNEL);
943 if (unlikely(ret)) {
944 rmcd_error("sg_alloc_table failed for internal buf");
945 goto err_req;
946 }
947
948 sg_set_buf(req->sgt.sgl,
949 map->virt_addr + (baddr - map->phys_addr) +
950 xfer->offset, xfer->length);
951 }
952
953 req->dir = dir;
954 req->filp = filp;
955 req->priv = priv;
956 chan = priv->dmach;
957
958 nents = dma_map_sg(chan->device->dev,
959 req->sgt.sgl, req->sgt.nents, dir);
960 if (nents == -EFAULT) {
961 rmcd_error("Failed to map SG list");
962 return -EFAULT;
963 }
964
965 ret = do_dma_request(req, xfer, sync, nents);
966
967 if (ret >= 0) {
968 if (sync == RIO_TRANSFER_SYNC)
969 goto sync_out;
970 return ret; /* return ASYNC cookie */
971 }
972
973 if (ret == -ETIMEDOUT || ret == -EINTR) {
974 /*
975 * This can happen only in case of SYNC transfer.
976 * Do not free unfinished request structure immediately.
977 * Place it into pending list and deal with it later
978 */
979 spin_lock(&priv->req_lock);
980 list_add_tail(&req->node, &priv->pend_list);
981 spin_unlock(&priv->req_lock);
982 return ret;
983 }
984
985
986 rmcd_debug(DMA, "do_dma_request failed with err=%d", ret);
987 sync_out:
988 dma_unmap_sg(chan->device->dev, req->sgt.sgl, req->sgt.nents, dir);
989 sg_free_table(&req->sgt);
990 err_pg:
991 if (page_list) {
992 for (i = 0; i < nr_pages; i++)
993 put_page(page_list[i]);
994 kfree(page_list);
995 }
996 err_req:
997 if (req->map) {
998 mutex_lock(&md->buf_mutex);
999 kref_put(&req->map->ref, mport_release_mapping);
1000 mutex_unlock(&md->buf_mutex);
1001 }
1002 put_dma_channel(priv);
1003 kfree(req);
1004 return ret;
1005 }
1006
1007 static int rio_mport_transfer_ioctl(struct file *filp, void __user *arg)
1008 {
1009 struct mport_cdev_priv *priv = filp->private_data;
1010 struct rio_transaction transaction;
1011 struct rio_transfer_io *transfer;
1012 enum dma_data_direction dir;
1013 int i, ret = 0;
1014
1015 if (unlikely(copy_from_user(&transaction, arg, sizeof(transaction))))
1016 return -EFAULT;
1017
1018 if (transaction.count != 1)
1019 return -EINVAL;
1020
1021 if ((transaction.transfer_mode &
1022 priv->md->properties.transfer_mode) == 0)
1023 return -ENODEV;
1024
1025 transfer = vmalloc(transaction.count * sizeof(struct rio_transfer_io));
1026 if (!transfer)
1027 return -ENOMEM;
1028
1029 if (unlikely(copy_from_user(transfer, transaction.block,
1030 transaction.count * sizeof(struct rio_transfer_io)))) {
1031 ret = -EFAULT;
1032 goto out_free;
1033 }
1034
1035 dir = (transaction.dir == RIO_TRANSFER_DIR_READ) ?
1036 DMA_FROM_DEVICE : DMA_TO_DEVICE;
1037 for (i = 0; i < transaction.count && ret == 0; i++)
1038 ret = rio_dma_transfer(filp, transaction.transfer_mode,
1039 transaction.sync, dir, &transfer[i]);
1040
1041 if (unlikely(copy_to_user(transaction.block, transfer,
1042 transaction.count * sizeof(struct rio_transfer_io))))
1043 ret = -EFAULT;
1044
1045 out_free:
1046 vfree(transfer);
1047
1048 return ret;
1049 }
1050
1051 static int rio_mport_wait_for_async_dma(struct file *filp, void __user *arg)
1052 {
1053 struct mport_cdev_priv *priv;
1054 struct mport_dev *md;
1055 struct rio_async_tx_wait w_param;
1056 struct mport_dma_req *req;
1057 dma_cookie_t cookie;
1058 unsigned long tmo;
1059 long wret;
1060 int found = 0;
1061 int ret;
1062
1063 priv = (struct mport_cdev_priv *)filp->private_data;
1064 md = priv->md;
1065
1066 if (unlikely(copy_from_user(&w_param, arg, sizeof(w_param))))
1067 return -EFAULT;
1068
1069 cookie = w_param.token;
1070 if (w_param.timeout)
1071 tmo = msecs_to_jiffies(w_param.timeout);
1072 else /* Use default DMA timeout */
1073 tmo = msecs_to_jiffies(dma_timeout);
1074
1075 spin_lock(&priv->req_lock);
1076 list_for_each_entry(req, &priv->async_list, node) {
1077 if (req->cookie == cookie) {
1078 list_del(&req->node);
1079 found = 1;
1080 break;
1081 }
1082 }
1083 spin_unlock(&priv->req_lock);
1084
1085 if (!found)
1086 return -EAGAIN;
1087
1088 wret = wait_for_completion_interruptible_timeout(&req->req_comp, tmo);
1089
1090 if (wret == 0) {
1091 /* Timeout on wait occurred */
1092 rmcd_error("%s(%d) timed out waiting for ASYNC DMA_%s",
1093 current->comm, task_pid_nr(current),
1094 (req->dir == DMA_FROM_DEVICE)?"READ":"WRITE");
1095 ret = -ETIMEDOUT;
1096 goto err_tmo;
1097 } else if (wret == -ERESTARTSYS) {
1098 /* Wait_for_completion was interrupted by a signal but DMA may
1099 * be still in progress
1100 */
1101 rmcd_error("%s(%d) wait for ASYNC DMA_%s was interrupted",
1102 current->comm, task_pid_nr(current),
1103 (req->dir == DMA_FROM_DEVICE)?"READ":"WRITE");
1104 ret = -EINTR;
1105 goto err_tmo;
1106 }
1107
1108 if (req->status != DMA_COMPLETE) {
1109 /* DMA transaction completion signaled with transfer error */
1110 rmcd_error("%s(%d) ASYNC DMA_%s completion with status %d",
1111 current->comm, task_pid_nr(current),
1112 (req->dir == DMA_FROM_DEVICE)?"READ":"WRITE",
1113 req->status);
1114 ret = -EIO;
1115 } else
1116 ret = 0;
1117
1118 if (req->status != DMA_IN_PROGRESS && req->status != DMA_PAUSED)
1119 dma_req_free(req);
1120
1121 return ret;
1122
1123 err_tmo:
1124 /* Return request back into async queue */
1125 spin_lock(&priv->req_lock);
1126 list_add_tail(&req->node, &priv->async_list);
1127 spin_unlock(&priv->req_lock);
1128 return ret;
1129 }
1130
1131 static int rio_mport_create_dma_mapping(struct mport_dev *md, struct file *filp,
1132 uint64_t size, struct rio_mport_mapping **mapping)
1133 {
1134 struct rio_mport_mapping *map;
1135
1136 map = kzalloc(sizeof(struct rio_mport_mapping), GFP_KERNEL);
1137 if (map == NULL)
1138 return -ENOMEM;
1139
1140 map->virt_addr = dma_alloc_coherent(md->mport->dev.parent, size,
1141 &map->phys_addr, GFP_KERNEL);
1142 if (map->virt_addr == NULL) {
1143 kfree(map);
1144 return -ENOMEM;
1145 }
1146
1147 map->dir = MAP_DMA;
1148 map->size = size;
1149 map->filp = filp;
1150 map->md = md;
1151 kref_init(&map->ref);
1152 mutex_lock(&md->buf_mutex);
1153 list_add_tail(&map->node, &md->mappings);
1154 mutex_unlock(&md->buf_mutex);
1155 *mapping = map;
1156
1157 return 0;
1158 }
1159
1160 static int rio_mport_alloc_dma(struct file *filp, void __user *arg)
1161 {
1162 struct mport_cdev_priv *priv = filp->private_data;
1163 struct mport_dev *md = priv->md;
1164 struct rio_dma_mem map;
1165 struct rio_mport_mapping *mapping = NULL;
1166 int ret;
1167
1168 if (unlikely(copy_from_user(&map, arg, sizeof(struct rio_dma_mem))))
1169 return -EFAULT;
1170
1171 ret = rio_mport_create_dma_mapping(md, filp, map.length, &mapping);
1172 if (ret)
1173 return ret;
1174
1175 map.dma_handle = mapping->phys_addr;
1176
1177 if (unlikely(copy_to_user(arg, &map, sizeof(struct rio_dma_mem)))) {
1178 mutex_lock(&md->buf_mutex);
1179 kref_put(&mapping->ref, mport_release_mapping);
1180 mutex_unlock(&md->buf_mutex);
1181 return -EFAULT;
1182 }
1183
1184 return 0;
1185 }
1186
1187 static int rio_mport_free_dma(struct file *filp, void __user *arg)
1188 {
1189 struct mport_cdev_priv *priv = filp->private_data;
1190 struct mport_dev *md = priv->md;
1191 u64 handle;
1192 int ret = -EFAULT;
1193 struct rio_mport_mapping *map, *_map;
1194
1195 if (copy_from_user(&handle, arg, sizeof(u64)))
1196 return -EFAULT;
1197 rmcd_debug(EXIT, "filp=%p", filp);
1198
1199 mutex_lock(&md->buf_mutex);
1200 list_for_each_entry_safe(map, _map, &md->mappings, node) {
1201 if (map->dir == MAP_DMA && map->phys_addr == handle &&
1202 map->filp == filp) {
1203 kref_put(&map->ref, mport_release_mapping);
1204 ret = 0;
1205 break;
1206 }
1207 }
1208 mutex_unlock(&md->buf_mutex);
1209
1210 if (ret == -EFAULT) {
1211 rmcd_debug(DMA, "ERR no matching mapping");
1212 return ret;
1213 }
1214
1215 return 0;
1216 }
1217 #else
1218 static int rio_mport_transfer_ioctl(struct file *filp, void *arg)
1219 {
1220 return -ENODEV;
1221 }
1222
1223 static int rio_mport_wait_for_async_dma(struct file *filp, void __user *arg)
1224 {
1225 return -ENODEV;
1226 }
1227
1228 static int rio_mport_alloc_dma(struct file *filp, void __user *arg)
1229 {
1230 return -ENODEV;
1231 }
1232
1233 static int rio_mport_free_dma(struct file *filp, void __user *arg)
1234 {
1235 return -ENODEV;
1236 }
1237 #endif /* CONFIG_RAPIDIO_DMA_ENGINE */
1238
1239 /*
1240 * Inbound/outbound memory mapping functions
1241 */
1242
1243 static int
1244 rio_mport_create_inbound_mapping(struct mport_dev *md, struct file *filp,
1245 u64 raddr, u32 size,
1246 struct rio_mport_mapping **mapping)
1247 {
1248 struct rio_mport *mport = md->mport;
1249 struct rio_mport_mapping *map;
1250 int ret;
1251
1252 map = kzalloc(sizeof(struct rio_mport_mapping), GFP_KERNEL);
1253 if (map == NULL)
1254 return -ENOMEM;
1255
1256 map->virt_addr = dma_alloc_coherent(mport->dev.parent, size,
1257 &map->phys_addr, GFP_KERNEL);
1258 if (map->virt_addr == NULL) {
1259 ret = -ENOMEM;
1260 goto err_dma_alloc;
1261 }
1262
1263 if (raddr == RIO_MAP_ANY_ADDR)
1264 raddr = map->phys_addr;
1265 ret = rio_map_inb_region(mport, map->phys_addr, raddr, size, 0);
1266 if (ret < 0)
1267 goto err_map_inb;
1268
1269 map->dir = MAP_INBOUND;
1270 map->rio_addr = raddr;
1271 map->size = size;
1272 map->filp = filp;
1273 map->md = md;
1274 kref_init(&map->ref);
1275 mutex_lock(&md->buf_mutex);
1276 list_add_tail(&map->node, &md->mappings);
1277 mutex_unlock(&md->buf_mutex);
1278 *mapping = map;
1279 return 0;
1280
1281 err_map_inb:
1282 dma_free_coherent(mport->dev.parent, size,
1283 map->virt_addr, map->phys_addr);
1284 err_dma_alloc:
1285 kfree(map);
1286 return ret;
1287 }
1288
1289 static int
1290 rio_mport_get_inbound_mapping(struct mport_dev *md, struct file *filp,
1291 u64 raddr, u32 size,
1292 struct rio_mport_mapping **mapping)
1293 {
1294 struct rio_mport_mapping *map;
1295 int err = -ENOMEM;
1296
1297 if (raddr == RIO_MAP_ANY_ADDR)
1298 goto get_new;
1299
1300 mutex_lock(&md->buf_mutex);
1301 list_for_each_entry(map, &md->mappings, node) {
1302 if (map->dir != MAP_INBOUND)
1303 continue;
1304 if (raddr == map->rio_addr && size == map->size) {
1305 /* allow exact match only */
1306 *mapping = map;
1307 err = 0;
1308 break;
1309 } else if (raddr < (map->rio_addr + map->size - 1) &&
1310 (raddr + size) > map->rio_addr) {
1311 err = -EBUSY;
1312 break;
1313 }
1314 }
1315 mutex_unlock(&md->buf_mutex);
1316
1317 if (err != -ENOMEM)
1318 return err;
1319 get_new:
1320 /* not found, create new */
1321 return rio_mport_create_inbound_mapping(md, filp, raddr, size, mapping);
1322 }
1323
1324 static int rio_mport_map_inbound(struct file *filp, void __user *arg)
1325 {
1326 struct mport_cdev_priv *priv = filp->private_data;
1327 struct mport_dev *md = priv->md;
1328 struct rio_mmap map;
1329 struct rio_mport_mapping *mapping = NULL;
1330 int ret;
1331
1332 if (!md->mport->ops->map_inb)
1333 return -EPROTONOSUPPORT;
1334 if (unlikely(copy_from_user(&map, arg, sizeof(struct rio_mmap))))
1335 return -EFAULT;
1336
1337 rmcd_debug(IBW, "%s filp=%p", dev_name(&priv->md->dev), filp);
1338
1339 ret = rio_mport_get_inbound_mapping(md, filp, map.rio_addr,
1340 map.length, &mapping);
1341 if (ret)
1342 return ret;
1343
1344 map.handle = mapping->phys_addr;
1345 map.rio_addr = mapping->rio_addr;
1346
1347 if (unlikely(copy_to_user(arg, &map, sizeof(struct rio_mmap)))) {
1348 /* Delete mapping if it was created by this request */
1349 if (ret == 0 && mapping->filp == filp) {
1350 mutex_lock(&md->buf_mutex);
1351 kref_put(&mapping->ref, mport_release_mapping);
1352 mutex_unlock(&md->buf_mutex);
1353 }
1354 return -EFAULT;
1355 }
1356
1357 return 0;
1358 }
1359
1360 /*
1361 * rio_mport_inbound_free() - unmap from RapidIO address space and free
1362 * previously allocated inbound DMA coherent buffer
1363 * @priv: driver private data
1364 * @arg: buffer handle returned by allocation routine
1365 */
1366 static int rio_mport_inbound_free(struct file *filp, void __user *arg)
1367 {
1368 struct mport_cdev_priv *priv = filp->private_data;
1369 struct mport_dev *md = priv->md;
1370 u64 handle;
1371 struct rio_mport_mapping *map, *_map;
1372
1373 rmcd_debug(IBW, "%s filp=%p", dev_name(&priv->md->dev), filp);
1374
1375 if (!md->mport->ops->unmap_inb)
1376 return -EPROTONOSUPPORT;
1377
1378 if (copy_from_user(&handle, arg, sizeof(u64)))
1379 return -EFAULT;
1380
1381 mutex_lock(&md->buf_mutex);
1382 list_for_each_entry_safe(map, _map, &md->mappings, node) {
1383 if (map->dir == MAP_INBOUND && map->phys_addr == handle) {
1384 if (map->filp == filp) {
1385 map->filp = NULL;
1386 kref_put(&map->ref, mport_release_mapping);
1387 }
1388 break;
1389 }
1390 }
1391 mutex_unlock(&md->buf_mutex);
1392
1393 return 0;
1394 }
1395
1396 /*
1397 * maint_port_idx_get() - Get the port index of the mport instance
1398 * @priv: driver private data
1399 * @arg: port index
1400 */
1401 static int maint_port_idx_get(struct mport_cdev_priv *priv, void __user *arg)
1402 {
1403 struct mport_dev *md = priv->md;
1404 uint32_t port_idx = md->mport->index;
1405
1406 rmcd_debug(MPORT, "port_index=%d", port_idx);
1407
1408 if (copy_to_user(arg, &port_idx, sizeof(port_idx)))
1409 return -EFAULT;
1410
1411 return 0;
1412 }
1413
1414 static int rio_mport_add_event(struct mport_cdev_priv *priv,
1415 struct rio_event *event)
1416 {
1417 int overflow;
1418
1419 if (!(priv->event_mask & event->header))
1420 return -EACCES;
1421
1422 spin_lock(&priv->fifo_lock);
1423 overflow = kfifo_avail(&priv->event_fifo) < sizeof(*event)
1424 || kfifo_in(&priv->event_fifo, (unsigned char *)event,
1425 sizeof(*event)) != sizeof(*event);
1426 spin_unlock(&priv->fifo_lock);
1427
1428 wake_up_interruptible(&priv->event_rx_wait);
1429
1430 if (overflow) {
1431 dev_warn(&priv->md->dev, DRV_NAME ": event fifo overflow\n");
1432 return -EBUSY;
1433 }
1434
1435 return 0;
1436 }
1437
1438 static void rio_mport_doorbell_handler(struct rio_mport *mport, void *dev_id,
1439 u16 src, u16 dst, u16 info)
1440 {
1441 struct mport_dev *data = dev_id;
1442 struct mport_cdev_priv *priv;
1443 struct rio_mport_db_filter *db_filter;
1444 struct rio_event event;
1445 int handled;
1446
1447 event.header = RIO_DOORBELL;
1448 event.u.doorbell.rioid = src;
1449 event.u.doorbell.payload = info;
1450
1451 handled = 0;
1452 spin_lock(&data->db_lock);
1453 list_for_each_entry(db_filter, &data->doorbells, data_node) {
1454 if (((db_filter->filter.rioid == 0xffffffff ||
1455 db_filter->filter.rioid == src)) &&
1456 info >= db_filter->filter.low &&
1457 info <= db_filter->filter.high) {
1458 priv = db_filter->priv;
1459 rio_mport_add_event(priv, &event);
1460 handled = 1;
1461 }
1462 }
1463 spin_unlock(&data->db_lock);
1464
1465 if (!handled)
1466 dev_warn(&data->dev,
1467 "%s: spurious DB received from 0x%x, info=0x%04x\n",
1468 __func__, src, info);
1469 }
1470
1471 static int rio_mport_add_db_filter(struct mport_cdev_priv *priv,
1472 void __user *arg)
1473 {
1474 struct mport_dev *md = priv->md;
1475 struct rio_mport_db_filter *db_filter;
1476 struct rio_doorbell_filter filter;
1477 unsigned long flags;
1478 int ret;
1479
1480 if (copy_from_user(&filter, arg, sizeof(filter)))
1481 return -EFAULT;
1482
1483 if (filter.low > filter.high)
1484 return -EINVAL;
1485
1486 ret = rio_request_inb_dbell(md->mport, md, filter.low, filter.high,
1487 rio_mport_doorbell_handler);
1488 if (ret) {
1489 rmcd_error("%s failed to register IBDB, err=%d",
1490 dev_name(&md->dev), ret);
1491 return ret;
1492 }
1493
1494 db_filter = kzalloc(sizeof(*db_filter), GFP_KERNEL);
1495 if (db_filter == NULL) {
1496 rio_release_inb_dbell(md->mport, filter.low, filter.high);
1497 return -ENOMEM;
1498 }
1499
1500 db_filter->filter = filter;
1501 db_filter->priv = priv;
1502 spin_lock_irqsave(&md->db_lock, flags);
1503 list_add_tail(&db_filter->priv_node, &priv->db_filters);
1504 list_add_tail(&db_filter->data_node, &md->doorbells);
1505 spin_unlock_irqrestore(&md->db_lock, flags);
1506
1507 return 0;
1508 }
1509
1510 static void rio_mport_delete_db_filter(struct rio_mport_db_filter *db_filter)
1511 {
1512 list_del(&db_filter->data_node);
1513 list_del(&db_filter->priv_node);
1514 kfree(db_filter);
1515 }
1516
1517 static int rio_mport_remove_db_filter(struct mport_cdev_priv *priv,
1518 void __user *arg)
1519 {
1520 struct rio_mport_db_filter *db_filter;
1521 struct rio_doorbell_filter filter;
1522 unsigned long flags;
1523 int ret = -EINVAL;
1524
1525 if (copy_from_user(&filter, arg, sizeof(filter)))
1526 return -EFAULT;
1527
1528 spin_lock_irqsave(&priv->md->db_lock, flags);
1529 list_for_each_entry(db_filter, &priv->db_filters, priv_node) {
1530 if (db_filter->filter.rioid == filter.rioid &&
1531 db_filter->filter.low == filter.low &&
1532 db_filter->filter.high == filter.high) {
1533 rio_mport_delete_db_filter(db_filter);
1534 ret = 0;
1535 break;
1536 }
1537 }
1538 spin_unlock_irqrestore(&priv->md->db_lock, flags);
1539
1540 if (!ret)
1541 rio_release_inb_dbell(priv->md->mport, filter.low, filter.high);
1542
1543 return ret;
1544 }
1545
1546 static int rio_mport_match_pw(union rio_pw_msg *msg,
1547 struct rio_pw_filter *filter)
1548 {
1549 if ((msg->em.comptag & filter->mask) < filter->low ||
1550 (msg->em.comptag & filter->mask) > filter->high)
1551 return 0;
1552 return 1;
1553 }
1554
1555 static int rio_mport_pw_handler(struct rio_mport *mport, void *context,
1556 union rio_pw_msg *msg, int step)
1557 {
1558 struct mport_dev *md = context;
1559 struct mport_cdev_priv *priv;
1560 struct rio_mport_pw_filter *pw_filter;
1561 struct rio_event event;
1562 int handled;
1563
1564 event.header = RIO_PORTWRITE;
1565 memcpy(event.u.portwrite.payload, msg->raw, RIO_PW_MSG_SIZE);
1566
1567 handled = 0;
1568 spin_lock(&md->pw_lock);
1569 list_for_each_entry(pw_filter, &md->portwrites, md_node) {
1570 if (rio_mport_match_pw(msg, &pw_filter->filter)) {
1571 priv = pw_filter->priv;
1572 rio_mport_add_event(priv, &event);
1573 handled = 1;
1574 }
1575 }
1576 spin_unlock(&md->pw_lock);
1577
1578 if (!handled) {
1579 printk_ratelimited(KERN_WARNING DRV_NAME
1580 ": mport%d received spurious PW from 0x%08x\n",
1581 mport->id, msg->em.comptag);
1582 }
1583
1584 return 0;
1585 }
1586
1587 static int rio_mport_add_pw_filter(struct mport_cdev_priv *priv,
1588 void __user *arg)
1589 {
1590 struct mport_dev *md = priv->md;
1591 struct rio_mport_pw_filter *pw_filter;
1592 struct rio_pw_filter filter;
1593 unsigned long flags;
1594 int hadd = 0;
1595
1596 if (copy_from_user(&filter, arg, sizeof(filter)))
1597 return -EFAULT;
1598
1599 pw_filter = kzalloc(sizeof(*pw_filter), GFP_KERNEL);
1600 if (pw_filter == NULL)
1601 return -ENOMEM;
1602
1603 pw_filter->filter = filter;
1604 pw_filter->priv = priv;
1605 spin_lock_irqsave(&md->pw_lock, flags);
1606 if (list_empty(&md->portwrites))
1607 hadd = 1;
1608 list_add_tail(&pw_filter->priv_node, &priv->pw_filters);
1609 list_add_tail(&pw_filter->md_node, &md->portwrites);
1610 spin_unlock_irqrestore(&md->pw_lock, flags);
1611
1612 if (hadd) {
1613 int ret;
1614
1615 ret = rio_add_mport_pw_handler(md->mport, md,
1616 rio_mport_pw_handler);
1617 if (ret) {
1618 dev_err(&md->dev,
1619 "%s: failed to add IB_PW handler, err=%d\n",
1620 __func__, ret);
1621 return ret;
1622 }
1623 rio_pw_enable(md->mport, 1);
1624 }
1625
1626 return 0;
1627 }
1628
1629 static void rio_mport_delete_pw_filter(struct rio_mport_pw_filter *pw_filter)
1630 {
1631 list_del(&pw_filter->md_node);
1632 list_del(&pw_filter->priv_node);
1633 kfree(pw_filter);
1634 }
1635
1636 static int rio_mport_match_pw_filter(struct rio_pw_filter *a,
1637 struct rio_pw_filter *b)
1638 {
1639 if ((a->mask == b->mask) && (a->low == b->low) && (a->high == b->high))
1640 return 1;
1641 return 0;
1642 }
1643
1644 static int rio_mport_remove_pw_filter(struct mport_cdev_priv *priv,
1645 void __user *arg)
1646 {
1647 struct mport_dev *md = priv->md;
1648 struct rio_mport_pw_filter *pw_filter;
1649 struct rio_pw_filter filter;
1650 unsigned long flags;
1651 int ret = -EINVAL;
1652 int hdel = 0;
1653
1654 if (copy_from_user(&filter, arg, sizeof(filter)))
1655 return -EFAULT;
1656
1657 spin_lock_irqsave(&md->pw_lock, flags);
1658 list_for_each_entry(pw_filter, &priv->pw_filters, priv_node) {
1659 if (rio_mport_match_pw_filter(&pw_filter->filter, &filter)) {
1660 rio_mport_delete_pw_filter(pw_filter);
1661 ret = 0;
1662 break;
1663 }
1664 }
1665
1666 if (list_empty(&md->portwrites))
1667 hdel = 1;
1668 spin_unlock_irqrestore(&md->pw_lock, flags);
1669
1670 if (hdel) {
1671 rio_del_mport_pw_handler(md->mport, priv->md,
1672 rio_mport_pw_handler);
1673 rio_pw_enable(md->mport, 0);
1674 }
1675
1676 return ret;
1677 }
1678
1679 /*
1680 * rio_release_dev - release routine for kernel RIO device object
1681 * @dev: kernel device object associated with a RIO device structure
1682 *
1683 * Frees a RIO device struct associated a RIO device struct.
1684 * The RIO device struct is freed.
1685 */
1686 static void rio_release_dev(struct device *dev)
1687 {
1688 struct rio_dev *rdev;
1689
1690 rdev = to_rio_dev(dev);
1691 pr_info(DRV_PREFIX "%s: %s\n", __func__, rio_name(rdev));
1692 kfree(rdev);
1693 }
1694
1695
1696 static void rio_release_net(struct device *dev)
1697 {
1698 struct rio_net *net;
1699
1700 net = to_rio_net(dev);
1701 rmcd_debug(RDEV, "net_%d", net->id);
1702 kfree(net);
1703 }
1704
1705
1706 /*
1707 * rio_mport_add_riodev - creates a kernel RIO device object
1708 *
1709 * Allocates a RIO device data structure and initializes required fields based
1710 * on device's configuration space contents.
1711 * If the device has switch capabilities, then a switch specific portion is
1712 * allocated and configured.
1713 */
1714 static int rio_mport_add_riodev(struct mport_cdev_priv *priv,
1715 void __user *arg)
1716 {
1717 struct mport_dev *md = priv->md;
1718 struct rio_rdev_info dev_info;
1719 struct rio_dev *rdev;
1720 struct rio_switch *rswitch = NULL;
1721 struct rio_mport *mport;
1722 size_t size;
1723 u32 rval;
1724 u32 swpinfo = 0;
1725 u16 destid;
1726 u8 hopcount;
1727 int err;
1728
1729 if (copy_from_user(&dev_info, arg, sizeof(dev_info)))
1730 return -EFAULT;
1731
1732 rmcd_debug(RDEV, "name:%s ct:0x%x did:0x%x hc:0x%x", dev_info.name,
1733 dev_info.comptag, dev_info.destid, dev_info.hopcount);
1734
1735 if (bus_find_device_by_name(&rio_bus_type, NULL, dev_info.name)) {
1736 rmcd_debug(RDEV, "device %s already exists", dev_info.name);
1737 return -EEXIST;
1738 }
1739
1740 size = sizeof(struct rio_dev);
1741 mport = md->mport;
1742 destid = (u16)dev_info.destid;
1743 hopcount = (u8)dev_info.hopcount;
1744
1745 if (rio_mport_read_config_32(mport, destid, hopcount,
1746 RIO_PEF_CAR, &rval))
1747 return -EIO;
1748
1749 if (rval & RIO_PEF_SWITCH) {
1750 rio_mport_read_config_32(mport, destid, hopcount,
1751 RIO_SWP_INFO_CAR, &swpinfo);
1752 size += (RIO_GET_TOTAL_PORTS(swpinfo) *
1753 sizeof(rswitch->nextdev[0])) + sizeof(*rswitch);
1754 }
1755
1756 rdev = kzalloc(size, GFP_KERNEL);
1757 if (rdev == NULL)
1758 return -ENOMEM;
1759
1760 if (mport->net == NULL) {
1761 struct rio_net *net;
1762
1763 net = rio_alloc_net(mport);
1764 if (!net) {
1765 err = -ENOMEM;
1766 rmcd_debug(RDEV, "failed to allocate net object");
1767 goto cleanup;
1768 }
1769
1770 net->id = mport->id;
1771 net->hport = mport;
1772 dev_set_name(&net->dev, "rnet_%d", net->id);
1773 net->dev.parent = &mport->dev;
1774 net->dev.release = rio_release_net;
1775 err = rio_add_net(net);
1776 if (err) {
1777 rmcd_debug(RDEV, "failed to register net, err=%d", err);
1778 kfree(net);
1779 goto cleanup;
1780 }
1781 }
1782
1783 rdev->net = mport->net;
1784 rdev->pef = rval;
1785 rdev->swpinfo = swpinfo;
1786 rio_mport_read_config_32(mport, destid, hopcount,
1787 RIO_DEV_ID_CAR, &rval);
1788 rdev->did = rval >> 16;
1789 rdev->vid = rval & 0xffff;
1790 rio_mport_read_config_32(mport, destid, hopcount, RIO_DEV_INFO_CAR,
1791 &rdev->device_rev);
1792 rio_mport_read_config_32(mport, destid, hopcount, RIO_ASM_ID_CAR,
1793 &rval);
1794 rdev->asm_did = rval >> 16;
1795 rdev->asm_vid = rval & 0xffff;
1796 rio_mport_read_config_32(mport, destid, hopcount, RIO_ASM_INFO_CAR,
1797 &rval);
1798 rdev->asm_rev = rval >> 16;
1799
1800 if (rdev->pef & RIO_PEF_EXT_FEATURES) {
1801 rdev->efptr = rval & 0xffff;
1802 rdev->phys_efptr = rio_mport_get_physefb(mport, 0, destid,
1803 hopcount);
1804
1805 rdev->em_efptr = rio_mport_get_feature(mport, 0, destid,
1806 hopcount, RIO_EFB_ERR_MGMNT);
1807 }
1808
1809 rio_mport_read_config_32(mport, destid, hopcount, RIO_SRC_OPS_CAR,
1810 &rdev->src_ops);
1811 rio_mport_read_config_32(mport, destid, hopcount, RIO_DST_OPS_CAR,
1812 &rdev->dst_ops);
1813
1814 rdev->comp_tag = dev_info.comptag;
1815 rdev->destid = destid;
1816 /* hopcount is stored as specified by a caller, regardles of EP or SW */
1817 rdev->hopcount = hopcount;
1818
1819 if (rdev->pef & RIO_PEF_SWITCH) {
1820 rswitch = rdev->rswitch;
1821 rswitch->route_table = NULL;
1822 }
1823
1824 if (strlen(dev_info.name))
1825 dev_set_name(&rdev->dev, "%s", dev_info.name);
1826 else if (rdev->pef & RIO_PEF_SWITCH)
1827 dev_set_name(&rdev->dev, "%02x:s:%04x", mport->id,
1828 rdev->comp_tag & RIO_CTAG_UDEVID);
1829 else
1830 dev_set_name(&rdev->dev, "%02x:e:%04x", mport->id,
1831 rdev->comp_tag & RIO_CTAG_UDEVID);
1832
1833 INIT_LIST_HEAD(&rdev->net_list);
1834 rdev->dev.parent = &mport->net->dev;
1835 rio_attach_device(rdev);
1836 rdev->dev.release = rio_release_dev;
1837
1838 if (rdev->dst_ops & RIO_DST_OPS_DOORBELL)
1839 rio_init_dbell_res(&rdev->riores[RIO_DOORBELL_RESOURCE],
1840 0, 0xffff);
1841 err = rio_add_device(rdev);
1842 if (err)
1843 goto cleanup;
1844 rio_dev_get(rdev);
1845
1846 return 0;
1847 cleanup:
1848 kfree(rdev);
1849 return err;
1850 }
1851
1852 static int rio_mport_del_riodev(struct mport_cdev_priv *priv, void __user *arg)
1853 {
1854 struct rio_rdev_info dev_info;
1855 struct rio_dev *rdev = NULL;
1856 struct device *dev;
1857 struct rio_mport *mport;
1858 struct rio_net *net;
1859
1860 if (copy_from_user(&dev_info, arg, sizeof(dev_info)))
1861 return -EFAULT;
1862
1863 mport = priv->md->mport;
1864
1865 /* If device name is specified, removal by name has priority */
1866 if (strlen(dev_info.name)) {
1867 dev = bus_find_device_by_name(&rio_bus_type, NULL,
1868 dev_info.name);
1869 if (dev)
1870 rdev = to_rio_dev(dev);
1871 } else {
1872 do {
1873 rdev = rio_get_comptag(dev_info.comptag, rdev);
1874 if (rdev && rdev->dev.parent == &mport->net->dev &&
1875 rdev->destid == (u16)dev_info.destid &&
1876 rdev->hopcount == (u8)dev_info.hopcount)
1877 break;
1878 } while (rdev);
1879 }
1880
1881 if (!rdev) {
1882 rmcd_debug(RDEV,
1883 "device name:%s ct:0x%x did:0x%x hc:0x%x not found",
1884 dev_info.name, dev_info.comptag, dev_info.destid,
1885 dev_info.hopcount);
1886 return -ENODEV;
1887 }
1888
1889 net = rdev->net;
1890 rio_dev_put(rdev);
1891 rio_del_device(rdev, RIO_DEVICE_SHUTDOWN);
1892
1893 if (list_empty(&net->devices)) {
1894 rio_free_net(net);
1895 mport->net = NULL;
1896 }
1897
1898 return 0;
1899 }
1900
1901 /*
1902 * Mport cdev management
1903 */
1904
1905 /*
1906 * mport_cdev_open() - Open character device (mport)
1907 */
1908 static int mport_cdev_open(struct inode *inode, struct file *filp)
1909 {
1910 int ret;
1911 int minor = iminor(inode);
1912 struct mport_dev *chdev;
1913 struct mport_cdev_priv *priv;
1914
1915 /* Test for valid device */
1916 if (minor >= RIO_MAX_MPORTS) {
1917 rmcd_error("Invalid minor device number");
1918 return -EINVAL;
1919 }
1920
1921 chdev = container_of(inode->i_cdev, struct mport_dev, cdev);
1922
1923 rmcd_debug(INIT, "%s filp=%p", dev_name(&chdev->dev), filp);
1924
1925 if (atomic_read(&chdev->active) == 0)
1926 return -ENODEV;
1927
1928 get_device(&chdev->dev);
1929
1930 priv = kzalloc(sizeof(*priv), GFP_KERNEL);
1931 if (!priv) {
1932 put_device(&chdev->dev);
1933 return -ENOMEM;
1934 }
1935
1936 priv->md = chdev;
1937
1938 mutex_lock(&chdev->file_mutex);
1939 list_add_tail(&priv->list, &chdev->file_list);
1940 mutex_unlock(&chdev->file_mutex);
1941
1942 INIT_LIST_HEAD(&priv->db_filters);
1943 INIT_LIST_HEAD(&priv->pw_filters);
1944 spin_lock_init(&priv->fifo_lock);
1945 init_waitqueue_head(&priv->event_rx_wait);
1946 ret = kfifo_alloc(&priv->event_fifo,
1947 sizeof(struct rio_event) * MPORT_EVENT_DEPTH,
1948 GFP_KERNEL);
1949 if (ret < 0) {
1950 dev_err(&chdev->dev, DRV_NAME ": kfifo_alloc failed\n");
1951 ret = -ENOMEM;
1952 goto err_fifo;
1953 }
1954
1955 #ifdef CONFIG_RAPIDIO_DMA_ENGINE
1956 INIT_LIST_HEAD(&priv->async_list);
1957 INIT_LIST_HEAD(&priv->pend_list);
1958 spin_lock_init(&priv->req_lock);
1959 mutex_init(&priv->dma_lock);
1960 #endif
1961
1962 filp->private_data = priv;
1963 goto out;
1964 err_fifo:
1965 kfree(priv);
1966 out:
1967 return ret;
1968 }
1969
1970 static int mport_cdev_fasync(int fd, struct file *filp, int mode)
1971 {
1972 struct mport_cdev_priv *priv = filp->private_data;
1973
1974 return fasync_helper(fd, filp, mode, &priv->async_queue);
1975 }
1976
1977 #ifdef CONFIG_RAPIDIO_DMA_ENGINE
1978 static void mport_cdev_release_dma(struct file *filp)
1979 {
1980 struct mport_cdev_priv *priv = filp->private_data;
1981 struct mport_dev *md;
1982 struct mport_dma_req *req, *req_next;
1983 unsigned long tmo = msecs_to_jiffies(dma_timeout);
1984 long wret;
1985 LIST_HEAD(list);
1986
1987 rmcd_debug(EXIT, "from filp=%p %s(%d)",
1988 filp, current->comm, task_pid_nr(current));
1989
1990 if (!priv->dmach) {
1991 rmcd_debug(EXIT, "No DMA channel for filp=%p", filp);
1992 return;
1993 }
1994
1995 md = priv->md;
1996
1997 flush_workqueue(dma_wq);
1998
1999 spin_lock(&priv->req_lock);
2000 if (!list_empty(&priv->async_list)) {
2001 rmcd_debug(EXIT, "async list not empty filp=%p %s(%d)",
2002 filp, current->comm, task_pid_nr(current));
2003 list_splice_init(&priv->async_list, &list);
2004 }
2005 spin_unlock(&priv->req_lock);
2006
2007 if (!list_empty(&list)) {
2008 rmcd_debug(EXIT, "temp list not empty");
2009 list_for_each_entry_safe(req, req_next, &list, node) {
2010 rmcd_debug(EXIT, "free req->filp=%p cookie=%d compl=%s",
2011 req->filp, req->cookie,
2012 completion_done(&req->req_comp)?"yes":"no");
2013 list_del(&req->node);
2014 dma_req_free(req);
2015 }
2016 }
2017
2018 if (!list_empty(&priv->pend_list)) {
2019 rmcd_debug(EXIT, "Free pending DMA requests for filp=%p %s(%d)",
2020 filp, current->comm, task_pid_nr(current));
2021 list_for_each_entry_safe(req,
2022 req_next, &priv->pend_list, node) {
2023 rmcd_debug(EXIT, "free req->filp=%p cookie=%d compl=%s",
2024 req->filp, req->cookie,
2025 completion_done(&req->req_comp)?"yes":"no");
2026 list_del(&req->node);
2027 dma_req_free(req);
2028 }
2029 }
2030
2031 put_dma_channel(priv);
2032 wret = wait_for_completion_interruptible_timeout(&priv->comp, tmo);
2033
2034 if (wret <= 0) {
2035 rmcd_error("%s(%d) failed waiting for DMA release err=%ld",
2036 current->comm, task_pid_nr(current), wret);
2037 }
2038
2039 spin_lock(&priv->req_lock);
2040
2041 if (!list_empty(&priv->pend_list)) {
2042 rmcd_debug(EXIT, "ATTN: pending DMA requests, filp=%p %s(%d)",
2043 filp, current->comm, task_pid_nr(current));
2044 }
2045
2046 spin_unlock(&priv->req_lock);
2047
2048 if (priv->dmach != priv->md->dma_chan) {
2049 rmcd_debug(EXIT, "Release DMA channel for filp=%p %s(%d)",
2050 filp, current->comm, task_pid_nr(current));
2051 rio_release_dma(priv->dmach);
2052 } else {
2053 rmcd_debug(EXIT, "Adjust default DMA channel refcount");
2054 kref_put(&md->dma_ref, mport_release_def_dma);
2055 }
2056
2057 priv->dmach = NULL;
2058 }
2059 #else
2060 #define mport_cdev_release_dma(priv) do {} while (0)
2061 #endif
2062
2063 /*
2064 * mport_cdev_release() - Release character device
2065 */
2066 static int mport_cdev_release(struct inode *inode, struct file *filp)
2067 {
2068 struct mport_cdev_priv *priv = filp->private_data;
2069 struct mport_dev *chdev;
2070 struct rio_mport_pw_filter *pw_filter, *pw_filter_next;
2071 struct rio_mport_db_filter *db_filter, *db_filter_next;
2072 struct rio_mport_mapping *map, *_map;
2073 unsigned long flags;
2074
2075 rmcd_debug(EXIT, "%s filp=%p", dev_name(&priv->md->dev), filp);
2076
2077 chdev = priv->md;
2078 mport_cdev_release_dma(filp);
2079
2080 priv->event_mask = 0;
2081
2082 spin_lock_irqsave(&chdev->pw_lock, flags);
2083 if (!list_empty(&priv->pw_filters)) {
2084 list_for_each_entry_safe(pw_filter, pw_filter_next,
2085 &priv->pw_filters, priv_node)
2086 rio_mport_delete_pw_filter(pw_filter);
2087 }
2088 spin_unlock_irqrestore(&chdev->pw_lock, flags);
2089
2090 spin_lock_irqsave(&chdev->db_lock, flags);
2091 list_for_each_entry_safe(db_filter, db_filter_next,
2092 &priv->db_filters, priv_node) {
2093 rio_mport_delete_db_filter(db_filter);
2094 }
2095 spin_unlock_irqrestore(&chdev->db_lock, flags);
2096
2097 kfifo_free(&priv->event_fifo);
2098
2099 mutex_lock(&chdev->buf_mutex);
2100 list_for_each_entry_safe(map, _map, &chdev->mappings, node) {
2101 if (map->filp == filp) {
2102 rmcd_debug(EXIT, "release mapping %p filp=%p",
2103 map->virt_addr, filp);
2104 kref_put(&map->ref, mport_release_mapping);
2105 }
2106 }
2107 mutex_unlock(&chdev->buf_mutex);
2108
2109 mport_cdev_fasync(-1, filp, 0);
2110 filp->private_data = NULL;
2111 mutex_lock(&chdev->file_mutex);
2112 list_del(&priv->list);
2113 mutex_unlock(&chdev->file_mutex);
2114 put_device(&chdev->dev);
2115 kfree(priv);
2116 return 0;
2117 }
2118
2119 /*
2120 * mport_cdev_ioctl() - IOCTLs for character device
2121 */
2122 static long mport_cdev_ioctl(struct file *filp,
2123 unsigned int cmd, unsigned long arg)
2124 {
2125 int err = -EINVAL;
2126 struct mport_cdev_priv *data = filp->private_data;
2127 struct mport_dev *md = data->md;
2128
2129 if (atomic_read(&md->active) == 0)
2130 return -ENODEV;
2131
2132 switch (cmd) {
2133 case RIO_MPORT_MAINT_READ_LOCAL:
2134 return rio_mport_maint_rd(data, (void __user *)arg, 1);
2135 case RIO_MPORT_MAINT_WRITE_LOCAL:
2136 return rio_mport_maint_wr(data, (void __user *)arg, 1);
2137 case RIO_MPORT_MAINT_READ_REMOTE:
2138 return rio_mport_maint_rd(data, (void __user *)arg, 0);
2139 case RIO_MPORT_MAINT_WRITE_REMOTE:
2140 return rio_mport_maint_wr(data, (void __user *)arg, 0);
2141 case RIO_MPORT_MAINT_HDID_SET:
2142 return maint_hdid_set(data, (void __user *)arg);
2143 case RIO_MPORT_MAINT_COMPTAG_SET:
2144 return maint_comptag_set(data, (void __user *)arg);
2145 case RIO_MPORT_MAINT_PORT_IDX_GET:
2146 return maint_port_idx_get(data, (void __user *)arg);
2147 case RIO_MPORT_GET_PROPERTIES:
2148 md->properties.hdid = md->mport->host_deviceid;
2149 if (copy_to_user((void __user *)arg, &(data->md->properties),
2150 sizeof(data->md->properties)))
2151 return -EFAULT;
2152 return 0;
2153 case RIO_ENABLE_DOORBELL_RANGE:
2154 return rio_mport_add_db_filter(data, (void __user *)arg);
2155 case RIO_DISABLE_DOORBELL_RANGE:
2156 return rio_mport_remove_db_filter(data, (void __user *)arg);
2157 case RIO_ENABLE_PORTWRITE_RANGE:
2158 return rio_mport_add_pw_filter(data, (void __user *)arg);
2159 case RIO_DISABLE_PORTWRITE_RANGE:
2160 return rio_mport_remove_pw_filter(data, (void __user *)arg);
2161 case RIO_SET_EVENT_MASK:
2162 data->event_mask = arg;
2163 return 0;
2164 case RIO_GET_EVENT_MASK:
2165 if (copy_to_user((void __user *)arg, &data->event_mask,
2166 sizeof(data->event_mask)))
2167 return -EFAULT;
2168 return 0;
2169 case RIO_MAP_OUTBOUND:
2170 return rio_mport_obw_map(filp, (void __user *)arg);
2171 case RIO_MAP_INBOUND:
2172 return rio_mport_map_inbound(filp, (void __user *)arg);
2173 case RIO_UNMAP_OUTBOUND:
2174 return rio_mport_obw_free(filp, (void __user *)arg);
2175 case RIO_UNMAP_INBOUND:
2176 return rio_mport_inbound_free(filp, (void __user *)arg);
2177 case RIO_ALLOC_DMA:
2178 return rio_mport_alloc_dma(filp, (void __user *)arg);
2179 case RIO_FREE_DMA:
2180 return rio_mport_free_dma(filp, (void __user *)arg);
2181 case RIO_WAIT_FOR_ASYNC:
2182 return rio_mport_wait_for_async_dma(filp, (void __user *)arg);
2183 case RIO_TRANSFER:
2184 return rio_mport_transfer_ioctl(filp, (void __user *)arg);
2185 case RIO_DEV_ADD:
2186 return rio_mport_add_riodev(data, (void __user *)arg);
2187 case RIO_DEV_DEL:
2188 return rio_mport_del_riodev(data, (void __user *)arg);
2189 default:
2190 break;
2191 }
2192
2193 return err;
2194 }
2195
2196 /*
2197 * mport_release_mapping - free mapping resources and info structure
2198 * @ref: a pointer to the kref within struct rio_mport_mapping
2199 *
2200 * NOTE: Shall be called while holding buf_mutex.
2201 */
2202 static void mport_release_mapping(struct kref *ref)
2203 {
2204 struct rio_mport_mapping *map =
2205 container_of(ref, struct rio_mport_mapping, ref);
2206 struct rio_mport *mport = map->md->mport;
2207
2208 rmcd_debug(MMAP, "type %d mapping @ %p (phys = %pad) for %s",
2209 map->dir, map->virt_addr,
2210 &map->phys_addr, mport->name);
2211
2212 list_del(&map->node);
2213
2214 switch (map->dir) {
2215 case MAP_INBOUND:
2216 rio_unmap_inb_region(mport, map->phys_addr);
2217 case MAP_DMA:
2218 dma_free_coherent(mport->dev.parent, map->size,
2219 map->virt_addr, map->phys_addr);
2220 break;
2221 case MAP_OUTBOUND:
2222 rio_unmap_outb_region(mport, map->rioid, map->rio_addr);
2223 break;
2224 }
2225 kfree(map);
2226 }
2227
2228 static void mport_mm_open(struct vm_area_struct *vma)
2229 {
2230 struct rio_mport_mapping *map = vma->vm_private_data;
2231
2232 rmcd_debug(MMAP, "0x%pad", &map->phys_addr);
2233 kref_get(&map->ref);
2234 }
2235
2236 static void mport_mm_close(struct vm_area_struct *vma)
2237 {
2238 struct rio_mport_mapping *map = vma->vm_private_data;
2239
2240 rmcd_debug(MMAP, "0x%pad", &map->phys_addr);
2241 mutex_lock(&map->md->buf_mutex);
2242 kref_put(&map->ref, mport_release_mapping);
2243 mutex_unlock(&map->md->buf_mutex);
2244 }
2245
2246 static const struct vm_operations_struct vm_ops = {
2247 .open = mport_mm_open,
2248 .close = mport_mm_close,
2249 };
2250
2251 static int mport_cdev_mmap(struct file *filp, struct vm_area_struct *vma)
2252 {
2253 struct mport_cdev_priv *priv = filp->private_data;
2254 struct mport_dev *md;
2255 size_t size = vma->vm_end - vma->vm_start;
2256 dma_addr_t baddr;
2257 unsigned long offset;
2258 int found = 0, ret;
2259 struct rio_mport_mapping *map;
2260
2261 rmcd_debug(MMAP, "0x%x bytes at offset 0x%lx",
2262 (unsigned int)size, vma->vm_pgoff);
2263
2264 md = priv->md;
2265 baddr = ((dma_addr_t)vma->vm_pgoff << PAGE_SHIFT);
2266
2267 mutex_lock(&md->buf_mutex);
2268 list_for_each_entry(map, &md->mappings, node) {
2269 if (baddr >= map->phys_addr &&
2270 baddr < (map->phys_addr + map->size)) {
2271 found = 1;
2272 break;
2273 }
2274 }
2275 mutex_unlock(&md->buf_mutex);
2276
2277 if (!found)
2278 return -ENOMEM;
2279
2280 offset = baddr - map->phys_addr;
2281
2282 if (size + offset > map->size)
2283 return -EINVAL;
2284
2285 vma->vm_pgoff = offset >> PAGE_SHIFT;
2286 rmcd_debug(MMAP, "MMAP adjusted offset = 0x%lx", vma->vm_pgoff);
2287
2288 if (map->dir == MAP_INBOUND || map->dir == MAP_DMA)
2289 ret = dma_mmap_coherent(md->mport->dev.parent, vma,
2290 map->virt_addr, map->phys_addr, map->size);
2291 else if (map->dir == MAP_OUTBOUND) {
2292 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
2293 ret = vm_iomap_memory(vma, map->phys_addr, map->size);
2294 } else {
2295 rmcd_error("Attempt to mmap unsupported mapping type");
2296 ret = -EIO;
2297 }
2298
2299 if (!ret) {
2300 vma->vm_private_data = map;
2301 vma->vm_ops = &vm_ops;
2302 mport_mm_open(vma);
2303 } else {
2304 rmcd_error("MMAP exit with err=%d", ret);
2305 }
2306
2307 return ret;
2308 }
2309
2310 static unsigned int mport_cdev_poll(struct file *filp, poll_table *wait)
2311 {
2312 struct mport_cdev_priv *priv = filp->private_data;
2313
2314 poll_wait(filp, &priv->event_rx_wait, wait);
2315 if (kfifo_len(&priv->event_fifo))
2316 return POLLIN | POLLRDNORM;
2317
2318 return 0;
2319 }
2320
2321 static ssize_t mport_read(struct file *filp, char __user *buf, size_t count,
2322 loff_t *ppos)
2323 {
2324 struct mport_cdev_priv *priv = filp->private_data;
2325 int copied;
2326 ssize_t ret;
2327
2328 if (!count)
2329 return 0;
2330
2331 if (kfifo_is_empty(&priv->event_fifo) &&
2332 (filp->f_flags & O_NONBLOCK))
2333 return -EAGAIN;
2334
2335 if (count % sizeof(struct rio_event))
2336 return -EINVAL;
2337
2338 ret = wait_event_interruptible(priv->event_rx_wait,
2339 kfifo_len(&priv->event_fifo) != 0);
2340 if (ret)
2341 return ret;
2342
2343 while (ret < count) {
2344 if (kfifo_to_user(&priv->event_fifo, buf,
2345 sizeof(struct rio_event), &copied))
2346 return -EFAULT;
2347 ret += copied;
2348 buf += copied;
2349 }
2350
2351 return ret;
2352 }
2353
2354 static ssize_t mport_write(struct file *filp, const char __user *buf,
2355 size_t count, loff_t *ppos)
2356 {
2357 struct mport_cdev_priv *priv = filp->private_data;
2358 struct rio_mport *mport = priv->md->mport;
2359 struct rio_event event;
2360 int len, ret;
2361
2362 if (!count)
2363 return 0;
2364
2365 if (count % sizeof(event))
2366 return -EINVAL;
2367
2368 len = 0;
2369 while ((count - len) >= (int)sizeof(event)) {
2370 if (copy_from_user(&event, buf, sizeof(event)))
2371 return -EFAULT;
2372
2373 if (event.header != RIO_DOORBELL)
2374 return -EINVAL;
2375
2376 ret = rio_mport_send_doorbell(mport,
2377 (u16)event.u.doorbell.rioid,
2378 event.u.doorbell.payload);
2379 if (ret < 0)
2380 return ret;
2381
2382 len += sizeof(event);
2383 buf += sizeof(event);
2384 }
2385
2386 return len;
2387 }
2388
2389 static const struct file_operations mport_fops = {
2390 .owner = THIS_MODULE,
2391 .open = mport_cdev_open,
2392 .release = mport_cdev_release,
2393 .poll = mport_cdev_poll,
2394 .read = mport_read,
2395 .write = mport_write,
2396 .mmap = mport_cdev_mmap,
2397 .fasync = mport_cdev_fasync,
2398 .unlocked_ioctl = mport_cdev_ioctl
2399 };
2400
2401 /*
2402 * Character device management
2403 */
2404
2405 static void mport_device_release(struct device *dev)
2406 {
2407 struct mport_dev *md;
2408
2409 rmcd_debug(EXIT, "%s", dev_name(dev));
2410 md = container_of(dev, struct mport_dev, dev);
2411 kfree(md);
2412 }
2413
2414 /*
2415 * mport_cdev_add() - Create mport_dev from rio_mport
2416 * @mport: RapidIO master port
2417 */
2418 static struct mport_dev *mport_cdev_add(struct rio_mport *mport)
2419 {
2420 int ret = 0;
2421 struct mport_dev *md;
2422 struct rio_mport_attr attr;
2423
2424 md = kzalloc(sizeof(struct mport_dev), GFP_KERNEL);
2425 if (!md) {
2426 rmcd_error("Unable allocate a device object");
2427 return NULL;
2428 }
2429
2430 md->mport = mport;
2431 mutex_init(&md->buf_mutex);
2432 mutex_init(&md->file_mutex);
2433 INIT_LIST_HEAD(&md->file_list);
2434 cdev_init(&md->cdev, &mport_fops);
2435 md->cdev.owner = THIS_MODULE;
2436 ret = cdev_add(&md->cdev, MKDEV(MAJOR(dev_number), mport->id), 1);
2437 if (ret < 0) {
2438 kfree(md);
2439 rmcd_error("Unable to register a device, err=%d", ret);
2440 return NULL;
2441 }
2442
2443 md->dev.devt = md->cdev.dev;
2444 md->dev.class = dev_class;
2445 md->dev.parent = &mport->dev;
2446 md->dev.release = mport_device_release;
2447 dev_set_name(&md->dev, DEV_NAME "%d", mport->id);
2448 atomic_set(&md->active, 1);
2449
2450 ret = device_register(&md->dev);
2451 if (ret) {
2452 rmcd_error("Failed to register mport %d (err=%d)",
2453 mport->id, ret);
2454 goto err_cdev;
2455 }
2456
2457 get_device(&md->dev);
2458
2459 INIT_LIST_HEAD(&md->doorbells);
2460 spin_lock_init(&md->db_lock);
2461 INIT_LIST_HEAD(&md->portwrites);
2462 spin_lock_init(&md->pw_lock);
2463 INIT_LIST_HEAD(&md->mappings);
2464
2465 md->properties.id = mport->id;
2466 md->properties.sys_size = mport->sys_size;
2467 md->properties.hdid = mport->host_deviceid;
2468 md->properties.index = mport->index;
2469
2470 /* The transfer_mode property will be returned through mport query
2471 * interface
2472 */
2473 #ifdef CONFIG_PPC /* for now: only on Freescale's SoCs */
2474 md->properties.transfer_mode |= RIO_TRANSFER_MODE_MAPPED;
2475 #else
2476 md->properties.transfer_mode |= RIO_TRANSFER_MODE_TRANSFER;
2477 #endif
2478 ret = rio_query_mport(mport, &attr);
2479 if (!ret) {
2480 md->properties.flags = attr.flags;
2481 md->properties.link_speed = attr.link_speed;
2482 md->properties.link_width = attr.link_width;
2483 md->properties.dma_max_sge = attr.dma_max_sge;
2484 md->properties.dma_max_size = attr.dma_max_size;
2485 md->properties.dma_align = attr.dma_align;
2486 md->properties.cap_sys_size = 0;
2487 md->properties.cap_transfer_mode = 0;
2488 md->properties.cap_addr_size = 0;
2489 } else
2490 pr_info(DRV_PREFIX "Failed to obtain info for %s cdev(%d:%d)\n",
2491 mport->name, MAJOR(dev_number), mport->id);
2492
2493 mutex_lock(&mport_devs_lock);
2494 list_add_tail(&md->node, &mport_devs);
2495 mutex_unlock(&mport_devs_lock);
2496
2497 pr_info(DRV_PREFIX "Added %s cdev(%d:%d)\n",
2498 mport->name, MAJOR(dev_number), mport->id);
2499
2500 return md;
2501
2502 err_cdev:
2503 cdev_del(&md->cdev);
2504 kfree(md);
2505 return NULL;
2506 }
2507
2508 /*
2509 * mport_cdev_terminate_dma() - Stop all active DMA data transfers and release
2510 * associated DMA channels.
2511 */
2512 static void mport_cdev_terminate_dma(struct mport_dev *md)
2513 {
2514 #ifdef CONFIG_RAPIDIO_DMA_ENGINE
2515 struct mport_cdev_priv *client;
2516
2517 rmcd_debug(DMA, "%s", dev_name(&md->dev));
2518
2519 mutex_lock(&md->file_mutex);
2520 list_for_each_entry(client, &md->file_list, list) {
2521 if (client->dmach) {
2522 dmaengine_terminate_all(client->dmach);
2523 rio_release_dma(client->dmach);
2524 }
2525 }
2526 mutex_unlock(&md->file_mutex);
2527
2528 if (md->dma_chan) {
2529 dmaengine_terminate_all(md->dma_chan);
2530 rio_release_dma(md->dma_chan);
2531 md->dma_chan = NULL;
2532 }
2533 #endif
2534 }
2535
2536
2537 /*
2538 * mport_cdev_kill_fasync() - Send SIGIO signal to all processes with open
2539 * mport_cdev files.
2540 */
2541 static int mport_cdev_kill_fasync(struct mport_dev *md)
2542 {
2543 unsigned int files = 0;
2544 struct mport_cdev_priv *client;
2545
2546 mutex_lock(&md->file_mutex);
2547 list_for_each_entry(client, &md->file_list, list) {
2548 if (client->async_queue)
2549 kill_fasync(&client->async_queue, SIGIO, POLL_HUP);
2550 files++;
2551 }
2552 mutex_unlock(&md->file_mutex);
2553 return files;
2554 }
2555
2556 /*
2557 * mport_cdev_remove() - Remove mport character device
2558 * @dev: Mport device to remove
2559 */
2560 static void mport_cdev_remove(struct mport_dev *md)
2561 {
2562 struct rio_mport_mapping *map, *_map;
2563
2564 rmcd_debug(EXIT, "Remove %s cdev", md->mport->name);
2565 atomic_set(&md->active, 0);
2566 mport_cdev_terminate_dma(md);
2567 rio_del_mport_pw_handler(md->mport, md, rio_mport_pw_handler);
2568 cdev_del(&(md->cdev));
2569 mport_cdev_kill_fasync(md);
2570
2571 flush_workqueue(dma_wq);
2572
2573 /* TODO: do we need to give clients some time to close file
2574 * descriptors? Simple wait for XX, or kref?
2575 */
2576
2577 /*
2578 * Release DMA buffers allocated for the mport device.
2579 * Disable associated inbound Rapidio requests mapping if applicable.
2580 */
2581 mutex_lock(&md->buf_mutex);
2582 list_for_each_entry_safe(map, _map, &md->mappings, node) {
2583 kref_put(&map->ref, mport_release_mapping);
2584 }
2585 mutex_unlock(&md->buf_mutex);
2586
2587 if (!list_empty(&md->mappings))
2588 rmcd_warn("WARNING: %s pending mappings on removal",
2589 md->mport->name);
2590
2591 rio_release_inb_dbell(md->mport, 0, 0x0fff);
2592
2593 device_unregister(&md->dev);
2594 put_device(&md->dev);
2595 }
2596
2597 /*
2598 * RIO rio_mport_interface driver
2599 */
2600
2601 /*
2602 * mport_add_mport() - Add rio_mport from LDM device struct
2603 * @dev: Linux device model struct
2604 * @class_intf: Linux class_interface
2605 */
2606 static int mport_add_mport(struct device *dev,
2607 struct class_interface *class_intf)
2608 {
2609 struct rio_mport *mport = NULL;
2610 struct mport_dev *chdev = NULL;
2611
2612 mport = to_rio_mport(dev);
2613 if (!mport)
2614 return -ENODEV;
2615
2616 chdev = mport_cdev_add(mport);
2617 if (!chdev)
2618 return -ENODEV;
2619
2620 return 0;
2621 }
2622
2623 /*
2624 * mport_remove_mport() - Remove rio_mport from global list
2625 * TODO remove device from global mport_dev list
2626 */
2627 static void mport_remove_mport(struct device *dev,
2628 struct class_interface *class_intf)
2629 {
2630 struct rio_mport *mport = NULL;
2631 struct mport_dev *chdev;
2632 int found = 0;
2633
2634 mport = to_rio_mport(dev);
2635 rmcd_debug(EXIT, "Remove %s", mport->name);
2636
2637 mutex_lock(&mport_devs_lock);
2638 list_for_each_entry(chdev, &mport_devs, node) {
2639 if (chdev->mport->id == mport->id) {
2640 atomic_set(&chdev->active, 0);
2641 list_del(&chdev->node);
2642 found = 1;
2643 break;
2644 }
2645 }
2646 mutex_unlock(&mport_devs_lock);
2647
2648 if (found)
2649 mport_cdev_remove(chdev);
2650 }
2651
2652 /* the rio_mport_interface is used to handle local mport devices */
2653 static struct class_interface rio_mport_interface __refdata = {
2654 .class = &rio_mport_class,
2655 .add_dev = mport_add_mport,
2656 .remove_dev = mport_remove_mport,
2657 };
2658
2659 /*
2660 * Linux kernel module
2661 */
2662
2663 /*
2664 * mport_init - Driver module loading
2665 */
2666 static int __init mport_init(void)
2667 {
2668 int ret;
2669
2670 /* Create device class needed by udev */
2671 dev_class = class_create(THIS_MODULE, DRV_NAME);
2672 if (IS_ERR(dev_class)) {
2673 rmcd_error("Unable to create " DRV_NAME " class");
2674 return PTR_ERR(dev_class);
2675 }
2676
2677 ret = alloc_chrdev_region(&dev_number, 0, RIO_MAX_MPORTS, DRV_NAME);
2678 if (ret < 0)
2679 goto err_chr;
2680
2681 rmcd_debug(INIT, "Registered class with major=%d", MAJOR(dev_number));
2682
2683 /* Register to rio_mport_interface */
2684 ret = class_interface_register(&rio_mport_interface);
2685 if (ret) {
2686 rmcd_error("class_interface_register() failed, err=%d", ret);
2687 goto err_cli;
2688 }
2689
2690 dma_wq = create_singlethread_workqueue("dma_wq");
2691 if (!dma_wq) {
2692 rmcd_error("failed to create DMA work queue");
2693 ret = -ENOMEM;
2694 goto err_wq;
2695 }
2696
2697 return 0;
2698
2699 err_wq:
2700 class_interface_unregister(&rio_mport_interface);
2701 err_cli:
2702 unregister_chrdev_region(dev_number, RIO_MAX_MPORTS);
2703 err_chr:
2704 class_destroy(dev_class);
2705 return ret;
2706 }
2707
2708 /**
2709 * mport_exit - Driver module unloading
2710 */
2711 static void __exit mport_exit(void)
2712 {
2713 class_interface_unregister(&rio_mport_interface);
2714 class_destroy(dev_class);
2715 unregister_chrdev_region(dev_number, RIO_MAX_MPORTS);
2716 destroy_workqueue(dma_wq);
2717 }
2718
2719 module_init(mport_init);
2720 module_exit(mport_exit);
Linux kernel - Deliverables
This page took 0.128572 seconds and 4 git commands to generate.