2 * linux/drivers/misc/xillybus_core.c
4 * Copyright 2011 Xillybus Ltd, http://xillybus.com
6 * Driver for the Xillybus FPGA/host framework.
8 * This driver interfaces with a special IP core in an FPGA, setting up
9 * a pipe between a hardware FIFO in the programmable logic and a device
10 * file in the host. The number of such pipes and their attributes are
11 * set up on the logic. This driver detects these automatically and
12 * creates the device files accordingly.
14 * This program is free software; you can redistribute it and/or modify
15 * it under the smems of the GNU General Public License as published by
16 * the Free Software Foundation; version 2 of the License.
19 #include <linux/list.h>
20 #include <linux/device.h>
21 #include <linux/module.h>
23 #include <linux/dma-mapping.h>
24 #include <linux/interrupt.h>
25 #include <linux/sched.h>
27 #include <linux/cdev.h>
28 #include <linux/spinlock.h>
29 #include <linux/mutex.h>
30 #include <linux/crc32.h>
31 #include <linux/poll.h>
32 #include <linux/delay.h>
33 #include <linux/slab.h>
34 #include <linux/workqueue.h>
37 MODULE_DESCRIPTION("Xillybus core functions");
38 MODULE_AUTHOR("Eli Billauer, Xillybus Ltd.");
39 MODULE_VERSION("1.07");
40 MODULE_ALIAS("xillybus_core");
41 MODULE_LICENSE("GPL v2");
43 /* General timeout is 100 ms, rx timeout is 10 ms */
44 #define XILLY_RX_TIMEOUT (10*HZ/1000)
45 #define XILLY_TIMEOUT (100*HZ/1000)
47 #define fpga_msg_ctrl_reg 0x0002
48 #define fpga_dma_control_reg 0x0008
49 #define fpga_dma_bufno_reg 0x0009
50 #define fpga_dma_bufaddr_lowaddr_reg 0x000a
51 #define fpga_dma_bufaddr_highaddr_reg 0x000b
52 #define fpga_buf_ctrl_reg 0x000c
53 #define fpga_buf_offset_reg 0x000d
54 #define fpga_endian_reg 0x0010
56 #define XILLYMSG_OPCODE_RELEASEBUF 1
57 #define XILLYMSG_OPCODE_QUIESCEACK 2
58 #define XILLYMSG_OPCODE_FIFOEOF 3
59 #define XILLYMSG_OPCODE_FATAL_ERROR 4
60 #define XILLYMSG_OPCODE_NONEMPTY 5
62 static const char xillyname
[] = "xillybus";
64 static struct class *xillybus_class
;
67 * ep_list_lock is the last lock to be taken; No other lock requests are
68 * allowed while holding it. It merely protects list_of_endpoints, and not
69 * the endpoints listed in it.
72 static LIST_HEAD(list_of_endpoints
);
73 static struct mutex ep_list_lock
;
74 static struct workqueue_struct
*xillybus_wq
;
77 * Locking scheme: Mutexes protect invocations of character device methods.
78 * If both locks are taken, wr_mutex is taken first, rd_mutex second.
80 * wr_spinlock protects wr_*_buf_idx, wr_empty, wr_sleepy, wr_ready and the
81 * buffers' end_offset fields against changes made by IRQ handler (and in
82 * theory, other file request handlers, but the mutex handles that). Nothing
84 * They are held for short direct memory manipulations. Needless to say,
85 * no mutex locking is allowed when a spinlock is held.
87 * rd_spinlock does the same with rd_*_buf_idx, rd_empty and end_offset.
89 * register_mutex is endpoint-specific, and is held when non-atomic
90 * register operations are performed. wr_mutex and rd_mutex may be
91 * held when register_mutex is taken, but none of the spinlocks. Note that
92 * register_mutex doesn't protect against sporadic buf_ctrl_reg writes
93 * which are unrelated to buf_offset_reg, since they are harmless.
95 * Blocking on the wait queues is allowed with mutexes held, but not with
98 * Only interruptible blocking is allowed on mutexes and wait queues.
100 * All in all, the locking order goes (with skips allowed, of course):
101 * wr_mutex -> rd_mutex -> register_mutex -> wr_spinlock -> rd_spinlock
104 static void malformed_message(struct xilly_endpoint
*endpoint
, u32
*buf
)
107 int msg_channel
, msg_bufno
, msg_data
, msg_dir
;
109 opcode
= (buf
[0] >> 24) & 0xff;
110 msg_dir
= buf
[0] & 1;
111 msg_channel
= (buf
[0] >> 1) & 0x7ff;
112 msg_bufno
= (buf
[0] >> 12) & 0x3ff;
113 msg_data
= buf
[1] & 0xfffffff;
115 dev_warn(endpoint
->dev
,
116 "Malformed message (skipping): opcode=%d, channel=%03x, dir=%d, bufno=%03x, data=%07x\n",
117 opcode
, msg_channel
, msg_dir
, msg_bufno
, msg_data
);
121 * xillybus_isr assumes the interrupt is allocated exclusively to it,
122 * which is the natural case MSI and several other hardware-oriented
123 * interrupts. Sharing is not allowed.
126 irqreturn_t
xillybus_isr(int irq
, void *data
)
128 struct xilly_endpoint
*ep
= data
;
130 unsigned int buf_size
;
133 unsigned int msg_channel
, msg_bufno
, msg_data
, msg_dir
;
134 struct xilly_channel
*channel
;
137 * The endpoint structure is altered during periods when it's
138 * guaranteed no interrupt will occur, but in theory, the cache
139 * lines may not be updated. So a memory barrier is issued.
144 buf
= ep
->msgbuf_addr
;
145 buf_size
= ep
->msg_buf_size
/sizeof(u32
);
148 ep
->ephw
->hw_sync_sgl_for_cpu(ep
,
153 for (i
= 0; i
< buf_size
; i
+= 2)
154 if (((buf
[i
+1] >> 28) & 0xf) != ep
->msg_counter
) {
155 malformed_message(ep
, &buf
[i
]);
157 "Sending a NACK on counter %x (instead of %x) on entry %d\n",
158 ((buf
[i
+1] >> 28) & 0xf),
162 if (++ep
->failed_messages
> 10)
164 "Lost sync with interrupt messages. Stopping.\n");
166 ep
->ephw
->hw_sync_sgl_for_device(
172 iowrite32(0x01, /* Message NACK */
173 &ep
->registers
[fpga_msg_ctrl_reg
]);
176 } else if (buf
[i
] & (1 << 22)) /* Last message */
180 dev_err(ep
->dev
, "Bad interrupt message. Stopping.\n");
186 for (i
= 0; i
<= buf_size
; i
+= 2) { /* Scan through messages */
187 opcode
= (buf
[i
] >> 24) & 0xff;
189 msg_dir
= buf
[i
] & 1;
190 msg_channel
= (buf
[i
] >> 1) & 0x7ff;
191 msg_bufno
= (buf
[i
] >> 12) & 0x3ff;
192 msg_data
= buf
[i
+1] & 0xfffffff;
195 case XILLYMSG_OPCODE_RELEASEBUF
:
197 if ((msg_channel
> ep
->num_channels
) ||
198 (msg_channel
== 0)) {
199 malformed_message(ep
, &buf
[i
]);
203 channel
= ep
->channels
[msg_channel
];
205 if (msg_dir
) { /* Write channel */
206 if (msg_bufno
>= channel
->num_wr_buffers
) {
207 malformed_message(ep
, &buf
[i
]);
210 spin_lock(&channel
->wr_spinlock
);
211 channel
->wr_buffers
[msg_bufno
]->end_offset
=
213 channel
->wr_fpga_buf_idx
= msg_bufno
;
214 channel
->wr_empty
= 0;
215 channel
->wr_sleepy
= 0;
216 spin_unlock(&channel
->wr_spinlock
);
218 wake_up_interruptible(&channel
->wr_wait
);
223 if (msg_bufno
>= channel
->num_rd_buffers
) {
224 malformed_message(ep
, &buf
[i
]);
228 spin_lock(&channel
->rd_spinlock
);
229 channel
->rd_fpga_buf_idx
= msg_bufno
;
230 channel
->rd_full
= 0;
231 spin_unlock(&channel
->rd_spinlock
);
233 wake_up_interruptible(&channel
->rd_wait
);
234 if (!channel
->rd_synchronous
)
237 &channel
->rd_workitem
,
242 case XILLYMSG_OPCODE_NONEMPTY
:
243 if ((msg_channel
> ep
->num_channels
) ||
244 (msg_channel
== 0) || (!msg_dir
) ||
245 !ep
->channels
[msg_channel
]->wr_supports_nonempty
) {
246 malformed_message(ep
, &buf
[i
]);
250 channel
= ep
->channels
[msg_channel
];
252 if (msg_bufno
>= channel
->num_wr_buffers
) {
253 malformed_message(ep
, &buf
[i
]);
256 spin_lock(&channel
->wr_spinlock
);
257 if (msg_bufno
== channel
->wr_host_buf_idx
)
258 channel
->wr_ready
= 1;
259 spin_unlock(&channel
->wr_spinlock
);
261 wake_up_interruptible(&channel
->wr_ready_wait
);
264 case XILLYMSG_OPCODE_QUIESCEACK
:
265 ep
->idtlen
= msg_data
;
266 wake_up_interruptible(&ep
->ep_wait
);
269 case XILLYMSG_OPCODE_FIFOEOF
:
270 channel
= ep
->channels
[msg_channel
];
271 spin_lock(&channel
->wr_spinlock
);
272 channel
->wr_eof
= msg_bufno
;
273 channel
->wr_sleepy
= 0;
275 channel
->wr_hangup
= channel
->wr_empty
&&
276 (channel
->wr_host_buf_idx
== msg_bufno
);
278 spin_unlock(&channel
->wr_spinlock
);
280 wake_up_interruptible(&channel
->wr_wait
);
283 case XILLYMSG_OPCODE_FATAL_ERROR
:
285 wake_up_interruptible(&ep
->ep_wait
); /* For select() */
287 "FPGA reported a fatal error. This means that the low-level communication with the device has failed. This hardware problem is most likely unrelated to Xillybus (neither kernel module nor FPGA core), but reports are still welcome. All I/O is aborted.\n");
290 malformed_message(ep
, &buf
[i
]);
295 ep
->ephw
->hw_sync_sgl_for_device(ep
,
300 ep
->msg_counter
= (ep
->msg_counter
+ 1) & 0xf;
301 ep
->failed_messages
= 0;
302 iowrite32(0x03, &ep
->registers
[fpga_msg_ctrl_reg
]); /* Message ACK */
306 EXPORT_SYMBOL(xillybus_isr
);
309 * A few trivial memory management functions.
310 * NOTE: These functions are used only on probe and remove, and therefore
311 * no locks are applied!
314 void xillybus_do_cleanup(struct xilly_cleanup
*mem
,
315 struct xilly_endpoint
*endpoint
)
317 struct list_head
*this, *next
;
319 list_for_each_safe(this, next
, &mem
->to_unmap
) {
320 struct xilly_dma
*entry
=
321 list_entry(this, struct xilly_dma
, node
);
323 endpoint
->ephw
->unmap_single(entry
);
327 INIT_LIST_HEAD(&mem
->to_unmap
);
329 list_for_each_safe(this, next
, &mem
->to_kfree
)
332 INIT_LIST_HEAD(&mem
->to_kfree
);
334 list_for_each_safe(this, next
, &mem
->to_pagefree
) {
335 struct xilly_page
*entry
=
336 list_entry(this, struct xilly_page
, node
);
338 free_pages(entry
->addr
, entry
->order
);
341 INIT_LIST_HEAD(&mem
->to_pagefree
);
343 EXPORT_SYMBOL(xillybus_do_cleanup
);
345 static void *xilly_malloc(struct xilly_cleanup
*mem
, size_t size
)
349 ptr
= kzalloc(sizeof(struct list_head
) + size
, GFP_KERNEL
);
354 list_add_tail((struct list_head
*) ptr
, &mem
->to_kfree
);
356 return ptr
+ sizeof(struct list_head
);
359 static unsigned long xilly_pagealloc(struct xilly_cleanup
*mem
,
363 struct xilly_page
*this;
365 this = kmalloc(sizeof(struct xilly_page
), GFP_KERNEL
);
369 addr
= __get_free_pages(GFP_KERNEL
| __GFP_DMA32
| __GFP_ZERO
, order
);
379 list_add_tail(&this->node
, &mem
->to_pagefree
);
385 static void xillybus_autoflush(struct work_struct
*work
);
387 static int xilly_setupchannels(struct xilly_endpoint
*ep
,
388 struct xilly_cleanup
*mem
,
389 unsigned char *chandesc
,
393 int i
, entry
, wr_nbuffer
, rd_nbuffer
;
394 struct xilly_channel
*channel
;
395 int channelnum
, bufnum
, bufsize
, format
, is_writebuf
;
397 int synchronous
, allowpartial
, exclusive_open
, seekable
;
398 int supports_nonempty
;
399 void *wr_salami
= NULL
;
400 void *rd_salami
= NULL
;
401 int left_of_wr_salami
= 0;
402 int left_of_rd_salami
= 0;
404 int msg_buf_done
= 0;
406 struct xilly_buffer
*this_buffer
= NULL
; /* Init to silence warning */
408 channel
= xilly_malloc(mem
, ep
->num_channels
*
409 sizeof(struct xilly_channel
));
414 ep
->channels
= xilly_malloc(mem
, (ep
->num_channels
+ 1) *
415 sizeof(struct xilly_channel
*));
420 ep
->channels
[0] = NULL
; /* Channel 0 is message buf. */
422 /* Initialize all channels with defaults */
424 for (i
= 1; i
<= ep
->num_channels
; i
++) {
425 channel
->wr_buffers
= NULL
;
426 channel
->rd_buffers
= NULL
;
427 channel
->num_wr_buffers
= 0;
428 channel
->num_rd_buffers
= 0;
429 channel
->wr_fpga_buf_idx
= -1;
430 channel
->wr_host_buf_idx
= 0;
431 channel
->wr_host_buf_pos
= 0;
432 channel
->wr_empty
= 1;
433 channel
->wr_ready
= 0;
434 channel
->wr_sleepy
= 1;
435 channel
->rd_fpga_buf_idx
= 0;
436 channel
->rd_host_buf_idx
= 0;
437 channel
->rd_host_buf_pos
= 0;
438 channel
->rd_full
= 0;
439 channel
->wr_ref_count
= 0;
440 channel
->rd_ref_count
= 0;
442 spin_lock_init(&channel
->wr_spinlock
);
443 spin_lock_init(&channel
->rd_spinlock
);
444 mutex_init(&channel
->wr_mutex
);
445 mutex_init(&channel
->rd_mutex
);
446 init_waitqueue_head(&channel
->rd_wait
);
447 init_waitqueue_head(&channel
->wr_wait
);
448 init_waitqueue_head(&channel
->wr_ready_wait
);
450 INIT_DELAYED_WORK(&channel
->rd_workitem
, xillybus_autoflush
);
452 channel
->endpoint
= ep
;
453 channel
->chan_num
= i
;
455 channel
->log2_element_size
= 0;
457 ep
->channels
[i
] = channel
++;
461 * The DMA buffer address update is atomic on the FPGA, so even if
462 * it was in the middle of sending messages to some buffer, changing
463 * the address is safe, since the data will go to either of the
464 * buffers. Not that this situation should occur at all anyhow.
468 rd_nbuffer
= 1; /* Buffer zero isn't used at all */
470 for (entry
= 0; entry
< entries
; entry
++, chandesc
+= 4) {
471 is_writebuf
= chandesc
[0] & 0x01;
472 channelnum
= (chandesc
[0] >> 1) | ((chandesc
[1] & 0x0f) << 7);
473 format
= (chandesc
[1] >> 4) & 0x03;
474 allowpartial
= (chandesc
[1] >> 6) & 0x01;
475 synchronous
= (chandesc
[1] >> 7) & 0x01;
476 bufsize
= 1 << (chandesc
[2] & 0x1f);
477 bufnum
= 1 << (chandesc
[3] & 0x0f);
478 exclusive_open
= (chandesc
[2] >> 7) & 0x01;
479 seekable
= (chandesc
[2] >> 6) & 0x01;
480 supports_nonempty
= (chandesc
[2] >> 5) & 0x01;
482 if ((channelnum
> ep
->num_channels
) ||
483 ((channelnum
== 0) && !is_writebuf
)) {
485 "IDT requests channel out of range. Aborting.\n");
489 channel
= ep
->channels
[channelnum
]; /* NULL for msg channel */
491 bytebufsize
= bufsize
<< 2; /* Overwritten just below */
494 channel
->num_rd_buffers
= bufnum
;
495 channel
->log2_element_size
= ((format
> 2) ?
497 bytebufsize
= channel
->rd_buf_size
= bufsize
*
498 (1 << channel
->log2_element_size
);
499 channel
->rd_allow_partial
= allowpartial
;
500 channel
->rd_synchronous
= synchronous
;
501 channel
->rd_exclusive_open
= exclusive_open
;
502 channel
->seekable
= seekable
;
504 channel
->rd_buffers
= xilly_malloc(
506 bufnum
* sizeof(struct xilly_buffer
*));
508 if (!channel
->rd_buffers
)
511 this_buffer
= xilly_malloc(
513 bufnum
* sizeof(struct xilly_buffer
));
519 else if (channelnum
> 0) {
520 channel
->num_wr_buffers
= bufnum
;
521 channel
->log2_element_size
= ((format
> 2) ?
523 bytebufsize
= channel
->wr_buf_size
= bufsize
*
524 (1 << channel
->log2_element_size
);
526 channel
->seekable
= seekable
;
527 channel
->wr_supports_nonempty
= supports_nonempty
;
529 channel
->wr_allow_partial
= allowpartial
;
530 channel
->wr_synchronous
= synchronous
;
531 channel
->wr_exclusive_open
= exclusive_open
;
533 channel
->wr_buffers
= xilly_malloc(
535 bufnum
* sizeof(struct xilly_buffer
*));
537 if (!channel
->wr_buffers
)
540 this_buffer
= xilly_malloc(
542 bufnum
* sizeof(struct xilly_buffer
));
549 * Although daunting, we cut the chunks for read buffers
550 * from a different salami than the write buffers',
551 * possibly improving performance.
555 for (i
= 0; i
< bufnum
; i
++) {
557 * Buffers are expected in descending
558 * byte-size order, so there is either
559 * enough for this buffer or none at all.
561 if ((left_of_wr_salami
< bytebufsize
) &&
562 (left_of_wr_salami
> 0)) {
564 "Corrupt buffer allocation in IDT. Aborting.\n");
568 if (left_of_wr_salami
== 0) {
569 int allocorder
, allocsize
;
571 allocsize
= PAGE_SIZE
;
573 while (bytebufsize
> allocsize
) {
583 left_of_wr_salami
= allocsize
;
586 dma_addr
= ep
->ephw
->map_single(
597 (u32
) (dma_addr
& 0xffffffff),
599 fpga_dma_bufaddr_lowaddr_reg
]
602 ((u32
) ((((u64
) dma_addr
) >> 32)
605 fpga_dma_bufaddr_highaddr_reg
]
609 if (channelnum
> 0) {
610 this_buffer
->addr
= wr_salami
;
611 this_buffer
->dma_addr
= dma_addr
;
612 channel
->wr_buffers
[i
] = this_buffer
++;
615 0x80000000 | wr_nbuffer
++,
617 fpga_dma_bufno_reg
]);
619 ep
->msgbuf_addr
= wr_salami
;
620 ep
->msgbuf_dma_addr
= dma_addr
;
621 ep
->msg_buf_size
= bytebufsize
;
625 0x80000000, &ep
->registers
[
626 fpga_dma_bufno_reg
]);
629 left_of_wr_salami
-= bytebufsize
;
630 wr_salami
+= bytebufsize
;
632 else /* Read buffers */
633 for (i
= 0; i
< bufnum
; i
++) {
635 * Buffers are expected in descending
636 * byte-size order, so there is either
637 * enough for this buffer or none at all.
639 if ((left_of_rd_salami
< bytebufsize
) &&
640 (left_of_rd_salami
> 0)) {
642 "Corrupt buffer allocation in IDT. Aborting.\n");
646 if (left_of_rd_salami
== 0) {
647 int allocorder
, allocsize
;
649 allocsize
= PAGE_SIZE
;
651 while (bytebufsize
> allocsize
) {
663 left_of_rd_salami
= allocsize
;
666 dma_addr
= ep
->ephw
->map_single(
677 (u32
) (dma_addr
& 0xffffffff),
679 fpga_dma_bufaddr_lowaddr_reg
]
682 ((u32
) ((((u64
) dma_addr
) >> 32)
685 fpga_dma_bufaddr_highaddr_reg
]
689 this_buffer
->addr
= rd_salami
;
690 this_buffer
->dma_addr
= dma_addr
;
691 channel
->rd_buffers
[i
] = this_buffer
++;
693 iowrite32(rd_nbuffer
++,
694 &ep
->registers
[fpga_dma_bufno_reg
]);
696 left_of_rd_salami
-= bytebufsize
;
697 rd_salami
+= bytebufsize
;
703 "Corrupt IDT: No message buffer. Aborting.\n");
711 "Failed to allocate write buffer memory. Aborting.\n");
714 dev_err(ep
->dev
, "Failed to map DMA memory!. Aborting.\n");
718 static void xilly_scan_idt(struct xilly_endpoint
*endpoint
,
719 struct xilly_idt_handle
*idt_handle
)
722 unsigned char *idt
= endpoint
->channels
[1]->wr_buffers
[0]->addr
;
723 unsigned char *end_of_idt
= idt
+ endpoint
->idtlen
- 4;
728 idt_handle
->idt
= idt
;
730 scan
++; /* Skip version number */
732 while ((scan
<= end_of_idt
) && *scan
) {
733 while ((scan
<= end_of_idt
) && *scan
++)
734 /* Do nothing, just scan thru string */;
740 if (scan
> end_of_idt
) {
741 dev_err(endpoint
->dev
,
742 "IDT device name list overflow. Aborting.\n");
743 idt_handle
->chandesc
= NULL
;
746 idt_handle
->chandesc
= scan
;
748 len
= endpoint
->idtlen
- (3 + ((int) (scan
- idt
)));
751 idt_handle
->chandesc
= NULL
;
753 dev_err(endpoint
->dev
,
754 "Corrupt IDT device name list. Aborting.\n");
757 idt_handle
->entries
= len
>> 2;
759 endpoint
->num_channels
= count
;
762 static int xilly_obtain_idt(struct xilly_endpoint
*endpoint
)
765 struct xilly_channel
*channel
;
766 unsigned char *version
;
768 channel
= endpoint
->channels
[1]; /* This should be generated ad-hoc */
770 channel
->wr_sleepy
= 1;
771 wmb(); /* Setting wr_sleepy must come before the command */
774 (3 << 24), /* Opcode 3 for channel 0 = Send IDT */
775 &endpoint
->registers
[fpga_buf_ctrl_reg
]);
776 mmiowb(); /* Just to appear safe */
778 wait_event_interruptible_timeout(channel
->wr_wait
,
779 (!channel
->wr_sleepy
),
782 if (channel
->wr_sleepy
) {
783 dev_err(endpoint
->dev
, "Failed to obtain IDT. Aborting.\n");
785 if (endpoint
->fatal_error
)
792 endpoint
->ephw
->hw_sync_sgl_for_cpu(
794 channel
->wr_buffers
[0]->dma_addr
,
795 channel
->wr_buf_size
,
798 if (channel
->wr_buffers
[0]->end_offset
!= endpoint
->idtlen
) {
799 dev_err(endpoint
->dev
,
800 "IDT length mismatch (%d != %d). Aborting.\n",
801 channel
->wr_buffers
[0]->end_offset
, endpoint
->idtlen
);
806 if (crc32_le(~0, channel
->wr_buffers
[0]->addr
,
807 endpoint
->idtlen
+1) != 0) {
808 dev_err(endpoint
->dev
, "IDT failed CRC check. Aborting.\n");
813 version
= channel
->wr_buffers
[0]->addr
;
815 /* Check version number. Accept anything below 0x82 for now. */
816 if (*version
> 0x82) {
817 dev_err(endpoint
->dev
,
818 "No support for IDT version 0x%02x. Maybe the xillybus driver needs an upgarde. Aborting.\n",
824 return 0; /* Success */
827 static ssize_t
xillybus_read(struct file
*filp
, char __user
*userbuf
,
828 size_t count
, loff_t
*f_pos
)
833 int no_time_left
= 0;
834 long deadline
, left_to_sleep
;
835 struct xilly_channel
*channel
= filp
->private_data
;
837 int empty
, reached_eof
, exhausted
, ready
;
838 /* Initializations are there only to silence warnings */
840 int howmany
= 0, bufpos
= 0, bufidx
= 0, bufferdone
= 0;
843 if (channel
->endpoint
->fatal_error
)
846 deadline
= jiffies
+ 1 + XILLY_RX_TIMEOUT
;
848 rc
= mutex_lock_interruptible(&channel
->wr_mutex
);
853 rc
= 0; /* Just to be clear about it. Compiler optimizes this out */
855 while (1) { /* Note that we may drop mutex within this loop */
856 int bytes_to_do
= count
- bytes_done
;
857 spin_lock_irqsave(&channel
->wr_spinlock
, flags
);
859 empty
= channel
->wr_empty
;
860 ready
= !empty
|| channel
->wr_ready
;
863 bufidx
= channel
->wr_host_buf_idx
;
864 bufpos
= channel
->wr_host_buf_pos
;
865 howmany
= ((channel
->wr_buffers
[bufidx
]->end_offset
866 + 1) << channel
->log2_element_size
)
869 /* Update wr_host_* to its post-operation state */
870 if (howmany
> bytes_to_do
) {
873 howmany
= bytes_to_do
;
874 channel
->wr_host_buf_pos
+= howmany
;
878 channel
->wr_host_buf_pos
= 0;
880 if (bufidx
== channel
->wr_fpga_buf_idx
) {
881 channel
->wr_empty
= 1;
882 channel
->wr_sleepy
= 1;
883 channel
->wr_ready
= 0;
886 if (bufidx
>= (channel
->num_wr_buffers
- 1))
887 channel
->wr_host_buf_idx
= 0;
889 channel
->wr_host_buf_idx
++;
894 * Marking our situation after the possible changes above,
895 * for use after releasing the spinlock.
897 * empty = empty before change
898 * exhasted = empty after possible change
901 reached_eof
= channel
->wr_empty
&&
902 (channel
->wr_host_buf_idx
== channel
->wr_eof
);
903 channel
->wr_hangup
= reached_eof
;
904 exhausted
= channel
->wr_empty
;
905 waiting_bufidx
= channel
->wr_host_buf_idx
;
907 spin_unlock_irqrestore(&channel
->wr_spinlock
, flags
);
909 if (!empty
) { /* Go on, now without the spinlock */
911 if (bufpos
== 0) /* Position zero means it's virgin */
912 channel
->endpoint
->ephw
->hw_sync_sgl_for_cpu(
914 channel
->wr_buffers
[bufidx
]->dma_addr
,
915 channel
->wr_buf_size
,
920 channel
->wr_buffers
[bufidx
]->addr
925 bytes_done
+= howmany
;
928 channel
->endpoint
->ephw
->
929 hw_sync_sgl_for_device
932 channel
->wr_buffers
[bufidx
]->
934 channel
->wr_buf_size
,
938 * Tell FPGA the buffer is done with. It's an
939 * atomic operation to the FPGA, so what
940 * happens with other channels doesn't matter,
941 * and the certain channel is protected with
942 * the channel-specific mutex.
945 iowrite32(1 | (channel
->chan_num
<< 1)
947 &channel
->endpoint
->registers
[
949 mmiowb(); /* Just to appear safe */
953 mutex_unlock(&channel
->wr_mutex
);
958 /* This includes a zero-count return = EOF */
959 if ((bytes_done
>= count
) || reached_eof
)
963 continue; /* More in RAM buffer(s)? Just go on. */
965 if ((bytes_done
> 0) &&
967 (channel
->wr_synchronous
&& channel
->wr_allow_partial
)))
971 * Nonblocking read: The "ready" flag tells us that the FPGA
972 * has data to send. In non-blocking mode, if it isn't on,
973 * just return. But if there is, we jump directly to the point
974 * where we ask for the FPGA to send all it has, and wait
975 * until that data arrives. So in a sense, we *do* block in
976 * nonblocking mode, but only for a very short time.
979 if (!no_time_left
&& (filp
->f_flags
& O_NONBLOCK
)) {
986 bytes_done
= -EAGAIN
;
990 if (!no_time_left
|| (bytes_done
> 0)) {
992 * Note that in case of an element-misaligned read
993 * request, offsetlimit will include the last element,
994 * which will be partially read from.
996 int offsetlimit
= ((count
- bytes_done
) - 1) >>
997 channel
->log2_element_size
;
998 int buf_elements
= channel
->wr_buf_size
>>
999 channel
->log2_element_size
;
1002 * In synchronous mode, always send an offset limit.
1003 * Just don't send a value too big.
1006 if (channel
->wr_synchronous
) {
1007 /* Don't request more than one buffer */
1008 if (channel
->wr_allow_partial
&&
1009 (offsetlimit
>= buf_elements
))
1010 offsetlimit
= buf_elements
- 1;
1012 /* Don't request more than all buffers */
1013 if (!channel
->wr_allow_partial
&&
1015 (buf_elements
* channel
->num_wr_buffers
)))
1016 offsetlimit
= buf_elements
*
1017 channel
->num_wr_buffers
- 1;
1021 * In asynchronous mode, force early flush of a buffer
1022 * only if that will allow returning a full count. The
1023 * "offsetlimit < ( ... )" rather than "<=" excludes
1024 * requesting a full buffer, which would obviously
1025 * cause a buffer transmission anyhow
1028 if (channel
->wr_synchronous
||
1029 (offsetlimit
< (buf_elements
- 1))) {
1031 mutex_lock(&channel
->endpoint
->register_mutex
);
1033 iowrite32(offsetlimit
,
1034 &channel
->endpoint
->registers
[
1035 fpga_buf_offset_reg
]);
1038 iowrite32(1 | (channel
->chan_num
<< 1) |
1039 (2 << 24) | /* 2 = offset limit */
1040 (waiting_bufidx
<< 12),
1041 &channel
->endpoint
->registers
[
1042 fpga_buf_ctrl_reg
]);
1044 mmiowb(); /* Just to appear safe */
1046 mutex_unlock(&channel
->endpoint
->
1053 * If partial completion is disallowed, there is no point in
1054 * timeout sleeping. Neither if no_time_left is set and
1058 if (!channel
->wr_allow_partial
||
1059 (no_time_left
&& (bytes_done
== 0))) {
1062 * This do-loop will run more than once if another
1063 * thread reasserted wr_sleepy before we got the mutex
1064 * back, so we try again.
1068 mutex_unlock(&channel
->wr_mutex
);
1070 if (wait_event_interruptible(
1072 (!channel
->wr_sleepy
)))
1075 if (mutex_lock_interruptible(
1076 &channel
->wr_mutex
))
1078 } while (channel
->wr_sleepy
);
1082 interrupted
: /* Mutex is not held if got here */
1083 if (channel
->endpoint
->fatal_error
)
1087 if (filp
->f_flags
& O_NONBLOCK
)
1088 return -EAGAIN
; /* Don't admit snoozing */
1092 left_to_sleep
= deadline
- ((long) jiffies
);
1095 * If our time is out, skip the waiting. We may miss wr_sleepy
1096 * being deasserted but hey, almost missing the train is like
1100 if (left_to_sleep
> 0) {
1102 wait_event_interruptible_timeout(
1104 (!channel
->wr_sleepy
),
1107 if (!channel
->wr_sleepy
)
1110 if (left_to_sleep
< 0) { /* Interrupt */
1111 mutex_unlock(&channel
->wr_mutex
);
1112 if (channel
->endpoint
->fatal_error
)
1121 no_time_left
= 1; /* We're out of sleeping time. Desperate! */
1123 if (bytes_done
== 0) {
1125 * Reaching here means that we allow partial return,
1126 * that we've run out of time, and that we have
1127 * nothing to return.
1128 * So tell the FPGA to send anything it has or gets.
1131 iowrite32(1 | (channel
->chan_num
<< 1) |
1132 (3 << 24) | /* Opcode 3, flush it all! */
1133 (waiting_bufidx
<< 12),
1134 &channel
->endpoint
->registers
[
1135 fpga_buf_ctrl_reg
]);
1136 mmiowb(); /* Just to appear safe */
1140 * Formally speaking, we should block for data at this point.
1141 * But to keep the code cleaner, we'll just finish the loop,
1142 * make the unlikely check for data, and then block at the
1147 mutex_unlock(&channel
->wr_mutex
);
1149 if (channel
->endpoint
->fatal_error
)
1156 * The timeout argument takes values as follows:
1157 * >0 : Flush with timeout
1158 * ==0 : Flush, and wait idefinitely for the flush to complete
1159 * <0 : Autoflush: Flush only if there's a single buffer occupied
1162 static int xillybus_myflush(struct xilly_channel
*channel
, long timeout
)
1165 unsigned long flags
;
1167 int end_offset_plus1
;
1168 int bufidx
, bufidx_minus1
;
1171 int new_rd_host_buf_pos
;
1173 if (channel
->endpoint
->fatal_error
)
1175 rc
= mutex_lock_interruptible(&channel
->rd_mutex
);
1181 * Don't flush a closed channel. This can happen when the work queued
1182 * autoflush thread fires off after the file has closed. This is not
1183 * an error, just something to dismiss.
1186 if (!channel
->rd_ref_count
)
1189 bufidx
= channel
->rd_host_buf_idx
;
1191 bufidx_minus1
= (bufidx
== 0) ? channel
->num_rd_buffers
- 1 : bufidx
-1;
1193 end_offset_plus1
= channel
->rd_host_buf_pos
>>
1194 channel
->log2_element_size
;
1196 new_rd_host_buf_pos
= channel
->rd_host_buf_pos
-
1197 (end_offset_plus1
<< channel
->log2_element_size
);
1199 /* Submit the current buffer if it's nonempty */
1200 if (end_offset_plus1
) {
1201 unsigned char *tail
= channel
->rd_buffers
[bufidx
]->addr
+
1202 (end_offset_plus1
<< channel
->log2_element_size
);
1204 /* Copy unflushed data, so we can put it in next buffer */
1205 for (i
= 0; i
< new_rd_host_buf_pos
; i
++)
1206 channel
->rd_leftovers
[i
] = *tail
++;
1208 spin_lock_irqsave(&channel
->rd_spinlock
, flags
);
1210 /* Autoflush only if a single buffer is occupied */
1212 if ((timeout
< 0) &&
1213 (channel
->rd_full
||
1214 (bufidx_minus1
!= channel
->rd_fpga_buf_idx
))) {
1215 spin_unlock_irqrestore(&channel
->rd_spinlock
, flags
);
1217 * A new work item may be queued by the ISR exactly
1218 * now, since the execution of a work item allows the
1219 * queuing of a new one while it's running.
1224 /* The 4th element is never needed for data, so it's a flag */
1225 channel
->rd_leftovers
[3] = (new_rd_host_buf_pos
!= 0);
1227 /* Set up rd_full to reflect a certain moment's state */
1229 if (bufidx
== channel
->rd_fpga_buf_idx
)
1230 channel
->rd_full
= 1;
1231 spin_unlock_irqrestore(&channel
->rd_spinlock
, flags
);
1233 if (bufidx
>= (channel
->num_rd_buffers
- 1))
1234 channel
->rd_host_buf_idx
= 0;
1236 channel
->rd_host_buf_idx
++;
1238 channel
->endpoint
->ephw
->hw_sync_sgl_for_device(
1240 channel
->rd_buffers
[bufidx
]->dma_addr
,
1241 channel
->rd_buf_size
,
1244 mutex_lock(&channel
->endpoint
->register_mutex
);
1246 iowrite32(end_offset_plus1
- 1,
1247 &channel
->endpoint
->registers
[fpga_buf_offset_reg
]);
1250 iowrite32((channel
->chan_num
<< 1) | /* Channel ID */
1251 (2 << 24) | /* Opcode 2, submit buffer */
1253 &channel
->endpoint
->registers
[fpga_buf_ctrl_reg
]);
1254 mmiowb(); /* Just to appear safe */
1256 mutex_unlock(&channel
->endpoint
->register_mutex
);
1257 } else if (bufidx
== 0)
1258 bufidx
= channel
->num_rd_buffers
- 1;
1262 channel
->rd_host_buf_pos
= new_rd_host_buf_pos
;
1265 goto done
; /* Autoflush */
1269 * bufidx is now the last buffer written to (or equal to
1270 * rd_fpga_buf_idx if buffer was never written to), and
1271 * channel->rd_host_buf_idx the one after it.
1273 * If bufidx == channel->rd_fpga_buf_idx we're either empty or full.
1278 while (1) { /* Loop waiting for draining of buffers */
1279 spin_lock_irqsave(&channel
->rd_spinlock
, flags
);
1281 if (bufidx
!= channel
->rd_fpga_buf_idx
)
1282 channel
->rd_full
= 1; /*
1284 * but needs waiting.
1287 empty
= !channel
->rd_full
;
1289 spin_unlock_irqrestore(&channel
->rd_spinlock
, flags
);
1295 * Indefinite sleep with mutex taken. With data waiting for
1296 * flushing user should not be surprised if open() for write
1300 wait_event_interruptible(channel
->rd_wait
,
1301 (!channel
->rd_full
));
1303 else if (wait_event_interruptible_timeout(
1305 (!channel
->rd_full
),
1307 dev_warn(channel
->endpoint
->dev
,
1308 "Timed out while flushing. Output data may be lost.\n");
1314 if (channel
->rd_full
) {
1321 mutex_unlock(&channel
->rd_mutex
);
1323 if (channel
->endpoint
->fatal_error
)
1329 static int xillybus_flush(struct file
*filp
, fl_owner_t id
)
1331 if (!(filp
->f_mode
& FMODE_WRITE
))
1334 return xillybus_myflush(filp
->private_data
, HZ
); /* 1 second timeout */
1337 static void xillybus_autoflush(struct work_struct
*work
)
1339 struct delayed_work
*workitem
= container_of(
1340 work
, struct delayed_work
, work
);
1341 struct xilly_channel
*channel
= container_of(
1342 workitem
, struct xilly_channel
, rd_workitem
);
1345 rc
= xillybus_myflush(channel
, -1);
1348 dev_warn(channel
->endpoint
->dev
,
1349 "Autoflush failed because work queue thread got a signal.\n");
1351 dev_err(channel
->endpoint
->dev
,
1352 "Autoflush failed under weird circumstances.\n");
1355 static ssize_t
xillybus_write(struct file
*filp
, const char __user
*userbuf
,
1356 size_t count
, loff_t
*f_pos
)
1359 unsigned long flags
;
1361 struct xilly_channel
*channel
= filp
->private_data
;
1363 int full
, exhausted
;
1364 /* Initializations are there only to silence warnings */
1366 int howmany
= 0, bufpos
= 0, bufidx
= 0, bufferdone
= 0;
1367 int end_offset_plus1
= 0;
1369 if (channel
->endpoint
->fatal_error
)
1372 rc
= mutex_lock_interruptible(&channel
->rd_mutex
);
1377 rc
= 0; /* Just to be clear about it. Compiler optimizes this out */
1380 int bytes_to_do
= count
- bytes_done
;
1382 spin_lock_irqsave(&channel
->rd_spinlock
, flags
);
1384 full
= channel
->rd_full
;
1387 bufidx
= channel
->rd_host_buf_idx
;
1388 bufpos
= channel
->rd_host_buf_pos
;
1389 howmany
= channel
->rd_buf_size
- bufpos
;
1392 * Update rd_host_* to its state after this operation.
1393 * count=0 means committing the buffer immediately,
1394 * which is like flushing, but not necessarily block.
1397 if ((howmany
> bytes_to_do
) &&
1399 ((bufpos
>> channel
->log2_element_size
) == 0))) {
1402 howmany
= bytes_to_do
;
1403 channel
->rd_host_buf_pos
+= howmany
;
1409 channel
->rd_buf_size
>>
1410 channel
->log2_element_size
;
1411 channel
->rd_host_buf_pos
= 0;
1413 unsigned char *tail
;
1416 end_offset_plus1
= bufpos
>>
1417 channel
->log2_element_size
;
1419 channel
->rd_host_buf_pos
-=
1421 channel
->log2_element_size
;
1424 rd_buffers
[bufidx
]->addr
+
1425 (end_offset_plus1
<<
1426 channel
->log2_element_size
);
1429 i
< channel
->rd_host_buf_pos
;
1431 channel
->rd_leftovers
[i
] =
1435 if (bufidx
== channel
->rd_fpga_buf_idx
)
1436 channel
->rd_full
= 1;
1438 if (bufidx
>= (channel
->num_rd_buffers
- 1))
1439 channel
->rd_host_buf_idx
= 0;
1441 channel
->rd_host_buf_idx
++;
1446 * Marking our situation after the possible changes above,
1447 * for use after releasing the spinlock.
1449 * full = full before change
1450 * exhasted = full after possible change
1453 exhausted
= channel
->rd_full
;
1455 spin_unlock_irqrestore(&channel
->rd_spinlock
, flags
);
1457 if (!full
) { /* Go on, now without the spinlock */
1458 unsigned char *head
=
1459 channel
->rd_buffers
[bufidx
]->addr
;
1462 if ((bufpos
== 0) || /* Zero means it's virgin */
1463 (channel
->rd_leftovers
[3] != 0)) {
1464 channel
->endpoint
->ephw
->hw_sync_sgl_for_cpu(
1466 channel
->rd_buffers
[bufidx
]->dma_addr
,
1467 channel
->rd_buf_size
,
1470 /* Virgin, but leftovers are due */
1471 for (i
= 0; i
< bufpos
; i
++)
1472 *head
++ = channel
->rd_leftovers
[i
];
1474 channel
->rd_leftovers
[3] = 0; /* Clear flag */
1478 channel
->rd_buffers
[bufidx
]->addr
+ bufpos
,
1483 bytes_done
+= howmany
;
1486 channel
->endpoint
->ephw
->
1487 hw_sync_sgl_for_device(
1489 channel
->rd_buffers
[bufidx
]->
1491 channel
->rd_buf_size
,
1494 mutex_lock(&channel
->endpoint
->register_mutex
);
1496 iowrite32(end_offset_plus1
- 1,
1497 &channel
->endpoint
->registers
[
1498 fpga_buf_offset_reg
]);
1500 iowrite32((channel
->chan_num
<< 1) |
1501 (2 << 24) | /* 2 = submit buffer */
1503 &channel
->endpoint
->registers
[
1504 fpga_buf_ctrl_reg
]);
1505 mmiowb(); /* Just to appear safe */
1507 mutex_unlock(&channel
->endpoint
->
1510 channel
->rd_leftovers
[3] =
1511 (channel
->rd_host_buf_pos
!= 0);
1515 mutex_unlock(&channel
->rd_mutex
);
1517 if (channel
->endpoint
->fatal_error
)
1520 if (!channel
->rd_synchronous
)
1523 &channel
->rd_workitem
,
1530 if (bytes_done
>= count
)
1534 continue; /* If there's more space, just go on */
1536 if ((bytes_done
> 0) && channel
->rd_allow_partial
)
1540 * Indefinite sleep with mutex taken. With data waiting for
1541 * flushing, user should not be surprised if open() for write
1545 if (filp
->f_flags
& O_NONBLOCK
) {
1546 bytes_done
= -EAGAIN
;
1550 wait_event_interruptible(channel
->rd_wait
,
1551 (!channel
->rd_full
));
1553 if (channel
->rd_full
) {
1554 mutex_unlock(&channel
->rd_mutex
);
1556 if (channel
->endpoint
->fatal_error
)
1565 mutex_unlock(&channel
->rd_mutex
);
1567 if (!channel
->rd_synchronous
)
1568 queue_delayed_work(xillybus_wq
,
1569 &channel
->rd_workitem
,
1572 if ((channel
->rd_synchronous
) && (bytes_done
> 0)) {
1573 rc
= xillybus_myflush(filp
->private_data
, 0); /* No timeout */
1575 if (rc
&& (rc
!= -EINTR
))
1579 if (channel
->endpoint
->fatal_error
)
1585 static int xillybus_open(struct inode
*inode
, struct file
*filp
)
1588 unsigned long flags
;
1589 int minor
= iminor(inode
);
1590 int major
= imajor(inode
);
1591 struct xilly_endpoint
*ep_iter
, *endpoint
= NULL
;
1592 struct xilly_channel
*channel
;
1594 mutex_lock(&ep_list_lock
);
1596 list_for_each_entry(ep_iter
, &list_of_endpoints
, ep_list
) {
1597 if ((ep_iter
->major
== major
) &&
1598 (minor
>= ep_iter
->lowest_minor
) &&
1599 (minor
< (ep_iter
->lowest_minor
+
1600 ep_iter
->num_channels
))) {
1605 mutex_unlock(&ep_list_lock
);
1608 pr_err("xillybus: open() failed to find a device for major=%d and minor=%d\n",
1613 if (endpoint
->fatal_error
)
1616 channel
= endpoint
->channels
[1 + minor
- endpoint
->lowest_minor
];
1617 filp
->private_data
= channel
;
1621 * It gets complicated because:
1622 * 1. We don't want to take a mutex we don't have to
1623 * 2. We don't want to open one direction if the other will fail.
1626 if ((filp
->f_mode
& FMODE_READ
) && (!channel
->num_wr_buffers
))
1629 if ((filp
->f_mode
& FMODE_WRITE
) && (!channel
->num_rd_buffers
))
1632 if ((filp
->f_mode
& FMODE_READ
) && (filp
->f_flags
& O_NONBLOCK
) &&
1633 (channel
->wr_synchronous
|| !channel
->wr_allow_partial
||
1634 !channel
->wr_supports_nonempty
)) {
1635 dev_err(endpoint
->dev
,
1636 "open() failed: O_NONBLOCK not allowed for read on this device\n");
1640 if ((filp
->f_mode
& FMODE_WRITE
) && (filp
->f_flags
& O_NONBLOCK
) &&
1641 (channel
->rd_synchronous
|| !channel
->rd_allow_partial
)) {
1642 dev_err(endpoint
->dev
,
1643 "open() failed: O_NONBLOCK not allowed for write on this device\n");
1648 * Note: open() may block on getting mutexes despite O_NONBLOCK.
1649 * This shouldn't occur normally, since multiple open of the same
1650 * file descriptor is almost always prohibited anyhow
1651 * (*_exclusive_open is normally set in real-life systems).
1654 if (filp
->f_mode
& FMODE_READ
) {
1655 rc
= mutex_lock_interruptible(&channel
->wr_mutex
);
1660 if (filp
->f_mode
& FMODE_WRITE
) {
1661 rc
= mutex_lock_interruptible(&channel
->rd_mutex
);
1666 if ((filp
->f_mode
& FMODE_READ
) &&
1667 (channel
->wr_ref_count
!= 0) &&
1668 (channel
->wr_exclusive_open
)) {
1673 if ((filp
->f_mode
& FMODE_WRITE
) &&
1674 (channel
->rd_ref_count
!= 0) &&
1675 (channel
->rd_exclusive_open
)) {
1681 if (filp
->f_mode
& FMODE_READ
) {
1682 if (channel
->wr_ref_count
== 0) { /* First open of file */
1683 /* Move the host to first buffer */
1684 spin_lock_irqsave(&channel
->wr_spinlock
, flags
);
1685 channel
->wr_host_buf_idx
= 0;
1686 channel
->wr_host_buf_pos
= 0;
1687 channel
->wr_fpga_buf_idx
= -1;
1688 channel
->wr_empty
= 1;
1689 channel
->wr_ready
= 0;
1690 channel
->wr_sleepy
= 1;
1691 channel
->wr_eof
= -1;
1692 channel
->wr_hangup
= 0;
1694 spin_unlock_irqrestore(&channel
->wr_spinlock
, flags
);
1696 iowrite32(1 | (channel
->chan_num
<< 1) |
1697 (4 << 24) | /* Opcode 4, open channel */
1698 ((channel
->wr_synchronous
& 1) << 23),
1699 &channel
->endpoint
->registers
[
1700 fpga_buf_ctrl_reg
]);
1701 mmiowb(); /* Just to appear safe */
1704 channel
->wr_ref_count
++;
1707 if (filp
->f_mode
& FMODE_WRITE
) {
1708 if (channel
->rd_ref_count
== 0) { /* First open of file */
1709 /* Move the host to first buffer */
1710 spin_lock_irqsave(&channel
->rd_spinlock
, flags
);
1711 channel
->rd_host_buf_idx
= 0;
1712 channel
->rd_host_buf_pos
= 0;
1713 channel
->rd_leftovers
[3] = 0; /* No leftovers. */
1714 channel
->rd_fpga_buf_idx
= channel
->num_rd_buffers
- 1;
1715 channel
->rd_full
= 0;
1717 spin_unlock_irqrestore(&channel
->rd_spinlock
, flags
);
1719 iowrite32((channel
->chan_num
<< 1) |
1720 (4 << 24), /* Opcode 4, open channel */
1721 &channel
->endpoint
->registers
[
1722 fpga_buf_ctrl_reg
]);
1723 mmiowb(); /* Just to appear safe */
1726 channel
->rd_ref_count
++;
1730 if (filp
->f_mode
& FMODE_WRITE
)
1731 mutex_unlock(&channel
->rd_mutex
);
1733 if (filp
->f_mode
& FMODE_READ
)
1734 mutex_unlock(&channel
->wr_mutex
);
1736 if (!rc
&& (!channel
->seekable
))
1737 return nonseekable_open(inode
, filp
);
1742 static int xillybus_release(struct inode
*inode
, struct file
*filp
)
1745 unsigned long flags
;
1746 struct xilly_channel
*channel
= filp
->private_data
;
1751 if (channel
->endpoint
->fatal_error
)
1754 if (filp
->f_mode
& FMODE_WRITE
) {
1755 rc
= mutex_lock_interruptible(&channel
->rd_mutex
);
1758 dev_warn(channel
->endpoint
->dev
,
1759 "Failed to close file. Hardware left in messy state.\n");
1763 channel
->rd_ref_count
--;
1765 if (channel
->rd_ref_count
== 0) {
1768 * We rely on the kernel calling flush()
1769 * before we get here.
1772 iowrite32((channel
->chan_num
<< 1) | /* Channel ID */
1773 (5 << 24), /* Opcode 5, close channel */
1774 &channel
->endpoint
->registers
[
1775 fpga_buf_ctrl_reg
]);
1776 mmiowb(); /* Just to appear safe */
1778 mutex_unlock(&channel
->rd_mutex
);
1781 if (filp
->f_mode
& FMODE_READ
) {
1782 rc
= mutex_lock_interruptible(&channel
->wr_mutex
);
1784 dev_warn(channel
->endpoint
->dev
,
1785 "Failed to close file. Hardware left in messy state.\n");
1789 channel
->wr_ref_count
--;
1791 if (channel
->wr_ref_count
== 0) {
1793 iowrite32(1 | (channel
->chan_num
<< 1) |
1794 (5 << 24), /* Opcode 5, close channel */
1795 &channel
->endpoint
->registers
[
1796 fpga_buf_ctrl_reg
]);
1797 mmiowb(); /* Just to appear safe */
1800 * This is crazily cautious: We make sure that not
1801 * only that we got an EOF (be it because we closed
1802 * the channel or because of a user's EOF), but verify
1803 * that it's one beyond the last buffer arrived, so
1804 * we have no leftover buffers pending before wrapping
1805 * up (which can only happen in asynchronous channels,
1810 spin_lock_irqsave(&channel
->wr_spinlock
,
1812 buf_idx
= channel
->wr_fpga_buf_idx
;
1813 eof
= channel
->wr_eof
;
1814 channel
->wr_sleepy
= 1;
1815 spin_unlock_irqrestore(&channel
->wr_spinlock
,
1819 * Check if eof points at the buffer after
1820 * the last one the FPGA submitted. Note that
1821 * no EOF is marked by negative eof.
1825 if (buf_idx
== channel
->num_wr_buffers
)
1832 * Steal extra 100 ms if awaken by interrupt.
1833 * This is a simple workaround for an
1834 * interrupt pending when entering, which would
1835 * otherwise result in declaring the hardware
1839 if (wait_event_interruptible(
1841 (!channel
->wr_sleepy
)))
1844 if (channel
->wr_sleepy
) {
1845 mutex_unlock(&channel
->wr_mutex
);
1846 dev_warn(channel
->endpoint
->dev
,
1847 "Hardware failed to respond to close command, therefore left in messy state.\n");
1853 mutex_unlock(&channel
->wr_mutex
);
1858 static loff_t
xillybus_llseek(struct file
*filp
, loff_t offset
, int whence
)
1860 struct xilly_channel
*channel
= filp
->private_data
;
1861 loff_t pos
= filp
->f_pos
;
1865 * Take both mutexes not allowing interrupts, since it seems like
1866 * common applications don't expect an -EINTR here. Besides, multiple
1867 * access to a single file descriptor on seekable devices is a mess
1871 if (channel
->endpoint
->fatal_error
)
1874 mutex_lock(&channel
->wr_mutex
);
1875 mutex_lock(&channel
->rd_mutex
);
1885 pos
= offset
; /* Going to the end => to the beginning */
1892 /* In any case, we must finish on an element boundary */
1893 if (pos
& ((1 << channel
->log2_element_size
) - 1)) {
1898 mutex_lock(&channel
->endpoint
->register_mutex
);
1900 iowrite32(pos
>> channel
->log2_element_size
,
1901 &channel
->endpoint
->registers
[fpga_buf_offset_reg
]);
1903 iowrite32((channel
->chan_num
<< 1) |
1904 (6 << 24), /* Opcode 6, set address */
1905 &channel
->endpoint
->registers
[fpga_buf_ctrl_reg
]);
1906 mmiowb(); /* Just to appear safe */
1908 mutex_unlock(&channel
->endpoint
->register_mutex
);
1911 mutex_unlock(&channel
->rd_mutex
);
1912 mutex_unlock(&channel
->wr_mutex
);
1914 if (rc
) /* Return error after releasing mutexes */
1920 * Since seekable devices are allowed only when the channel is
1921 * synchronous, we assume that there is no data pending in either
1922 * direction (which holds true as long as no concurrent access on the
1923 * file descriptor takes place).
1924 * The only thing we may need to throw away is leftovers from partial
1928 channel
->rd_leftovers
[3] = 0;
1933 static unsigned int xillybus_poll(struct file
*filp
, poll_table
*wait
)
1935 struct xilly_channel
*channel
= filp
->private_data
;
1936 unsigned int mask
= 0;
1937 unsigned long flags
;
1939 poll_wait(filp
, &channel
->endpoint
->ep_wait
, wait
);
1942 * poll() won't play ball regarding read() channels which
1943 * aren't asynchronous and support the nonempty message. Allowing
1944 * that will create situations where data has been delivered at
1945 * the FPGA, and users expecting select() to wake up, which it may
1949 if (!channel
->wr_synchronous
&& channel
->wr_supports_nonempty
) {
1950 poll_wait(filp
, &channel
->wr_wait
, wait
);
1951 poll_wait(filp
, &channel
->wr_ready_wait
, wait
);
1953 spin_lock_irqsave(&channel
->wr_spinlock
, flags
);
1954 if (!channel
->wr_empty
|| channel
->wr_ready
)
1955 mask
|= POLLIN
| POLLRDNORM
;
1957 if (channel
->wr_hangup
)
1959 * Not POLLHUP, because its behavior is in the
1960 * mist, and POLLIN does what we want: Wake up
1961 * the read file descriptor so it sees EOF.
1963 mask
|= POLLIN
| POLLRDNORM
;
1964 spin_unlock_irqrestore(&channel
->wr_spinlock
, flags
);
1968 * If partial data write is disallowed on a write() channel,
1969 * it's pointless to ever signal OK to write, because is could
1970 * block despite some space being available.
1973 if (channel
->rd_allow_partial
) {
1974 poll_wait(filp
, &channel
->rd_wait
, wait
);
1976 spin_lock_irqsave(&channel
->rd_spinlock
, flags
);
1977 if (!channel
->rd_full
)
1978 mask
|= POLLOUT
| POLLWRNORM
;
1979 spin_unlock_irqrestore(&channel
->rd_spinlock
, flags
);
1982 if (channel
->endpoint
->fatal_error
)
1988 static const struct file_operations xillybus_fops
= {
1989 .owner
= THIS_MODULE
,
1990 .read
= xillybus_read
,
1991 .write
= xillybus_write
,
1992 .open
= xillybus_open
,
1993 .flush
= xillybus_flush
,
1994 .release
= xillybus_release
,
1995 .llseek
= xillybus_llseek
,
1996 .poll
= xillybus_poll
,
1999 static int xillybus_init_chrdev(struct xilly_endpoint
*endpoint
,
2000 const unsigned char *idt
)
2004 int devnum
, i
, minor
, major
;
2006 struct device
*device
;
2008 rc
= alloc_chrdev_region(&dev
, 0, /* minor start */
2009 endpoint
->num_channels
,
2013 dev_warn(endpoint
->dev
, "Failed to obtain major/minors");
2017 endpoint
->major
= major
= MAJOR(dev
);
2018 endpoint
->lowest_minor
= minor
= MINOR(dev
);
2020 cdev_init(&endpoint
->cdev
, &xillybus_fops
);
2021 endpoint
->cdev
.owner
= endpoint
->ephw
->owner
;
2022 rc
= cdev_add(&endpoint
->cdev
, MKDEV(major
, minor
),
2023 endpoint
->num_channels
);
2025 dev_warn(endpoint
->dev
, "Failed to add cdev. Aborting.\n");
2031 for (i
= minor
, devnum
= 0;
2032 devnum
< endpoint
->num_channels
;
2034 snprintf(devname
, sizeof(devname
)-1, "xillybus_%s", idt
);
2036 devname
[sizeof(devname
)-1] = 0; /* Should never matter */
2041 device
= device_create(xillybus_class
,
2047 if (IS_ERR(device
)) {
2048 dev_warn(endpoint
->dev
,
2049 "Failed to create %s device. Aborting.\n",
2055 dev_info(endpoint
->dev
, "Created %d device files.\n",
2056 endpoint
->num_channels
);
2057 return 0; /* succeed */
2061 for (; devnum
>= 0; devnum
--, i
--)
2062 device_destroy(xillybus_class
, MKDEV(major
, i
));
2064 cdev_del(&endpoint
->cdev
);
2066 unregister_chrdev_region(MKDEV(major
, minor
), endpoint
->num_channels
);
2072 static void xillybus_cleanup_chrdev(struct xilly_endpoint
*endpoint
)
2076 for (minor
= endpoint
->lowest_minor
;
2077 minor
< (endpoint
->lowest_minor
+ endpoint
->num_channels
);
2079 device_destroy(xillybus_class
, MKDEV(endpoint
->major
, minor
));
2080 cdev_del(&endpoint
->cdev
);
2081 unregister_chrdev_region(MKDEV(endpoint
->major
,
2082 endpoint
->lowest_minor
),
2083 endpoint
->num_channels
);
2085 dev_info(endpoint
->dev
, "Removed %d device files.\n",
2086 endpoint
->num_channels
);
2090 struct xilly_endpoint
*xillybus_init_endpoint(struct pci_dev
*pdev
,
2092 struct xilly_endpoint_hardware
2095 struct xilly_endpoint
*endpoint
;
2097 endpoint
= devm_kzalloc(dev
, sizeof(*endpoint
), GFP_KERNEL
);
2099 dev_err(dev
, "Failed to allocate memory. Aborting.\n");
2103 endpoint
->pdev
= pdev
;
2104 endpoint
->dev
= dev
;
2105 endpoint
->ephw
= ephw
;
2106 INIT_LIST_HEAD(&endpoint
->cleanup
.to_kfree
);
2107 INIT_LIST_HEAD(&endpoint
->cleanup
.to_pagefree
);
2108 INIT_LIST_HEAD(&endpoint
->cleanup
.to_unmap
);
2109 endpoint
->msg_counter
= 0x0b;
2110 endpoint
->failed_messages
= 0;
2111 endpoint
->fatal_error
= 0;
2113 init_waitqueue_head(&endpoint
->ep_wait
);
2114 mutex_init(&endpoint
->register_mutex
);
2118 EXPORT_SYMBOL(xillybus_init_endpoint
);
2120 static int xilly_quiesce(struct xilly_endpoint
*endpoint
)
2122 endpoint
->idtlen
= -1;
2123 wmb(); /* Make sure idtlen is set before sending command */
2124 iowrite32((u32
) (endpoint
->dma_using_dac
& 0x0001),
2125 &endpoint
->registers
[fpga_dma_control_reg
]);
2128 wait_event_interruptible_timeout(endpoint
->ep_wait
,
2129 (endpoint
->idtlen
>= 0),
2132 if (endpoint
->idtlen
< 0) {
2133 dev_err(endpoint
->dev
,
2134 "Failed to quiesce the device on exit. Quitting while leaving a mess.\n");
2137 return 0; /* Success */
2140 int xillybus_endpoint_discovery(struct xilly_endpoint
*endpoint
)
2144 struct xilly_cleanup tmpmem
;
2145 int idtbuffersize
= (1 << PAGE_SHIFT
);
2148 * The bogus IDT is used during bootstrap for allocating the initial
2149 * message buffer, and then the message buffer and space for the IDT
2150 * itself. The initial message buffer is of a single page's size, but
2151 * it's soon replaced with a more modest one (and memory is freed).
2154 unsigned char bogus_idt
[8] = { 1, 224, (PAGE_SHIFT
)-2, 0,
2155 3, 192, PAGE_SHIFT
, 0 };
2156 struct xilly_idt_handle idt_handle
;
2158 INIT_LIST_HEAD(&tmpmem
.to_kfree
);
2159 INIT_LIST_HEAD(&tmpmem
.to_pagefree
);
2160 INIT_LIST_HEAD(&tmpmem
.to_unmap
);
2163 * Writing the value 0x00000001 to Endianness register signals which
2164 * endianness this processor is using, so the FPGA can swap words as
2168 iowrite32(1, &endpoint
->registers
[fpga_endian_reg
]);
2169 mmiowb(); /* Writes below are affected by the one above. */
2171 /* Bootstrap phase I: Allocate temporary message buffer */
2173 endpoint
->num_channels
= 0;
2175 rc
= xilly_setupchannels(endpoint
, &tmpmem
, bogus_idt
, 1);
2178 goto failed_buffers
;
2180 /* Clear the message subsystem (and counter in particular) */
2181 iowrite32(0x04, &endpoint
->registers
[fpga_msg_ctrl_reg
]);
2184 endpoint
->idtlen
= -1;
2189 * Set DMA 32/64 bit mode, quiesce the device (?!) and get IDT
2192 iowrite32((u32
) (endpoint
->dma_using_dac
& 0x0001),
2193 &endpoint
->registers
[fpga_dma_control_reg
]);
2196 wait_event_interruptible_timeout(endpoint
->ep_wait
,
2197 (endpoint
->idtlen
>= 0),
2200 if (endpoint
->idtlen
< 0) {
2201 dev_err(endpoint
->dev
, "No response from FPGA. Aborting.\n");
2203 goto failed_quiesce
;
2207 iowrite32((u32
) (0x0002 | (endpoint
->dma_using_dac
& 0x0001)),
2208 &endpoint
->registers
[fpga_dma_control_reg
]);
2211 /* Bootstrap phase II: Allocate buffer for IDT and obtain it */
2212 while (endpoint
->idtlen
>= idtbuffersize
) {
2217 endpoint
->num_channels
= 1;
2219 rc
= xilly_setupchannels(endpoint
, &tmpmem
, bogus_idt
, 2);
2226 rc
= xilly_obtain_idt(endpoint
);
2231 xilly_scan_idt(endpoint
, &idt_handle
);
2233 if (!idt_handle
.chandesc
) {
2237 /* Bootstrap phase III: Allocate buffers according to IDT */
2239 rc
= xilly_setupchannels(endpoint
,
2241 idt_handle
.chandesc
,
2242 idt_handle
.entries
);
2247 smp_wmb(); /* mutex_lock below should suffice, but won't hurt.*/
2250 * endpoint is now completely configured. We put it on the list
2251 * available to open() before registering the char device(s)
2254 mutex_lock(&ep_list_lock
);
2255 list_add_tail(&endpoint
->ep_list
, &list_of_endpoints
);
2256 mutex_unlock(&ep_list_lock
);
2258 rc
= xillybus_init_chrdev(endpoint
, idt_handle
.idt
);
2261 goto failed_chrdevs
;
2263 xillybus_do_cleanup(&tmpmem
, endpoint
);
2268 mutex_lock(&ep_list_lock
);
2269 list_del(&endpoint
->ep_list
);
2270 mutex_unlock(&ep_list_lock
);
2273 /* Quiesce the device. Now it's serious to do it */
2274 rc
= xilly_quiesce(endpoint
);
2277 return rc
; /* FPGA may still DMA, so no release */
2279 flush_workqueue(xillybus_wq
);
2282 xillybus_do_cleanup(&tmpmem
, endpoint
);
2286 EXPORT_SYMBOL(xillybus_endpoint_discovery
);
2288 void xillybus_endpoint_remove(struct xilly_endpoint
*endpoint
)
2290 xillybus_cleanup_chrdev(endpoint
);
2292 mutex_lock(&ep_list_lock
);
2293 list_del(&endpoint
->ep_list
);
2294 mutex_unlock(&ep_list_lock
);
2296 xilly_quiesce(endpoint
);
2299 * Flushing is done upon endpoint release to prevent access to memory
2300 * just about to be released. This makes the quiesce complete.
2302 flush_workqueue(xillybus_wq
);
2304 EXPORT_SYMBOL(xillybus_endpoint_remove
);
2306 static int __init
xillybus_init(void)
2310 mutex_init(&ep_list_lock
);
2312 xillybus_class
= class_create(THIS_MODULE
, xillyname
);
2313 if (IS_ERR(xillybus_class
)) {
2314 rc
= PTR_ERR(xillybus_class
);
2315 pr_warn("Failed to register class xillybus\n");
2320 xillybus_wq
= alloc_workqueue(xillyname
, 0, 0);
2322 class_destroy(xillybus_class
);
2329 static void __exit
xillybus_exit(void)
2331 /* flush_workqueue() was called for each endpoint released */
2332 destroy_workqueue(xillybus_wq
);
2334 class_destroy(xillybus_class
);
2337 module_init(xillybus_init
);
2338 module_exit(xillybus_exit
);