2 * Copyright (c) 2001-2004 by David Brownell
3 * Copyright (c) 2003 Michal Sojka, for high-speed iso transfers
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License as published by the
7 * Free Software Foundation; either version 2 of the License, or (at your
8 * option) any later version.
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
12 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software Foundation,
17 * Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
20 /* this file is part of ehci-hcd.c */
22 /*-------------------------------------------------------------------------*/
25 * EHCI scheduled transaction support: interrupt, iso, split iso
26 * These are called "periodic" transactions in the EHCI spec.
28 * Note that for interrupt transfers, the QH/QTD manipulation is shared
29 * with the "asynchronous" transaction support (control/bulk transfers).
30 * The only real difference is in how interrupt transfers are scheduled.
32 * For ISO, we make an "iso_stream" head to serve the same role as a QH.
33 * It keeps track of every ITD (or SITD) that's linked, and holds enough
34 * pre-calculated schedule data to make appending to the queue be quick.
37 static int ehci_get_frame (struct usb_hcd
*hcd
);
39 /*-------------------------------------------------------------------------*/
42 * periodic_next_shadow - return "next" pointer on shadow list
43 * @periodic: host pointer to qh/itd/sitd
44 * @tag: hardware tag for type of this record
46 static union ehci_shadow
*
47 periodic_next_shadow (union ehci_shadow
*periodic
, __le32 tag
)
51 return &periodic
->qh
->qh_next
;
53 return &periodic
->fstn
->fstn_next
;
55 return &periodic
->itd
->itd_next
;
58 return &periodic
->sitd
->sitd_next
;
62 /* caller must hold ehci->lock */
63 static void periodic_unlink (struct ehci_hcd
*ehci
, unsigned frame
, void *ptr
)
65 union ehci_shadow
*prev_p
= &ehci
->pshadow
[frame
];
66 __le32
*hw_p
= &ehci
->periodic
[frame
];
67 union ehci_shadow here
= *prev_p
;
69 /* find predecessor of "ptr"; hw and shadow lists are in sync */
70 while (here
.ptr
&& here
.ptr
!= ptr
) {
71 prev_p
= periodic_next_shadow (prev_p
, Q_NEXT_TYPE (*hw_p
));
75 /* an interrupt entry (at list end) could have been shared */
79 /* update shadow and hardware lists ... the old "next" pointers
80 * from ptr may still be in use, the caller updates them.
82 *prev_p
= *periodic_next_shadow (&here
, Q_NEXT_TYPE (*hw_p
));
83 *hw_p
= *here
.hw_next
;
86 /* how many of the uframe's 125 usecs are allocated? */
88 periodic_usecs (struct ehci_hcd
*ehci
, unsigned frame
, unsigned uframe
)
90 __le32
*hw_p
= &ehci
->periodic
[frame
];
91 union ehci_shadow
*q
= &ehci
->pshadow
[frame
];
95 switch (Q_NEXT_TYPE (*hw_p
)) {
97 /* is it in the S-mask? */
98 if (q
->qh
->hw_info2
& cpu_to_le32 (1 << uframe
))
99 usecs
+= q
->qh
->usecs
;
101 if (q
->qh
->hw_info2
& cpu_to_le32 (1 << (8 + uframe
)))
102 usecs
+= q
->qh
->c_usecs
;
103 hw_p
= &q
->qh
->hw_next
;
108 /* for "save place" FSTNs, count the relevant INTR
109 * bandwidth from the previous frame
111 if (q
->fstn
->hw_prev
!= EHCI_LIST_END
) {
112 ehci_dbg (ehci
, "ignoring FSTN cost ...\n");
114 hw_p
= &q
->fstn
->hw_next
;
115 q
= &q
->fstn
->fstn_next
;
118 usecs
+= q
->itd
->usecs
[uframe
];
119 hw_p
= &q
->itd
->hw_next
;
120 q
= &q
->itd
->itd_next
;
123 /* is it in the S-mask? (count SPLIT, DATA) */
124 if (q
->sitd
->hw_uframe
& cpu_to_le32 (1 << uframe
)) {
125 if (q
->sitd
->hw_fullspeed_ep
&
126 __constant_cpu_to_le32 (1<<31))
127 usecs
+= q
->sitd
->stream
->usecs
;
128 else /* worst case for OUT start-split */
129 usecs
+= HS_USECS_ISO (188);
132 /* ... C-mask? (count CSPLIT, DATA) */
133 if (q
->sitd
->hw_uframe
&
134 cpu_to_le32 (1 << (8 + uframe
))) {
135 /* worst case for IN complete-split */
136 usecs
+= q
->sitd
->stream
->c_usecs
;
139 hw_p
= &q
->sitd
->hw_next
;
140 q
= &q
->sitd
->sitd_next
;
146 ehci_err (ehci
, "uframe %d sched overrun: %d usecs\n",
147 frame
* 8 + uframe
, usecs
);
152 /*-------------------------------------------------------------------------*/
154 static int same_tt (struct usb_device
*dev1
, struct usb_device
*dev2
)
156 if (!dev1
->tt
|| !dev2
->tt
)
158 if (dev1
->tt
!= dev2
->tt
)
161 return dev1
->ttport
== dev2
->ttport
;
166 #ifdef CONFIG_USB_EHCI_TT_NEWSCHED
168 /* Which uframe does the low/fullspeed transfer start in?
170 * The parameter is the mask of ssplits in "H-frame" terms
171 * and this returns the transfer start uframe in "B-frame" terms,
172 * which allows both to match, e.g. a ssplit in "H-frame" uframe 0
173 * will cause a transfer in "B-frame" uframe 0. "B-frames" lag
174 * "H-frames" by 1 uframe. See the EHCI spec sec 4.5 and figure 4.7.
176 static inline unsigned char tt_start_uframe(struct ehci_hcd
*ehci
, __le32 mask
)
178 unsigned char smask
= QH_SMASK
& le32_to_cpu(mask
);
180 ehci_err(ehci
, "invalid empty smask!\n");
181 /* uframe 7 can't have bw so this will indicate failure */
184 return ffs(smask
) - 1;
187 static const unsigned char
188 max_tt_usecs
[] = { 125, 125, 125, 125, 125, 125, 30, 0 };
190 /* carryover low/fullspeed bandwidth that crosses uframe boundries */
191 static inline void carryover_tt_bandwidth(unsigned short tt_usecs
[8])
194 for (i
=0; i
<7; i
++) {
195 if (max_tt_usecs
[i
] < tt_usecs
[i
]) {
196 tt_usecs
[i
+1] += tt_usecs
[i
] - max_tt_usecs
[i
];
197 tt_usecs
[i
] = max_tt_usecs
[i
];
202 /* How many of the tt's periodic downstream 1000 usecs are allocated?
204 * While this measures the bandwidth in terms of usecs/uframe,
205 * the low/fullspeed bus has no notion of uframes, so any particular
206 * low/fullspeed transfer can "carry over" from one uframe to the next,
207 * since the TT just performs downstream transfers in sequence.
209 * For example two seperate 100 usec transfers can start in the same uframe,
210 * and the second one would "carry over" 75 usecs into the next uframe.
214 struct ehci_hcd
*ehci
,
215 struct usb_device
*dev
,
217 unsigned short tt_usecs
[8]
220 __le32
*hw_p
= &ehci
->periodic
[frame
];
221 union ehci_shadow
*q
= &ehci
->pshadow
[frame
];
224 memset(tt_usecs
, 0, 16);
227 switch (Q_NEXT_TYPE(*hw_p
)) {
229 hw_p
= &q
->itd
->hw_next
;
230 q
= &q
->itd
->itd_next
;
233 if (same_tt(dev
, q
->qh
->dev
)) {
234 uf
= tt_start_uframe(ehci
, q
->qh
->hw_info2
);
235 tt_usecs
[uf
] += q
->qh
->tt_usecs
;
237 hw_p
= &q
->qh
->hw_next
;
241 if (same_tt(dev
, q
->sitd
->urb
->dev
)) {
242 uf
= tt_start_uframe(ehci
, q
->sitd
->hw_uframe
);
243 tt_usecs
[uf
] += q
->sitd
->stream
->tt_usecs
;
245 hw_p
= &q
->sitd
->hw_next
;
246 q
= &q
->sitd
->sitd_next
;
251 "ignoring periodic frame %d FSTN\n", frame
);
252 hw_p
= &q
->fstn
->hw_next
;
253 q
= &q
->fstn
->fstn_next
;
257 carryover_tt_bandwidth(tt_usecs
);
259 if (max_tt_usecs
[7] < tt_usecs
[7])
260 ehci_err(ehci
, "frame %d tt sched overrun: %d usecs\n",
261 frame
, tt_usecs
[7] - max_tt_usecs
[7]);
265 * Return true if the device's tt's downstream bus is available for a
266 * periodic transfer of the specified length (usecs), starting at the
267 * specified frame/uframe. Note that (as summarized in section 11.19
268 * of the usb 2.0 spec) TTs can buffer multiple transactions for each
271 * The uframe parameter is when the fullspeed/lowspeed transfer
272 * should be executed in "B-frame" terms, which is the same as the
273 * highspeed ssplit's uframe (which is in "H-frame" terms). For example
274 * a ssplit in "H-frame" 0 causes a transfer in "B-frame" 0.
275 * See the EHCI spec sec 4.5 and fig 4.7.
277 * This checks if the full/lowspeed bus, at the specified starting uframe,
278 * has the specified bandwidth available, according to rules listed
279 * in USB 2.0 spec section 11.18.1 fig 11-60.
281 * This does not check if the transfer would exceed the max ssplit
282 * limit of 16, specified in USB 2.0 spec section 11.18.4 requirement #4,
283 * since proper scheduling limits ssplits to less than 16 per uframe.
285 static int tt_available (
286 struct ehci_hcd
*ehci
,
288 struct usb_device
*dev
,
294 if ((period
== 0) || (uframe
>= 7)) /* error */
297 for (; frame
< ehci
->periodic_size
; frame
+= period
) {
298 unsigned short tt_usecs
[8];
300 periodic_tt_usecs (ehci
, dev
, frame
, tt_usecs
);
302 ehci_vdbg(ehci
, "tt frame %d check %d usecs start uframe %d in"
303 " schedule %d/%d/%d/%d/%d/%d/%d/%d\n",
304 frame
, usecs
, uframe
,
305 tt_usecs
[0], tt_usecs
[1], tt_usecs
[2], tt_usecs
[3],
306 tt_usecs
[4], tt_usecs
[5], tt_usecs
[6], tt_usecs
[7]);
308 if (max_tt_usecs
[uframe
] <= tt_usecs
[uframe
]) {
309 ehci_vdbg(ehci
, "frame %d uframe %d fully scheduled\n",
314 /* special case for isoc transfers larger than 125us:
315 * the first and each subsequent fully used uframe
316 * must be empty, so as to not illegally delay
317 * already scheduled transactions
320 int ufs
= (usecs
/ 125) - 1;
322 for (i
= uframe
; i
< (uframe
+ ufs
) && i
< 8; i
++)
323 if (0 < tt_usecs
[i
]) {
325 "multi-uframe xfer can't fit "
326 "in frame %d uframe %d\n",
332 tt_usecs
[uframe
] += usecs
;
334 carryover_tt_bandwidth(tt_usecs
);
336 /* fail if the carryover pushed bw past the last uframe's limit */
337 if (max_tt_usecs
[7] < tt_usecs
[7]) {
339 "tt unavailable usecs %d frame %d uframe %d\n",
340 usecs
, frame
, uframe
);
350 /* return true iff the device's transaction translator is available
351 * for a periodic transfer starting at the specified frame, using
352 * all the uframes in the mask.
354 static int tt_no_collision (
355 struct ehci_hcd
*ehci
,
357 struct usb_device
*dev
,
362 if (period
== 0) /* error */
365 /* note bandwidth wastage: split never follows csplit
366 * (different dev or endpoint) until the next uframe.
367 * calling convention doesn't make that distinction.
369 for (; frame
< ehci
->periodic_size
; frame
+= period
) {
370 union ehci_shadow here
;
373 here
= ehci
->pshadow
[frame
];
374 type
= Q_NEXT_TYPE (ehci
->periodic
[frame
]);
378 type
= Q_NEXT_TYPE (here
.itd
->hw_next
);
379 here
= here
.itd
->itd_next
;
382 if (same_tt (dev
, here
.qh
->dev
)) {
385 mask
= le32_to_cpu (here
.qh
->hw_info2
);
386 /* "knows" no gap is needed */
391 type
= Q_NEXT_TYPE (here
.qh
->hw_next
);
392 here
= here
.qh
->qh_next
;
395 if (same_tt (dev
, here
.sitd
->urb
->dev
)) {
398 mask
= le32_to_cpu (here
.sitd
400 /* FIXME assumes no gap for IN! */
405 type
= Q_NEXT_TYPE (here
.sitd
->hw_next
);
406 here
= here
.sitd
->sitd_next
;
411 "periodic frame %d bogus type %d\n",
415 /* collision or error */
424 #endif /* CONFIG_USB_EHCI_TT_NEWSCHED */
426 /*-------------------------------------------------------------------------*/
428 static int enable_periodic (struct ehci_hcd
*ehci
)
433 /* did clearing PSE did take effect yet?
434 * takes effect only at frame boundaries...
436 status
= handshake(ehci
, &ehci
->regs
->status
, STS_PSS
, 0, 9 * 125);
438 ehci_to_hcd(ehci
)->state
= HC_STATE_HALT
;
442 cmd
= ehci_readl(ehci
, &ehci
->regs
->command
) | CMD_PSE
;
443 ehci_writel(ehci
, cmd
, &ehci
->regs
->command
);
444 /* posted write ... PSS happens later */
445 ehci_to_hcd(ehci
)->state
= HC_STATE_RUNNING
;
447 /* make sure ehci_work scans these */
448 ehci
->next_uframe
= ehci_readl(ehci
, &ehci
->regs
->frame_index
)
449 % (ehci
->periodic_size
<< 3);
453 static int disable_periodic (struct ehci_hcd
*ehci
)
458 /* did setting PSE not take effect yet?
459 * takes effect only at frame boundaries...
461 status
= handshake(ehci
, &ehci
->regs
->status
, STS_PSS
, STS_PSS
, 9 * 125);
463 ehci_to_hcd(ehci
)->state
= HC_STATE_HALT
;
467 cmd
= ehci_readl(ehci
, &ehci
->regs
->command
) & ~CMD_PSE
;
468 ehci_writel(ehci
, cmd
, &ehci
->regs
->command
);
469 /* posted write ... */
471 ehci
->next_uframe
= -1;
475 /*-------------------------------------------------------------------------*/
476 #ifdef CONFIG_CPU_FREQ
478 /* ignore/inactivate bit in QH hw_info1 */
479 #define INACTIVATE_BIT __constant_cpu_to_le32(QH_INACTIVATE)
481 #define HALT_BIT __constant_cpu_to_le32(QTD_STS_HALT)
482 #define ACTIVE_BIT __constant_cpu_to_le32(QTD_STS_ACTIVE)
483 #define STATUS_BIT __constant_cpu_to_le32(QTD_STS_STS)
485 static int safe_to_modify_i (struct ehci_hcd
*ehci
, struct ehci_qh
*qh
)
487 int now
; /* current (frame * 8) + uframe */
488 int prev_start
, next_start
; /* uframes from/to split start */
489 int start_uframe
= ffs(le32_to_cpup (&qh
->hw_info2
) & QH_SMASK
);
490 int end_uframe
= fls((le32_to_cpup (&qh
->hw_info2
) & QH_CMASK
) >> 8);
491 int split_duration
= end_uframe
- start_uframe
;
493 now
= readl(&ehci
->regs
->frame_index
) % (ehci
->periodic_size
<< 3);
495 next_start
= ((1024 << 3) + (qh
->start
<< 3) + start_uframe
- now
) %
497 prev_start
= (qh
->period
<< 3) - next_start
;
500 * Make sure there will be at least one uframe when qh is safe.
502 if ((qh
->period
<< 3) <= (ehci
->i_thresh
+ 2 + split_duration
))
507 * Wait 1 uframe after transaction should have started, to make
508 * sure controller has time to write back overlay, so we can
509 * check QTD_STS_STS to see if transaction is in progress.
511 if ((next_start
> ehci
->i_thresh
) && (prev_start
> 1))
512 /* safe to set "i" bit if split isn't in progress */
513 return (qh
->hw_token
& STATUS_BIT
) ? 0 : 1;
518 /* Set inactivate bit for all the split interrupt QHs. */
519 static void qh_inactivate_split_intr_qhs (struct ehci_hcd
*ehci
)
526 list_for_each_entry(qh
, &ehci
->split_intr_qhs
,
528 if (qh
->hw_info1
& INACTIVATE_BIT
)
532 * To avoid setting "I" after the start split happens,
533 * don't set it if the QH might be cached in the
534 * controller. Some HCs (Broadcom/ServerWorks HT1000)
535 * will stop in the middle of a split transaction when
536 * the "I" bit is set.
538 safe
= safe_to_modify_i(ehci
, qh
);
541 } else if (safe
> 0) {
542 qh
->was_active
= qh
->hw_token
& ACTIVE_BIT
;
543 qh
->hw_info1
|= INACTIVATE_BIT
;
550 static void qh_reactivate_split_intr_qhs (struct ehci_hcd
*ehci
)
558 list_for_each_entry(qh
, &ehci
->split_intr_qhs
, split_intr_qhs
) {
559 if (!(qh
->hw_info1
& INACTIVATE_BIT
)) /* already on */
562 * Don't reactivate if cached, or controller might
563 * overwrite overlay after we modify it!
565 safe
= safe_to_modify_i(ehci
, qh
);
568 } else if (safe
> 0) {
569 /* See EHCI 1.0 section 4.15.2.4. */
570 token
= qh
->hw_token
;
571 qh
->hw_token
= (token
| HALT_BIT
) & ~ACTIVE_BIT
;
573 qh
->hw_info1
&= ~INACTIVATE_BIT
;
575 qh
->hw_token
= (token
& ~HALT_BIT
) | qh
->was_active
;
582 /* periodic schedule slots have iso tds (normal or split) first, then a
583 * sparse tree for active interrupt transfers.
585 * this just links in a qh; caller guarantees uframe masks are set right.
586 * no FSTN support (yet; ehci 0.96+)
588 static int qh_link_periodic (struct ehci_hcd
*ehci
, struct ehci_qh
*qh
)
591 unsigned period
= qh
->period
;
593 dev_dbg (&qh
->dev
->dev
,
594 "link qh%d-%04x/%p start %d [%d/%d us]\n",
595 period
, le32_to_cpup (&qh
->hw_info2
) & (QH_CMASK
| QH_SMASK
),
596 qh
, qh
->start
, qh
->usecs
, qh
->c_usecs
);
598 #ifdef CONFIG_CPU_FREQ
600 * If low/full speed interrupt QHs are inactive (because of
601 * cpufreq changing processor speeds), start QH with I flag set--
602 * it will automatically be cleared when cpufreq is done.
604 if (ehci
->cpufreq_changing
)
605 if (!(qh
->hw_info1
& (cpu_to_le32(1 << 13))))
606 qh
->hw_info1
|= INACTIVATE_BIT
;
609 /* high bandwidth, or otherwise every microframe */
613 for (i
= qh
->start
; i
< ehci
->periodic_size
; i
+= period
) {
614 union ehci_shadow
*prev
= &ehci
->pshadow
[i
];
615 __le32
*hw_p
= &ehci
->periodic
[i
];
616 union ehci_shadow here
= *prev
;
619 /* skip the iso nodes at list head */
621 type
= Q_NEXT_TYPE (*hw_p
);
622 if (type
== Q_TYPE_QH
)
624 prev
= periodic_next_shadow (prev
, type
);
625 hw_p
= &here
.qh
->hw_next
;
629 /* sorting each branch by period (slow-->fast)
630 * enables sharing interior tree nodes
632 while (here
.ptr
&& qh
!= here
.qh
) {
633 if (qh
->period
> here
.qh
->period
)
635 prev
= &here
.qh
->qh_next
;
636 hw_p
= &here
.qh
->hw_next
;
639 /* link in this qh, unless some earlier pass did that */
646 *hw_p
= QH_NEXT (qh
->qh_dma
);
649 qh
->qh_state
= QH_STATE_LINKED
;
652 /* update per-qh bandwidth for usbfs */
653 ehci_to_hcd(ehci
)->self
.bandwidth_allocated
+= qh
->period
654 ? ((qh
->usecs
+ qh
->c_usecs
) / qh
->period
)
657 #ifdef CONFIG_CPU_FREQ
658 /* add qh to list of low/full speed interrupt QHs, if applicable */
659 if (!(qh
->hw_info1
& (cpu_to_le32(1 << 13)))) {
660 list_add(&qh
->split_intr_qhs
, &ehci
->split_intr_qhs
);
663 /* maybe enable periodic schedule processing */
664 if (!ehci
->periodic_sched
++)
665 return enable_periodic (ehci
);
670 static void qh_unlink_periodic (struct ehci_hcd
*ehci
, struct ehci_qh
*qh
)
676 // IF this isn't high speed
677 // and this qh is active in the current uframe
678 // (and overlay token SplitXstate is false?)
680 // qh->hw_info1 |= __constant_cpu_to_le32 (1 << 7 /* "ignore" */);
682 #ifdef CONFIG_CPU_FREQ
683 /* remove qh from list of low/full speed interrupt QHs */
684 if (!(qh
->hw_info1
& (cpu_to_le32(1 << 13)))) {
685 list_del_init(&qh
->split_intr_qhs
);
689 /* high bandwidth, or otherwise part of every microframe */
690 if ((period
= qh
->period
) == 0)
693 for (i
= qh
->start
; i
< ehci
->periodic_size
; i
+= period
)
694 periodic_unlink (ehci
, i
, qh
);
696 /* update per-qh bandwidth for usbfs */
697 ehci_to_hcd(ehci
)->self
.bandwidth_allocated
-= qh
->period
698 ? ((qh
->usecs
+ qh
->c_usecs
) / qh
->period
)
701 dev_dbg (&qh
->dev
->dev
,
702 "unlink qh%d-%04x/%p start %d [%d/%d us]\n",
704 le32_to_cpup (&qh
->hw_info2
) & (QH_CMASK
| QH_SMASK
),
705 qh
, qh
->start
, qh
->usecs
, qh
->c_usecs
);
707 /* qh->qh_next still "live" to HC */
708 qh
->qh_state
= QH_STATE_UNLINK
;
709 qh
->qh_next
.ptr
= NULL
;
712 /* maybe turn off periodic schedule */
713 ehci
->periodic_sched
--;
714 if (!ehci
->periodic_sched
)
715 (void) disable_periodic (ehci
);
718 static void intr_deschedule (struct ehci_hcd
*ehci
, struct ehci_qh
*qh
)
722 qh_unlink_periodic (ehci
, qh
);
724 /* simple/paranoid: always delay, expecting the HC needs to read
725 * qh->hw_next or finish a writeback after SPLIT/CSPLIT ... and
726 * expect khubd to clean up after any CSPLITs we won't issue.
727 * active high speed queues may need bigger delays...
729 if (list_empty (&qh
->qtd_list
)
730 || (__constant_cpu_to_le32 (QH_CMASK
)
731 & qh
->hw_info2
) != 0)
734 wait
= 55; /* worst case: 3 * 1024 */
737 qh
->qh_state
= QH_STATE_IDLE
;
738 qh
->hw_next
= EHCI_LIST_END
;
742 /*-------------------------------------------------------------------------*/
744 static int check_period (
745 struct ehci_hcd
*ehci
,
753 /* complete split running into next frame?
754 * given FSTN support, we could sometimes check...
760 * 80% periodic == 100 usec/uframe available
761 * convert "usecs we need" to "max already claimed"
765 /* we "know" 2 and 4 uframe intervals were rejected; so
766 * for period 0, check _every_ microframe in the schedule.
768 if (unlikely (period
== 0)) {
770 for (uframe
= 0; uframe
< 7; uframe
++) {
771 claimed
= periodic_usecs (ehci
, frame
, uframe
);
775 } while ((frame
+= 1) < ehci
->periodic_size
);
777 /* just check the specified uframe, at that period */
780 claimed
= periodic_usecs (ehci
, frame
, uframe
);
783 } while ((frame
+= period
) < ehci
->periodic_size
);
790 static int check_intr_schedule (
791 struct ehci_hcd
*ehci
,
794 const struct ehci_qh
*qh
,
798 int retval
= -ENOSPC
;
801 if (qh
->c_usecs
&& uframe
>= 6) /* FSTN territory? */
804 if (!check_period (ehci
, frame
, uframe
, qh
->period
, qh
->usecs
))
812 #ifdef CONFIG_USB_EHCI_TT_NEWSCHED
813 if (tt_available (ehci
, qh
->period
, qh
->dev
, frame
, uframe
,
817 /* TODO : this may need FSTN for SSPLIT in uframe 5. */
818 for (i
=uframe
+1; i
<8 && i
<uframe
+4; i
++)
819 if (!check_period (ehci
, frame
, i
,
820 qh
->period
, qh
->c_usecs
))
827 *c_maskp
= cpu_to_le32 (mask
<< 8);
830 /* Make sure this tt's buffer is also available for CSPLITs.
831 * We pessimize a bit; probably the typical full speed case
832 * doesn't need the second CSPLIT.
834 * NOTE: both SPLIT and CSPLIT could be checked in just
837 mask
= 0x03 << (uframe
+ qh
->gap_uf
);
838 *c_maskp
= cpu_to_le32 (mask
<< 8);
841 if (tt_no_collision (ehci
, qh
->period
, qh
->dev
, frame
, mask
)) {
842 if (!check_period (ehci
, frame
, uframe
+ qh
->gap_uf
+ 1,
843 qh
->period
, qh
->c_usecs
))
845 if (!check_period (ehci
, frame
, uframe
+ qh
->gap_uf
,
846 qh
->period
, qh
->c_usecs
))
855 /* "first fit" scheduling policy used the first time through,
856 * or when the previous schedule slot can't be re-used.
858 static int qh_schedule (struct ehci_hcd
*ehci
, struct ehci_qh
*qh
)
863 unsigned frame
; /* 0..(qh->period - 1), or NO_FRAME */
865 qh_refresh(ehci
, qh
);
866 qh
->hw_next
= EHCI_LIST_END
;
869 /* reuse the previous schedule slots, if we can */
870 if (frame
< qh
->period
) {
871 uframe
= ffs (le32_to_cpup (&qh
->hw_info2
) & QH_SMASK
);
872 status
= check_intr_schedule (ehci
, frame
, --uframe
,
880 /* else scan the schedule to find a group of slots such that all
881 * uframes have enough periodic bandwidth available.
884 /* "normal" case, uframing flexible except with splits */
886 frame
= qh
->period
- 1;
888 for (uframe
= 0; uframe
< 8; uframe
++) {
889 status
= check_intr_schedule (ehci
,
895 } while (status
&& frame
--);
897 /* qh->period == 0 means every uframe */
900 status
= check_intr_schedule (ehci
, 0, 0, qh
, &c_mask
);
906 /* reset S-frame and (maybe) C-frame masks */
907 qh
->hw_info2
&= __constant_cpu_to_le32(~(QH_CMASK
| QH_SMASK
));
908 qh
->hw_info2
|= qh
->period
909 ? cpu_to_le32 (1 << uframe
)
910 : __constant_cpu_to_le32 (QH_SMASK
);
911 qh
->hw_info2
|= c_mask
;
913 ehci_dbg (ehci
, "reused qh %p schedule\n", qh
);
915 /* stuff into the periodic schedule */
916 status
= qh_link_periodic (ehci
, qh
);
921 static int intr_submit (
922 struct ehci_hcd
*ehci
,
923 struct usb_host_endpoint
*ep
,
925 struct list_head
*qtd_list
,
932 struct list_head empty
;
934 /* get endpoint and transfer/schedule data */
935 epnum
= ep
->desc
.bEndpointAddress
;
937 spin_lock_irqsave (&ehci
->lock
, flags
);
939 if (unlikely(!test_bit(HCD_FLAG_HW_ACCESSIBLE
,
940 &ehci_to_hcd(ehci
)->flags
))) {
945 /* get qh and force any scheduling errors */
946 INIT_LIST_HEAD (&empty
);
947 qh
= qh_append_tds (ehci
, urb
, &empty
, epnum
, &ep
->hcpriv
);
952 if (qh
->qh_state
== QH_STATE_IDLE
) {
953 if ((status
= qh_schedule (ehci
, qh
)) != 0)
957 /* then queue the urb's tds to the qh */
958 qh
= qh_append_tds (ehci
, urb
, qtd_list
, epnum
, &ep
->hcpriv
);
961 /* ... update usbfs periodic stats */
962 ehci_to_hcd(ehci
)->self
.bandwidth_int_reqs
++;
965 spin_unlock_irqrestore (&ehci
->lock
, flags
);
967 qtd_list_free (ehci
, urb
, qtd_list
);
972 /*-------------------------------------------------------------------------*/
974 /* ehci_iso_stream ops work with both ITD and SITD */
976 static struct ehci_iso_stream
*
977 iso_stream_alloc (gfp_t mem_flags
)
979 struct ehci_iso_stream
*stream
;
981 stream
= kzalloc(sizeof *stream
, mem_flags
);
982 if (likely (stream
!= NULL
)) {
983 INIT_LIST_HEAD(&stream
->td_list
);
984 INIT_LIST_HEAD(&stream
->free_list
);
985 stream
->next_uframe
= -1;
986 stream
->refcount
= 1;
993 struct ehci_hcd
*ehci
,
994 struct ehci_iso_stream
*stream
,
995 struct usb_device
*dev
,
1000 static const u8 smask_out
[] = { 0x01, 0x03, 0x07, 0x0f, 0x1f, 0x3f };
1003 unsigned epnum
, maxp
;
1008 * this might be a "high bandwidth" highspeed endpoint,
1009 * as encoded in the ep descriptor's wMaxPacket field
1011 epnum
= usb_pipeendpoint (pipe
);
1012 is_input
= usb_pipein (pipe
) ? USB_DIR_IN
: 0;
1013 maxp
= usb_maxpacket(dev
, pipe
, !is_input
);
1020 /* knows about ITD vs SITD */
1021 if (dev
->speed
== USB_SPEED_HIGH
) {
1022 unsigned multi
= hb_mult(maxp
);
1024 stream
->highspeed
= 1;
1026 maxp
= max_packet(maxp
);
1030 stream
->buf0
= cpu_to_le32 ((epnum
<< 8) | dev
->devnum
);
1031 stream
->buf1
= cpu_to_le32 (buf1
);
1032 stream
->buf2
= cpu_to_le32 (multi
);
1034 /* usbfs wants to report the average usecs per frame tied up
1035 * when transfers on this endpoint are scheduled ...
1037 stream
->usecs
= HS_USECS_ISO (maxp
);
1038 bandwidth
= stream
->usecs
* 8;
1039 bandwidth
/= 1 << (interval
- 1);
1046 addr
= dev
->ttport
<< 24;
1047 if (!ehci_is_TDI(ehci
)
1049 ehci_to_hcd(ehci
)->self
.root_hub
))
1050 addr
|= dev
->tt
->hub
->devnum
<< 16;
1052 addr
|= dev
->devnum
;
1053 stream
->usecs
= HS_USECS_ISO (maxp
);
1054 think_time
= dev
->tt
? dev
->tt
->think_time
: 0;
1055 stream
->tt_usecs
= NS_TO_US (think_time
+ usb_calc_bus_time (
1056 dev
->speed
, is_input
, 1, maxp
));
1057 hs_transfers
= max (1u, (maxp
+ 187) / 188);
1062 stream
->c_usecs
= stream
->usecs
;
1063 stream
->usecs
= HS_USECS_ISO (1);
1064 stream
->raw_mask
= 1;
1066 /* c-mask as specified in USB 2.0 11.18.4 3.c */
1067 tmp
= (1 << (hs_transfers
+ 2)) - 1;
1068 stream
->raw_mask
|= tmp
<< (8 + 2);
1070 stream
->raw_mask
= smask_out
[hs_transfers
- 1];
1071 bandwidth
= stream
->usecs
+ stream
->c_usecs
;
1072 bandwidth
/= 1 << (interval
+ 2);
1074 /* stream->splits gets created from raw_mask later */
1075 stream
->address
= cpu_to_le32 (addr
);
1077 stream
->bandwidth
= bandwidth
;
1081 stream
->bEndpointAddress
= is_input
| epnum
;
1082 stream
->interval
= interval
;
1083 stream
->maxp
= maxp
;
1087 iso_stream_put(struct ehci_hcd
*ehci
, struct ehci_iso_stream
*stream
)
1091 /* free whenever just a dev->ep reference remains.
1092 * not like a QH -- no persistent state (toggle, halt)
1094 if (stream
->refcount
== 1) {
1097 // BUG_ON (!list_empty(&stream->td_list));
1099 while (!list_empty (&stream
->free_list
)) {
1100 struct list_head
*entry
;
1102 entry
= stream
->free_list
.next
;
1105 /* knows about ITD vs SITD */
1106 if (stream
->highspeed
) {
1107 struct ehci_itd
*itd
;
1109 itd
= list_entry (entry
, struct ehci_itd
,
1111 dma_pool_free (ehci
->itd_pool
, itd
,
1114 struct ehci_sitd
*sitd
;
1116 sitd
= list_entry (entry
, struct ehci_sitd
,
1118 dma_pool_free (ehci
->sitd_pool
, sitd
,
1123 is_in
= (stream
->bEndpointAddress
& USB_DIR_IN
) ? 0x10 : 0;
1124 stream
->bEndpointAddress
&= 0x0f;
1125 stream
->ep
->hcpriv
= NULL
;
1127 if (stream
->rescheduled
) {
1128 ehci_info (ehci
, "ep%d%s-iso rescheduled "
1129 "%lu times in %lu seconds\n",
1130 stream
->bEndpointAddress
, is_in
? "in" : "out",
1131 stream
->rescheduled
,
1132 ((jiffies
- stream
->start
)/HZ
)
1140 static inline struct ehci_iso_stream
*
1141 iso_stream_get (struct ehci_iso_stream
*stream
)
1143 if (likely (stream
!= NULL
))
1148 static struct ehci_iso_stream
*
1149 iso_stream_find (struct ehci_hcd
*ehci
, struct urb
*urb
)
1152 struct ehci_iso_stream
*stream
;
1153 struct usb_host_endpoint
*ep
;
1154 unsigned long flags
;
1156 epnum
= usb_pipeendpoint (urb
->pipe
);
1157 if (usb_pipein(urb
->pipe
))
1158 ep
= urb
->dev
->ep_in
[epnum
];
1160 ep
= urb
->dev
->ep_out
[epnum
];
1162 spin_lock_irqsave (&ehci
->lock
, flags
);
1163 stream
= ep
->hcpriv
;
1165 if (unlikely (stream
== NULL
)) {
1166 stream
= iso_stream_alloc(GFP_ATOMIC
);
1167 if (likely (stream
!= NULL
)) {
1168 /* dev->ep owns the initial refcount */
1169 ep
->hcpriv
= stream
;
1171 iso_stream_init(ehci
, stream
, urb
->dev
, urb
->pipe
,
1175 /* if dev->ep [epnum] is a QH, info1.maxpacket is nonzero */
1176 } else if (unlikely (stream
->hw_info1
!= 0)) {
1177 ehci_dbg (ehci
, "dev %s ep%d%s, not iso??\n",
1178 urb
->dev
->devpath
, epnum
,
1179 usb_pipein(urb
->pipe
) ? "in" : "out");
1183 /* caller guarantees an eventual matching iso_stream_put */
1184 stream
= iso_stream_get (stream
);
1186 spin_unlock_irqrestore (&ehci
->lock
, flags
);
1190 /*-------------------------------------------------------------------------*/
1192 /* ehci_iso_sched ops can be ITD-only or SITD-only */
1194 static struct ehci_iso_sched
*
1195 iso_sched_alloc (unsigned packets
, gfp_t mem_flags
)
1197 struct ehci_iso_sched
*iso_sched
;
1198 int size
= sizeof *iso_sched
;
1200 size
+= packets
* sizeof (struct ehci_iso_packet
);
1201 iso_sched
= kzalloc(size
, mem_flags
);
1202 if (likely (iso_sched
!= NULL
)) {
1203 INIT_LIST_HEAD (&iso_sched
->td_list
);
1210 struct ehci_iso_sched
*iso_sched
,
1211 struct ehci_iso_stream
*stream
,
1216 dma_addr_t dma
= urb
->transfer_dma
;
1218 /* how many uframes are needed for these transfers */
1219 iso_sched
->span
= urb
->number_of_packets
* stream
->interval
;
1221 /* figure out per-uframe itd fields that we'll need later
1222 * when we fit new itds into the schedule.
1224 for (i
= 0; i
< urb
->number_of_packets
; i
++) {
1225 struct ehci_iso_packet
*uframe
= &iso_sched
->packet
[i
];
1230 length
= urb
->iso_frame_desc
[i
].length
;
1231 buf
= dma
+ urb
->iso_frame_desc
[i
].offset
;
1233 trans
= EHCI_ISOC_ACTIVE
;
1234 trans
|= buf
& 0x0fff;
1235 if (unlikely (((i
+ 1) == urb
->number_of_packets
))
1236 && !(urb
->transfer_flags
& URB_NO_INTERRUPT
))
1237 trans
|= EHCI_ITD_IOC
;
1238 trans
|= length
<< 16;
1239 uframe
->transaction
= cpu_to_le32 (trans
);
1241 /* might need to cross a buffer page within a uframe */
1242 uframe
->bufp
= (buf
& ~(u64
)0x0fff);
1244 if (unlikely ((uframe
->bufp
!= (buf
& ~(u64
)0x0fff))))
1251 struct ehci_iso_stream
*stream
,
1252 struct ehci_iso_sched
*iso_sched
1257 // caller must hold ehci->lock!
1258 list_splice (&iso_sched
->td_list
, &stream
->free_list
);
1263 itd_urb_transaction (
1264 struct ehci_iso_stream
*stream
,
1265 struct ehci_hcd
*ehci
,
1270 struct ehci_itd
*itd
;
1274 struct ehci_iso_sched
*sched
;
1275 unsigned long flags
;
1277 sched
= iso_sched_alloc (urb
->number_of_packets
, mem_flags
);
1278 if (unlikely (sched
== NULL
))
1281 itd_sched_init (sched
, stream
, urb
);
1283 if (urb
->interval
< 8)
1284 num_itds
= 1 + (sched
->span
+ 7) / 8;
1286 num_itds
= urb
->number_of_packets
;
1288 /* allocate/init ITDs */
1289 spin_lock_irqsave (&ehci
->lock
, flags
);
1290 for (i
= 0; i
< num_itds
; i
++) {
1292 /* free_list.next might be cache-hot ... but maybe
1293 * the HC caches it too. avoid that issue for now.
1296 /* prefer previously-allocated itds */
1297 if (likely (!list_empty(&stream
->free_list
))) {
1298 itd
= list_entry (stream
->free_list
.prev
,
1299 struct ehci_itd
, itd_list
);
1300 list_del (&itd
->itd_list
);
1301 itd_dma
= itd
->itd_dma
;
1306 spin_unlock_irqrestore (&ehci
->lock
, flags
);
1307 itd
= dma_pool_alloc (ehci
->itd_pool
, mem_flags
,
1309 spin_lock_irqsave (&ehci
->lock
, flags
);
1312 if (unlikely (NULL
== itd
)) {
1313 iso_sched_free (stream
, sched
);
1314 spin_unlock_irqrestore (&ehci
->lock
, flags
);
1317 memset (itd
, 0, sizeof *itd
);
1318 itd
->itd_dma
= itd_dma
;
1319 list_add (&itd
->itd_list
, &sched
->td_list
);
1321 spin_unlock_irqrestore (&ehci
->lock
, flags
);
1323 /* temporarily store schedule info in hcpriv */
1324 urb
->hcpriv
= sched
;
1325 urb
->error_count
= 0;
1329 /*-------------------------------------------------------------------------*/
1333 struct ehci_hcd
*ehci
,
1342 /* can't commit more than 80% periodic == 100 usec */
1343 if (periodic_usecs (ehci
, uframe
>> 3, uframe
& 0x7)
1347 /* we know urb->interval is 2^N uframes */
1349 } while (uframe
< mod
);
1355 struct ehci_hcd
*ehci
,
1357 struct ehci_iso_stream
*stream
,
1359 struct ehci_iso_sched
*sched
,
1366 mask
= stream
->raw_mask
<< (uframe
& 7);
1368 /* for IN, don't wrap CSPLIT into the next frame */
1372 /* this multi-pass logic is simple, but performance may
1373 * suffer when the schedule data isn't cached.
1376 /* check bandwidth */
1377 uframe
%= period_uframes
;
1381 frame
= uframe
>> 3;
1384 #ifdef CONFIG_USB_EHCI_TT_NEWSCHED
1385 /* The tt's fullspeed bus bandwidth must be available.
1386 * tt_available scheduling guarantees 10+% for control/bulk.
1388 if (!tt_available (ehci
, period_uframes
<< 3,
1389 stream
->udev
, frame
, uf
, stream
->tt_usecs
))
1392 /* tt must be idle for start(s), any gap, and csplit.
1393 * assume scheduling slop leaves 10+% for control/bulk.
1395 if (!tt_no_collision (ehci
, period_uframes
<< 3,
1396 stream
->udev
, frame
, mask
))
1400 /* check starts (OUT uses more than one) */
1401 max_used
= 100 - stream
->usecs
;
1402 for (tmp
= stream
->raw_mask
& 0xff; tmp
; tmp
>>= 1, uf
++) {
1403 if (periodic_usecs (ehci
, frame
, uf
) > max_used
)
1407 /* for IN, check CSPLIT */
1408 if (stream
->c_usecs
) {
1410 max_used
= 100 - stream
->c_usecs
;
1414 if ((stream
->raw_mask
& tmp
) == 0)
1416 if (periodic_usecs (ehci
, frame
, uf
)
1422 /* we know urb->interval is 2^N uframes */
1423 uframe
+= period_uframes
;
1424 } while (uframe
< mod
);
1426 stream
->splits
= cpu_to_le32(stream
->raw_mask
<< (uframe
& 7));
1431 * This scheduler plans almost as far into the future as it has actual
1432 * periodic schedule slots. (Affected by TUNE_FLS, which defaults to
1433 * "as small as possible" to be cache-friendlier.) That limits the size
1434 * transfers you can stream reliably; avoid more than 64 msec per urb.
1435 * Also avoid queue depths of less than ehci's worst irq latency (affected
1436 * by the per-urb URB_NO_INTERRUPT hint, the log2_irq_thresh module parameter,
1437 * and other factors); or more than about 230 msec total (for portability,
1438 * given EHCI_TUNE_FLS and the slop). Or, write a smarter scheduler!
1441 #define SCHEDULE_SLOP 10 /* frames */
1444 iso_stream_schedule (
1445 struct ehci_hcd
*ehci
,
1447 struct ehci_iso_stream
*stream
1450 u32 now
, start
, max
, period
;
1452 unsigned mod
= ehci
->periodic_size
<< 3;
1453 struct ehci_iso_sched
*sched
= urb
->hcpriv
;
1455 if (sched
->span
> (mod
- 8 * SCHEDULE_SLOP
)) {
1456 ehci_dbg (ehci
, "iso request %p too long\n", urb
);
1461 if ((stream
->depth
+ sched
->span
) > mod
) {
1462 ehci_dbg (ehci
, "request %p would overflow (%d+%d>%d)\n",
1463 urb
, stream
->depth
, sched
->span
, mod
);
1468 now
= ehci_readl(ehci
, &ehci
->regs
->frame_index
) % mod
;
1470 /* when's the last uframe this urb could start? */
1473 /* typical case: reuse current schedule. stream is still active,
1474 * and no gaps from host falling behind (irq delays etc)
1476 if (likely (!list_empty (&stream
->td_list
))) {
1477 start
= stream
->next_uframe
;
1480 if (likely ((start
+ sched
->span
) < max
))
1482 /* else fell behind; someday, try to reschedule */
1487 /* need to schedule; when's the next (u)frame we could start?
1488 * this is bigger than ehci->i_thresh allows; scheduling itself
1489 * isn't free, the slop should handle reasonably slow cpus. it
1490 * can also help high bandwidth if the dma and irq loads don't
1491 * jump until after the queue is primed.
1493 start
= SCHEDULE_SLOP
* 8 + (now
& ~0x07);
1495 stream
->next_uframe
= start
;
1497 /* NOTE: assumes URB_ISO_ASAP, to limit complexity/bugs */
1499 period
= urb
->interval
;
1500 if (!stream
->highspeed
)
1503 /* find a uframe slot with enough bandwidth */
1504 for (; start
< (stream
->next_uframe
+ period
); start
++) {
1507 /* check schedule: enough space? */
1508 if (stream
->highspeed
)
1509 enough_space
= itd_slot_ok (ehci
, mod
, start
,
1510 stream
->usecs
, period
);
1512 if ((start
% 8) >= 6)
1514 enough_space
= sitd_slot_ok (ehci
, mod
, stream
,
1515 start
, sched
, period
);
1518 /* schedule it here if there's enough bandwidth */
1520 stream
->next_uframe
= start
% mod
;
1525 /* no room in the schedule */
1526 ehci_dbg (ehci
, "iso %ssched full %p (now %d max %d)\n",
1527 list_empty (&stream
->td_list
) ? "" : "re",
1532 iso_sched_free (stream
, sched
);
1537 /* report high speed start in uframes; full speed, in frames */
1538 urb
->start_frame
= stream
->next_uframe
;
1539 if (!stream
->highspeed
)
1540 urb
->start_frame
>>= 3;
1544 /*-------------------------------------------------------------------------*/
1547 itd_init (struct ehci_iso_stream
*stream
, struct ehci_itd
*itd
)
1551 /* it's been recently zeroed */
1552 itd
->hw_next
= EHCI_LIST_END
;
1553 itd
->hw_bufp
[0] = stream
->buf0
;
1554 itd
->hw_bufp
[1] = stream
->buf1
;
1555 itd
->hw_bufp
[2] = stream
->buf2
;
1557 for (i
= 0; i
< 8; i
++)
1560 /* All other fields are filled when scheduling */
1565 struct ehci_itd
*itd
,
1566 struct ehci_iso_sched
*iso_sched
,
1571 struct ehci_iso_packet
*uf
= &iso_sched
->packet
[index
];
1572 unsigned pg
= itd
->pg
;
1574 // BUG_ON (pg == 6 && uf->cross);
1577 itd
->index
[uframe
] = index
;
1579 itd
->hw_transaction
[uframe
] = uf
->transaction
;
1580 itd
->hw_transaction
[uframe
] |= cpu_to_le32 (pg
<< 12);
1581 itd
->hw_bufp
[pg
] |= cpu_to_le32 (uf
->bufp
& ~(u32
)0);
1582 itd
->hw_bufp_hi
[pg
] |= cpu_to_le32 ((u32
)(uf
->bufp
>> 32));
1584 /* iso_frame_desc[].offset must be strictly increasing */
1585 if (unlikely (uf
->cross
)) {
1586 u64 bufp
= uf
->bufp
+ 4096;
1588 itd
->hw_bufp
[pg
] |= cpu_to_le32 (bufp
& ~(u32
)0);
1589 itd
->hw_bufp_hi
[pg
] |= cpu_to_le32 ((u32
)(bufp
>> 32));
1594 itd_link (struct ehci_hcd
*ehci
, unsigned frame
, struct ehci_itd
*itd
)
1596 /* always prepend ITD/SITD ... only QH tree is order-sensitive */
1597 itd
->itd_next
= ehci
->pshadow
[frame
];
1598 itd
->hw_next
= ehci
->periodic
[frame
];
1599 ehci
->pshadow
[frame
].itd
= itd
;
1602 ehci
->periodic
[frame
] = cpu_to_le32 (itd
->itd_dma
) | Q_TYPE_ITD
;
1605 /* fit urb's itds into the selected schedule slot; activate as needed */
1608 struct ehci_hcd
*ehci
,
1611 struct ehci_iso_stream
*stream
1615 unsigned next_uframe
, uframe
, frame
;
1616 struct ehci_iso_sched
*iso_sched
= urb
->hcpriv
;
1617 struct ehci_itd
*itd
;
1619 next_uframe
= stream
->next_uframe
% mod
;
1621 if (unlikely (list_empty(&stream
->td_list
))) {
1622 ehci_to_hcd(ehci
)->self
.bandwidth_allocated
1623 += stream
->bandwidth
;
1625 "schedule devp %s ep%d%s-iso period %d start %d.%d\n",
1626 urb
->dev
->devpath
, stream
->bEndpointAddress
& 0x0f,
1627 (stream
->bEndpointAddress
& USB_DIR_IN
) ? "in" : "out",
1629 next_uframe
>> 3, next_uframe
& 0x7);
1630 stream
->start
= jiffies
;
1632 ehci_to_hcd(ehci
)->self
.bandwidth_isoc_reqs
++;
1634 /* fill iTDs uframe by uframe */
1635 for (packet
= 0, itd
= NULL
; packet
< urb
->number_of_packets
; ) {
1637 /* ASSERT: we have all necessary itds */
1638 // BUG_ON (list_empty (&iso_sched->td_list));
1640 /* ASSERT: no itds for this endpoint in this uframe */
1642 itd
= list_entry (iso_sched
->td_list
.next
,
1643 struct ehci_itd
, itd_list
);
1644 list_move_tail (&itd
->itd_list
, &stream
->td_list
);
1645 itd
->stream
= iso_stream_get (stream
);
1646 itd
->urb
= usb_get_urb (urb
);
1647 itd_init (stream
, itd
);
1650 uframe
= next_uframe
& 0x07;
1651 frame
= next_uframe
>> 3;
1653 itd
->usecs
[uframe
] = stream
->usecs
;
1654 itd_patch (itd
, iso_sched
, packet
, uframe
);
1656 next_uframe
+= stream
->interval
;
1657 stream
->depth
+= stream
->interval
;
1661 /* link completed itds into the schedule */
1662 if (((next_uframe
>> 3) != frame
)
1663 || packet
== urb
->number_of_packets
) {
1664 itd_link (ehci
, frame
% ehci
->periodic_size
, itd
);
1668 stream
->next_uframe
= next_uframe
;
1670 /* don't need that schedule data any more */
1671 iso_sched_free (stream
, iso_sched
);
1674 timer_action (ehci
, TIMER_IO_WATCHDOG
);
1675 if (unlikely (!ehci
->periodic_sched
++))
1676 return enable_periodic (ehci
);
1680 #define ISO_ERRS (EHCI_ISOC_BUF_ERR | EHCI_ISOC_BABBLE | EHCI_ISOC_XACTERR)
1684 struct ehci_hcd
*ehci
,
1685 struct ehci_itd
*itd
1687 struct urb
*urb
= itd
->urb
;
1688 struct usb_iso_packet_descriptor
*desc
;
1692 struct ehci_iso_stream
*stream
= itd
->stream
;
1693 struct usb_device
*dev
;
1695 /* for each uframe with a packet */
1696 for (uframe
= 0; uframe
< 8; uframe
++) {
1697 if (likely (itd
->index
[uframe
] == -1))
1699 urb_index
= itd
->index
[uframe
];
1700 desc
= &urb
->iso_frame_desc
[urb_index
];
1702 t
= le32_to_cpup (&itd
->hw_transaction
[uframe
]);
1703 itd
->hw_transaction
[uframe
] = 0;
1704 stream
->depth
-= stream
->interval
;
1706 /* report transfer status */
1707 if (unlikely (t
& ISO_ERRS
)) {
1709 if (t
& EHCI_ISOC_BUF_ERR
)
1710 desc
->status
= usb_pipein (urb
->pipe
)
1711 ? -ENOSR
/* hc couldn't read */
1712 : -ECOMM
; /* hc couldn't write */
1713 else if (t
& EHCI_ISOC_BABBLE
)
1714 desc
->status
= -EOVERFLOW
;
1715 else /* (t & EHCI_ISOC_XACTERR) */
1716 desc
->status
= -EPROTO
;
1718 /* HC need not update length with this error */
1719 if (!(t
& EHCI_ISOC_BABBLE
))
1720 desc
->actual_length
= EHCI_ITD_LENGTH (t
);
1721 } else if (likely ((t
& EHCI_ISOC_ACTIVE
) == 0)) {
1723 desc
->actual_length
= EHCI_ITD_LENGTH (t
);
1730 list_move (&itd
->itd_list
, &stream
->free_list
);
1731 iso_stream_put (ehci
, stream
);
1733 /* handle completion now? */
1734 if (likely ((urb_index
+ 1) != urb
->number_of_packets
))
1737 /* ASSERT: it's really the last itd for this urb
1738 list_for_each_entry (itd, &stream->td_list, itd_list)
1739 BUG_ON (itd->urb == urb);
1742 /* give urb back to the driver ... can be out-of-order */
1744 ehci_urb_done (ehci
, urb
);
1747 /* defer stopping schedule; completion can submit */
1748 ehci
->periodic_sched
--;
1749 if (unlikely (!ehci
->periodic_sched
))
1750 (void) disable_periodic (ehci
);
1751 ehci_to_hcd(ehci
)->self
.bandwidth_isoc_reqs
--;
1753 if (unlikely (list_empty (&stream
->td_list
))) {
1754 ehci_to_hcd(ehci
)->self
.bandwidth_allocated
1755 -= stream
->bandwidth
;
1757 "deschedule devp %s ep%d%s-iso\n",
1758 dev
->devpath
, stream
->bEndpointAddress
& 0x0f,
1759 (stream
->bEndpointAddress
& USB_DIR_IN
) ? "in" : "out");
1761 iso_stream_put (ehci
, stream
);
1766 /*-------------------------------------------------------------------------*/
1768 static int itd_submit (struct ehci_hcd
*ehci
, struct urb
*urb
,
1771 int status
= -EINVAL
;
1772 unsigned long flags
;
1773 struct ehci_iso_stream
*stream
;
1775 /* Get iso_stream head */
1776 stream
= iso_stream_find (ehci
, urb
);
1777 if (unlikely (stream
== NULL
)) {
1778 ehci_dbg (ehci
, "can't get iso stream\n");
1781 if (unlikely (urb
->interval
!= stream
->interval
)) {
1782 ehci_dbg (ehci
, "can't change iso interval %d --> %d\n",
1783 stream
->interval
, urb
->interval
);
1787 #ifdef EHCI_URB_TRACE
1789 "%s %s urb %p ep%d%s len %d, %d pkts %d uframes [%p]\n",
1790 __FUNCTION__
, urb
->dev
->devpath
, urb
,
1791 usb_pipeendpoint (urb
->pipe
),
1792 usb_pipein (urb
->pipe
) ? "in" : "out",
1793 urb
->transfer_buffer_length
,
1794 urb
->number_of_packets
, urb
->interval
,
1798 /* allocate ITDs w/o locking anything */
1799 status
= itd_urb_transaction (stream
, ehci
, urb
, mem_flags
);
1800 if (unlikely (status
< 0)) {
1801 ehci_dbg (ehci
, "can't init itds\n");
1805 /* schedule ... need to lock */
1806 spin_lock_irqsave (&ehci
->lock
, flags
);
1807 if (unlikely(!test_bit(HCD_FLAG_HW_ACCESSIBLE
,
1808 &ehci_to_hcd(ehci
)->flags
)))
1809 status
= -ESHUTDOWN
;
1811 status
= iso_stream_schedule (ehci
, urb
, stream
);
1812 if (likely (status
== 0))
1813 itd_link_urb (ehci
, urb
, ehci
->periodic_size
<< 3, stream
);
1814 spin_unlock_irqrestore (&ehci
->lock
, flags
);
1817 if (unlikely (status
< 0))
1818 iso_stream_put (ehci
, stream
);
1822 #ifdef CONFIG_USB_EHCI_SPLIT_ISO
1824 /*-------------------------------------------------------------------------*/
1827 * "Split ISO TDs" ... used for USB 1.1 devices going through the
1828 * TTs in USB 2.0 hubs. These need microframe scheduling.
1833 struct ehci_iso_sched
*iso_sched
,
1834 struct ehci_iso_stream
*stream
,
1839 dma_addr_t dma
= urb
->transfer_dma
;
1841 /* how many frames are needed for these transfers */
1842 iso_sched
->span
= urb
->number_of_packets
* stream
->interval
;
1844 /* figure out per-frame sitd fields that we'll need later
1845 * when we fit new sitds into the schedule.
1847 for (i
= 0; i
< urb
->number_of_packets
; i
++) {
1848 struct ehci_iso_packet
*packet
= &iso_sched
->packet
[i
];
1853 length
= urb
->iso_frame_desc
[i
].length
& 0x03ff;
1854 buf
= dma
+ urb
->iso_frame_desc
[i
].offset
;
1856 trans
= SITD_STS_ACTIVE
;
1857 if (((i
+ 1) == urb
->number_of_packets
)
1858 && !(urb
->transfer_flags
& URB_NO_INTERRUPT
))
1860 trans
|= length
<< 16;
1861 packet
->transaction
= cpu_to_le32 (trans
);
1863 /* might need to cross a buffer page within a td */
1865 packet
->buf1
= (buf
+ length
) & ~0x0fff;
1866 if (packet
->buf1
!= (buf
& ~(u64
)0x0fff))
1869 /* OUT uses multiple start-splits */
1870 if (stream
->bEndpointAddress
& USB_DIR_IN
)
1872 length
= (length
+ 187) / 188;
1873 if (length
> 1) /* BEGIN vs ALL */
1875 packet
->buf1
|= length
;
1880 sitd_urb_transaction (
1881 struct ehci_iso_stream
*stream
,
1882 struct ehci_hcd
*ehci
,
1887 struct ehci_sitd
*sitd
;
1888 dma_addr_t sitd_dma
;
1890 struct ehci_iso_sched
*iso_sched
;
1891 unsigned long flags
;
1893 iso_sched
= iso_sched_alloc (urb
->number_of_packets
, mem_flags
);
1894 if (iso_sched
== NULL
)
1897 sitd_sched_init (iso_sched
, stream
, urb
);
1899 /* allocate/init sITDs */
1900 spin_lock_irqsave (&ehci
->lock
, flags
);
1901 for (i
= 0; i
< urb
->number_of_packets
; i
++) {
1903 /* NOTE: for now, we don't try to handle wraparound cases
1904 * for IN (using sitd->hw_backpointer, like a FSTN), which
1905 * means we never need two sitds for full speed packets.
1908 /* free_list.next might be cache-hot ... but maybe
1909 * the HC caches it too. avoid that issue for now.
1912 /* prefer previously-allocated sitds */
1913 if (!list_empty(&stream
->free_list
)) {
1914 sitd
= list_entry (stream
->free_list
.prev
,
1915 struct ehci_sitd
, sitd_list
);
1916 list_del (&sitd
->sitd_list
);
1917 sitd_dma
= sitd
->sitd_dma
;
1922 spin_unlock_irqrestore (&ehci
->lock
, flags
);
1923 sitd
= dma_pool_alloc (ehci
->sitd_pool
, mem_flags
,
1925 spin_lock_irqsave (&ehci
->lock
, flags
);
1929 iso_sched_free (stream
, iso_sched
);
1930 spin_unlock_irqrestore (&ehci
->lock
, flags
);
1933 memset (sitd
, 0, sizeof *sitd
);
1934 sitd
->sitd_dma
= sitd_dma
;
1935 list_add (&sitd
->sitd_list
, &iso_sched
->td_list
);
1938 /* temporarily store schedule info in hcpriv */
1939 urb
->hcpriv
= iso_sched
;
1940 urb
->error_count
= 0;
1942 spin_unlock_irqrestore (&ehci
->lock
, flags
);
1946 /*-------------------------------------------------------------------------*/
1950 struct ehci_iso_stream
*stream
,
1951 struct ehci_sitd
*sitd
,
1952 struct ehci_iso_sched
*iso_sched
,
1956 struct ehci_iso_packet
*uf
= &iso_sched
->packet
[index
];
1957 u64 bufp
= uf
->bufp
;
1959 sitd
->hw_next
= EHCI_LIST_END
;
1960 sitd
->hw_fullspeed_ep
= stream
->address
;
1961 sitd
->hw_uframe
= stream
->splits
;
1962 sitd
->hw_results
= uf
->transaction
;
1963 sitd
->hw_backpointer
= EHCI_LIST_END
;
1966 sitd
->hw_buf
[0] = cpu_to_le32 (bufp
);
1967 sitd
->hw_buf_hi
[0] = cpu_to_le32 (bufp
>> 32);
1969 sitd
->hw_buf
[1] = cpu_to_le32 (uf
->buf1
);
1972 sitd
->hw_buf_hi
[1] = cpu_to_le32 (bufp
>> 32);
1973 sitd
->index
= index
;
1977 sitd_link (struct ehci_hcd
*ehci
, unsigned frame
, struct ehci_sitd
*sitd
)
1979 /* note: sitd ordering could matter (CSPLIT then SSPLIT) */
1980 sitd
->sitd_next
= ehci
->pshadow
[frame
];
1981 sitd
->hw_next
= ehci
->periodic
[frame
];
1982 ehci
->pshadow
[frame
].sitd
= sitd
;
1983 sitd
->frame
= frame
;
1985 ehci
->periodic
[frame
] = cpu_to_le32 (sitd
->sitd_dma
) | Q_TYPE_SITD
;
1988 /* fit urb's sitds into the selected schedule slot; activate as needed */
1991 struct ehci_hcd
*ehci
,
1994 struct ehci_iso_stream
*stream
1998 unsigned next_uframe
;
1999 struct ehci_iso_sched
*sched
= urb
->hcpriv
;
2000 struct ehci_sitd
*sitd
;
2002 next_uframe
= stream
->next_uframe
;
2004 if (list_empty(&stream
->td_list
)) {
2005 /* usbfs ignores TT bandwidth */
2006 ehci_to_hcd(ehci
)->self
.bandwidth_allocated
2007 += stream
->bandwidth
;
2009 "sched devp %s ep%d%s-iso [%d] %dms/%04x\n",
2010 urb
->dev
->devpath
, stream
->bEndpointAddress
& 0x0f,
2011 (stream
->bEndpointAddress
& USB_DIR_IN
) ? "in" : "out",
2012 (next_uframe
>> 3) % ehci
->periodic_size
,
2013 stream
->interval
, le32_to_cpu (stream
->splits
));
2014 stream
->start
= jiffies
;
2016 ehci_to_hcd(ehci
)->self
.bandwidth_isoc_reqs
++;
2018 /* fill sITDs frame by frame */
2019 for (packet
= 0, sitd
= NULL
;
2020 packet
< urb
->number_of_packets
;
2023 /* ASSERT: we have all necessary sitds */
2024 BUG_ON (list_empty (&sched
->td_list
));
2026 /* ASSERT: no itds for this endpoint in this frame */
2028 sitd
= list_entry (sched
->td_list
.next
,
2029 struct ehci_sitd
, sitd_list
);
2030 list_move_tail (&sitd
->sitd_list
, &stream
->td_list
);
2031 sitd
->stream
= iso_stream_get (stream
);
2032 sitd
->urb
= usb_get_urb (urb
);
2034 sitd_patch (stream
, sitd
, sched
, packet
);
2035 sitd_link (ehci
, (next_uframe
>> 3) % ehci
->periodic_size
,
2038 next_uframe
+= stream
->interval
<< 3;
2039 stream
->depth
+= stream
->interval
<< 3;
2041 stream
->next_uframe
= next_uframe
% mod
;
2043 /* don't need that schedule data any more */
2044 iso_sched_free (stream
, sched
);
2047 timer_action (ehci
, TIMER_IO_WATCHDOG
);
2048 if (!ehci
->periodic_sched
++)
2049 return enable_periodic (ehci
);
2053 /*-------------------------------------------------------------------------*/
2055 #define SITD_ERRS (SITD_STS_ERR | SITD_STS_DBE | SITD_STS_BABBLE \
2056 | SITD_STS_XACT | SITD_STS_MMF)
2060 struct ehci_hcd
*ehci
,
2061 struct ehci_sitd
*sitd
2063 struct urb
*urb
= sitd
->urb
;
2064 struct usb_iso_packet_descriptor
*desc
;
2067 struct ehci_iso_stream
*stream
= sitd
->stream
;
2068 struct usb_device
*dev
;
2070 urb_index
= sitd
->index
;
2071 desc
= &urb
->iso_frame_desc
[urb_index
];
2072 t
= le32_to_cpup (&sitd
->hw_results
);
2074 /* report transfer status */
2075 if (t
& SITD_ERRS
) {
2077 if (t
& SITD_STS_DBE
)
2078 desc
->status
= usb_pipein (urb
->pipe
)
2079 ? -ENOSR
/* hc couldn't read */
2080 : -ECOMM
; /* hc couldn't write */
2081 else if (t
& SITD_STS_BABBLE
)
2082 desc
->status
= -EOVERFLOW
;
2083 else /* XACT, MMF, etc */
2084 desc
->status
= -EPROTO
;
2087 desc
->actual_length
= desc
->length
- SITD_LENGTH (t
);
2092 sitd
->stream
= NULL
;
2093 list_move (&sitd
->sitd_list
, &stream
->free_list
);
2094 stream
->depth
-= stream
->interval
<< 3;
2095 iso_stream_put (ehci
, stream
);
2097 /* handle completion now? */
2098 if ((urb_index
+ 1) != urb
->number_of_packets
)
2101 /* ASSERT: it's really the last sitd for this urb
2102 list_for_each_entry (sitd, &stream->td_list, sitd_list)
2103 BUG_ON (sitd->urb == urb);
2106 /* give urb back to the driver */
2108 ehci_urb_done (ehci
, urb
);
2111 /* defer stopping schedule; completion can submit */
2112 ehci
->periodic_sched
--;
2113 if (!ehci
->periodic_sched
)
2114 (void) disable_periodic (ehci
);
2115 ehci_to_hcd(ehci
)->self
.bandwidth_isoc_reqs
--;
2117 if (list_empty (&stream
->td_list
)) {
2118 ehci_to_hcd(ehci
)->self
.bandwidth_allocated
2119 -= stream
->bandwidth
;
2121 "deschedule devp %s ep%d%s-iso\n",
2122 dev
->devpath
, stream
->bEndpointAddress
& 0x0f,
2123 (stream
->bEndpointAddress
& USB_DIR_IN
) ? "in" : "out");
2125 iso_stream_put (ehci
, stream
);
2131 static int sitd_submit (struct ehci_hcd
*ehci
, struct urb
*urb
,
2134 int status
= -EINVAL
;
2135 unsigned long flags
;
2136 struct ehci_iso_stream
*stream
;
2138 /* Get iso_stream head */
2139 stream
= iso_stream_find (ehci
, urb
);
2140 if (stream
== NULL
) {
2141 ehci_dbg (ehci
, "can't get iso stream\n");
2144 if (urb
->interval
!= stream
->interval
) {
2145 ehci_dbg (ehci
, "can't change iso interval %d --> %d\n",
2146 stream
->interval
, urb
->interval
);
2150 #ifdef EHCI_URB_TRACE
2152 "submit %p dev%s ep%d%s-iso len %d\n",
2153 urb
, urb
->dev
->devpath
,
2154 usb_pipeendpoint (urb
->pipe
),
2155 usb_pipein (urb
->pipe
) ? "in" : "out",
2156 urb
->transfer_buffer_length
);
2159 /* allocate SITDs */
2160 status
= sitd_urb_transaction (stream
, ehci
, urb
, mem_flags
);
2162 ehci_dbg (ehci
, "can't init sitds\n");
2166 /* schedule ... need to lock */
2167 spin_lock_irqsave (&ehci
->lock
, flags
);
2168 if (unlikely(!test_bit(HCD_FLAG_HW_ACCESSIBLE
,
2169 &ehci_to_hcd(ehci
)->flags
)))
2170 status
= -ESHUTDOWN
;
2172 status
= iso_stream_schedule (ehci
, urb
, stream
);
2174 sitd_link_urb (ehci
, urb
, ehci
->periodic_size
<< 3, stream
);
2175 spin_unlock_irqrestore (&ehci
->lock
, flags
);
2179 iso_stream_put (ehci
, stream
);
2186 sitd_submit (struct ehci_hcd
*ehci
, struct urb
*urb
, gfp_t mem_flags
)
2188 ehci_dbg (ehci
, "split iso support is disabled\n");
2192 static inline unsigned
2194 struct ehci_hcd
*ehci
,
2195 struct ehci_sitd
*sitd
2197 ehci_err (ehci
, "sitd_complete %p?\n", sitd
);
2201 #endif /* USB_EHCI_SPLIT_ISO */
2203 /*-------------------------------------------------------------------------*/
2206 scan_periodic (struct ehci_hcd
*ehci
)
2208 unsigned frame
, clock
, now_uframe
, mod
;
2211 mod
= ehci
->periodic_size
<< 3;
2214 * When running, scan from last scan point up to "now"
2215 * else clean up by scanning everything that's left.
2216 * Touches as few pages as possible: cache-friendly.
2218 now_uframe
= ehci
->next_uframe
;
2219 if (HC_IS_RUNNING (ehci_to_hcd(ehci
)->state
))
2220 clock
= ehci_readl(ehci
, &ehci
->regs
->frame_index
);
2222 clock
= now_uframe
+ mod
- 1;
2226 union ehci_shadow q
, *q_p
;
2230 /* don't scan past the live uframe */
2231 frame
= now_uframe
>> 3;
2232 if (frame
== (clock
>> 3))
2233 uframes
= now_uframe
& 0x07;
2235 /* safe to scan the whole frame at once */
2241 /* scan each element in frame's queue for completions */
2242 q_p
= &ehci
->pshadow
[frame
];
2243 hw_p
= &ehci
->periodic
[frame
];
2245 type
= Q_NEXT_TYPE (*hw_p
);
2248 while (q
.ptr
!= NULL
) {
2250 union ehci_shadow temp
;
2253 live
= HC_IS_RUNNING (ehci_to_hcd(ehci
)->state
);
2256 /* handle any completions */
2257 temp
.qh
= qh_get (q
.qh
);
2258 type
= Q_NEXT_TYPE (q
.qh
->hw_next
);
2260 modified
= qh_completions (ehci
, temp
.qh
);
2261 if (unlikely (list_empty (&temp
.qh
->qtd_list
)))
2262 intr_deschedule (ehci
, temp
.qh
);
2266 /* for "save place" FSTNs, look at QH entries
2267 * in the previous frame for completions.
2269 if (q
.fstn
->hw_prev
!= EHCI_LIST_END
) {
2270 dbg ("ignoring completions from FSTNs");
2272 type
= Q_NEXT_TYPE (q
.fstn
->hw_next
);
2273 q
= q
.fstn
->fstn_next
;
2276 /* skip itds for later in the frame */
2278 for (uf
= live
? uframes
: 8; uf
< 8; uf
++) {
2279 if (0 == (q
.itd
->hw_transaction
[uf
]
2282 q_p
= &q
.itd
->itd_next
;
2283 hw_p
= &q
.itd
->hw_next
;
2284 type
= Q_NEXT_TYPE (q
.itd
->hw_next
);
2291 /* this one's ready ... HC won't cache the
2292 * pointer for much longer, if at all.
2294 *q_p
= q
.itd
->itd_next
;
2295 *hw_p
= q
.itd
->hw_next
;
2296 type
= Q_NEXT_TYPE (q
.itd
->hw_next
);
2298 modified
= itd_complete (ehci
, q
.itd
);
2302 if ((q
.sitd
->hw_results
& SITD_ACTIVE
)
2304 q_p
= &q
.sitd
->sitd_next
;
2305 hw_p
= &q
.sitd
->hw_next
;
2306 type
= Q_NEXT_TYPE (q
.sitd
->hw_next
);
2310 *q_p
= q
.sitd
->sitd_next
;
2311 *hw_p
= q
.sitd
->hw_next
;
2312 type
= Q_NEXT_TYPE (q
.sitd
->hw_next
);
2314 modified
= sitd_complete (ehci
, q
.sitd
);
2318 dbg ("corrupt type %d frame %d shadow %p",
2319 type
, frame
, q
.ptr
);
2324 /* assume completion callbacks modify the queue */
2325 if (unlikely (modified
))
2329 /* stop when we catch up to the HC */
2331 // FIXME: this assumes we won't get lapped when
2332 // latencies climb; that should be rare, but...
2333 // detect it, and just go all the way around.
2334 // FLR might help detect this case, so long as latencies
2335 // don't exceed periodic_size msec (default 1.024 sec).
2337 // FIXME: likewise assumes HC doesn't halt mid-scan
2339 if (now_uframe
== clock
) {
2342 if (!HC_IS_RUNNING (ehci_to_hcd(ehci
)->state
))
2344 ehci
->next_uframe
= now_uframe
;
2345 now
= ehci_readl(ehci
, &ehci
->regs
->frame_index
) % mod
;
2346 if (now_uframe
== now
)
2349 /* rescan the rest of this frame, then ... */