projects / deliverable / linux.git / blob
? search:


9a7d0bd15dc311c2a697678f05a7cc67ca3ef2ad
[deliverable/linux.git] / drivers / usb / dwc3 / gadget.c
1 /**
2 * gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link
3 *
4 * Copyright (C) 2010-2011 Texas Instruments Incorporated - http://www.ti.com
5 *
6 * Authors: Felipe Balbi <balbi@ti.com>,
7 * Sebastian Andrzej Siewior <bigeasy@linutronix.de>
8 *
9 * This program is free software: you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 of
11 * the License as published by the Free Software Foundation.
12 *
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
17 */
18
19 #include <linux/kernel.h>
20 #include <linux/delay.h>
21 #include <linux/slab.h>
22 #include <linux/spinlock.h>
23 #include <linux/platform_device.h>
24 #include <linux/pm_runtime.h>
25 #include <linux/interrupt.h>
26 #include <linux/io.h>
27 #include <linux/list.h>
28 #include <linux/dma-mapping.h>
29
30 #include <linux/usb/ch9.h>
31 #include <linux/usb/gadget.h>
32
33 #include "debug.h"
34 #include "core.h"
35 #include "gadget.h"
36 #include "io.h"
37
38 /**
39 * dwc3_gadget_set_test_mode - Enables USB2 Test Modes
40 * @dwc: pointer to our context structure
41 * @mode: the mode to set (J, K SE0 NAK, Force Enable)
42 *
43 * Caller should take care of locking. This function will
44 * return 0 on success or -EINVAL if wrong Test Selector
45 * is passed
46 */
47 int dwc3_gadget_set_test_mode(struct dwc3 *dwc, int mode)
48 {
49 u32 reg;
50
51 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
52 reg &= ~DWC3_DCTL_TSTCTRL_MASK;
53
54 switch (mode) {
55 case TEST_J:
56 case TEST_K:
57 case TEST_SE0_NAK:
58 case TEST_PACKET:
59 case TEST_FORCE_EN:
60 reg |= mode << 1;
61 break;
62 default:
63 return -EINVAL;
64 }
65
66 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
67
68 return 0;
69 }
70
71 /**
72 * dwc3_gadget_get_link_state - Gets current state of USB Link
73 * @dwc: pointer to our context structure
74 *
75 * Caller should take care of locking. This function will
76 * return the link state on success (>= 0) or -ETIMEDOUT.
77 */
78 int dwc3_gadget_get_link_state(struct dwc3 *dwc)
79 {
80 u32 reg;
81
82 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
83
84 return DWC3_DSTS_USBLNKST(reg);
85 }
86
87 /**
88 * dwc3_gadget_set_link_state - Sets USB Link to a particular State
89 * @dwc: pointer to our context structure
90 * @state: the state to put link into
91 *
92 * Caller should take care of locking. This function will
93 * return 0 on success or -ETIMEDOUT.
94 */
95 int dwc3_gadget_set_link_state(struct dwc3 *dwc, enum dwc3_link_state state)
96 {
97 int retries = 10000;
98 u32 reg;
99
100 /*
101 * Wait until device controller is ready. Only applies to 1.94a and
102 * later RTL.
103 */
104 if (dwc->revision >= DWC3_REVISION_194A) {
105 while (--retries) {
106 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
107 if (reg & DWC3_DSTS_DCNRD)
108 udelay(5);
109 else
110 break;
111 }
112
113 if (retries <= 0)
114 return -ETIMEDOUT;
115 }
116
117 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
118 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
119
120 /* set requested state */
121 reg |= DWC3_DCTL_ULSTCHNGREQ(state);
122 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
123
124 /*
125 * The following code is racy when called from dwc3_gadget_wakeup,
126 * and is not needed, at least on newer versions
127 */
128 if (dwc->revision >= DWC3_REVISION_194A)
129 return 0;
130
131 /* wait for a change in DSTS */
132 retries = 10000;
133 while (--retries) {
134 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
135
136 if (DWC3_DSTS_USBLNKST(reg) == state)
137 return 0;
138
139 udelay(5);
140 }
141
142 dwc3_trace(trace_dwc3_gadget,
143 "link state change request timed out");
144
145 return -ETIMEDOUT;
146 }
147
148 static void dwc3_ep_inc_enq(struct dwc3_ep *dep)
149 {
150 dep->trb_enqueue++;
151 dep->trb_enqueue %= DWC3_TRB_NUM;
152 }
153
154 static void dwc3_ep_inc_deq(struct dwc3_ep *dep)
155 {
156 dep->trb_dequeue++;
157 dep->trb_dequeue %= DWC3_TRB_NUM;
158 }
159
160 static int dwc3_ep_is_last_trb(unsigned int index)
161 {
162 return index == DWC3_TRB_NUM - 1;
163 }
164
165 void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req,
166 int status)
167 {
168 struct dwc3 *dwc = dep->dwc;
169 int i;
170
171 if (req->started) {
172 i = 0;
173 do {
174 dwc3_ep_inc_deq(dep);
175 /*
176 * Skip LINK TRB. We can't use req->trb and check for
177 * DWC3_TRBCTL_LINK_TRB because it points the TRB we
178 * just completed (not the LINK TRB).
179 */
180 if (dwc3_ep_is_last_trb(dep->trb_dequeue))
181 dwc3_ep_inc_deq(dep);
182 } while(++i < req->request.num_mapped_sgs);
183 req->started = false;
184 }
185 list_del(&req->list);
186 req->trb = NULL;
187
188 if (req->request.status == -EINPROGRESS)
189 req->request.status = status;
190
191 if (dwc->ep0_bounced && dep->number == 0)
192 dwc->ep0_bounced = false;
193 else
194 usb_gadget_unmap_request(&dwc->gadget, &req->request,
195 req->direction);
196
197 trace_dwc3_gadget_giveback(req);
198
199 spin_unlock(&dwc->lock);
200 usb_gadget_giveback_request(&dep->endpoint, &req->request);
201 spin_lock(&dwc->lock);
202 }
203
204 int dwc3_send_gadget_generic_command(struct dwc3 *dwc, unsigned cmd, u32 param)
205 {
206 u32 timeout = 500;
207 u32 reg;
208
209 trace_dwc3_gadget_generic_cmd(cmd, param);
210
211 dwc3_writel(dwc->regs, DWC3_DGCMDPAR, param);
212 dwc3_writel(dwc->regs, DWC3_DGCMD, cmd | DWC3_DGCMD_CMDACT);
213
214 do {
215 reg = dwc3_readl(dwc->regs, DWC3_DGCMD);
216 if (!(reg & DWC3_DGCMD_CMDACT)) {
217 dwc3_trace(trace_dwc3_gadget,
218 "Command Complete --> %d",
219 DWC3_DGCMD_STATUS(reg));
220 if (DWC3_DGCMD_STATUS(reg))
221 return -EINVAL;
222 return 0;
223 }
224
225 /*
226 * We can't sleep here, because it's also called from
227 * interrupt context.
228 */
229 timeout--;
230 if (!timeout) {
231 dwc3_trace(trace_dwc3_gadget,
232 "Command Timed Out");
233 return -ETIMEDOUT;
234 }
235 udelay(1);
236 } while (1);
237 }
238
239 static int __dwc3_gadget_wakeup(struct dwc3 *dwc);
240
241 int dwc3_send_gadget_ep_cmd(struct dwc3 *dwc, unsigned ep,
242 unsigned cmd, struct dwc3_gadget_ep_cmd_params *params)
243 {
244 struct dwc3_ep *dep = dwc->eps[ep];
245 u32 timeout = 500;
246 u32 reg;
247
248 int susphy = false;
249 int ret = -EINVAL;
250
251 trace_dwc3_gadget_ep_cmd(dep, cmd, params);
252
253 /*
254 * Synopsys Databook 2.60a states, on section 6.3.2.5.[1-8], that if
255 * we're issuing an endpoint command, we must check if
256 * GUSB2PHYCFG.SUSPHY bit is set. If it is, then we need to clear it.
257 *
258 * We will also set SUSPHY bit to what it was before returning as stated
259 * by the same section on Synopsys databook.
260 */
261 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
262 if (unlikely(reg & DWC3_GUSB2PHYCFG_SUSPHY)) {
263 susphy = true;
264 reg &= ~DWC3_GUSB2PHYCFG_SUSPHY;
265 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
266 }
267
268 if (cmd == DWC3_DEPCMD_STARTTRANSFER) {
269 int needs_wakeup;
270
271 needs_wakeup = (dwc->link_state == DWC3_LINK_STATE_U1 ||
272 dwc->link_state == DWC3_LINK_STATE_U2 ||
273 dwc->link_state == DWC3_LINK_STATE_U3);
274
275 if (unlikely(needs_wakeup)) {
276 ret = __dwc3_gadget_wakeup(dwc);
277 dev_WARN_ONCE(dwc->dev, ret, "wakeup failed --> %d\n",
278 ret);
279 }
280 }
281
282 dwc3_writel(dwc->regs, DWC3_DEPCMDPAR0(ep), params->param0);
283 dwc3_writel(dwc->regs, DWC3_DEPCMDPAR1(ep), params->param1);
284 dwc3_writel(dwc->regs, DWC3_DEPCMDPAR2(ep), params->param2);
285
286 dwc3_writel(dwc->regs, DWC3_DEPCMD(ep), cmd | DWC3_DEPCMD_CMDACT);
287 do {
288 reg = dwc3_readl(dwc->regs, DWC3_DEPCMD(ep));
289 if (!(reg & DWC3_DEPCMD_CMDACT)) {
290 int cmd_status = DWC3_DEPCMD_STATUS(reg);
291
292 dwc3_trace(trace_dwc3_gadget,
293 "Command Complete --> %d",
294 cmd_status);
295
296 switch (cmd_status) {
297 case 0:
298 ret = 0;
299 break;
300 case DEPEVT_TRANSFER_NO_RESOURCE:
301 dwc3_trace(trace_dwc3_gadget, "%s: no resource available");
302 ret = -EINVAL;
303 break;
304 case DEPEVT_TRANSFER_BUS_EXPIRY:
305 /*
306 * SW issues START TRANSFER command to
307 * isochronous ep with future frame interval. If
308 * future interval time has already passed when
309 * core receives the command, it will respond
310 * with an error status of 'Bus Expiry'.
311 *
312 * Instead of always returning -EINVAL, let's
313 * give a hint to the gadget driver that this is
314 * the case by returning -EAGAIN.
315 */
316 dwc3_trace(trace_dwc3_gadget, "%s: bus expiry");
317 ret = -EAGAIN;
318 break;
319 default:
320 dev_WARN(dwc->dev, "UNKNOWN cmd status\n");
321 }
322
323 break;
324 }
325
326 /*
327 * We can't sleep here, because it is also called from
328 * interrupt context.
329 */
330 timeout--;
331 if (!timeout) {
332 dwc3_trace(trace_dwc3_gadget,
333 "Command Timed Out");
334 ret = -ETIMEDOUT;
335 break;
336 }
337
338 udelay(1);
339 } while (1);
340
341 if (unlikely(susphy)) {
342 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
343 reg |= DWC3_GUSB2PHYCFG_SUSPHY;
344 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
345 }
346
347 return ret;
348 }
349
350 static dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep,
351 struct dwc3_trb *trb)
352 {
353 u32 offset = (char *) trb - (char *) dep->trb_pool;
354
355 return dep->trb_pool_dma + offset;
356 }
357
358 static int dwc3_alloc_trb_pool(struct dwc3_ep *dep)
359 {
360 struct dwc3 *dwc = dep->dwc;
361
362 if (dep->trb_pool)
363 return 0;
364
365 dep->trb_pool = dma_alloc_coherent(dwc->dev,
366 sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
367 &dep->trb_pool_dma, GFP_KERNEL);
368 if (!dep->trb_pool) {
369 dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n",
370 dep->name);
371 return -ENOMEM;
372 }
373
374 return 0;
375 }
376
377 static void dwc3_free_trb_pool(struct dwc3_ep *dep)
378 {
379 struct dwc3 *dwc = dep->dwc;
380
381 dma_free_coherent(dwc->dev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
382 dep->trb_pool, dep->trb_pool_dma);
383
384 dep->trb_pool = NULL;
385 dep->trb_pool_dma = 0;
386 }
387
388 static int dwc3_gadget_set_xfer_resource(struct dwc3 *dwc, struct dwc3_ep *dep);
389
390 /**
391 * dwc3_gadget_start_config - Configure EP resources
392 * @dwc: pointer to our controller context structure
393 * @dep: endpoint that is being enabled
394 *
395 * The assignment of transfer resources cannot perfectly follow the
396 * data book due to the fact that the controller driver does not have
397 * all knowledge of the configuration in advance. It is given this
398 * information piecemeal by the composite gadget framework after every
399 * SET_CONFIGURATION and SET_INTERFACE. Trying to follow the databook
400 * programming model in this scenario can cause errors. For two
401 * reasons:
402 *
403 * 1) The databook says to do DEPSTARTCFG for every SET_CONFIGURATION
404 * and SET_INTERFACE (8.1.5). This is incorrect in the scenario of
405 * multiple interfaces.
406 *
407 * 2) The databook does not mention doing more DEPXFERCFG for new
408 * endpoint on alt setting (8.1.6).
409 *
410 * The following simplified method is used instead:
411 *
412 * All hardware endpoints can be assigned a transfer resource and this
413 * setting will stay persistent until either a core reset or
414 * hibernation. So whenever we do a DEPSTARTCFG(0) we can go ahead and
415 * do DEPXFERCFG for every hardware endpoint as well. We are
416 * guaranteed that there are as many transfer resources as endpoints.
417 *
418 * This function is called for each endpoint when it is being enabled
419 * but is triggered only when called for EP0-out, which always happens
420 * first, and which should only happen in one of the above conditions.
421 */
422 static int dwc3_gadget_start_config(struct dwc3 *dwc, struct dwc3_ep *dep)
423 {
424 struct dwc3_gadget_ep_cmd_params params;
425 u32 cmd;
426 int i;
427 int ret;
428
429 if (dep->number)
430 return 0;
431
432 memset(&params, 0x00, sizeof(params));
433 cmd = DWC3_DEPCMD_DEPSTARTCFG;
434
435 ret = dwc3_send_gadget_ep_cmd(dwc, 0, cmd, &params);
436 if (ret)
437 return ret;
438
439 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
440 struct dwc3_ep *dep = dwc->eps[i];
441
442 if (!dep)
443 continue;
444
445 ret = dwc3_gadget_set_xfer_resource(dwc, dep);
446 if (ret)
447 return ret;
448 }
449
450 return 0;
451 }
452
453 static int dwc3_gadget_set_ep_config(struct dwc3 *dwc, struct dwc3_ep *dep,
454 const struct usb_endpoint_descriptor *desc,
455 const struct usb_ss_ep_comp_descriptor *comp_desc,
456 bool ignore, bool restore)
457 {
458 struct dwc3_gadget_ep_cmd_params params;
459
460 memset(&params, 0x00, sizeof(params));
461
462 params.param0 = DWC3_DEPCFG_EP_TYPE(usb_endpoint_type(desc))
463 | DWC3_DEPCFG_MAX_PACKET_SIZE(usb_endpoint_maxp(desc));
464
465 /* Burst size is only needed in SuperSpeed mode */
466 if (dwc->gadget.speed >= USB_SPEED_SUPER) {
467 u32 burst = dep->endpoint.maxburst;
468 u32 nump;
469 u32 reg;
470
471 /* update NumP */
472 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
473 nump = DWC3_DCFG_NUMP(reg);
474 nump = max(nump, burst);
475 reg &= ~DWC3_DCFG_NUMP_MASK;
476 reg |= nump << DWC3_DCFG_NUMP_SHIFT;
477 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
478
479 params.param0 |= DWC3_DEPCFG_BURST_SIZE(burst - 1);
480 }
481
482 if (ignore)
483 params.param0 |= DWC3_DEPCFG_IGN_SEQ_NUM;
484
485 if (restore) {
486 params.param0 |= DWC3_DEPCFG_ACTION_RESTORE;
487 params.param2 |= dep->saved_state;
488 }
489
490 params.param1 = DWC3_DEPCFG_XFER_COMPLETE_EN
491 | DWC3_DEPCFG_XFER_NOT_READY_EN;
492
493 if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) {
494 params.param1 |= DWC3_DEPCFG_STREAM_CAPABLE
495 | DWC3_DEPCFG_STREAM_EVENT_EN;
496 dep->stream_capable = true;
497 }
498
499 if (!usb_endpoint_xfer_control(desc))
500 params.param1 |= DWC3_DEPCFG_XFER_IN_PROGRESS_EN;
501
502 /*
503 * We are doing 1:1 mapping for endpoints, meaning
504 * Physical Endpoints 2 maps to Logical Endpoint 2 and
505 * so on. We consider the direction bit as part of the physical
506 * endpoint number. So USB endpoint 0x81 is 0x03.
507 */
508 params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number);
509
510 /*
511 * We must use the lower 16 TX FIFOs even though
512 * HW might have more
513 */
514 if (dep->direction)
515 params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1);
516
517 if (desc->bInterval) {
518 params.param1 |= DWC3_DEPCFG_BINTERVAL_M1(desc->bInterval - 1);
519 dep->interval = 1 << (desc->bInterval - 1);
520 }
521
522 return dwc3_send_gadget_ep_cmd(dwc, dep->number,
523 DWC3_DEPCMD_SETEPCONFIG, &params);
524 }
525
526 static int dwc3_gadget_set_xfer_resource(struct dwc3 *dwc, struct dwc3_ep *dep)
527 {
528 struct dwc3_gadget_ep_cmd_params params;
529
530 memset(&params, 0x00, sizeof(params));
531
532 params.param0 = DWC3_DEPXFERCFG_NUM_XFER_RES(1);
533
534 return dwc3_send_gadget_ep_cmd(dwc, dep->number,
535 DWC3_DEPCMD_SETTRANSFRESOURCE, &params);
536 }
537
538 /**
539 * __dwc3_gadget_ep_enable - Initializes a HW endpoint
540 * @dep: endpoint to be initialized
541 * @desc: USB Endpoint Descriptor
542 *
543 * Caller should take care of locking
544 */
545 static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep,
546 const struct usb_endpoint_descriptor *desc,
547 const struct usb_ss_ep_comp_descriptor *comp_desc,
548 bool ignore, bool restore)
549 {
550 struct dwc3 *dwc = dep->dwc;
551 u32 reg;
552 int ret;
553
554 dwc3_trace(trace_dwc3_gadget, "Enabling %s", dep->name);
555
556 if (!(dep->flags & DWC3_EP_ENABLED)) {
557 ret = dwc3_gadget_start_config(dwc, dep);
558 if (ret)
559 return ret;
560 }
561
562 ret = dwc3_gadget_set_ep_config(dwc, dep, desc, comp_desc, ignore,
563 restore);
564 if (ret)
565 return ret;
566
567 if (!(dep->flags & DWC3_EP_ENABLED)) {
568 struct dwc3_trb *trb_st_hw;
569 struct dwc3_trb *trb_link;
570
571 dep->endpoint.desc = desc;
572 dep->comp_desc = comp_desc;
573 dep->type = usb_endpoint_type(desc);
574 dep->flags |= DWC3_EP_ENABLED;
575
576 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
577 reg |= DWC3_DALEPENA_EP(dep->number);
578 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
579
580 if (usb_endpoint_xfer_control(desc))
581 goto out;
582
583 /* Link TRB. The HWO bit is never reset */
584 trb_st_hw = &dep->trb_pool[0];
585
586 trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1];
587 memset(trb_link, 0, sizeof(*trb_link));
588
589 trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
590 trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
591 trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB;
592 trb_link->ctrl |= DWC3_TRB_CTRL_HWO;
593 }
594
595 out:
596 switch (usb_endpoint_type(desc)) {
597 case USB_ENDPOINT_XFER_CONTROL:
598 /* don't change name */
599 break;
600 case USB_ENDPOINT_XFER_ISOC:
601 strlcat(dep->name, "-isoc", sizeof(dep->name));
602 break;
603 case USB_ENDPOINT_XFER_BULK:
604 strlcat(dep->name, "-bulk", sizeof(dep->name));
605 break;
606 case USB_ENDPOINT_XFER_INT:
607 strlcat(dep->name, "-int", sizeof(dep->name));
608 break;
609 default:
610 dev_err(dwc->dev, "invalid endpoint transfer type\n");
611 }
612
613 return 0;
614 }
615
616 static void dwc3_stop_active_transfer(struct dwc3 *dwc, u32 epnum, bool force);
617 static void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep)
618 {
619 struct dwc3_request *req;
620
621 if (!list_empty(&dep->started_list)) {
622 dwc3_stop_active_transfer(dwc, dep->number, true);
623
624 /* - giveback all requests to gadget driver */
625 while (!list_empty(&dep->started_list)) {
626 req = next_request(&dep->started_list);
627
628 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
629 }
630 }
631
632 while (!list_empty(&dep->pending_list)) {
633 req = next_request(&dep->pending_list);
634
635 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
636 }
637 }
638
639 /**
640 * __dwc3_gadget_ep_disable - Disables a HW endpoint
641 * @dep: the endpoint to disable
642 *
643 * This function also removes requests which are currently processed ny the
644 * hardware and those which are not yet scheduled.
645 * Caller should take care of locking.
646 */
647 static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep)
648 {
649 struct dwc3 *dwc = dep->dwc;
650 u32 reg;
651
652 dwc3_trace(trace_dwc3_gadget, "Disabling %s", dep->name);
653
654 dwc3_remove_requests(dwc, dep);
655
656 /* make sure HW endpoint isn't stalled */
657 if (dep->flags & DWC3_EP_STALL)
658 __dwc3_gadget_ep_set_halt(dep, 0, false);
659
660 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
661 reg &= ~DWC3_DALEPENA_EP(dep->number);
662 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
663
664 dep->stream_capable = false;
665 dep->endpoint.desc = NULL;
666 dep->comp_desc = NULL;
667 dep->type = 0;
668 dep->flags = 0;
669
670 snprintf(dep->name, sizeof(dep->name), "ep%d%s",
671 dep->number >> 1,
672 (dep->number & 1) ? "in" : "out");
673
674 return 0;
675 }
676
677 /* -------------------------------------------------------------------------- */
678
679 static int dwc3_gadget_ep0_enable(struct usb_ep *ep,
680 const struct usb_endpoint_descriptor *desc)
681 {
682 return -EINVAL;
683 }
684
685 static int dwc3_gadget_ep0_disable(struct usb_ep *ep)
686 {
687 return -EINVAL;
688 }
689
690 /* -------------------------------------------------------------------------- */
691
692 static int dwc3_gadget_ep_enable(struct usb_ep *ep,
693 const struct usb_endpoint_descriptor *desc)
694 {
695 struct dwc3_ep *dep;
696 struct dwc3 *dwc;
697 unsigned long flags;
698 int ret;
699
700 if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
701 pr_debug("dwc3: invalid parameters\n");
702 return -EINVAL;
703 }
704
705 if (!desc->wMaxPacketSize) {
706 pr_debug("dwc3: missing wMaxPacketSize\n");
707 return -EINVAL;
708 }
709
710 dep = to_dwc3_ep(ep);
711 dwc = dep->dwc;
712
713 if (dev_WARN_ONCE(dwc->dev, dep->flags & DWC3_EP_ENABLED,
714 "%s is already enabled\n",
715 dep->name))
716 return 0;
717
718 spin_lock_irqsave(&dwc->lock, flags);
719 ret = __dwc3_gadget_ep_enable(dep, desc, ep->comp_desc, false, false);
720 spin_unlock_irqrestore(&dwc->lock, flags);
721
722 return ret;
723 }
724
725 static int dwc3_gadget_ep_disable(struct usb_ep *ep)
726 {
727 struct dwc3_ep *dep;
728 struct dwc3 *dwc;
729 unsigned long flags;
730 int ret;
731
732 if (!ep) {
733 pr_debug("dwc3: invalid parameters\n");
734 return -EINVAL;
735 }
736
737 dep = to_dwc3_ep(ep);
738 dwc = dep->dwc;
739
740 if (dev_WARN_ONCE(dwc->dev, !(dep->flags & DWC3_EP_ENABLED),
741 "%s is already disabled\n",
742 dep->name))
743 return 0;
744
745 spin_lock_irqsave(&dwc->lock, flags);
746 ret = __dwc3_gadget_ep_disable(dep);
747 spin_unlock_irqrestore(&dwc->lock, flags);
748
749 return ret;
750 }
751
752 static struct usb_request *dwc3_gadget_ep_alloc_request(struct usb_ep *ep,
753 gfp_t gfp_flags)
754 {
755 struct dwc3_request *req;
756 struct dwc3_ep *dep = to_dwc3_ep(ep);
757
758 req = kzalloc(sizeof(*req), gfp_flags);
759 if (!req)
760 return NULL;
761
762 req->epnum = dep->number;
763 req->dep = dep;
764
765 trace_dwc3_alloc_request(req);
766
767 return &req->request;
768 }
769
770 static void dwc3_gadget_ep_free_request(struct usb_ep *ep,
771 struct usb_request *request)
772 {
773 struct dwc3_request *req = to_dwc3_request(request);
774
775 trace_dwc3_free_request(req);
776 kfree(req);
777 }
778
779 /**
780 * dwc3_prepare_one_trb - setup one TRB from one request
781 * @dep: endpoint for which this request is prepared
782 * @req: dwc3_request pointer
783 */
784 static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
785 struct dwc3_request *req, dma_addr_t dma,
786 unsigned length, unsigned last, unsigned chain, unsigned node)
787 {
788 struct dwc3_trb *trb;
789
790 dwc3_trace(trace_dwc3_gadget, "%s: req %p dma %08llx length %d%s%s",
791 dep->name, req, (unsigned long long) dma,
792 length, last ? " last" : "",
793 chain ? " chain" : "");
794
795
796 trb = &dep->trb_pool[dep->trb_enqueue];
797
798 if (!req->trb) {
799 dwc3_gadget_move_started_request(req);
800 req->trb = trb;
801 req->trb_dma = dwc3_trb_dma_offset(dep, trb);
802 req->first_trb_index = dep->trb_enqueue;
803 }
804
805 dwc3_ep_inc_enq(dep);
806 /* Skip the LINK-TRB */
807 if (dwc3_ep_is_last_trb(dep->trb_enqueue))
808 dwc3_ep_inc_enq(dep);
809
810 trb->size = DWC3_TRB_SIZE_LENGTH(length);
811 trb->bpl = lower_32_bits(dma);
812 trb->bph = upper_32_bits(dma);
813
814 switch (usb_endpoint_type(dep->endpoint.desc)) {
815 case USB_ENDPOINT_XFER_CONTROL:
816 trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP;
817 break;
818
819 case USB_ENDPOINT_XFER_ISOC:
820 if (!node)
821 trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST;
822 else
823 trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS;
824
825 /* always enable Interrupt on Missed ISOC */
826 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
827 break;
828
829 case USB_ENDPOINT_XFER_BULK:
830 case USB_ENDPOINT_XFER_INT:
831 trb->ctrl = DWC3_TRBCTL_NORMAL;
832 break;
833 default:
834 /*
835 * This is only possible with faulty memory because we
836 * checked it already :)
837 */
838 BUG();
839 }
840
841 /* always enable Continue on Short Packet */
842 trb->ctrl |= DWC3_TRB_CTRL_CSP;
843
844 if (!req->request.no_interrupt && !chain)
845 trb->ctrl |= DWC3_TRB_CTRL_IOC | DWC3_TRB_CTRL_ISP_IMI;
846
847 if (last)
848 trb->ctrl |= DWC3_TRB_CTRL_LST;
849
850 if (chain)
851 trb->ctrl |= DWC3_TRB_CTRL_CHN;
852
853 if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable)
854 trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(req->request.stream_id);
855
856 trb->ctrl |= DWC3_TRB_CTRL_HWO;
857
858 trace_dwc3_prepare_trb(dep, trb);
859 }
860
861 /*
862 * dwc3_prepare_trbs - setup TRBs from requests
863 * @dep: endpoint for which requests are being prepared
864 * @starting: true if the endpoint is idle and no requests are queued.
865 *
866 * The function goes through the requests list and sets up TRBs for the
867 * transfers. The function returns once there are no more TRBs available or
868 * it runs out of requests.
869 */
870 static void dwc3_prepare_trbs(struct dwc3_ep *dep, bool starting)
871 {
872 struct dwc3_request *req, *n;
873 u32 trbs_left;
874 unsigned int last_one = 0;
875
876 BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM);
877
878 trbs_left = dep->trb_dequeue - dep->trb_enqueue;
879
880 /*
881 * If enqueue & dequeue are equal than it is either full or empty. If we
882 * are starting to process requests then we are empty. Otherwise we are
883 * full and don't do anything
884 */
885 if (!trbs_left) {
886 if (!starting)
887 return;
888
889 trbs_left = DWC3_TRB_NUM;
890 }
891
892 /* The last TRB is a link TRB, not used for xfer */
893 if (trbs_left <= 1)
894 return;
895
896 list_for_each_entry_safe(req, n, &dep->pending_list, list) {
897 unsigned length;
898 dma_addr_t dma;
899 last_one = false;
900
901 if (req->request.num_mapped_sgs > 0) {
902 struct usb_request *request = &req->request;
903 struct scatterlist *sg = request->sg;
904 struct scatterlist *s;
905 int i;
906
907 for_each_sg(sg, s, request->num_mapped_sgs, i) {
908 unsigned chain = true;
909
910 length = sg_dma_len(s);
911 dma = sg_dma_address(s);
912
913 if (i == (request->num_mapped_sgs - 1) ||
914 sg_is_last(s)) {
915 if (list_empty(&dep->pending_list))
916 last_one = true;
917 chain = false;
918 }
919
920 trbs_left--;
921 if (!trbs_left)
922 last_one = true;
923
924 if (last_one)
925 chain = false;
926
927 dwc3_prepare_one_trb(dep, req, dma, length,
928 last_one, chain, i);
929
930 if (last_one)
931 break;
932 }
933
934 if (last_one)
935 break;
936 } else {
937 dma = req->request.dma;
938 length = req->request.length;
939 trbs_left--;
940
941 if (!trbs_left)
942 last_one = 1;
943
944 /* Is this the last request? */
945 if (list_is_last(&req->list, &dep->pending_list))
946 last_one = 1;
947
948 dwc3_prepare_one_trb(dep, req, dma, length,
949 last_one, false, 0);
950
951 if (last_one)
952 break;
953 }
954 }
955 }
956
957 static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep, u16 cmd_param,
958 int start_new)
959 {
960 struct dwc3_gadget_ep_cmd_params params;
961 struct dwc3_request *req;
962 struct dwc3 *dwc = dep->dwc;
963 int ret;
964 u32 cmd;
965
966 if (start_new && (dep->flags & DWC3_EP_BUSY)) {
967 dwc3_trace(trace_dwc3_gadget, "%s: endpoint busy", dep->name);
968 return -EBUSY;
969 }
970
971 /*
972 * If we are getting here after a short-out-packet we don't enqueue any
973 * new requests as we try to set the IOC bit only on the last request.
974 */
975 if (start_new) {
976 if (list_empty(&dep->started_list))
977 dwc3_prepare_trbs(dep, start_new);
978
979 /* req points to the first request which will be sent */
980 req = next_request(&dep->started_list);
981 } else {
982 dwc3_prepare_trbs(dep, start_new);
983
984 /*
985 * req points to the first request where HWO changed from 0 to 1
986 */
987 req = next_request(&dep->started_list);
988 }
989 if (!req) {
990 dep->flags |= DWC3_EP_PENDING_REQUEST;
991 return 0;
992 }
993
994 memset(&params, 0, sizeof(params));
995
996 if (start_new) {
997 params.param0 = upper_32_bits(req->trb_dma);
998 params.param1 = lower_32_bits(req->trb_dma);
999 cmd = DWC3_DEPCMD_STARTTRANSFER;
1000 } else {
1001 cmd = DWC3_DEPCMD_UPDATETRANSFER;
1002 }
1003
1004 cmd |= DWC3_DEPCMD_PARAM(cmd_param);
1005 ret = dwc3_send_gadget_ep_cmd(dwc, dep->number, cmd, &params);
1006 if (ret < 0) {
1007 /*
1008 * FIXME we need to iterate over the list of requests
1009 * here and stop, unmap, free and del each of the linked
1010 * requests instead of what we do now.
1011 */
1012 usb_gadget_unmap_request(&dwc->gadget, &req->request,
1013 req->direction);
1014 list_del(&req->list);
1015 return ret;
1016 }
1017
1018 dep->flags |= DWC3_EP_BUSY;
1019
1020 if (start_new) {
1021 dep->resource_index = dwc3_gadget_ep_get_transfer_index(dwc,
1022 dep->number);
1023 WARN_ON_ONCE(!dep->resource_index);
1024 }
1025
1026 return 0;
1027 }
1028
1029 static void __dwc3_gadget_start_isoc(struct dwc3 *dwc,
1030 struct dwc3_ep *dep, u32 cur_uf)
1031 {
1032 u32 uf;
1033
1034 if (list_empty(&dep->pending_list)) {
1035 dwc3_trace(trace_dwc3_gadget,
1036 "ISOC ep %s run out for requests",
1037 dep->name);
1038 dep->flags |= DWC3_EP_PENDING_REQUEST;
1039 return;
1040 }
1041
1042 /* 4 micro frames in the future */
1043 uf = cur_uf + dep->interval * 4;
1044
1045 __dwc3_gadget_kick_transfer(dep, uf, 1);
1046 }
1047
1048 static void dwc3_gadget_start_isoc(struct dwc3 *dwc,
1049 struct dwc3_ep *dep, const struct dwc3_event_depevt *event)
1050 {
1051 u32 cur_uf, mask;
1052
1053 mask = ~(dep->interval - 1);
1054 cur_uf = event->parameters & mask;
1055
1056 __dwc3_gadget_start_isoc(dwc, dep, cur_uf);
1057 }
1058
1059 static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
1060 {
1061 struct dwc3 *dwc = dep->dwc;
1062 int ret;
1063
1064 if (!dep->endpoint.desc) {
1065 dwc3_trace(trace_dwc3_gadget,
1066 "trying to queue request %p to disabled %s\n",
1067 &req->request, dep->endpoint.name);
1068 return -ESHUTDOWN;
1069 }
1070
1071 if (WARN(req->dep != dep, "request %p belongs to '%s'\n",
1072 &req->request, req->dep->name)) {
1073 dwc3_trace(trace_dwc3_gadget, "request %p belongs to '%s'\n",
1074 &req->request, req->dep->name);
1075 return -EINVAL;
1076 }
1077
1078 req->request.actual = 0;
1079 req->request.status = -EINPROGRESS;
1080 req->direction = dep->direction;
1081 req->epnum = dep->number;
1082
1083 trace_dwc3_ep_queue(req);
1084
1085 /*
1086 * We only add to our list of requests now and
1087 * start consuming the list once we get XferNotReady
1088 * IRQ.
1089 *
1090 * That way, we avoid doing anything that we don't need
1091 * to do now and defer it until the point we receive a
1092 * particular token from the Host side.
1093 *
1094 * This will also avoid Host cancelling URBs due to too
1095 * many NAKs.
1096 */
1097 ret = usb_gadget_map_request(&dwc->gadget, &req->request,
1098 dep->direction);
1099 if (ret)
1100 return ret;
1101
1102 list_add_tail(&req->list, &dep->pending_list);
1103
1104 /*
1105 * If there are no pending requests and the endpoint isn't already
1106 * busy, we will just start the request straight away.
1107 *
1108 * This will save one IRQ (XFER_NOT_READY) and possibly make it a
1109 * little bit faster.
1110 */
1111 if (!usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
1112 !usb_endpoint_xfer_int(dep->endpoint.desc) &&
1113 !(dep->flags & DWC3_EP_BUSY)) {
1114 ret = __dwc3_gadget_kick_transfer(dep, 0, true);
1115 goto out;
1116 }
1117
1118 /*
1119 * There are a few special cases:
1120 *
1121 * 1. XferNotReady with empty list of requests. We need to kick the
1122 * transfer here in that situation, otherwise we will be NAKing
1123 * forever. If we get XferNotReady before gadget driver has a
1124 * chance to queue a request, we will ACK the IRQ but won't be
1125 * able to receive the data until the next request is queued.
1126 * The following code is handling exactly that.
1127 *
1128 */
1129 if (dep->flags & DWC3_EP_PENDING_REQUEST) {
1130 /*
1131 * If xfernotready is already elapsed and it is a case
1132 * of isoc transfer, then issue END TRANSFER, so that
1133 * you can receive xfernotready again and can have
1134 * notion of current microframe.
1135 */
1136 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1137 if (list_empty(&dep->started_list)) {
1138 dwc3_stop_active_transfer(dwc, dep->number, true);
1139 dep->flags = DWC3_EP_ENABLED;
1140 }
1141 return 0;
1142 }
1143
1144 ret = __dwc3_gadget_kick_transfer(dep, 0, true);
1145 if (!ret)
1146 dep->flags &= ~DWC3_EP_PENDING_REQUEST;
1147
1148 goto out;
1149 }
1150
1151 /*
1152 * 2. XferInProgress on Isoc EP with an active transfer. We need to
1153 * kick the transfer here after queuing a request, otherwise the
1154 * core may not see the modified TRB(s).
1155 */
1156 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
1157 (dep->flags & DWC3_EP_BUSY) &&
1158 !(dep->flags & DWC3_EP_MISSED_ISOC)) {
1159 WARN_ON_ONCE(!dep->resource_index);
1160 ret = __dwc3_gadget_kick_transfer(dep, dep->resource_index,
1161 false);
1162 goto out;
1163 }
1164
1165 /*
1166 * 4. Stream Capable Bulk Endpoints. We need to start the transfer
1167 * right away, otherwise host will not know we have streams to be
1168 * handled.
1169 */
1170 if (dep->stream_capable)
1171 ret = __dwc3_gadget_kick_transfer(dep, 0, true);
1172
1173 out:
1174 if (ret && ret != -EBUSY)
1175 dwc3_trace(trace_dwc3_gadget,
1176 "%s: failed to kick transfers\n",
1177 dep->name);
1178 if (ret == -EBUSY)
1179 ret = 0;
1180
1181 return ret;
1182 }
1183
1184 static void __dwc3_gadget_ep_zlp_complete(struct usb_ep *ep,
1185 struct usb_request *request)
1186 {
1187 dwc3_gadget_ep_free_request(ep, request);
1188 }
1189
1190 static int __dwc3_gadget_ep_queue_zlp(struct dwc3 *dwc, struct dwc3_ep *dep)
1191 {
1192 struct dwc3_request *req;
1193 struct usb_request *request;
1194 struct usb_ep *ep = &dep->endpoint;
1195
1196 dwc3_trace(trace_dwc3_gadget, "queueing ZLP\n");
1197 request = dwc3_gadget_ep_alloc_request(ep, GFP_ATOMIC);
1198 if (!request)
1199 return -ENOMEM;
1200
1201 request->length = 0;
1202 request->buf = dwc->zlp_buf;
1203 request->complete = __dwc3_gadget_ep_zlp_complete;
1204
1205 req = to_dwc3_request(request);
1206
1207 return __dwc3_gadget_ep_queue(dep, req);
1208 }
1209
1210 static int dwc3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
1211 gfp_t gfp_flags)
1212 {
1213 struct dwc3_request *req = to_dwc3_request(request);
1214 struct dwc3_ep *dep = to_dwc3_ep(ep);
1215 struct dwc3 *dwc = dep->dwc;
1216
1217 unsigned long flags;
1218
1219 int ret;
1220
1221 spin_lock_irqsave(&dwc->lock, flags);
1222 ret = __dwc3_gadget_ep_queue(dep, req);
1223
1224 /*
1225 * Okay, here's the thing, if gadget driver has requested for a ZLP by
1226 * setting request->zero, instead of doing magic, we will just queue an
1227 * extra usb_request ourselves so that it gets handled the same way as
1228 * any other request.
1229 */
1230 if (ret == 0 && request->zero && request->length &&
1231 (request->length % ep->maxpacket == 0))
1232 ret = __dwc3_gadget_ep_queue_zlp(dwc, dep);
1233
1234 spin_unlock_irqrestore(&dwc->lock, flags);
1235
1236 return ret;
1237 }
1238
1239 static int dwc3_gadget_ep_dequeue(struct usb_ep *ep,
1240 struct usb_request *request)
1241 {
1242 struct dwc3_request *req = to_dwc3_request(request);
1243 struct dwc3_request *r = NULL;
1244
1245 struct dwc3_ep *dep = to_dwc3_ep(ep);
1246 struct dwc3 *dwc = dep->dwc;
1247
1248 unsigned long flags;
1249 int ret = 0;
1250
1251 trace_dwc3_ep_dequeue(req);
1252
1253 spin_lock_irqsave(&dwc->lock, flags);
1254
1255 list_for_each_entry(r, &dep->pending_list, list) {
1256 if (r == req)
1257 break;
1258 }
1259
1260 if (r != req) {
1261 list_for_each_entry(r, &dep->started_list, list) {
1262 if (r == req)
1263 break;
1264 }
1265 if (r == req) {
1266 /* wait until it is processed */
1267 dwc3_stop_active_transfer(dwc, dep->number, true);
1268 goto out1;
1269 }
1270 dev_err(dwc->dev, "request %p was not queued to %s\n",
1271 request, ep->name);
1272 ret = -EINVAL;
1273 goto out0;
1274 }
1275
1276 out1:
1277 /* giveback the request */
1278 dwc3_gadget_giveback(dep, req, -ECONNRESET);
1279
1280 out0:
1281 spin_unlock_irqrestore(&dwc->lock, flags);
1282
1283 return ret;
1284 }
1285
1286 int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol)
1287 {
1288 struct dwc3_gadget_ep_cmd_params params;
1289 struct dwc3 *dwc = dep->dwc;
1290 int ret;
1291
1292 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1293 dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name);
1294 return -EINVAL;
1295 }
1296
1297 memset(&params, 0x00, sizeof(params));
1298
1299 if (value) {
1300 if (!protocol && ((dep->direction && dep->flags & DWC3_EP_BUSY) ||
1301 (!list_empty(&dep->started_list) ||
1302 !list_empty(&dep->pending_list)))) {
1303 dwc3_trace(trace_dwc3_gadget,
1304 "%s: pending request, cannot halt",
1305 dep->name);
1306 return -EAGAIN;
1307 }
1308
1309 ret = dwc3_send_gadget_ep_cmd(dwc, dep->number,
1310 DWC3_DEPCMD_SETSTALL, &params);
1311 if (ret)
1312 dev_err(dwc->dev, "failed to set STALL on %s\n",
1313 dep->name);
1314 else
1315 dep->flags |= DWC3_EP_STALL;
1316 } else {
1317 ret = dwc3_send_gadget_ep_cmd(dwc, dep->number,
1318 DWC3_DEPCMD_CLEARSTALL, &params);
1319 if (ret)
1320 dev_err(dwc->dev, "failed to clear STALL on %s\n",
1321 dep->name);
1322 else
1323 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
1324 }
1325
1326 return ret;
1327 }
1328
1329 static int dwc3_gadget_ep_set_halt(struct usb_ep *ep, int value)
1330 {
1331 struct dwc3_ep *dep = to_dwc3_ep(ep);
1332 struct dwc3 *dwc = dep->dwc;
1333
1334 unsigned long flags;
1335
1336 int ret;
1337
1338 spin_lock_irqsave(&dwc->lock, flags);
1339 ret = __dwc3_gadget_ep_set_halt(dep, value, false);
1340 spin_unlock_irqrestore(&dwc->lock, flags);
1341
1342 return ret;
1343 }
1344
1345 static int dwc3_gadget_ep_set_wedge(struct usb_ep *ep)
1346 {
1347 struct dwc3_ep *dep = to_dwc3_ep(ep);
1348 struct dwc3 *dwc = dep->dwc;
1349 unsigned long flags;
1350 int ret;
1351
1352 spin_lock_irqsave(&dwc->lock, flags);
1353 dep->flags |= DWC3_EP_WEDGE;
1354
1355 if (dep->number == 0 || dep->number == 1)
1356 ret = __dwc3_gadget_ep0_set_halt(ep, 1);
1357 else
1358 ret = __dwc3_gadget_ep_set_halt(dep, 1, false);
1359 spin_unlock_irqrestore(&dwc->lock, flags);
1360
1361 return ret;
1362 }
1363
1364 /* -------------------------------------------------------------------------- */
1365
1366 static struct usb_endpoint_descriptor dwc3_gadget_ep0_desc = {
1367 .bLength = USB_DT_ENDPOINT_SIZE,
1368 .bDescriptorType = USB_DT_ENDPOINT,
1369 .bmAttributes = USB_ENDPOINT_XFER_CONTROL,
1370 };
1371
1372 static const struct usb_ep_ops dwc3_gadget_ep0_ops = {
1373 .enable = dwc3_gadget_ep0_enable,
1374 .disable = dwc3_gadget_ep0_disable,
1375 .alloc_request = dwc3_gadget_ep_alloc_request,
1376 .free_request = dwc3_gadget_ep_free_request,
1377 .queue = dwc3_gadget_ep0_queue,
1378 .dequeue = dwc3_gadget_ep_dequeue,
1379 .set_halt = dwc3_gadget_ep0_set_halt,
1380 .set_wedge = dwc3_gadget_ep_set_wedge,
1381 };
1382
1383 static const struct usb_ep_ops dwc3_gadget_ep_ops = {
1384 .enable = dwc3_gadget_ep_enable,
1385 .disable = dwc3_gadget_ep_disable,
1386 .alloc_request = dwc3_gadget_ep_alloc_request,
1387 .free_request = dwc3_gadget_ep_free_request,
1388 .queue = dwc3_gadget_ep_queue,
1389 .dequeue = dwc3_gadget_ep_dequeue,
1390 .set_halt = dwc3_gadget_ep_set_halt,
1391 .set_wedge = dwc3_gadget_ep_set_wedge,
1392 };
1393
1394 /* -------------------------------------------------------------------------- */
1395
1396 static int dwc3_gadget_get_frame(struct usb_gadget *g)
1397 {
1398 struct dwc3 *dwc = gadget_to_dwc(g);
1399 u32 reg;
1400
1401 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1402 return DWC3_DSTS_SOFFN(reg);
1403 }
1404
1405 static int __dwc3_gadget_wakeup(struct dwc3 *dwc)
1406 {
1407 unsigned long timeout;
1408
1409 int ret;
1410 u32 reg;
1411
1412 u8 link_state;
1413 u8 speed;
1414
1415 /*
1416 * According to the Databook Remote wakeup request should
1417 * be issued only when the device is in early suspend state.
1418 *
1419 * We can check that via USB Link State bits in DSTS register.
1420 */
1421 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1422
1423 speed = reg & DWC3_DSTS_CONNECTSPD;
1424 if ((speed == DWC3_DSTS_SUPERSPEED) ||
1425 (speed == DWC3_DSTS_SUPERSPEED_PLUS)) {
1426 dwc3_trace(trace_dwc3_gadget, "no wakeup on SuperSpeed\n");
1427 return -EINVAL;
1428 }
1429
1430 link_state = DWC3_DSTS_USBLNKST(reg);
1431
1432 switch (link_state) {
1433 case DWC3_LINK_STATE_RX_DET: /* in HS, means Early Suspend */
1434 case DWC3_LINK_STATE_U3: /* in HS, means SUSPEND */
1435 break;
1436 default:
1437 dwc3_trace(trace_dwc3_gadget,
1438 "can't wakeup from '%s'\n",
1439 dwc3_gadget_link_string(link_state));
1440 return -EINVAL;
1441 }
1442
1443 ret = dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RECOV);
1444 if (ret < 0) {
1445 dev_err(dwc->dev, "failed to put link in Recovery\n");
1446 return ret;
1447 }
1448
1449 /* Recent versions do this automatically */
1450 if (dwc->revision < DWC3_REVISION_194A) {
1451 /* write zeroes to Link Change Request */
1452 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1453 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
1454 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1455 }
1456
1457 /* poll until Link State changes to ON */
1458 timeout = jiffies + msecs_to_jiffies(100);
1459
1460 while (!time_after(jiffies, timeout)) {
1461 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1462
1463 /* in HS, means ON */
1464 if (DWC3_DSTS_USBLNKST(reg) == DWC3_LINK_STATE_U0)
1465 break;
1466 }
1467
1468 if (DWC3_DSTS_USBLNKST(reg) != DWC3_LINK_STATE_U0) {
1469 dev_err(dwc->dev, "failed to send remote wakeup\n");
1470 return -EINVAL;
1471 }
1472
1473 return 0;
1474 }
1475
1476 static int dwc3_gadget_wakeup(struct usb_gadget *g)
1477 {
1478 struct dwc3 *dwc = gadget_to_dwc(g);
1479 unsigned long flags;
1480 int ret;
1481
1482 spin_lock_irqsave(&dwc->lock, flags);
1483 ret = __dwc3_gadget_wakeup(dwc);
1484 spin_unlock_irqrestore(&dwc->lock, flags);
1485
1486 return ret;
1487 }
1488
1489 static int dwc3_gadget_set_selfpowered(struct usb_gadget *g,
1490 int is_selfpowered)
1491 {
1492 struct dwc3 *dwc = gadget_to_dwc(g);
1493 unsigned long flags;
1494
1495 spin_lock_irqsave(&dwc->lock, flags);
1496 g->is_selfpowered = !!is_selfpowered;
1497 spin_unlock_irqrestore(&dwc->lock, flags);
1498
1499 return 0;
1500 }
1501
1502 static int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on, int suspend)
1503 {
1504 u32 reg;
1505 u32 timeout = 500;
1506
1507 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1508 if (is_on) {
1509 if (dwc->revision <= DWC3_REVISION_187A) {
1510 reg &= ~DWC3_DCTL_TRGTULST_MASK;
1511 reg |= DWC3_DCTL_TRGTULST_RX_DET;
1512 }
1513
1514 if (dwc->revision >= DWC3_REVISION_194A)
1515 reg &= ~DWC3_DCTL_KEEP_CONNECT;
1516 reg |= DWC3_DCTL_RUN_STOP;
1517
1518 if (dwc->has_hibernation)
1519 reg |= DWC3_DCTL_KEEP_CONNECT;
1520
1521 dwc->pullups_connected = true;
1522 } else {
1523 reg &= ~DWC3_DCTL_RUN_STOP;
1524
1525 if (dwc->has_hibernation && !suspend)
1526 reg &= ~DWC3_DCTL_KEEP_CONNECT;
1527
1528 dwc->pullups_connected = false;
1529 }
1530
1531 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1532
1533 do {
1534 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1535 if (is_on) {
1536 if (!(reg & DWC3_DSTS_DEVCTRLHLT))
1537 break;
1538 } else {
1539 if (reg & DWC3_DSTS_DEVCTRLHLT)
1540 break;
1541 }
1542 timeout--;
1543 if (!timeout)
1544 return -ETIMEDOUT;
1545 udelay(1);
1546 } while (1);
1547
1548 dwc3_trace(trace_dwc3_gadget, "gadget %s data soft-%s",
1549 dwc->gadget_driver
1550 ? dwc->gadget_driver->function : "no-function",
1551 is_on ? "connect" : "disconnect");
1552
1553 return 0;
1554 }
1555
1556 static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
1557 {
1558 struct dwc3 *dwc = gadget_to_dwc(g);
1559 unsigned long flags;
1560 int ret;
1561
1562 is_on = !!is_on;
1563
1564 spin_lock_irqsave(&dwc->lock, flags);
1565 ret = dwc3_gadget_run_stop(dwc, is_on, false);
1566 spin_unlock_irqrestore(&dwc->lock, flags);
1567
1568 return ret;
1569 }
1570
1571 static void dwc3_gadget_enable_irq(struct dwc3 *dwc)
1572 {
1573 u32 reg;
1574
1575 /* Enable all but Start and End of Frame IRQs */
1576 reg = (DWC3_DEVTEN_VNDRDEVTSTRCVEDEN |
1577 DWC3_DEVTEN_EVNTOVERFLOWEN |
1578 DWC3_DEVTEN_CMDCMPLTEN |
1579 DWC3_DEVTEN_ERRTICERREN |
1580 DWC3_DEVTEN_WKUPEVTEN |
1581 DWC3_DEVTEN_ULSTCNGEN |
1582 DWC3_DEVTEN_CONNECTDONEEN |
1583 DWC3_DEVTEN_USBRSTEN |
1584 DWC3_DEVTEN_DISCONNEVTEN);
1585
1586 dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);
1587 }
1588
1589 static void dwc3_gadget_disable_irq(struct dwc3 *dwc)
1590 {
1591 /* mask all interrupts */
1592 dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00);
1593 }
1594
1595 static irqreturn_t dwc3_interrupt(int irq, void *_dwc);
1596 static irqreturn_t dwc3_thread_interrupt(int irq, void *_dwc);
1597
1598 static int dwc3_gadget_start(struct usb_gadget *g,
1599 struct usb_gadget_driver *driver)
1600 {
1601 struct dwc3 *dwc = gadget_to_dwc(g);
1602 struct dwc3_ep *dep;
1603 unsigned long flags;
1604 int ret = 0;
1605 int irq;
1606 u32 reg;
1607
1608 irq = platform_get_irq(to_platform_device(dwc->dev), 0);
1609 ret = request_threaded_irq(irq, dwc3_interrupt, dwc3_thread_interrupt,
1610 IRQF_SHARED, "dwc3", dwc->ev_buf);
1611 if (ret) {
1612 dev_err(dwc->dev, "failed to request irq #%d --> %d\n",
1613 irq, ret);
1614 goto err0;
1615 }
1616
1617 spin_lock_irqsave(&dwc->lock, flags);
1618
1619 if (dwc->gadget_driver) {
1620 dev_err(dwc->dev, "%s is already bound to %s\n",
1621 dwc->gadget.name,
1622 dwc->gadget_driver->driver.name);
1623 ret = -EBUSY;
1624 goto err1;
1625 }
1626
1627 dwc->gadget_driver = driver;
1628
1629 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
1630 reg &= ~(DWC3_DCFG_SPEED_MASK);
1631
1632 /**
1633 * WORKAROUND: DWC3 revision < 2.20a have an issue
1634 * which would cause metastability state on Run/Stop
1635 * bit if we try to force the IP to USB2-only mode.
1636 *
1637 * Because of that, we cannot configure the IP to any
1638 * speed other than the SuperSpeed
1639 *
1640 * Refers to:
1641 *
1642 * STAR#9000525659: Clock Domain Crossing on DCTL in
1643 * USB 2.0 Mode
1644 */
1645 if (dwc->revision < DWC3_REVISION_220A) {
1646 reg |= DWC3_DCFG_SUPERSPEED;
1647 } else {
1648 switch (dwc->maximum_speed) {
1649 case USB_SPEED_LOW:
1650 reg |= DWC3_DSTS_LOWSPEED;
1651 break;
1652 case USB_SPEED_FULL:
1653 reg |= DWC3_DSTS_FULLSPEED1;
1654 break;
1655 case USB_SPEED_HIGH:
1656 reg |= DWC3_DSTS_HIGHSPEED;
1657 break;
1658 case USB_SPEED_SUPER_PLUS:
1659 reg |= DWC3_DSTS_SUPERSPEED_PLUS;
1660 break;
1661 default:
1662 dev_err(dwc->dev, "invalid dwc->maximum_speed (%d)\n",
1663 dwc->maximum_speed);
1664 /* fall through */
1665 case USB_SPEED_SUPER:
1666 reg |= DWC3_DCFG_SUPERSPEED;
1667 break;
1668 }
1669 }
1670 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
1671
1672 /*
1673 * We are telling dwc3 that we want to use DCFG.NUMP as ACK TP's NUMP
1674 * field instead of letting dwc3 itself calculate that automatically.
1675 *
1676 * This way, we maximize the chances that we'll be able to get several
1677 * bursts of data without going through any sort of endpoint throttling.
1678 */
1679 reg = dwc3_readl(dwc->regs, DWC3_GRXTHRCFG);
1680 reg &= ~DWC3_GRXTHRCFG_PKTCNTSEL;
1681 dwc3_writel(dwc->regs, DWC3_GRXTHRCFG, reg);
1682
1683 /* Start with SuperSpeed Default */
1684 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
1685
1686 dep = dwc->eps[0];
1687 ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, false,
1688 false);
1689 if (ret) {
1690 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
1691 goto err2;
1692 }
1693
1694 dep = dwc->eps[1];
1695 ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, false,
1696 false);
1697 if (ret) {
1698 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
1699 goto err3;
1700 }
1701
1702 /* begin to receive SETUP packets */
1703 dwc->ep0state = EP0_SETUP_PHASE;
1704 dwc3_ep0_out_start(dwc);
1705
1706 dwc3_gadget_enable_irq(dwc);
1707
1708 spin_unlock_irqrestore(&dwc->lock, flags);
1709
1710 return 0;
1711
1712 err3:
1713 __dwc3_gadget_ep_disable(dwc->eps[0]);
1714
1715 err2:
1716 dwc->gadget_driver = NULL;
1717
1718 err1:
1719 spin_unlock_irqrestore(&dwc->lock, flags);
1720
1721 free_irq(irq, dwc->ev_buf);
1722
1723 err0:
1724 return ret;
1725 }
1726
1727 static int dwc3_gadget_stop(struct usb_gadget *g)
1728 {
1729 struct dwc3 *dwc = gadget_to_dwc(g);
1730 unsigned long flags;
1731 int irq;
1732
1733 spin_lock_irqsave(&dwc->lock, flags);
1734
1735 dwc3_gadget_disable_irq(dwc);
1736 __dwc3_gadget_ep_disable(dwc->eps[0]);
1737 __dwc3_gadget_ep_disable(dwc->eps[1]);
1738
1739 dwc->gadget_driver = NULL;
1740
1741 spin_unlock_irqrestore(&dwc->lock, flags);
1742
1743 irq = platform_get_irq(to_platform_device(dwc->dev), 0);
1744 free_irq(irq, dwc->ev_buf);
1745
1746 return 0;
1747 }
1748
1749 static const struct usb_gadget_ops dwc3_gadget_ops = {
1750 .get_frame = dwc3_gadget_get_frame,
1751 .wakeup = dwc3_gadget_wakeup,
1752 .set_selfpowered = dwc3_gadget_set_selfpowered,
1753 .pullup = dwc3_gadget_pullup,
1754 .udc_start = dwc3_gadget_start,
1755 .udc_stop = dwc3_gadget_stop,
1756 };
1757
1758 /* -------------------------------------------------------------------------- */
1759
1760 static int dwc3_gadget_init_hw_endpoints(struct dwc3 *dwc,
1761 u8 num, u32 direction)
1762 {
1763 struct dwc3_ep *dep;
1764 u8 i;
1765
1766 for (i = 0; i < num; i++) {
1767 u8 epnum = (i << 1) | (!!direction);
1768
1769 dep = kzalloc(sizeof(*dep), GFP_KERNEL);
1770 if (!dep)
1771 return -ENOMEM;
1772
1773 dep->dwc = dwc;
1774 dep->number = epnum;
1775 dep->direction = !!direction;
1776 dwc->eps[epnum] = dep;
1777
1778 snprintf(dep->name, sizeof(dep->name), "ep%d%s", epnum >> 1,
1779 (epnum & 1) ? "in" : "out");
1780
1781 dep->endpoint.name = dep->name;
1782
1783 dwc3_trace(trace_dwc3_gadget, "initializing %s", dep->name);
1784
1785 if (epnum == 0 || epnum == 1) {
1786 usb_ep_set_maxpacket_limit(&dep->endpoint, 512);
1787 dep->endpoint.maxburst = 1;
1788 dep->endpoint.ops = &dwc3_gadget_ep0_ops;
1789 if (!epnum)
1790 dwc->gadget.ep0 = &dep->endpoint;
1791 } else {
1792 int ret;
1793
1794 usb_ep_set_maxpacket_limit(&dep->endpoint, 1024);
1795 dep->endpoint.max_streams = 15;
1796 dep->endpoint.ops = &dwc3_gadget_ep_ops;
1797 list_add_tail(&dep->endpoint.ep_list,
1798 &dwc->gadget.ep_list);
1799
1800 ret = dwc3_alloc_trb_pool(dep);
1801 if (ret)
1802 return ret;
1803 }
1804
1805 if (epnum == 0 || epnum == 1) {
1806 dep->endpoint.caps.type_control = true;
1807 } else {
1808 dep->endpoint.caps.type_iso = true;
1809 dep->endpoint.caps.type_bulk = true;
1810 dep->endpoint.caps.type_int = true;
1811 }
1812
1813 dep->endpoint.caps.dir_in = !!direction;
1814 dep->endpoint.caps.dir_out = !direction;
1815
1816 INIT_LIST_HEAD(&dep->pending_list);
1817 INIT_LIST_HEAD(&dep->started_list);
1818 }
1819
1820 return 0;
1821 }
1822
1823 static int dwc3_gadget_init_endpoints(struct dwc3 *dwc)
1824 {
1825 int ret;
1826
1827 INIT_LIST_HEAD(&dwc->gadget.ep_list);
1828
1829 ret = dwc3_gadget_init_hw_endpoints(dwc, dwc->num_out_eps, 0);
1830 if (ret < 0) {
1831 dwc3_trace(trace_dwc3_gadget,
1832 "failed to allocate OUT endpoints");
1833 return ret;
1834 }
1835
1836 ret = dwc3_gadget_init_hw_endpoints(dwc, dwc->num_in_eps, 1);
1837 if (ret < 0) {
1838 dwc3_trace(trace_dwc3_gadget,
1839 "failed to allocate IN endpoints");
1840 return ret;
1841 }
1842
1843 return 0;
1844 }
1845
1846 static void dwc3_gadget_free_endpoints(struct dwc3 *dwc)
1847 {
1848 struct dwc3_ep *dep;
1849 u8 epnum;
1850
1851 for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
1852 dep = dwc->eps[epnum];
1853 if (!dep)
1854 continue;
1855 /*
1856 * Physical endpoints 0 and 1 are special; they form the
1857 * bi-directional USB endpoint 0.
1858 *
1859 * For those two physical endpoints, we don't allocate a TRB
1860 * pool nor do we add them the endpoints list. Due to that, we
1861 * shouldn't do these two operations otherwise we would end up
1862 * with all sorts of bugs when removing dwc3.ko.
1863 */
1864 if (epnum != 0 && epnum != 1) {
1865 dwc3_free_trb_pool(dep);
1866 list_del(&dep->endpoint.ep_list);
1867 }
1868
1869 kfree(dep);
1870 }
1871 }
1872
1873 /* -------------------------------------------------------------------------- */
1874
1875 static int __dwc3_cleanup_done_trbs(struct dwc3 *dwc, struct dwc3_ep *dep,
1876 struct dwc3_request *req, struct dwc3_trb *trb,
1877 const struct dwc3_event_depevt *event, int status)
1878 {
1879 unsigned int count;
1880 unsigned int s_pkt = 0;
1881 unsigned int trb_status;
1882
1883 trace_dwc3_complete_trb(dep, trb);
1884
1885 if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN)
1886 /*
1887 * We continue despite the error. There is not much we
1888 * can do. If we don't clean it up we loop forever. If
1889 * we skip the TRB then it gets overwritten after a
1890 * while since we use them in a ring buffer. A BUG()
1891 * would help. Lets hope that if this occurs, someone
1892 * fixes the root cause instead of looking away :)
1893 */
1894 dev_err(dwc->dev, "%s's TRB (%p) still owned by HW\n",
1895 dep->name, trb);
1896 count = trb->size & DWC3_TRB_SIZE_MASK;
1897
1898 if (dep->direction) {
1899 if (count) {
1900 trb_status = DWC3_TRB_SIZE_TRBSTS(trb->size);
1901 if (trb_status == DWC3_TRBSTS_MISSED_ISOC) {
1902 dwc3_trace(trace_dwc3_gadget,
1903 "%s: incomplete IN transfer\n",
1904 dep->name);
1905 /*
1906 * If missed isoc occurred and there is
1907 * no request queued then issue END
1908 * TRANSFER, so that core generates
1909 * next xfernotready and we will issue
1910 * a fresh START TRANSFER.
1911 * If there are still queued request
1912 * then wait, do not issue either END
1913 * or UPDATE TRANSFER, just attach next
1914 * request in pending_list during
1915 * giveback.If any future queued request
1916 * is successfully transferred then we
1917 * will issue UPDATE TRANSFER for all
1918 * request in the pending_list.
1919 */
1920 dep->flags |= DWC3_EP_MISSED_ISOC;
1921 } else {
1922 dev_err(dwc->dev, "incomplete IN transfer %s\n",
1923 dep->name);
1924 status = -ECONNRESET;
1925 }
1926 } else {
1927 dep->flags &= ~DWC3_EP_MISSED_ISOC;
1928 }
1929 } else {
1930 if (count && (event->status & DEPEVT_STATUS_SHORT))
1931 s_pkt = 1;
1932 }
1933
1934 /*
1935 * We assume here we will always receive the entire data block
1936 * which we should receive. Meaning, if we program RX to
1937 * receive 4K but we receive only 2K, we assume that's all we
1938 * should receive and we simply bounce the request back to the
1939 * gadget driver for further processing.
1940 */
1941 req->request.actual += req->request.length - count;
1942 if (s_pkt)
1943 return 1;
1944 if ((event->status & DEPEVT_STATUS_LST) &&
1945 (trb->ctrl & (DWC3_TRB_CTRL_LST |
1946 DWC3_TRB_CTRL_HWO)))
1947 return 1;
1948 if ((event->status & DEPEVT_STATUS_IOC) &&
1949 (trb->ctrl & DWC3_TRB_CTRL_IOC))
1950 return 1;
1951 return 0;
1952 }
1953
1954 static int dwc3_cleanup_done_reqs(struct dwc3 *dwc, struct dwc3_ep *dep,
1955 const struct dwc3_event_depevt *event, int status)
1956 {
1957 struct dwc3_request *req;
1958 struct dwc3_trb *trb;
1959 unsigned int slot;
1960 unsigned int i;
1961 int ret;
1962
1963 do {
1964 req = next_request(&dep->started_list);
1965 if (WARN_ON_ONCE(!req))
1966 return 1;
1967
1968 i = 0;
1969 do {
1970 slot = req->first_trb_index + i;
1971 if (slot == DWC3_TRB_NUM - 1)
1972 slot++;
1973 slot %= DWC3_TRB_NUM;
1974 trb = &dep->trb_pool[slot];
1975
1976 ret = __dwc3_cleanup_done_trbs(dwc, dep, req, trb,
1977 event, status);
1978 if (ret)
1979 break;
1980 } while (++i < req->request.num_mapped_sgs);
1981
1982 dwc3_gadget_giveback(dep, req, status);
1983
1984 if (ret)
1985 break;
1986 } while (1);
1987
1988 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
1989 list_empty(&dep->started_list)) {
1990 if (list_empty(&dep->pending_list)) {
1991 /*
1992 * If there is no entry in request list then do
1993 * not issue END TRANSFER now. Just set PENDING
1994 * flag, so that END TRANSFER is issued when an
1995 * entry is added into request list.
1996 */
1997 dep->flags = DWC3_EP_PENDING_REQUEST;
1998 } else {
1999 dwc3_stop_active_transfer(dwc, dep->number, true);
2000 dep->flags = DWC3_EP_ENABLED;
2001 }
2002 return 1;
2003 }
2004
2005 return 1;
2006 }
2007
2008 static void dwc3_endpoint_transfer_complete(struct dwc3 *dwc,
2009 struct dwc3_ep *dep, const struct dwc3_event_depevt *event)
2010 {
2011 unsigned status = 0;
2012 int clean_busy;
2013 u32 is_xfer_complete;
2014
2015 is_xfer_complete = (event->endpoint_event == DWC3_DEPEVT_XFERCOMPLETE);
2016
2017 if (event->status & DEPEVT_STATUS_BUSERR)
2018 status = -ECONNRESET;
2019
2020 clean_busy = dwc3_cleanup_done_reqs(dwc, dep, event, status);
2021 if (clean_busy && (is_xfer_complete ||
2022 usb_endpoint_xfer_isoc(dep->endpoint.desc)))
2023 dep->flags &= ~DWC3_EP_BUSY;
2024
2025 /*
2026 * WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround.
2027 * See dwc3_gadget_linksts_change_interrupt() for 1st half.
2028 */
2029 if (dwc->revision < DWC3_REVISION_183A) {
2030 u32 reg;
2031 int i;
2032
2033 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
2034 dep = dwc->eps[i];
2035
2036 if (!(dep->flags & DWC3_EP_ENABLED))
2037 continue;
2038
2039 if (!list_empty(&dep->started_list))
2040 return;
2041 }
2042
2043 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2044 reg |= dwc->u1u2;
2045 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2046
2047 dwc->u1u2 = 0;
2048 }
2049
2050 if (!usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
2051 int ret;
2052
2053 ret = __dwc3_gadget_kick_transfer(dep, 0, is_xfer_complete);
2054 if (!ret || ret == -EBUSY)
2055 return;
2056 }
2057 }
2058
2059 static void dwc3_endpoint_interrupt(struct dwc3 *dwc,
2060 const struct dwc3_event_depevt *event)
2061 {
2062 struct dwc3_ep *dep;
2063 u8 epnum = event->endpoint_number;
2064
2065 dep = dwc->eps[epnum];
2066
2067 if (!(dep->flags & DWC3_EP_ENABLED))
2068 return;
2069
2070 if (epnum == 0 || epnum == 1) {
2071 dwc3_ep0_interrupt(dwc, event);
2072 return;
2073 }
2074
2075 switch (event->endpoint_event) {
2076 case DWC3_DEPEVT_XFERCOMPLETE:
2077 dep->resource_index = 0;
2078
2079 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
2080 dwc3_trace(trace_dwc3_gadget,
2081 "%s is an Isochronous endpoint\n",
2082 dep->name);
2083 return;
2084 }
2085
2086 dwc3_endpoint_transfer_complete(dwc, dep, event);
2087 break;
2088 case DWC3_DEPEVT_XFERINPROGRESS:
2089 dwc3_endpoint_transfer_complete(dwc, dep, event);
2090 break;
2091 case DWC3_DEPEVT_XFERNOTREADY:
2092 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
2093 dwc3_gadget_start_isoc(dwc, dep, event);
2094 } else {
2095 int active;
2096 int ret;
2097
2098 active = event->status & DEPEVT_STATUS_TRANSFER_ACTIVE;
2099
2100 dwc3_trace(trace_dwc3_gadget, "%s: reason %s",
2101 dep->name, active ? "Transfer Active"
2102 : "Transfer Not Active");
2103
2104 ret = __dwc3_gadget_kick_transfer(dep, 0, !active);
2105 if (!ret || ret == -EBUSY)
2106 return;
2107
2108 dwc3_trace(trace_dwc3_gadget,
2109 "%s: failed to kick transfers\n",
2110 dep->name);
2111 }
2112
2113 break;
2114 case DWC3_DEPEVT_STREAMEVT:
2115 if (!usb_endpoint_xfer_bulk(dep->endpoint.desc)) {
2116 dev_err(dwc->dev, "Stream event for non-Bulk %s\n",
2117 dep->name);
2118 return;
2119 }
2120
2121 switch (event->status) {
2122 case DEPEVT_STREAMEVT_FOUND:
2123 dwc3_trace(trace_dwc3_gadget,
2124 "Stream %d found and started",
2125 event->parameters);
2126
2127 break;
2128 case DEPEVT_STREAMEVT_NOTFOUND:
2129 /* FALLTHROUGH */
2130 default:
2131 dwc3_trace(trace_dwc3_gadget,
2132 "unable to find suitable stream\n");
2133 }
2134 break;
2135 case DWC3_DEPEVT_RXTXFIFOEVT:
2136 dwc3_trace(trace_dwc3_gadget, "%s FIFO Overrun\n", dep->name);
2137 break;
2138 case DWC3_DEPEVT_EPCMDCMPLT:
2139 dwc3_trace(trace_dwc3_gadget, "Endpoint Command Complete");
2140 break;
2141 }
2142 }
2143
2144 static void dwc3_disconnect_gadget(struct dwc3 *dwc)
2145 {
2146 if (dwc->gadget_driver && dwc->gadget_driver->disconnect) {
2147 spin_unlock(&dwc->lock);
2148 dwc->gadget_driver->disconnect(&dwc->gadget);
2149 spin_lock(&dwc->lock);
2150 }
2151 }
2152
2153 static void dwc3_suspend_gadget(struct dwc3 *dwc)
2154 {
2155 if (dwc->gadget_driver && dwc->gadget_driver->suspend) {
2156 spin_unlock(&dwc->lock);
2157 dwc->gadget_driver->suspend(&dwc->gadget);
2158 spin_lock(&dwc->lock);
2159 }
2160 }
2161
2162 static void dwc3_resume_gadget(struct dwc3 *dwc)
2163 {
2164 if (dwc->gadget_driver && dwc->gadget_driver->resume) {
2165 spin_unlock(&dwc->lock);
2166 dwc->gadget_driver->resume(&dwc->gadget);
2167 spin_lock(&dwc->lock);
2168 }
2169 }
2170
2171 static void dwc3_reset_gadget(struct dwc3 *dwc)
2172 {
2173 if (!dwc->gadget_driver)
2174 return;
2175
2176 if (dwc->gadget.speed != USB_SPEED_UNKNOWN) {
2177 spin_unlock(&dwc->lock);
2178 usb_gadget_udc_reset(&dwc->gadget, dwc->gadget_driver);
2179 spin_lock(&dwc->lock);
2180 }
2181 }
2182
2183 static void dwc3_stop_active_transfer(struct dwc3 *dwc, u32 epnum, bool force)
2184 {
2185 struct dwc3_ep *dep;
2186 struct dwc3_gadget_ep_cmd_params params;
2187 u32 cmd;
2188 int ret;
2189
2190 dep = dwc->eps[epnum];
2191
2192 if (!dep->resource_index)
2193 return;
2194
2195 /*
2196 * NOTICE: We are violating what the Databook says about the
2197 * EndTransfer command. Ideally we would _always_ wait for the
2198 * EndTransfer Command Completion IRQ, but that's causing too
2199 * much trouble synchronizing between us and gadget driver.
2200 *
2201 * We have discussed this with the IP Provider and it was
2202 * suggested to giveback all requests here, but give HW some
2203 * extra time to synchronize with the interconnect. We're using
2204 * an arbitrary 100us delay for that.
2205 *
2206 * Note also that a similar handling was tested by Synopsys
2207 * (thanks a lot Paul) and nothing bad has come out of it.
2208 * In short, what we're doing is:
2209 *
2210 * - Issue EndTransfer WITH CMDIOC bit set
2211 * - Wait 100us
2212 */
2213
2214 cmd = DWC3_DEPCMD_ENDTRANSFER;
2215 cmd |= force ? DWC3_DEPCMD_HIPRI_FORCERM : 0;
2216 cmd |= DWC3_DEPCMD_CMDIOC;
2217 cmd |= DWC3_DEPCMD_PARAM(dep->resource_index);
2218 memset(&params, 0, sizeof(params));
2219 ret = dwc3_send_gadget_ep_cmd(dwc, dep->number, cmd, &params);
2220 WARN_ON_ONCE(ret);
2221 dep->resource_index = 0;
2222 dep->flags &= ~DWC3_EP_BUSY;
2223 udelay(100);
2224 }
2225
2226 static void dwc3_stop_active_transfers(struct dwc3 *dwc)
2227 {
2228 u32 epnum;
2229
2230 for (epnum = 2; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
2231 struct dwc3_ep *dep;
2232
2233 dep = dwc->eps[epnum];
2234 if (!dep)
2235 continue;
2236
2237 if (!(dep->flags & DWC3_EP_ENABLED))
2238 continue;
2239
2240 dwc3_remove_requests(dwc, dep);
2241 }
2242 }
2243
2244 static void dwc3_clear_stall_all_ep(struct dwc3 *dwc)
2245 {
2246 u32 epnum;
2247
2248 for (epnum = 1; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
2249 struct dwc3_ep *dep;
2250 struct dwc3_gadget_ep_cmd_params params;
2251 int ret;
2252
2253 dep = dwc->eps[epnum];
2254 if (!dep)
2255 continue;
2256
2257 if (!(dep->flags & DWC3_EP_STALL))
2258 continue;
2259
2260 dep->flags &= ~DWC3_EP_STALL;
2261
2262 memset(&params, 0, sizeof(params));
2263 ret = dwc3_send_gadget_ep_cmd(dwc, dep->number,
2264 DWC3_DEPCMD_CLEARSTALL, &params);
2265 WARN_ON_ONCE(ret);
2266 }
2267 }
2268
2269 static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc)
2270 {
2271 int reg;
2272
2273 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2274 reg &= ~DWC3_DCTL_INITU1ENA;
2275 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2276
2277 reg &= ~DWC3_DCTL_INITU2ENA;
2278 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2279
2280 dwc3_disconnect_gadget(dwc);
2281
2282 dwc->gadget.speed = USB_SPEED_UNKNOWN;
2283 dwc->setup_packet_pending = false;
2284 usb_gadget_set_state(&dwc->gadget, USB_STATE_NOTATTACHED);
2285 }
2286
2287 static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
2288 {
2289 u32 reg;
2290
2291 /*
2292 * WORKAROUND: DWC3 revisions <1.88a have an issue which
2293 * would cause a missing Disconnect Event if there's a
2294 * pending Setup Packet in the FIFO.
2295 *
2296 * There's no suggested workaround on the official Bug
2297 * report, which states that "unless the driver/application
2298 * is doing any special handling of a disconnect event,
2299 * there is no functional issue".
2300 *
2301 * Unfortunately, it turns out that we _do_ some special
2302 * handling of a disconnect event, namely complete all
2303 * pending transfers, notify gadget driver of the
2304 * disconnection, and so on.
2305 *
2306 * Our suggested workaround is to follow the Disconnect
2307 * Event steps here, instead, based on a setup_packet_pending
2308 * flag. Such flag gets set whenever we have a SETUP_PENDING
2309 * status for EP0 TRBs and gets cleared on XferComplete for the
2310 * same endpoint.
2311 *
2312 * Refers to:
2313 *
2314 * STAR#9000466709: RTL: Device : Disconnect event not
2315 * generated if setup packet pending in FIFO
2316 */
2317 if (dwc->revision < DWC3_REVISION_188A) {
2318 if (dwc->setup_packet_pending)
2319 dwc3_gadget_disconnect_interrupt(dwc);
2320 }
2321
2322 dwc3_reset_gadget(dwc);
2323
2324 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2325 reg &= ~DWC3_DCTL_TSTCTRL_MASK;
2326 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2327 dwc->test_mode = false;
2328
2329 dwc3_stop_active_transfers(dwc);
2330 dwc3_clear_stall_all_ep(dwc);
2331
2332 /* Reset device address to zero */
2333 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2334 reg &= ~(DWC3_DCFG_DEVADDR_MASK);
2335 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2336 }
2337
2338 static void dwc3_update_ram_clk_sel(struct dwc3 *dwc, u32 speed)
2339 {
2340 u32 reg;
2341 u32 usb30_clock = DWC3_GCTL_CLK_BUS;
2342
2343 /*
2344 * We change the clock only at SS but I dunno why I would want to do
2345 * this. Maybe it becomes part of the power saving plan.
2346 */
2347
2348 if ((speed != DWC3_DSTS_SUPERSPEED) &&
2349 (speed != DWC3_DSTS_SUPERSPEED_PLUS))
2350 return;
2351
2352 /*
2353 * RAMClkSel is reset to 0 after USB reset, so it must be reprogrammed
2354 * each time on Connect Done.
2355 */
2356 if (!usb30_clock)
2357 return;
2358
2359 reg = dwc3_readl(dwc->regs, DWC3_GCTL);
2360 reg |= DWC3_GCTL_RAMCLKSEL(usb30_clock);
2361 dwc3_writel(dwc->regs, DWC3_GCTL, reg);
2362 }
2363
2364 static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
2365 {
2366 struct dwc3_ep *dep;
2367 int ret;
2368 u32 reg;
2369 u8 speed;
2370
2371 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2372 speed = reg & DWC3_DSTS_CONNECTSPD;
2373 dwc->speed = speed;
2374
2375 dwc3_update_ram_clk_sel(dwc, speed);
2376
2377 switch (speed) {
2378 case DWC3_DCFG_SUPERSPEED_PLUS:
2379 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2380 dwc->gadget.ep0->maxpacket = 512;
2381 dwc->gadget.speed = USB_SPEED_SUPER_PLUS;
2382 break;
2383 case DWC3_DCFG_SUPERSPEED:
2384 /*
2385 * WORKAROUND: DWC3 revisions <1.90a have an issue which
2386 * would cause a missing USB3 Reset event.
2387 *
2388 * In such situations, we should force a USB3 Reset
2389 * event by calling our dwc3_gadget_reset_interrupt()
2390 * routine.
2391 *
2392 * Refers to:
2393 *
2394 * STAR#9000483510: RTL: SS : USB3 reset event may
2395 * not be generated always when the link enters poll
2396 */
2397 if (dwc->revision < DWC3_REVISION_190A)
2398 dwc3_gadget_reset_interrupt(dwc);
2399
2400 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2401 dwc->gadget.ep0->maxpacket = 512;
2402 dwc->gadget.speed = USB_SPEED_SUPER;
2403 break;
2404 case DWC3_DCFG_HIGHSPEED:
2405 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
2406 dwc->gadget.ep0->maxpacket = 64;
2407 dwc->gadget.speed = USB_SPEED_HIGH;
2408 break;
2409 case DWC3_DCFG_FULLSPEED2:
2410 case DWC3_DCFG_FULLSPEED1:
2411 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
2412 dwc->gadget.ep0->maxpacket = 64;
2413 dwc->gadget.speed = USB_SPEED_FULL;
2414 break;
2415 case DWC3_DCFG_LOWSPEED:
2416 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(8);
2417 dwc->gadget.ep0->maxpacket = 8;
2418 dwc->gadget.speed = USB_SPEED_LOW;
2419 break;
2420 }
2421
2422 /* Enable USB2 LPM Capability */
2423
2424 if ((dwc->revision > DWC3_REVISION_194A) &&
2425 (speed != DWC3_DCFG_SUPERSPEED) &&
2426 (speed != DWC3_DCFG_SUPERSPEED_PLUS)) {
2427 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2428 reg |= DWC3_DCFG_LPM_CAP;
2429 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2430
2431 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2432 reg &= ~(DWC3_DCTL_HIRD_THRES_MASK | DWC3_DCTL_L1_HIBER_EN);
2433
2434 reg |= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold);
2435
2436 /*
2437 * When dwc3 revisions >= 2.40a, LPM Erratum is enabled and
2438 * DCFG.LPMCap is set, core responses with an ACK and the
2439 * BESL value in the LPM token is less than or equal to LPM
2440 * NYET threshold.
2441 */
2442 WARN_ONCE(dwc->revision < DWC3_REVISION_240A
2443 && dwc->has_lpm_erratum,
2444 "LPM Erratum not available on dwc3 revisisions < 2.40a\n");
2445
2446 if (dwc->has_lpm_erratum && dwc->revision >= DWC3_REVISION_240A)
2447 reg |= DWC3_DCTL_LPM_ERRATA(dwc->lpm_nyet_threshold);
2448
2449 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2450 } else {
2451 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2452 reg &= ~DWC3_DCTL_HIRD_THRES_MASK;
2453 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2454 }
2455
2456 dep = dwc->eps[0];
2457 ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, true,
2458 false);
2459 if (ret) {
2460 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2461 return;
2462 }
2463
2464 dep = dwc->eps[1];
2465 ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, true,
2466 false);
2467 if (ret) {
2468 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2469 return;
2470 }
2471
2472 /*
2473 * Configure PHY via GUSB3PIPECTLn if required.
2474 *
2475 * Update GTXFIFOSIZn
2476 *
2477 * In both cases reset values should be sufficient.
2478 */
2479 }
2480
2481 static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc)
2482 {
2483 /*
2484 * TODO take core out of low power mode when that's
2485 * implemented.
2486 */
2487
2488 if (dwc->gadget_driver && dwc->gadget_driver->resume) {
2489 spin_unlock(&dwc->lock);
2490 dwc->gadget_driver->resume(&dwc->gadget);
2491 spin_lock(&dwc->lock);
2492 }
2493 }
2494
2495 static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc,
2496 unsigned int evtinfo)
2497 {
2498 enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK;
2499 unsigned int pwropt;
2500
2501 /*
2502 * WORKAROUND: DWC3 < 2.50a have an issue when configured without
2503 * Hibernation mode enabled which would show up when device detects
2504 * host-initiated U3 exit.
2505 *
2506 * In that case, device will generate a Link State Change Interrupt
2507 * from U3 to RESUME which is only necessary if Hibernation is
2508 * configured in.
2509 *
2510 * There are no functional changes due to such spurious event and we
2511 * just need to ignore it.
2512 *
2513 * Refers to:
2514 *
2515 * STAR#9000570034 RTL: SS Resume event generated in non-Hibernation
2516 * operational mode
2517 */
2518 pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1);
2519 if ((dwc->revision < DWC3_REVISION_250A) &&
2520 (pwropt != DWC3_GHWPARAMS1_EN_PWROPT_HIB)) {
2521 if ((dwc->link_state == DWC3_LINK_STATE_U3) &&
2522 (next == DWC3_LINK_STATE_RESUME)) {
2523 dwc3_trace(trace_dwc3_gadget,
2524 "ignoring transition U3 -> Resume");
2525 return;
2526 }
2527 }
2528
2529 /*
2530 * WORKAROUND: DWC3 Revisions <1.83a have an issue which, depending
2531 * on the link partner, the USB session might do multiple entry/exit
2532 * of low power states before a transfer takes place.
2533 *
2534 * Due to this problem, we might experience lower throughput. The
2535 * suggested workaround is to disable DCTL[12:9] bits if we're
2536 * transitioning from U1/U2 to U0 and enable those bits again
2537 * after a transfer completes and there are no pending transfers
2538 * on any of the enabled endpoints.
2539 *
2540 * This is the first half of that workaround.
2541 *
2542 * Refers to:
2543 *
2544 * STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us
2545 * core send LGO_Ux entering U0
2546 */
2547 if (dwc->revision < DWC3_REVISION_183A) {
2548 if (next == DWC3_LINK_STATE_U0) {
2549 u32 u1u2;
2550 u32 reg;
2551
2552 switch (dwc->link_state) {
2553 case DWC3_LINK_STATE_U1:
2554 case DWC3_LINK_STATE_U2:
2555 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2556 u1u2 = reg & (DWC3_DCTL_INITU2ENA
2557 | DWC3_DCTL_ACCEPTU2ENA
2558 | DWC3_DCTL_INITU1ENA
2559 | DWC3_DCTL_ACCEPTU1ENA);
2560
2561 if (!dwc->u1u2)
2562 dwc->u1u2 = reg & u1u2;
2563
2564 reg &= ~u1u2;
2565
2566 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2567 break;
2568 default:
2569 /* do nothing */
2570 break;
2571 }
2572 }
2573 }
2574
2575 switch (next) {
2576 case DWC3_LINK_STATE_U1:
2577 if (dwc->speed == USB_SPEED_SUPER)
2578 dwc3_suspend_gadget(dwc);
2579 break;
2580 case DWC3_LINK_STATE_U2:
2581 case DWC3_LINK_STATE_U3:
2582 dwc3_suspend_gadget(dwc);
2583 break;
2584 case DWC3_LINK_STATE_RESUME:
2585 dwc3_resume_gadget(dwc);
2586 break;
2587 default:
2588 /* do nothing */
2589 break;
2590 }
2591
2592 dwc->link_state = next;
2593 }
2594
2595 static void dwc3_gadget_hibernation_interrupt(struct dwc3 *dwc,
2596 unsigned int evtinfo)
2597 {
2598 unsigned int is_ss = evtinfo & BIT(4);
2599
2600 /**
2601 * WORKAROUND: DWC3 revison 2.20a with hibernation support
2602 * have a known issue which can cause USB CV TD.9.23 to fail
2603 * randomly.
2604 *
2605 * Because of this issue, core could generate bogus hibernation
2606 * events which SW needs to ignore.
2607 *
2608 * Refers to:
2609 *
2610 * STAR#9000546576: Device Mode Hibernation: Issue in USB 2.0
2611 * Device Fallback from SuperSpeed
2612 */
2613 if (is_ss ^ (dwc->speed == USB_SPEED_SUPER))
2614 return;
2615
2616 /* enter hibernation here */
2617 }
2618
2619 static void dwc3_gadget_interrupt(struct dwc3 *dwc,
2620 const struct dwc3_event_devt *event)
2621 {
2622 switch (event->type) {
2623 case DWC3_DEVICE_EVENT_DISCONNECT:
2624 dwc3_gadget_disconnect_interrupt(dwc);
2625 break;
2626 case DWC3_DEVICE_EVENT_RESET:
2627 dwc3_gadget_reset_interrupt(dwc);
2628 break;
2629 case DWC3_DEVICE_EVENT_CONNECT_DONE:
2630 dwc3_gadget_conndone_interrupt(dwc);
2631 break;
2632 case DWC3_DEVICE_EVENT_WAKEUP:
2633 dwc3_gadget_wakeup_interrupt(dwc);
2634 break;
2635 case DWC3_DEVICE_EVENT_HIBER_REQ:
2636 if (dev_WARN_ONCE(dwc->dev, !dwc->has_hibernation,
2637 "unexpected hibernation event\n"))
2638 break;
2639
2640 dwc3_gadget_hibernation_interrupt(dwc, event->event_info);
2641 break;
2642 case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE:
2643 dwc3_gadget_linksts_change_interrupt(dwc, event->event_info);
2644 break;
2645 case DWC3_DEVICE_EVENT_EOPF:
2646 dwc3_trace(trace_dwc3_gadget, "End of Periodic Frame");
2647 break;
2648 case DWC3_DEVICE_EVENT_SOF:
2649 dwc3_trace(trace_dwc3_gadget, "Start of Periodic Frame");
2650 break;
2651 case DWC3_DEVICE_EVENT_ERRATIC_ERROR:
2652 dwc3_trace(trace_dwc3_gadget, "Erratic Error");
2653 break;
2654 case DWC3_DEVICE_EVENT_CMD_CMPL:
2655 dwc3_trace(trace_dwc3_gadget, "Command Complete");
2656 break;
2657 case DWC3_DEVICE_EVENT_OVERFLOW:
2658 dwc3_trace(trace_dwc3_gadget, "Overflow");
2659 break;
2660 default:
2661 dev_WARN(dwc->dev, "UNKNOWN IRQ %d\n", event->type);
2662 }
2663 }
2664
2665 static void dwc3_process_event_entry(struct dwc3 *dwc,
2666 const union dwc3_event *event)
2667 {
2668 trace_dwc3_event(event->raw);
2669
2670 /* Endpoint IRQ, handle it and return early */
2671 if (event->type.is_devspec == 0) {
2672 /* depevt */
2673 return dwc3_endpoint_interrupt(dwc, &event->depevt);
2674 }
2675
2676 switch (event->type.type) {
2677 case DWC3_EVENT_TYPE_DEV:
2678 dwc3_gadget_interrupt(dwc, &event->devt);
2679 break;
2680 /* REVISIT what to do with Carkit and I2C events ? */
2681 default:
2682 dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw);
2683 }
2684 }
2685
2686 static irqreturn_t dwc3_process_event_buf(struct dwc3_event_buffer *evt)
2687 {
2688 struct dwc3 *dwc = evt->dwc;
2689 irqreturn_t ret = IRQ_NONE;
2690 int left;
2691 u32 reg;
2692
2693 left = evt->count;
2694
2695 if (!(evt->flags & DWC3_EVENT_PENDING))
2696 return IRQ_NONE;
2697
2698 while (left > 0) {
2699 union dwc3_event event;
2700
2701 event.raw = *(u32 *) (evt->buf + evt->lpos);
2702
2703 dwc3_process_event_entry(dwc, &event);
2704
2705 /*
2706 * FIXME we wrap around correctly to the next entry as
2707 * almost all entries are 4 bytes in size. There is one
2708 * entry which has 12 bytes which is a regular entry
2709 * followed by 8 bytes data. ATM I don't know how
2710 * things are organized if we get next to the a
2711 * boundary so I worry about that once we try to handle
2712 * that.
2713 */
2714 evt->lpos = (evt->lpos + 4) % DWC3_EVENT_BUFFERS_SIZE;
2715 left -= 4;
2716
2717 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), 4);
2718 }
2719
2720 evt->count = 0;
2721 evt->flags &= ~DWC3_EVENT_PENDING;
2722 ret = IRQ_HANDLED;
2723
2724 /* Unmask interrupt */
2725 reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0));
2726 reg &= ~DWC3_GEVNTSIZ_INTMASK;
2727 dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg);
2728
2729 return ret;
2730 }
2731
2732 static irqreturn_t dwc3_thread_interrupt(int irq, void *_evt)
2733 {
2734 struct dwc3_event_buffer *evt = _evt;
2735 struct dwc3 *dwc = evt->dwc;
2736 unsigned long flags;
2737 irqreturn_t ret = IRQ_NONE;
2738
2739 spin_lock_irqsave(&dwc->lock, flags);
2740 ret = dwc3_process_event_buf(evt);
2741 spin_unlock_irqrestore(&dwc->lock, flags);
2742
2743 return ret;
2744 }
2745
2746 static irqreturn_t dwc3_check_event_buf(struct dwc3_event_buffer *evt)
2747 {
2748 struct dwc3 *dwc = evt->dwc;
2749 u32 count;
2750 u32 reg;
2751
2752 count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(0));
2753 count &= DWC3_GEVNTCOUNT_MASK;
2754 if (!count)
2755 return IRQ_NONE;
2756
2757 evt->count = count;
2758 evt->flags |= DWC3_EVENT_PENDING;
2759
2760 /* Mask interrupt */
2761 reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0));
2762 reg |= DWC3_GEVNTSIZ_INTMASK;
2763 dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg);
2764
2765 return IRQ_WAKE_THREAD;
2766 }
2767
2768 static irqreturn_t dwc3_interrupt(int irq, void *_evt)
2769 {
2770 struct dwc3_event_buffer *evt = _evt;
2771
2772 return dwc3_check_event_buf(evt);
2773 }
2774
2775 /**
2776 * dwc3_gadget_init - Initializes gadget related registers
2777 * @dwc: pointer to our controller context structure
2778 *
2779 * Returns 0 on success otherwise negative errno.
2780 */
2781 int dwc3_gadget_init(struct dwc3 *dwc)
2782 {
2783 int ret;
2784
2785 dwc->ctrl_req = dma_alloc_coherent(dwc->dev, sizeof(*dwc->ctrl_req),
2786 &dwc->ctrl_req_addr, GFP_KERNEL);
2787 if (!dwc->ctrl_req) {
2788 dev_err(dwc->dev, "failed to allocate ctrl request\n");
2789 ret = -ENOMEM;
2790 goto err0;
2791 }
2792
2793 dwc->ep0_trb = dma_alloc_coherent(dwc->dev, sizeof(*dwc->ep0_trb) * 2,
2794 &dwc->ep0_trb_addr, GFP_KERNEL);
2795 if (!dwc->ep0_trb) {
2796 dev_err(dwc->dev, "failed to allocate ep0 trb\n");
2797 ret = -ENOMEM;
2798 goto err1;
2799 }
2800
2801 dwc->setup_buf = kzalloc(DWC3_EP0_BOUNCE_SIZE, GFP_KERNEL);
2802 if (!dwc->setup_buf) {
2803 ret = -ENOMEM;
2804 goto err2;
2805 }
2806
2807 dwc->ep0_bounce = dma_alloc_coherent(dwc->dev,
2808 DWC3_EP0_BOUNCE_SIZE, &dwc->ep0_bounce_addr,
2809 GFP_KERNEL);
2810 if (!dwc->ep0_bounce) {
2811 dev_err(dwc->dev, "failed to allocate ep0 bounce buffer\n");
2812 ret = -ENOMEM;
2813 goto err3;
2814 }
2815
2816 dwc->zlp_buf = kzalloc(DWC3_ZLP_BUF_SIZE, GFP_KERNEL);
2817 if (!dwc->zlp_buf) {
2818 ret = -ENOMEM;
2819 goto err4;
2820 }
2821
2822 dwc->gadget.ops = &dwc3_gadget_ops;
2823 dwc->gadget.speed = USB_SPEED_UNKNOWN;
2824 dwc->gadget.sg_supported = true;
2825 dwc->gadget.name = "dwc3-gadget";
2826 dwc->gadget.is_otg = dwc->dr_mode == USB_DR_MODE_OTG;
2827
2828 /*
2829 * FIXME We might be setting max_speed to <SUPER, however versions
2830 * <2.20a of dwc3 have an issue with metastability (documented
2831 * elsewhere in this driver) which tells us we can't set max speed to
2832 * anything lower than SUPER.
2833 *
2834 * Because gadget.max_speed is only used by composite.c and function
2835 * drivers (i.e. it won't go into dwc3's registers) we are allowing this
2836 * to happen so we avoid sending SuperSpeed Capability descriptor
2837 * together with our BOS descriptor as that could confuse host into
2838 * thinking we can handle super speed.
2839 *
2840 * Note that, in fact, we won't even support GetBOS requests when speed
2841 * is less than super speed because we don't have means, yet, to tell
2842 * composite.c that we are USB 2.0 + LPM ECN.
2843 */
2844 if (dwc->revision < DWC3_REVISION_220A)
2845 dwc3_trace(trace_dwc3_gadget,
2846 "Changing max_speed on rev %08x\n",
2847 dwc->revision);
2848
2849 dwc->gadget.max_speed = dwc->maximum_speed;
2850
2851 /*
2852 * Per databook, DWC3 needs buffer size to be aligned to MaxPacketSize
2853 * on ep out.
2854 */
2855 dwc->gadget.quirk_ep_out_aligned_size = true;
2856
2857 /*
2858 * REVISIT: Here we should clear all pending IRQs to be
2859 * sure we're starting from a well known location.
2860 */
2861
2862 ret = dwc3_gadget_init_endpoints(dwc);
2863 if (ret)
2864 goto err5;
2865
2866 ret = usb_add_gadget_udc(dwc->dev, &dwc->gadget);
2867 if (ret) {
2868 dev_err(dwc->dev, "failed to register udc\n");
2869 goto err5;
2870 }
2871
2872 return 0;
2873
2874 err5:
2875 kfree(dwc->zlp_buf);
2876
2877 err4:
2878 dwc3_gadget_free_endpoints(dwc);
2879 dma_free_coherent(dwc->dev, DWC3_EP0_BOUNCE_SIZE,
2880 dwc->ep0_bounce, dwc->ep0_bounce_addr);
2881
2882 err3:
2883 kfree(dwc->setup_buf);
2884
2885 err2:
2886 dma_free_coherent(dwc->dev, sizeof(*dwc->ep0_trb),
2887 dwc->ep0_trb, dwc->ep0_trb_addr);
2888
2889 err1:
2890 dma_free_coherent(dwc->dev, sizeof(*dwc->ctrl_req),
2891 dwc->ctrl_req, dwc->ctrl_req_addr);
2892
2893 err0:
2894 return ret;
2895 }
2896
2897 /* -------------------------------------------------------------------------- */
2898
2899 void dwc3_gadget_exit(struct dwc3 *dwc)
2900 {
2901 usb_del_gadget_udc(&dwc->gadget);
2902
2903 dwc3_gadget_free_endpoints(dwc);
2904
2905 dma_free_coherent(dwc->dev, DWC3_EP0_BOUNCE_SIZE,
2906 dwc->ep0_bounce, dwc->ep0_bounce_addr);
2907
2908 kfree(dwc->setup_buf);
2909 kfree(dwc->zlp_buf);
2910
2911 dma_free_coherent(dwc->dev, sizeof(*dwc->ep0_trb),
2912 dwc->ep0_trb, dwc->ep0_trb_addr);
2913
2914 dma_free_coherent(dwc->dev, sizeof(*dwc->ctrl_req),
2915 dwc->ctrl_req, dwc->ctrl_req_addr);
2916 }
2917
2918 int dwc3_gadget_suspend(struct dwc3 *dwc)
2919 {
2920 if (!dwc->gadget_driver)
2921 return 0;
2922
2923 if (dwc->pullups_connected) {
2924 dwc3_gadget_disable_irq(dwc);
2925 dwc3_gadget_run_stop(dwc, true, true);
2926 }
2927
2928 __dwc3_gadget_ep_disable(dwc->eps[0]);
2929 __dwc3_gadget_ep_disable(dwc->eps[1]);
2930
2931 dwc->dcfg = dwc3_readl(dwc->regs, DWC3_DCFG);
2932
2933 return 0;
2934 }
2935
2936 int dwc3_gadget_resume(struct dwc3 *dwc)
2937 {
2938 struct dwc3_ep *dep;
2939 int ret;
2940
2941 if (!dwc->gadget_driver)
2942 return 0;
2943
2944 /* Start with SuperSpeed Default */
2945 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2946
2947 dep = dwc->eps[0];
2948 ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, false,
2949 false);
2950 if (ret)
2951 goto err0;
2952
2953 dep = dwc->eps[1];
2954 ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, false,
2955 false);
2956 if (ret)
2957 goto err1;
2958
2959 /* begin to receive SETUP packets */
2960 dwc->ep0state = EP0_SETUP_PHASE;
2961 dwc3_ep0_out_start(dwc);
2962
2963 dwc3_writel(dwc->regs, DWC3_DCFG, dwc->dcfg);
2964
2965 if (dwc->pullups_connected) {
2966 dwc3_gadget_enable_irq(dwc);
2967 dwc3_gadget_run_stop(dwc, true, false);
2968 }
2969
2970 return 0;
2971
2972 err1:
2973 __dwc3_gadget_ep_disable(dwc->eps[0]);
2974
2975 err0:
2976 return ret;
2977 }
Linux kernel - Deliverables
This page took 0.138206 seconds and 5 git commands to generate.