2 * gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link
4 * Copyright (C) 2010-2011 Texas Instruments Incorporated - http://www.ti.com
6 * Authors: Felipe Balbi <balbi@ti.com>,
7 * Sebastian Andrzej Siewior <bigeasy@linutronix.de>
9 * This program is free software: you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 of
11 * the License as published by the Free Software Foundation.
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
19 #include <linux/kernel.h>
20 #include <linux/delay.h>
21 #include <linux/slab.h>
22 #include <linux/spinlock.h>
23 #include <linux/platform_device.h>
24 #include <linux/pm_runtime.h>
25 #include <linux/interrupt.h>
27 #include <linux/list.h>
28 #include <linux/dma-mapping.h>
30 #include <linux/usb/ch9.h>
31 #include <linux/usb/gadget.h>
39 * dwc3_gadget_set_test_mode - Enables USB2 Test Modes
40 * @dwc: pointer to our context structure
41 * @mode: the mode to set (J, K SE0 NAK, Force Enable)
43 * Caller should take care of locking. This function will
44 * return 0 on success or -EINVAL if wrong Test Selector
47 int dwc3_gadget_set_test_mode(struct dwc3
*dwc
, int mode
)
51 reg
= dwc3_readl(dwc
->regs
, DWC3_DCTL
);
52 reg
&= ~DWC3_DCTL_TSTCTRL_MASK
;
66 dwc3_writel(dwc
->regs
, DWC3_DCTL
, reg
);
72 * dwc3_gadget_get_link_state - Gets current state of USB Link
73 * @dwc: pointer to our context structure
75 * Caller should take care of locking. This function will
76 * return the link state on success (>= 0) or -ETIMEDOUT.
78 int dwc3_gadget_get_link_state(struct dwc3
*dwc
)
82 reg
= dwc3_readl(dwc
->regs
, DWC3_DSTS
);
84 return DWC3_DSTS_USBLNKST(reg
);
88 * dwc3_gadget_set_link_state - Sets USB Link to a particular State
89 * @dwc: pointer to our context structure
90 * @state: the state to put link into
92 * Caller should take care of locking. This function will
93 * return 0 on success or -ETIMEDOUT.
95 int dwc3_gadget_set_link_state(struct dwc3
*dwc
, enum dwc3_link_state state
)
101 * Wait until device controller is ready. Only applies to 1.94a and
104 if (dwc
->revision
>= DWC3_REVISION_194A
) {
106 reg
= dwc3_readl(dwc
->regs
, DWC3_DSTS
);
107 if (reg
& DWC3_DSTS_DCNRD
)
117 reg
= dwc3_readl(dwc
->regs
, DWC3_DCTL
);
118 reg
&= ~DWC3_DCTL_ULSTCHNGREQ_MASK
;
120 /* set requested state */
121 reg
|= DWC3_DCTL_ULSTCHNGREQ(state
);
122 dwc3_writel(dwc
->regs
, DWC3_DCTL
, reg
);
125 * The following code is racy when called from dwc3_gadget_wakeup,
126 * and is not needed, at least on newer versions
128 if (dwc
->revision
>= DWC3_REVISION_194A
)
131 /* wait for a change in DSTS */
134 reg
= dwc3_readl(dwc
->regs
, DWC3_DSTS
);
136 if (DWC3_DSTS_USBLNKST(reg
) == state
)
142 dwc3_trace(trace_dwc3_gadget
,
143 "link state change request timed out");
149 * dwc3_ep_inc_trb() - Increment a TRB index.
150 * @index - Pointer to the TRB index to increment.
152 * The index should never point to the link TRB. After incrementing,
153 * if it is point to the link TRB, wrap around to the beginning. The
154 * link TRB is always at the last TRB entry.
156 static void dwc3_ep_inc_trb(u8
*index
)
159 if (*index
== (DWC3_TRB_NUM
- 1))
163 static void dwc3_ep_inc_enq(struct dwc3_ep
*dep
)
165 dwc3_ep_inc_trb(&dep
->trb_enqueue
);
168 static void dwc3_ep_inc_deq(struct dwc3_ep
*dep
)
170 dwc3_ep_inc_trb(&dep
->trb_dequeue
);
173 void dwc3_gadget_giveback(struct dwc3_ep
*dep
, struct dwc3_request
*req
,
176 struct dwc3
*dwc
= dep
->dwc
;
182 dwc3_ep_inc_deq(dep
);
183 } while(++i
< req
->request
.num_mapped_sgs
);
184 req
->started
= false;
186 list_del(&req
->list
);
189 if (req
->request
.status
== -EINPROGRESS
)
190 req
->request
.status
= status
;
192 if (dwc
->ep0_bounced
&& dep
->number
== 0)
193 dwc
->ep0_bounced
= false;
195 usb_gadget_unmap_request(&dwc
->gadget
, &req
->request
,
198 trace_dwc3_gadget_giveback(req
);
200 spin_unlock(&dwc
->lock
);
201 usb_gadget_giveback_request(&dep
->endpoint
, &req
->request
);
202 spin_lock(&dwc
->lock
);
205 pm_runtime_put(dwc
->dev
);
208 int dwc3_send_gadget_generic_command(struct dwc3
*dwc
, unsigned cmd
, u32 param
)
215 dwc3_writel(dwc
->regs
, DWC3_DGCMDPAR
, param
);
216 dwc3_writel(dwc
->regs
, DWC3_DGCMD
, cmd
| DWC3_DGCMD_CMDACT
);
219 reg
= dwc3_readl(dwc
->regs
, DWC3_DGCMD
);
220 if (!(reg
& DWC3_DGCMD_CMDACT
)) {
221 status
= DWC3_DGCMD_STATUS(reg
);
233 trace_dwc3_gadget_generic_cmd(cmd
, param
, status
);
238 static int __dwc3_gadget_wakeup(struct dwc3
*dwc
);
240 int dwc3_send_gadget_ep_cmd(struct dwc3_ep
*dep
, unsigned cmd
,
241 struct dwc3_gadget_ep_cmd_params
*params
)
243 struct dwc3
*dwc
= dep
->dwc
;
252 * Synopsys Databook 2.60a states, on section 6.3.2.5.[1-8], that if
253 * we're issuing an endpoint command, we must check if
254 * GUSB2PHYCFG.SUSPHY bit is set. If it is, then we need to clear it.
256 * We will also set SUSPHY bit to what it was before returning as stated
257 * by the same section on Synopsys databook.
259 if (dwc
->gadget
.speed
<= USB_SPEED_HIGH
) {
260 reg
= dwc3_readl(dwc
->regs
, DWC3_GUSB2PHYCFG(0));
261 if (unlikely(reg
& DWC3_GUSB2PHYCFG_SUSPHY
)) {
263 reg
&= ~DWC3_GUSB2PHYCFG_SUSPHY
;
264 dwc3_writel(dwc
->regs
, DWC3_GUSB2PHYCFG(0), reg
);
268 if (cmd
== DWC3_DEPCMD_STARTTRANSFER
) {
271 needs_wakeup
= (dwc
->link_state
== DWC3_LINK_STATE_U1
||
272 dwc
->link_state
== DWC3_LINK_STATE_U2
||
273 dwc
->link_state
== DWC3_LINK_STATE_U3
);
275 if (unlikely(needs_wakeup
)) {
276 ret
= __dwc3_gadget_wakeup(dwc
);
277 dev_WARN_ONCE(dwc
->dev
, ret
, "wakeup failed --> %d\n",
282 dwc3_writel(dep
->regs
, DWC3_DEPCMDPAR0
, params
->param0
);
283 dwc3_writel(dep
->regs
, DWC3_DEPCMDPAR1
, params
->param1
);
284 dwc3_writel(dep
->regs
, DWC3_DEPCMDPAR2
, params
->param2
);
286 dwc3_writel(dep
->regs
, DWC3_DEPCMD
, cmd
| DWC3_DEPCMD_CMDACT
);
288 reg
= dwc3_readl(dep
->regs
, DWC3_DEPCMD
);
289 if (!(reg
& DWC3_DEPCMD_CMDACT
)) {
290 cmd_status
= DWC3_DEPCMD_STATUS(reg
);
292 dwc3_trace(trace_dwc3_gadget
,
293 "Command Complete --> %d",
296 switch (cmd_status
) {
300 case DEPEVT_TRANSFER_NO_RESOURCE
:
301 dwc3_trace(trace_dwc3_gadget
, "no resource available");
304 case DEPEVT_TRANSFER_BUS_EXPIRY
:
306 * SW issues START TRANSFER command to
307 * isochronous ep with future frame interval. If
308 * future interval time has already passed when
309 * core receives the command, it will respond
310 * with an error status of 'Bus Expiry'.
312 * Instead of always returning -EINVAL, let's
313 * give a hint to the gadget driver that this is
314 * the case by returning -EAGAIN.
316 dwc3_trace(trace_dwc3_gadget
, "bus expiry");
320 dev_WARN(dwc
->dev
, "UNKNOWN cmd status\n");
328 dwc3_trace(trace_dwc3_gadget
,
329 "Command Timed Out");
331 cmd_status
= -ETIMEDOUT
;
334 trace_dwc3_gadget_ep_cmd(dep
, cmd
, params
, cmd_status
);
336 if (unlikely(susphy
)) {
337 reg
= dwc3_readl(dwc
->regs
, DWC3_GUSB2PHYCFG(0));
338 reg
|= DWC3_GUSB2PHYCFG_SUSPHY
;
339 dwc3_writel(dwc
->regs
, DWC3_GUSB2PHYCFG(0), reg
);
345 static int dwc3_send_clear_stall_ep_cmd(struct dwc3_ep
*dep
)
347 struct dwc3
*dwc
= dep
->dwc
;
348 struct dwc3_gadget_ep_cmd_params params
;
349 u32 cmd
= DWC3_DEPCMD_CLEARSTALL
;
352 * As of core revision 2.60a the recommended programming model
353 * is to set the ClearPendIN bit when issuing a Clear Stall EP
354 * command for IN endpoints. This is to prevent an issue where
355 * some (non-compliant) hosts may not send ACK TPs for pending
356 * IN transfers due to a mishandled error condition. Synopsys
359 if (dep
->direction
&& (dwc
->revision
>= DWC3_REVISION_260A
))
360 cmd
|= DWC3_DEPCMD_CLEARPENDIN
;
362 memset(¶ms
, 0, sizeof(params
));
364 return dwc3_send_gadget_ep_cmd(dep
, cmd
, ¶ms
);
367 static dma_addr_t
dwc3_trb_dma_offset(struct dwc3_ep
*dep
,
368 struct dwc3_trb
*trb
)
370 u32 offset
= (char *) trb
- (char *) dep
->trb_pool
;
372 return dep
->trb_pool_dma
+ offset
;
375 static int dwc3_alloc_trb_pool(struct dwc3_ep
*dep
)
377 struct dwc3
*dwc
= dep
->dwc
;
382 dep
->trb_pool
= dma_alloc_coherent(dwc
->dev
,
383 sizeof(struct dwc3_trb
) * DWC3_TRB_NUM
,
384 &dep
->trb_pool_dma
, GFP_KERNEL
);
385 if (!dep
->trb_pool
) {
386 dev_err(dep
->dwc
->dev
, "failed to allocate trb pool for %s\n",
394 static void dwc3_free_trb_pool(struct dwc3_ep
*dep
)
396 struct dwc3
*dwc
= dep
->dwc
;
398 dma_free_coherent(dwc
->dev
, sizeof(struct dwc3_trb
) * DWC3_TRB_NUM
,
399 dep
->trb_pool
, dep
->trb_pool_dma
);
401 dep
->trb_pool
= NULL
;
402 dep
->trb_pool_dma
= 0;
405 static int dwc3_gadget_set_xfer_resource(struct dwc3
*dwc
, struct dwc3_ep
*dep
);
408 * dwc3_gadget_start_config - Configure EP resources
409 * @dwc: pointer to our controller context structure
410 * @dep: endpoint that is being enabled
412 * The assignment of transfer resources cannot perfectly follow the
413 * data book due to the fact that the controller driver does not have
414 * all knowledge of the configuration in advance. It is given this
415 * information piecemeal by the composite gadget framework after every
416 * SET_CONFIGURATION and SET_INTERFACE. Trying to follow the databook
417 * programming model in this scenario can cause errors. For two
420 * 1) The databook says to do DEPSTARTCFG for every SET_CONFIGURATION
421 * and SET_INTERFACE (8.1.5). This is incorrect in the scenario of
422 * multiple interfaces.
424 * 2) The databook does not mention doing more DEPXFERCFG for new
425 * endpoint on alt setting (8.1.6).
427 * The following simplified method is used instead:
429 * All hardware endpoints can be assigned a transfer resource and this
430 * setting will stay persistent until either a core reset or
431 * hibernation. So whenever we do a DEPSTARTCFG(0) we can go ahead and
432 * do DEPXFERCFG for every hardware endpoint as well. We are
433 * guaranteed that there are as many transfer resources as endpoints.
435 * This function is called for each endpoint when it is being enabled
436 * but is triggered only when called for EP0-out, which always happens
437 * first, and which should only happen in one of the above conditions.
439 static int dwc3_gadget_start_config(struct dwc3
*dwc
, struct dwc3_ep
*dep
)
441 struct dwc3_gadget_ep_cmd_params params
;
449 memset(¶ms
, 0x00, sizeof(params
));
450 cmd
= DWC3_DEPCMD_DEPSTARTCFG
;
452 ret
= dwc3_send_gadget_ep_cmd(dep
, cmd
, ¶ms
);
456 for (i
= 0; i
< DWC3_ENDPOINTS_NUM
; i
++) {
457 struct dwc3_ep
*dep
= dwc
->eps
[i
];
462 ret
= dwc3_gadget_set_xfer_resource(dwc
, dep
);
470 static int dwc3_gadget_set_ep_config(struct dwc3
*dwc
, struct dwc3_ep
*dep
,
471 const struct usb_endpoint_descriptor
*desc
,
472 const struct usb_ss_ep_comp_descriptor
*comp_desc
,
473 bool ignore
, bool restore
)
475 struct dwc3_gadget_ep_cmd_params params
;
477 memset(¶ms
, 0x00, sizeof(params
));
479 params
.param0
= DWC3_DEPCFG_EP_TYPE(usb_endpoint_type(desc
))
480 | DWC3_DEPCFG_MAX_PACKET_SIZE(usb_endpoint_maxp(desc
));
482 /* Burst size is only needed in SuperSpeed mode */
483 if (dwc
->gadget
.speed
>= USB_SPEED_SUPER
) {
484 u32 burst
= dep
->endpoint
.maxburst
;
485 params
.param0
|= DWC3_DEPCFG_BURST_SIZE(burst
- 1);
489 params
.param0
|= DWC3_DEPCFG_IGN_SEQ_NUM
;
492 params
.param0
|= DWC3_DEPCFG_ACTION_RESTORE
;
493 params
.param2
|= dep
->saved_state
;
496 params
.param1
= DWC3_DEPCFG_XFER_COMPLETE_EN
497 | DWC3_DEPCFG_XFER_NOT_READY_EN
;
499 if (usb_ss_max_streams(comp_desc
) && usb_endpoint_xfer_bulk(desc
)) {
500 params
.param1
|= DWC3_DEPCFG_STREAM_CAPABLE
501 | DWC3_DEPCFG_STREAM_EVENT_EN
;
502 dep
->stream_capable
= true;
505 if (!usb_endpoint_xfer_control(desc
))
506 params
.param1
|= DWC3_DEPCFG_XFER_IN_PROGRESS_EN
;
509 * We are doing 1:1 mapping for endpoints, meaning
510 * Physical Endpoints 2 maps to Logical Endpoint 2 and
511 * so on. We consider the direction bit as part of the physical
512 * endpoint number. So USB endpoint 0x81 is 0x03.
514 params
.param1
|= DWC3_DEPCFG_EP_NUMBER(dep
->number
);
517 * We must use the lower 16 TX FIFOs even though
521 params
.param0
|= DWC3_DEPCFG_FIFO_NUMBER(dep
->number
>> 1);
523 if (desc
->bInterval
) {
524 params
.param1
|= DWC3_DEPCFG_BINTERVAL_M1(desc
->bInterval
- 1);
525 dep
->interval
= 1 << (desc
->bInterval
- 1);
528 return dwc3_send_gadget_ep_cmd(dep
, DWC3_DEPCMD_SETEPCONFIG
, ¶ms
);
531 static int dwc3_gadget_set_xfer_resource(struct dwc3
*dwc
, struct dwc3_ep
*dep
)
533 struct dwc3_gadget_ep_cmd_params params
;
535 memset(¶ms
, 0x00, sizeof(params
));
537 params
.param0
= DWC3_DEPXFERCFG_NUM_XFER_RES(1);
539 return dwc3_send_gadget_ep_cmd(dep
, DWC3_DEPCMD_SETTRANSFRESOURCE
,
544 * __dwc3_gadget_ep_enable - Initializes a HW endpoint
545 * @dep: endpoint to be initialized
546 * @desc: USB Endpoint Descriptor
548 * Caller should take care of locking
550 static int __dwc3_gadget_ep_enable(struct dwc3_ep
*dep
,
551 const struct usb_endpoint_descriptor
*desc
,
552 const struct usb_ss_ep_comp_descriptor
*comp_desc
,
553 bool ignore
, bool restore
)
555 struct dwc3
*dwc
= dep
->dwc
;
559 dwc3_trace(trace_dwc3_gadget
, "Enabling %s", dep
->name
);
561 if (!(dep
->flags
& DWC3_EP_ENABLED
)) {
562 ret
= dwc3_gadget_start_config(dwc
, dep
);
567 ret
= dwc3_gadget_set_ep_config(dwc
, dep
, desc
, comp_desc
, ignore
,
572 if (!(dep
->flags
& DWC3_EP_ENABLED
)) {
573 struct dwc3_trb
*trb_st_hw
;
574 struct dwc3_trb
*trb_link
;
576 dep
->endpoint
.desc
= desc
;
577 dep
->comp_desc
= comp_desc
;
578 dep
->type
= usb_endpoint_type(desc
);
579 dep
->flags
|= DWC3_EP_ENABLED
;
581 reg
= dwc3_readl(dwc
->regs
, DWC3_DALEPENA
);
582 reg
|= DWC3_DALEPENA_EP(dep
->number
);
583 dwc3_writel(dwc
->regs
, DWC3_DALEPENA
, reg
);
585 if (usb_endpoint_xfer_control(desc
))
588 /* Initialize the TRB ring */
589 dep
->trb_dequeue
= 0;
590 dep
->trb_enqueue
= 0;
591 memset(dep
->trb_pool
, 0,
592 sizeof(struct dwc3_trb
) * DWC3_TRB_NUM
);
594 /* Link TRB. The HWO bit is never reset */
595 trb_st_hw
= &dep
->trb_pool
[0];
597 trb_link
= &dep
->trb_pool
[DWC3_TRB_NUM
- 1];
598 trb_link
->bpl
= lower_32_bits(dwc3_trb_dma_offset(dep
, trb_st_hw
));
599 trb_link
->bph
= upper_32_bits(dwc3_trb_dma_offset(dep
, trb_st_hw
));
600 trb_link
->ctrl
|= DWC3_TRBCTL_LINK_TRB
;
601 trb_link
->ctrl
|= DWC3_TRB_CTRL_HWO
;
607 static void dwc3_stop_active_transfer(struct dwc3
*dwc
, u32 epnum
, bool force
);
608 static void dwc3_remove_requests(struct dwc3
*dwc
, struct dwc3_ep
*dep
)
610 struct dwc3_request
*req
;
612 if (!list_empty(&dep
->started_list
)) {
613 dwc3_stop_active_transfer(dwc
, dep
->number
, true);
615 /* - giveback all requests to gadget driver */
616 while (!list_empty(&dep
->started_list
)) {
617 req
= next_request(&dep
->started_list
);
619 dwc3_gadget_giveback(dep
, req
, -ESHUTDOWN
);
623 while (!list_empty(&dep
->pending_list
)) {
624 req
= next_request(&dep
->pending_list
);
626 dwc3_gadget_giveback(dep
, req
, -ESHUTDOWN
);
631 * __dwc3_gadget_ep_disable - Disables a HW endpoint
632 * @dep: the endpoint to disable
634 * This function also removes requests which are currently processed ny the
635 * hardware and those which are not yet scheduled.
636 * Caller should take care of locking.
638 static int __dwc3_gadget_ep_disable(struct dwc3_ep
*dep
)
640 struct dwc3
*dwc
= dep
->dwc
;
643 dwc3_trace(trace_dwc3_gadget
, "Disabling %s", dep
->name
);
645 dwc3_remove_requests(dwc
, dep
);
647 /* make sure HW endpoint isn't stalled */
648 if (dep
->flags
& DWC3_EP_STALL
)
649 __dwc3_gadget_ep_set_halt(dep
, 0, false);
651 reg
= dwc3_readl(dwc
->regs
, DWC3_DALEPENA
);
652 reg
&= ~DWC3_DALEPENA_EP(dep
->number
);
653 dwc3_writel(dwc
->regs
, DWC3_DALEPENA
, reg
);
655 dep
->stream_capable
= false;
656 dep
->endpoint
.desc
= NULL
;
657 dep
->comp_desc
= NULL
;
664 /* -------------------------------------------------------------------------- */
666 static int dwc3_gadget_ep0_enable(struct usb_ep
*ep
,
667 const struct usb_endpoint_descriptor
*desc
)
672 static int dwc3_gadget_ep0_disable(struct usb_ep
*ep
)
677 /* -------------------------------------------------------------------------- */
679 static int dwc3_gadget_ep_enable(struct usb_ep
*ep
,
680 const struct usb_endpoint_descriptor
*desc
)
687 if (!ep
|| !desc
|| desc
->bDescriptorType
!= USB_DT_ENDPOINT
) {
688 pr_debug("dwc3: invalid parameters\n");
692 if (!desc
->wMaxPacketSize
) {
693 pr_debug("dwc3: missing wMaxPacketSize\n");
697 dep
= to_dwc3_ep(ep
);
700 if (dev_WARN_ONCE(dwc
->dev
, dep
->flags
& DWC3_EP_ENABLED
,
701 "%s is already enabled\n",
705 spin_lock_irqsave(&dwc
->lock
, flags
);
706 ret
= __dwc3_gadget_ep_enable(dep
, desc
, ep
->comp_desc
, false, false);
707 spin_unlock_irqrestore(&dwc
->lock
, flags
);
712 static int dwc3_gadget_ep_disable(struct usb_ep
*ep
)
720 pr_debug("dwc3: invalid parameters\n");
724 dep
= to_dwc3_ep(ep
);
727 if (dev_WARN_ONCE(dwc
->dev
, !(dep
->flags
& DWC3_EP_ENABLED
),
728 "%s is already disabled\n",
732 spin_lock_irqsave(&dwc
->lock
, flags
);
733 ret
= __dwc3_gadget_ep_disable(dep
);
734 spin_unlock_irqrestore(&dwc
->lock
, flags
);
739 static struct usb_request
*dwc3_gadget_ep_alloc_request(struct usb_ep
*ep
,
742 struct dwc3_request
*req
;
743 struct dwc3_ep
*dep
= to_dwc3_ep(ep
);
745 req
= kzalloc(sizeof(*req
), gfp_flags
);
749 req
->epnum
= dep
->number
;
752 trace_dwc3_alloc_request(req
);
754 return &req
->request
;
757 static void dwc3_gadget_ep_free_request(struct usb_ep
*ep
,
758 struct usb_request
*request
)
760 struct dwc3_request
*req
= to_dwc3_request(request
);
762 trace_dwc3_free_request(req
);
767 * dwc3_prepare_one_trb - setup one TRB from one request
768 * @dep: endpoint for which this request is prepared
769 * @req: dwc3_request pointer
771 static void dwc3_prepare_one_trb(struct dwc3_ep
*dep
,
772 struct dwc3_request
*req
, dma_addr_t dma
,
773 unsigned length
, unsigned last
, unsigned chain
, unsigned node
)
775 struct dwc3_trb
*trb
;
777 dwc3_trace(trace_dwc3_gadget
, "%s: req %p dma %08llx length %d%s%s",
778 dep
->name
, req
, (unsigned long long) dma
,
779 length
, last
? " last" : "",
780 chain
? " chain" : "");
783 trb
= &dep
->trb_pool
[dep
->trb_enqueue
];
786 dwc3_gadget_move_started_request(req
);
788 req
->trb_dma
= dwc3_trb_dma_offset(dep
, trb
);
789 req
->first_trb_index
= dep
->trb_enqueue
;
792 dwc3_ep_inc_enq(dep
);
794 trb
->size
= DWC3_TRB_SIZE_LENGTH(length
);
795 trb
->bpl
= lower_32_bits(dma
);
796 trb
->bph
= upper_32_bits(dma
);
798 switch (usb_endpoint_type(dep
->endpoint
.desc
)) {
799 case USB_ENDPOINT_XFER_CONTROL
:
800 trb
->ctrl
= DWC3_TRBCTL_CONTROL_SETUP
;
803 case USB_ENDPOINT_XFER_ISOC
:
805 trb
->ctrl
= DWC3_TRBCTL_ISOCHRONOUS_FIRST
;
807 trb
->ctrl
= DWC3_TRBCTL_ISOCHRONOUS
;
809 /* always enable Interrupt on Missed ISOC */
810 trb
->ctrl
|= DWC3_TRB_CTRL_ISP_IMI
;
813 case USB_ENDPOINT_XFER_BULK
:
814 case USB_ENDPOINT_XFER_INT
:
815 trb
->ctrl
= DWC3_TRBCTL_NORMAL
;
819 * This is only possible with faulty memory because we
820 * checked it already :)
825 /* always enable Continue on Short Packet */
826 trb
->ctrl
|= DWC3_TRB_CTRL_CSP
;
828 if (!req
->request
.no_interrupt
&& !chain
)
829 trb
->ctrl
|= DWC3_TRB_CTRL_IOC
| DWC3_TRB_CTRL_ISP_IMI
;
832 trb
->ctrl
|= DWC3_TRB_CTRL_LST
;
835 trb
->ctrl
|= DWC3_TRB_CTRL_CHN
;
837 if (usb_endpoint_xfer_bulk(dep
->endpoint
.desc
) && dep
->stream_capable
)
838 trb
->ctrl
|= DWC3_TRB_CTRL_SID_SOFN(req
->request
.stream_id
);
840 trb
->ctrl
|= DWC3_TRB_CTRL_HWO
;
842 trace_dwc3_prepare_trb(dep
, trb
);
846 * dwc3_ep_prev_trb() - Returns the previous TRB in the ring
847 * @dep: The endpoint with the TRB ring
848 * @index: The index of the current TRB in the ring
850 * Returns the TRB prior to the one pointed to by the index. If the
851 * index is 0, we will wrap backwards, skip the link TRB, and return
852 * the one just before that.
854 static struct dwc3_trb
*dwc3_ep_prev_trb(struct dwc3_ep
*dep
, u8 index
)
857 index
= DWC3_TRB_NUM
- 2;
859 index
= dep
->trb_enqueue
- 1;
861 return &dep
->trb_pool
[index
];
864 static u32
dwc3_calc_trbs_left(struct dwc3_ep
*dep
)
866 struct dwc3_trb
*tmp
;
870 * If enqueue & dequeue are equal than it is either full or empty.
872 * One way to know for sure is if the TRB right before us has HWO bit
873 * set or not. If it has, then we're definitely full and can't fit any
874 * more transfers in our ring.
876 if (dep
->trb_enqueue
== dep
->trb_dequeue
) {
877 tmp
= dwc3_ep_prev_trb(dep
, dep
->trb_enqueue
);
878 if (tmp
->ctrl
& DWC3_TRB_CTRL_HWO
)
881 return DWC3_TRB_NUM
- 1;
884 trbs_left
= dep
->trb_dequeue
- dep
->trb_enqueue
;
885 trbs_left
&= (DWC3_TRB_NUM
- 1);
887 if (dep
->trb_dequeue
< dep
->trb_enqueue
)
893 static void dwc3_prepare_one_trb_sg(struct dwc3_ep
*dep
,
894 struct dwc3_request
*req
, unsigned int trbs_left
)
896 struct usb_request
*request
= &req
->request
;
897 struct scatterlist
*sg
= request
->sg
;
898 struct scatterlist
*s
;
899 unsigned int last
= false;
904 for_each_sg(sg
, s
, request
->num_mapped_sgs
, i
) {
905 unsigned chain
= true;
907 length
= sg_dma_len(s
);
908 dma
= sg_dma_address(s
);
911 if (list_is_last(&req
->list
, &dep
->pending_list
))
923 dwc3_prepare_one_trb(dep
, req
, dma
, length
,
931 static void dwc3_prepare_one_trb_linear(struct dwc3_ep
*dep
,
932 struct dwc3_request
*req
, unsigned int trbs_left
)
934 unsigned int last
= false;
938 dma
= req
->request
.dma
;
939 length
= req
->request
.length
;
944 /* Is this the last request? */
945 if (list_is_last(&req
->list
, &dep
->pending_list
))
948 dwc3_prepare_one_trb(dep
, req
, dma
, length
,
953 * dwc3_prepare_trbs - setup TRBs from requests
954 * @dep: endpoint for which requests are being prepared
956 * The function goes through the requests list and sets up TRBs for the
957 * transfers. The function returns once there are no more TRBs available or
958 * it runs out of requests.
960 static void dwc3_prepare_trbs(struct dwc3_ep
*dep
)
962 struct dwc3_request
*req
, *n
;
965 BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM
);
967 trbs_left
= dwc3_calc_trbs_left(dep
);
971 list_for_each_entry_safe(req
, n
, &dep
->pending_list
, list
) {
972 if (req
->request
.num_mapped_sgs
> 0)
973 dwc3_prepare_one_trb_sg(dep
, req
, trbs_left
--);
975 dwc3_prepare_one_trb_linear(dep
, req
, trbs_left
--);
982 static int __dwc3_gadget_kick_transfer(struct dwc3_ep
*dep
, u16 cmd_param
)
984 struct dwc3_gadget_ep_cmd_params params
;
985 struct dwc3_request
*req
;
986 struct dwc3
*dwc
= dep
->dwc
;
991 starting
= !(dep
->flags
& DWC3_EP_BUSY
);
993 dwc3_prepare_trbs(dep
);
994 req
= next_request(&dep
->started_list
);
996 dep
->flags
|= DWC3_EP_PENDING_REQUEST
;
1000 memset(¶ms
, 0, sizeof(params
));
1003 params
.param0
= upper_32_bits(req
->trb_dma
);
1004 params
.param1
= lower_32_bits(req
->trb_dma
);
1005 cmd
= DWC3_DEPCMD_STARTTRANSFER
;
1007 cmd
= DWC3_DEPCMD_UPDATETRANSFER
;
1010 cmd
|= DWC3_DEPCMD_PARAM(cmd_param
);
1011 ret
= dwc3_send_gadget_ep_cmd(dep
, cmd
, ¶ms
);
1014 * FIXME we need to iterate over the list of requests
1015 * here and stop, unmap, free and del each of the linked
1016 * requests instead of what we do now.
1018 usb_gadget_unmap_request(&dwc
->gadget
, &req
->request
,
1020 list_del(&req
->list
);
1024 dep
->flags
|= DWC3_EP_BUSY
;
1027 dep
->resource_index
= dwc3_gadget_ep_get_transfer_index(dep
);
1028 WARN_ON_ONCE(!dep
->resource_index
);
1034 static void __dwc3_gadget_start_isoc(struct dwc3
*dwc
,
1035 struct dwc3_ep
*dep
, u32 cur_uf
)
1039 if (list_empty(&dep
->pending_list
)) {
1040 dwc3_trace(trace_dwc3_gadget
,
1041 "ISOC ep %s run out for requests",
1043 dep
->flags
|= DWC3_EP_PENDING_REQUEST
;
1047 /* 4 micro frames in the future */
1048 uf
= cur_uf
+ dep
->interval
* 4;
1050 __dwc3_gadget_kick_transfer(dep
, uf
);
1053 static void dwc3_gadget_start_isoc(struct dwc3
*dwc
,
1054 struct dwc3_ep
*dep
, const struct dwc3_event_depevt
*event
)
1058 mask
= ~(dep
->interval
- 1);
1059 cur_uf
= event
->parameters
& mask
;
1061 __dwc3_gadget_start_isoc(dwc
, dep
, cur_uf
);
1064 static int __dwc3_gadget_ep_queue(struct dwc3_ep
*dep
, struct dwc3_request
*req
)
1066 struct dwc3
*dwc
= dep
->dwc
;
1069 if (!dep
->endpoint
.desc
) {
1070 dwc3_trace(trace_dwc3_gadget
,
1071 "trying to queue request %p to disabled %s\n",
1072 &req
->request
, dep
->endpoint
.name
);
1076 if (WARN(req
->dep
!= dep
, "request %p belongs to '%s'\n",
1077 &req
->request
, req
->dep
->name
)) {
1078 dwc3_trace(trace_dwc3_gadget
, "request %p belongs to '%s'\n",
1079 &req
->request
, req
->dep
->name
);
1083 pm_runtime_get(dwc
->dev
);
1085 req
->request
.actual
= 0;
1086 req
->request
.status
= -EINPROGRESS
;
1087 req
->direction
= dep
->direction
;
1088 req
->epnum
= dep
->number
;
1090 trace_dwc3_ep_queue(req
);
1093 * We only add to our list of requests now and
1094 * start consuming the list once we get XferNotReady
1097 * That way, we avoid doing anything that we don't need
1098 * to do now and defer it until the point we receive a
1099 * particular token from the Host side.
1101 * This will also avoid Host cancelling URBs due to too
1104 ret
= usb_gadget_map_request(&dwc
->gadget
, &req
->request
,
1109 list_add_tail(&req
->list
, &dep
->pending_list
);
1112 * If there are no pending requests and the endpoint isn't already
1113 * busy, we will just start the request straight away.
1115 * This will save one IRQ (XFER_NOT_READY) and possibly make it a
1116 * little bit faster.
1118 if (!usb_endpoint_xfer_isoc(dep
->endpoint
.desc
) &&
1119 !usb_endpoint_xfer_int(dep
->endpoint
.desc
) &&
1120 !(dep
->flags
& DWC3_EP_BUSY
)) {
1121 ret
= __dwc3_gadget_kick_transfer(dep
, 0);
1126 * There are a few special cases:
1128 * 1. XferNotReady with empty list of requests. We need to kick the
1129 * transfer here in that situation, otherwise we will be NAKing
1130 * forever. If we get XferNotReady before gadget driver has a
1131 * chance to queue a request, we will ACK the IRQ but won't be
1132 * able to receive the data until the next request is queued.
1133 * The following code is handling exactly that.
1136 if (dep
->flags
& DWC3_EP_PENDING_REQUEST
) {
1138 * If xfernotready is already elapsed and it is a case
1139 * of isoc transfer, then issue END TRANSFER, so that
1140 * you can receive xfernotready again and can have
1141 * notion of current microframe.
1143 if (usb_endpoint_xfer_isoc(dep
->endpoint
.desc
)) {
1144 if (list_empty(&dep
->started_list
)) {
1145 dwc3_stop_active_transfer(dwc
, dep
->number
, true);
1146 dep
->flags
= DWC3_EP_ENABLED
;
1151 ret
= __dwc3_gadget_kick_transfer(dep
, 0);
1153 dep
->flags
&= ~DWC3_EP_PENDING_REQUEST
;
1159 * 2. XferInProgress on Isoc EP with an active transfer. We need to
1160 * kick the transfer here after queuing a request, otherwise the
1161 * core may not see the modified TRB(s).
1163 if (usb_endpoint_xfer_isoc(dep
->endpoint
.desc
) &&
1164 (dep
->flags
& DWC3_EP_BUSY
) &&
1165 !(dep
->flags
& DWC3_EP_MISSED_ISOC
)) {
1166 WARN_ON_ONCE(!dep
->resource_index
);
1167 ret
= __dwc3_gadget_kick_transfer(dep
, dep
->resource_index
);
1172 * 4. Stream Capable Bulk Endpoints. We need to start the transfer
1173 * right away, otherwise host will not know we have streams to be
1176 if (dep
->stream_capable
)
1177 ret
= __dwc3_gadget_kick_transfer(dep
, 0);
1180 if (ret
&& ret
!= -EBUSY
)
1181 dwc3_trace(trace_dwc3_gadget
,
1182 "%s: failed to kick transfers\n",
1190 static void __dwc3_gadget_ep_zlp_complete(struct usb_ep
*ep
,
1191 struct usb_request
*request
)
1193 dwc3_gadget_ep_free_request(ep
, request
);
1196 static int __dwc3_gadget_ep_queue_zlp(struct dwc3
*dwc
, struct dwc3_ep
*dep
)
1198 struct dwc3_request
*req
;
1199 struct usb_request
*request
;
1200 struct usb_ep
*ep
= &dep
->endpoint
;
1202 dwc3_trace(trace_dwc3_gadget
, "queueing ZLP\n");
1203 request
= dwc3_gadget_ep_alloc_request(ep
, GFP_ATOMIC
);
1207 request
->length
= 0;
1208 request
->buf
= dwc
->zlp_buf
;
1209 request
->complete
= __dwc3_gadget_ep_zlp_complete
;
1211 req
= to_dwc3_request(request
);
1213 return __dwc3_gadget_ep_queue(dep
, req
);
1216 static int dwc3_gadget_ep_queue(struct usb_ep
*ep
, struct usb_request
*request
,
1219 struct dwc3_request
*req
= to_dwc3_request(request
);
1220 struct dwc3_ep
*dep
= to_dwc3_ep(ep
);
1221 struct dwc3
*dwc
= dep
->dwc
;
1223 unsigned long flags
;
1227 spin_lock_irqsave(&dwc
->lock
, flags
);
1228 ret
= __dwc3_gadget_ep_queue(dep
, req
);
1231 * Okay, here's the thing, if gadget driver has requested for a ZLP by
1232 * setting request->zero, instead of doing magic, we will just queue an
1233 * extra usb_request ourselves so that it gets handled the same way as
1234 * any other request.
1236 if (ret
== 0 && request
->zero
&& request
->length
&&
1237 (request
->length
% ep
->maxpacket
== 0))
1238 ret
= __dwc3_gadget_ep_queue_zlp(dwc
, dep
);
1240 spin_unlock_irqrestore(&dwc
->lock
, flags
);
1245 static int dwc3_gadget_ep_dequeue(struct usb_ep
*ep
,
1246 struct usb_request
*request
)
1248 struct dwc3_request
*req
= to_dwc3_request(request
);
1249 struct dwc3_request
*r
= NULL
;
1251 struct dwc3_ep
*dep
= to_dwc3_ep(ep
);
1252 struct dwc3
*dwc
= dep
->dwc
;
1254 unsigned long flags
;
1257 trace_dwc3_ep_dequeue(req
);
1259 spin_lock_irqsave(&dwc
->lock
, flags
);
1261 list_for_each_entry(r
, &dep
->pending_list
, list
) {
1267 list_for_each_entry(r
, &dep
->started_list
, list
) {
1272 /* wait until it is processed */
1273 dwc3_stop_active_transfer(dwc
, dep
->number
, true);
1276 dev_err(dwc
->dev
, "request %p was not queued to %s\n",
1283 /* giveback the request */
1284 dwc3_gadget_giveback(dep
, req
, -ECONNRESET
);
1287 spin_unlock_irqrestore(&dwc
->lock
, flags
);
1292 int __dwc3_gadget_ep_set_halt(struct dwc3_ep
*dep
, int value
, int protocol
)
1294 struct dwc3_gadget_ep_cmd_params params
;
1295 struct dwc3
*dwc
= dep
->dwc
;
1298 if (usb_endpoint_xfer_isoc(dep
->endpoint
.desc
)) {
1299 dev_err(dwc
->dev
, "%s is of Isochronous type\n", dep
->name
);
1303 memset(¶ms
, 0x00, sizeof(params
));
1306 if (!protocol
&& ((dep
->direction
&& dep
->flags
& DWC3_EP_BUSY
) ||
1307 (!list_empty(&dep
->started_list
) ||
1308 !list_empty(&dep
->pending_list
)))) {
1309 dwc3_trace(trace_dwc3_gadget
,
1310 "%s: pending request, cannot halt",
1315 ret
= dwc3_send_gadget_ep_cmd(dep
, DWC3_DEPCMD_SETSTALL
,
1318 dev_err(dwc
->dev
, "failed to set STALL on %s\n",
1321 dep
->flags
|= DWC3_EP_STALL
;
1324 ret
= dwc3_send_clear_stall_ep_cmd(dep
);
1326 dev_err(dwc
->dev
, "failed to clear STALL on %s\n",
1329 dep
->flags
&= ~(DWC3_EP_STALL
| DWC3_EP_WEDGE
);
1335 static int dwc3_gadget_ep_set_halt(struct usb_ep
*ep
, int value
)
1337 struct dwc3_ep
*dep
= to_dwc3_ep(ep
);
1338 struct dwc3
*dwc
= dep
->dwc
;
1340 unsigned long flags
;
1344 spin_lock_irqsave(&dwc
->lock
, flags
);
1345 ret
= __dwc3_gadget_ep_set_halt(dep
, value
, false);
1346 spin_unlock_irqrestore(&dwc
->lock
, flags
);
1351 static int dwc3_gadget_ep_set_wedge(struct usb_ep
*ep
)
1353 struct dwc3_ep
*dep
= to_dwc3_ep(ep
);
1354 struct dwc3
*dwc
= dep
->dwc
;
1355 unsigned long flags
;
1358 spin_lock_irqsave(&dwc
->lock
, flags
);
1359 dep
->flags
|= DWC3_EP_WEDGE
;
1361 if (dep
->number
== 0 || dep
->number
== 1)
1362 ret
= __dwc3_gadget_ep0_set_halt(ep
, 1);
1364 ret
= __dwc3_gadget_ep_set_halt(dep
, 1, false);
1365 spin_unlock_irqrestore(&dwc
->lock
, flags
);
1370 /* -------------------------------------------------------------------------- */
1372 static struct usb_endpoint_descriptor dwc3_gadget_ep0_desc
= {
1373 .bLength
= USB_DT_ENDPOINT_SIZE
,
1374 .bDescriptorType
= USB_DT_ENDPOINT
,
1375 .bmAttributes
= USB_ENDPOINT_XFER_CONTROL
,
1378 static const struct usb_ep_ops dwc3_gadget_ep0_ops
= {
1379 .enable
= dwc3_gadget_ep0_enable
,
1380 .disable
= dwc3_gadget_ep0_disable
,
1381 .alloc_request
= dwc3_gadget_ep_alloc_request
,
1382 .free_request
= dwc3_gadget_ep_free_request
,
1383 .queue
= dwc3_gadget_ep0_queue
,
1384 .dequeue
= dwc3_gadget_ep_dequeue
,
1385 .set_halt
= dwc3_gadget_ep0_set_halt
,
1386 .set_wedge
= dwc3_gadget_ep_set_wedge
,
1389 static const struct usb_ep_ops dwc3_gadget_ep_ops
= {
1390 .enable
= dwc3_gadget_ep_enable
,
1391 .disable
= dwc3_gadget_ep_disable
,
1392 .alloc_request
= dwc3_gadget_ep_alloc_request
,
1393 .free_request
= dwc3_gadget_ep_free_request
,
1394 .queue
= dwc3_gadget_ep_queue
,
1395 .dequeue
= dwc3_gadget_ep_dequeue
,
1396 .set_halt
= dwc3_gadget_ep_set_halt
,
1397 .set_wedge
= dwc3_gadget_ep_set_wedge
,
1400 /* -------------------------------------------------------------------------- */
1402 static int dwc3_gadget_get_frame(struct usb_gadget
*g
)
1404 struct dwc3
*dwc
= gadget_to_dwc(g
);
1407 reg
= dwc3_readl(dwc
->regs
, DWC3_DSTS
);
1408 return DWC3_DSTS_SOFFN(reg
);
1411 static int __dwc3_gadget_wakeup(struct dwc3
*dwc
)
1413 unsigned long timeout
;
1422 * According to the Databook Remote wakeup request should
1423 * be issued only when the device is in early suspend state.
1425 * We can check that via USB Link State bits in DSTS register.
1427 reg
= dwc3_readl(dwc
->regs
, DWC3_DSTS
);
1429 speed
= reg
& DWC3_DSTS_CONNECTSPD
;
1430 if ((speed
== DWC3_DSTS_SUPERSPEED
) ||
1431 (speed
== DWC3_DSTS_SUPERSPEED_PLUS
)) {
1432 dwc3_trace(trace_dwc3_gadget
, "no wakeup on SuperSpeed\n");
1436 link_state
= DWC3_DSTS_USBLNKST(reg
);
1438 switch (link_state
) {
1439 case DWC3_LINK_STATE_RX_DET
: /* in HS, means Early Suspend */
1440 case DWC3_LINK_STATE_U3
: /* in HS, means SUSPEND */
1443 dwc3_trace(trace_dwc3_gadget
,
1444 "can't wakeup from '%s'\n",
1445 dwc3_gadget_link_string(link_state
));
1449 ret
= dwc3_gadget_set_link_state(dwc
, DWC3_LINK_STATE_RECOV
);
1451 dev_err(dwc
->dev
, "failed to put link in Recovery\n");
1455 /* Recent versions do this automatically */
1456 if (dwc
->revision
< DWC3_REVISION_194A
) {
1457 /* write zeroes to Link Change Request */
1458 reg
= dwc3_readl(dwc
->regs
, DWC3_DCTL
);
1459 reg
&= ~DWC3_DCTL_ULSTCHNGREQ_MASK
;
1460 dwc3_writel(dwc
->regs
, DWC3_DCTL
, reg
);
1463 /* poll until Link State changes to ON */
1464 timeout
= jiffies
+ msecs_to_jiffies(100);
1466 while (!time_after(jiffies
, timeout
)) {
1467 reg
= dwc3_readl(dwc
->regs
, DWC3_DSTS
);
1469 /* in HS, means ON */
1470 if (DWC3_DSTS_USBLNKST(reg
) == DWC3_LINK_STATE_U0
)
1474 if (DWC3_DSTS_USBLNKST(reg
) != DWC3_LINK_STATE_U0
) {
1475 dev_err(dwc
->dev
, "failed to send remote wakeup\n");
1482 static int dwc3_gadget_wakeup(struct usb_gadget
*g
)
1484 struct dwc3
*dwc
= gadget_to_dwc(g
);
1485 unsigned long flags
;
1488 spin_lock_irqsave(&dwc
->lock
, flags
);
1489 ret
= __dwc3_gadget_wakeup(dwc
);
1490 spin_unlock_irqrestore(&dwc
->lock
, flags
);
1495 static int dwc3_gadget_set_selfpowered(struct usb_gadget
*g
,
1498 struct dwc3
*dwc
= gadget_to_dwc(g
);
1499 unsigned long flags
;
1501 spin_lock_irqsave(&dwc
->lock
, flags
);
1502 g
->is_selfpowered
= !!is_selfpowered
;
1503 spin_unlock_irqrestore(&dwc
->lock
, flags
);
1508 static int dwc3_gadget_run_stop(struct dwc3
*dwc
, int is_on
, int suspend
)
1513 if (pm_runtime_suspended(dwc
->dev
))
1516 reg
= dwc3_readl(dwc
->regs
, DWC3_DCTL
);
1518 if (dwc
->revision
<= DWC3_REVISION_187A
) {
1519 reg
&= ~DWC3_DCTL_TRGTULST_MASK
;
1520 reg
|= DWC3_DCTL_TRGTULST_RX_DET
;
1523 if (dwc
->revision
>= DWC3_REVISION_194A
)
1524 reg
&= ~DWC3_DCTL_KEEP_CONNECT
;
1525 reg
|= DWC3_DCTL_RUN_STOP
;
1527 if (dwc
->has_hibernation
)
1528 reg
|= DWC3_DCTL_KEEP_CONNECT
;
1530 dwc
->pullups_connected
= true;
1532 reg
&= ~DWC3_DCTL_RUN_STOP
;
1534 if (dwc
->has_hibernation
&& !suspend
)
1535 reg
&= ~DWC3_DCTL_KEEP_CONNECT
;
1537 dwc
->pullups_connected
= false;
1540 dwc3_writel(dwc
->regs
, DWC3_DCTL
, reg
);
1543 reg
= dwc3_readl(dwc
->regs
, DWC3_DSTS
);
1545 if (!(reg
& DWC3_DSTS_DEVCTRLHLT
))
1548 if (reg
& DWC3_DSTS_DEVCTRLHLT
)
1557 dwc3_trace(trace_dwc3_gadget
, "gadget %s data soft-%s",
1559 ? dwc
->gadget_driver
->function
: "no-function",
1560 is_on
? "connect" : "disconnect");
1565 static int dwc3_gadget_pullup(struct usb_gadget
*g
, int is_on
)
1567 struct dwc3
*dwc
= gadget_to_dwc(g
);
1568 unsigned long flags
;
1573 spin_lock_irqsave(&dwc
->lock
, flags
);
1574 ret
= dwc3_gadget_run_stop(dwc
, is_on
, false);
1575 spin_unlock_irqrestore(&dwc
->lock
, flags
);
1580 static void dwc3_gadget_enable_irq(struct dwc3
*dwc
)
1584 /* Enable all but Start and End of Frame IRQs */
1585 reg
= (DWC3_DEVTEN_VNDRDEVTSTRCVEDEN
|
1586 DWC3_DEVTEN_EVNTOVERFLOWEN
|
1587 DWC3_DEVTEN_CMDCMPLTEN
|
1588 DWC3_DEVTEN_ERRTICERREN
|
1589 DWC3_DEVTEN_WKUPEVTEN
|
1590 DWC3_DEVTEN_ULSTCNGEN
|
1591 DWC3_DEVTEN_CONNECTDONEEN
|
1592 DWC3_DEVTEN_USBRSTEN
|
1593 DWC3_DEVTEN_DISCONNEVTEN
);
1595 dwc3_writel(dwc
->regs
, DWC3_DEVTEN
, reg
);
1598 static void dwc3_gadget_disable_irq(struct dwc3
*dwc
)
1600 /* mask all interrupts */
1601 dwc3_writel(dwc
->regs
, DWC3_DEVTEN
, 0x00);
1604 static irqreturn_t
dwc3_interrupt(int irq
, void *_dwc
);
1605 static irqreturn_t
dwc3_thread_interrupt(int irq
, void *_dwc
);
1608 * dwc3_gadget_setup_nump - Calculate and initialize NUMP field of DCFG
1609 * dwc: pointer to our context structure
1611 * The following looks like complex but it's actually very simple. In order to
1612 * calculate the number of packets we can burst at once on OUT transfers, we're
1613 * gonna use RxFIFO size.
1615 * To calculate RxFIFO size we need two numbers:
1616 * MDWIDTH = size, in bits, of the internal memory bus
1617 * RAM2_DEPTH = depth, in MDWIDTH, of internal RAM2 (where RxFIFO sits)
1619 * Given these two numbers, the formula is simple:
1621 * RxFIFO Size = (RAM2_DEPTH * MDWIDTH / 8) - 24 - 16;
1623 * 24 bytes is for 3x SETUP packets
1624 * 16 bytes is a clock domain crossing tolerance
1626 * Given RxFIFO Size, NUMP = RxFIFOSize / 1024;
1628 static void dwc3_gadget_setup_nump(struct dwc3
*dwc
)
1635 ram2_depth
= DWC3_GHWPARAMS7_RAM2_DEPTH(dwc
->hwparams
.hwparams7
);
1636 mdwidth
= DWC3_GHWPARAMS0_MDWIDTH(dwc
->hwparams
.hwparams0
);
1638 nump
= ((ram2_depth
* mdwidth
/ 8) - 24 - 16) / 1024;
1639 nump
= min_t(u32
, nump
, 16);
1642 reg
= dwc3_readl(dwc
->regs
, DWC3_DCFG
);
1643 reg
&= ~DWC3_DCFG_NUMP_MASK
;
1644 reg
|= nump
<< DWC3_DCFG_NUMP_SHIFT
;
1645 dwc3_writel(dwc
->regs
, DWC3_DCFG
, reg
);
1648 static int __dwc3_gadget_start(struct dwc3
*dwc
)
1650 struct dwc3_ep
*dep
;
1654 reg
= dwc3_readl(dwc
->regs
, DWC3_DCFG
);
1655 reg
&= ~(DWC3_DCFG_SPEED_MASK
);
1658 * WORKAROUND: DWC3 revision < 2.20a have an issue
1659 * which would cause metastability state on Run/Stop
1660 * bit if we try to force the IP to USB2-only mode.
1662 * Because of that, we cannot configure the IP to any
1663 * speed other than the SuperSpeed
1667 * STAR#9000525659: Clock Domain Crossing on DCTL in
1670 if (dwc
->revision
< DWC3_REVISION_220A
) {
1671 reg
|= DWC3_DCFG_SUPERSPEED
;
1673 switch (dwc
->maximum_speed
) {
1675 reg
|= DWC3_DCFG_LOWSPEED
;
1677 case USB_SPEED_FULL
:
1678 reg
|= DWC3_DCFG_FULLSPEED1
;
1680 case USB_SPEED_HIGH
:
1681 reg
|= DWC3_DCFG_HIGHSPEED
;
1683 case USB_SPEED_SUPER_PLUS
:
1684 reg
|= DWC3_DCFG_SUPERSPEED_PLUS
;
1687 dev_err(dwc
->dev
, "invalid dwc->maximum_speed (%d)\n",
1688 dwc
->maximum_speed
);
1690 case USB_SPEED_SUPER
:
1691 reg
|= DWC3_DCFG_SUPERSPEED
;
1695 dwc3_writel(dwc
->regs
, DWC3_DCFG
, reg
);
1698 * We are telling dwc3 that we want to use DCFG.NUMP as ACK TP's NUMP
1699 * field instead of letting dwc3 itself calculate that automatically.
1701 * This way, we maximize the chances that we'll be able to get several
1702 * bursts of data without going through any sort of endpoint throttling.
1704 reg
= dwc3_readl(dwc
->regs
, DWC3_GRXTHRCFG
);
1705 reg
&= ~DWC3_GRXTHRCFG_PKTCNTSEL
;
1706 dwc3_writel(dwc
->regs
, DWC3_GRXTHRCFG
, reg
);
1708 dwc3_gadget_setup_nump(dwc
);
1710 /* Start with SuperSpeed Default */
1711 dwc3_gadget_ep0_desc
.wMaxPacketSize
= cpu_to_le16(512);
1714 ret
= __dwc3_gadget_ep_enable(dep
, &dwc3_gadget_ep0_desc
, NULL
, false,
1717 dev_err(dwc
->dev
, "failed to enable %s\n", dep
->name
);
1722 ret
= __dwc3_gadget_ep_enable(dep
, &dwc3_gadget_ep0_desc
, NULL
, false,
1725 dev_err(dwc
->dev
, "failed to enable %s\n", dep
->name
);
1729 /* begin to receive SETUP packets */
1730 dwc
->ep0state
= EP0_SETUP_PHASE
;
1731 dwc3_ep0_out_start(dwc
);
1733 dwc3_gadget_enable_irq(dwc
);
1738 __dwc3_gadget_ep_disable(dwc
->eps
[0]);
1744 static int dwc3_gadget_start(struct usb_gadget
*g
,
1745 struct usb_gadget_driver
*driver
)
1747 struct dwc3
*dwc
= gadget_to_dwc(g
);
1748 unsigned long flags
;
1752 irq
= platform_get_irq(to_platform_device(dwc
->dev
), 0);
1753 ret
= request_threaded_irq(irq
, dwc3_interrupt
, dwc3_thread_interrupt
,
1754 IRQF_SHARED
, "dwc3", dwc
->ev_buf
);
1756 dev_err(dwc
->dev
, "failed to request irq #%d --> %d\n",
1760 dwc
->irq_gadget
= irq
;
1762 spin_lock_irqsave(&dwc
->lock
, flags
);
1763 if (dwc
->gadget_driver
) {
1764 dev_err(dwc
->dev
, "%s is already bound to %s\n",
1766 dwc
->gadget_driver
->driver
.name
);
1771 dwc
->gadget_driver
= driver
;
1773 if (pm_runtime_active(dwc
->dev
))
1774 __dwc3_gadget_start(dwc
);
1776 spin_unlock_irqrestore(&dwc
->lock
, flags
);
1781 spin_unlock_irqrestore(&dwc
->lock
, flags
);
1788 static void __dwc3_gadget_stop(struct dwc3
*dwc
)
1790 dwc3_gadget_disable_irq(dwc
);
1791 __dwc3_gadget_ep_disable(dwc
->eps
[0]);
1792 __dwc3_gadget_ep_disable(dwc
->eps
[1]);
1795 static int dwc3_gadget_stop(struct usb_gadget
*g
)
1797 struct dwc3
*dwc
= gadget_to_dwc(g
);
1798 unsigned long flags
;
1800 spin_lock_irqsave(&dwc
->lock
, flags
);
1801 __dwc3_gadget_stop(dwc
);
1802 dwc
->gadget_driver
= NULL
;
1803 spin_unlock_irqrestore(&dwc
->lock
, flags
);
1805 free_irq(dwc
->irq_gadget
, dwc
->ev_buf
);
1810 static const struct usb_gadget_ops dwc3_gadget_ops
= {
1811 .get_frame
= dwc3_gadget_get_frame
,
1812 .wakeup
= dwc3_gadget_wakeup
,
1813 .set_selfpowered
= dwc3_gadget_set_selfpowered
,
1814 .pullup
= dwc3_gadget_pullup
,
1815 .udc_start
= dwc3_gadget_start
,
1816 .udc_stop
= dwc3_gadget_stop
,
1819 /* -------------------------------------------------------------------------- */
1821 static int dwc3_gadget_init_hw_endpoints(struct dwc3
*dwc
,
1822 u8 num
, u32 direction
)
1824 struct dwc3_ep
*dep
;
1827 for (i
= 0; i
< num
; i
++) {
1828 u8 epnum
= (i
<< 1) | (direction
? 1 : 0);
1830 dep
= kzalloc(sizeof(*dep
), GFP_KERNEL
);
1835 dep
->number
= epnum
;
1836 dep
->direction
= !!direction
;
1837 dep
->regs
= dwc
->regs
+ DWC3_DEP_BASE(epnum
);
1838 dwc
->eps
[epnum
] = dep
;
1840 snprintf(dep
->name
, sizeof(dep
->name
), "ep%d%s", epnum
>> 1,
1841 (epnum
& 1) ? "in" : "out");
1843 dep
->endpoint
.name
= dep
->name
;
1844 spin_lock_init(&dep
->lock
);
1846 dwc3_trace(trace_dwc3_gadget
, "initializing %s", dep
->name
);
1848 if (epnum
== 0 || epnum
== 1) {
1849 usb_ep_set_maxpacket_limit(&dep
->endpoint
, 512);
1850 dep
->endpoint
.maxburst
= 1;
1851 dep
->endpoint
.ops
= &dwc3_gadget_ep0_ops
;
1853 dwc
->gadget
.ep0
= &dep
->endpoint
;
1857 usb_ep_set_maxpacket_limit(&dep
->endpoint
, 1024);
1858 dep
->endpoint
.max_streams
= 15;
1859 dep
->endpoint
.ops
= &dwc3_gadget_ep_ops
;
1860 list_add_tail(&dep
->endpoint
.ep_list
,
1861 &dwc
->gadget
.ep_list
);
1863 ret
= dwc3_alloc_trb_pool(dep
);
1868 if (epnum
== 0 || epnum
== 1) {
1869 dep
->endpoint
.caps
.type_control
= true;
1871 dep
->endpoint
.caps
.type_iso
= true;
1872 dep
->endpoint
.caps
.type_bulk
= true;
1873 dep
->endpoint
.caps
.type_int
= true;
1876 dep
->endpoint
.caps
.dir_in
= !!direction
;
1877 dep
->endpoint
.caps
.dir_out
= !direction
;
1879 INIT_LIST_HEAD(&dep
->pending_list
);
1880 INIT_LIST_HEAD(&dep
->started_list
);
1886 static int dwc3_gadget_init_endpoints(struct dwc3
*dwc
)
1890 INIT_LIST_HEAD(&dwc
->gadget
.ep_list
);
1892 ret
= dwc3_gadget_init_hw_endpoints(dwc
, dwc
->num_out_eps
, 0);
1894 dwc3_trace(trace_dwc3_gadget
,
1895 "failed to allocate OUT endpoints");
1899 ret
= dwc3_gadget_init_hw_endpoints(dwc
, dwc
->num_in_eps
, 1);
1901 dwc3_trace(trace_dwc3_gadget
,
1902 "failed to allocate IN endpoints");
1909 static void dwc3_gadget_free_endpoints(struct dwc3
*dwc
)
1911 struct dwc3_ep
*dep
;
1914 for (epnum
= 0; epnum
< DWC3_ENDPOINTS_NUM
; epnum
++) {
1915 dep
= dwc
->eps
[epnum
];
1919 * Physical endpoints 0 and 1 are special; they form the
1920 * bi-directional USB endpoint 0.
1922 * For those two physical endpoints, we don't allocate a TRB
1923 * pool nor do we add them the endpoints list. Due to that, we
1924 * shouldn't do these two operations otherwise we would end up
1925 * with all sorts of bugs when removing dwc3.ko.
1927 if (epnum
!= 0 && epnum
!= 1) {
1928 dwc3_free_trb_pool(dep
);
1929 list_del(&dep
->endpoint
.ep_list
);
1936 /* -------------------------------------------------------------------------- */
1938 static int __dwc3_cleanup_done_trbs(struct dwc3
*dwc
, struct dwc3_ep
*dep
,
1939 struct dwc3_request
*req
, struct dwc3_trb
*trb
,
1940 const struct dwc3_event_depevt
*event
, int status
)
1943 unsigned int s_pkt
= 0;
1944 unsigned int trb_status
;
1946 trace_dwc3_complete_trb(dep
, trb
);
1948 if ((trb
->ctrl
& DWC3_TRB_CTRL_HWO
) && status
!= -ESHUTDOWN
)
1950 * We continue despite the error. There is not much we
1951 * can do. If we don't clean it up we loop forever. If
1952 * we skip the TRB then it gets overwritten after a
1953 * while since we use them in a ring buffer. A BUG()
1954 * would help. Lets hope that if this occurs, someone
1955 * fixes the root cause instead of looking away :)
1957 dev_err(dwc
->dev
, "%s's TRB (%p) still owned by HW\n",
1959 count
= trb
->size
& DWC3_TRB_SIZE_MASK
;
1961 if (dep
->direction
) {
1963 trb_status
= DWC3_TRB_SIZE_TRBSTS(trb
->size
);
1964 if (trb_status
== DWC3_TRBSTS_MISSED_ISOC
) {
1965 dwc3_trace(trace_dwc3_gadget
,
1966 "%s: incomplete IN transfer\n",
1969 * If missed isoc occurred and there is
1970 * no request queued then issue END
1971 * TRANSFER, so that core generates
1972 * next xfernotready and we will issue
1973 * a fresh START TRANSFER.
1974 * If there are still queued request
1975 * then wait, do not issue either END
1976 * or UPDATE TRANSFER, just attach next
1977 * request in pending_list during
1978 * giveback.If any future queued request
1979 * is successfully transferred then we
1980 * will issue UPDATE TRANSFER for all
1981 * request in the pending_list.
1983 dep
->flags
|= DWC3_EP_MISSED_ISOC
;
1985 dev_err(dwc
->dev
, "incomplete IN transfer %s\n",
1987 status
= -ECONNRESET
;
1990 dep
->flags
&= ~DWC3_EP_MISSED_ISOC
;
1993 if (count
&& (event
->status
& DEPEVT_STATUS_SHORT
))
1998 * We assume here we will always receive the entire data block
1999 * which we should receive. Meaning, if we program RX to
2000 * receive 4K but we receive only 2K, we assume that's all we
2001 * should receive and we simply bounce the request back to the
2002 * gadget driver for further processing.
2004 req
->request
.actual
+= req
->request
.length
- count
;
2007 if ((event
->status
& DEPEVT_STATUS_LST
) &&
2008 (trb
->ctrl
& (DWC3_TRB_CTRL_LST
|
2009 DWC3_TRB_CTRL_HWO
)))
2011 if ((event
->status
& DEPEVT_STATUS_IOC
) &&
2012 (trb
->ctrl
& DWC3_TRB_CTRL_IOC
))
2017 static int dwc3_cleanup_done_reqs(struct dwc3
*dwc
, struct dwc3_ep
*dep
,
2018 const struct dwc3_event_depevt
*event
, int status
)
2020 struct dwc3_request
*req
;
2021 struct dwc3_trb
*trb
;
2027 req
= next_request(&dep
->started_list
);
2028 if (WARN_ON_ONCE(!req
))
2033 slot
= req
->first_trb_index
+ i
;
2034 if (slot
== DWC3_TRB_NUM
- 1)
2036 slot
%= DWC3_TRB_NUM
;
2037 trb
= &dep
->trb_pool
[slot
];
2039 ret
= __dwc3_cleanup_done_trbs(dwc
, dep
, req
, trb
,
2043 } while (++i
< req
->request
.num_mapped_sgs
);
2045 dwc3_gadget_giveback(dep
, req
, status
);
2052 * Our endpoint might get disabled by another thread during
2053 * dwc3_gadget_giveback(). If that happens, we're just gonna return 1
2054 * early on so DWC3_EP_BUSY flag gets cleared
2056 if (!dep
->endpoint
.desc
)
2059 if (usb_endpoint_xfer_isoc(dep
->endpoint
.desc
) &&
2060 list_empty(&dep
->started_list
)) {
2061 if (list_empty(&dep
->pending_list
)) {
2063 * If there is no entry in request list then do
2064 * not issue END TRANSFER now. Just set PENDING
2065 * flag, so that END TRANSFER is issued when an
2066 * entry is added into request list.
2068 dep
->flags
= DWC3_EP_PENDING_REQUEST
;
2070 dwc3_stop_active_transfer(dwc
, dep
->number
, true);
2071 dep
->flags
= DWC3_EP_ENABLED
;
2076 if (usb_endpoint_xfer_isoc(dep
->endpoint
.desc
))
2077 if ((event
->status
& DEPEVT_STATUS_IOC
) &&
2078 (trb
->ctrl
& DWC3_TRB_CTRL_IOC
))
2083 static void dwc3_endpoint_transfer_complete(struct dwc3
*dwc
,
2084 struct dwc3_ep
*dep
, const struct dwc3_event_depevt
*event
)
2086 unsigned status
= 0;
2088 u32 is_xfer_complete
;
2090 is_xfer_complete
= (event
->endpoint_event
== DWC3_DEPEVT_XFERCOMPLETE
);
2092 if (event
->status
& DEPEVT_STATUS_BUSERR
)
2093 status
= -ECONNRESET
;
2095 clean_busy
= dwc3_cleanup_done_reqs(dwc
, dep
, event
, status
);
2096 if (clean_busy
&& (!dep
->endpoint
.desc
|| is_xfer_complete
||
2097 usb_endpoint_xfer_isoc(dep
->endpoint
.desc
)))
2098 dep
->flags
&= ~DWC3_EP_BUSY
;
2101 * WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround.
2102 * See dwc3_gadget_linksts_change_interrupt() for 1st half.
2104 if (dwc
->revision
< DWC3_REVISION_183A
) {
2108 for (i
= 0; i
< DWC3_ENDPOINTS_NUM
; i
++) {
2111 if (!(dep
->flags
& DWC3_EP_ENABLED
))
2114 if (!list_empty(&dep
->started_list
))
2118 reg
= dwc3_readl(dwc
->regs
, DWC3_DCTL
);
2120 dwc3_writel(dwc
->regs
, DWC3_DCTL
, reg
);
2126 * Our endpoint might get disabled by another thread during
2127 * dwc3_gadget_giveback(). If that happens, we're just gonna return 1
2128 * early on so DWC3_EP_BUSY flag gets cleared
2130 if (!dep
->endpoint
.desc
)
2133 if (!usb_endpoint_xfer_isoc(dep
->endpoint
.desc
)) {
2136 ret
= __dwc3_gadget_kick_transfer(dep
, 0);
2137 if (!ret
|| ret
== -EBUSY
)
2142 static void dwc3_endpoint_interrupt(struct dwc3
*dwc
,
2143 const struct dwc3_event_depevt
*event
)
2145 struct dwc3_ep
*dep
;
2146 u8 epnum
= event
->endpoint_number
;
2148 dep
= dwc
->eps
[epnum
];
2150 if (!(dep
->flags
& DWC3_EP_ENABLED
))
2153 if (epnum
== 0 || epnum
== 1) {
2154 dwc3_ep0_interrupt(dwc
, event
);
2158 switch (event
->endpoint_event
) {
2159 case DWC3_DEPEVT_XFERCOMPLETE
:
2160 dep
->resource_index
= 0;
2162 if (usb_endpoint_xfer_isoc(dep
->endpoint
.desc
)) {
2163 dwc3_trace(trace_dwc3_gadget
,
2164 "%s is an Isochronous endpoint\n",
2169 dwc3_endpoint_transfer_complete(dwc
, dep
, event
);
2171 case DWC3_DEPEVT_XFERINPROGRESS
:
2172 dwc3_endpoint_transfer_complete(dwc
, dep
, event
);
2174 case DWC3_DEPEVT_XFERNOTREADY
:
2175 if (usb_endpoint_xfer_isoc(dep
->endpoint
.desc
)) {
2176 dwc3_gadget_start_isoc(dwc
, dep
, event
);
2181 active
= event
->status
& DEPEVT_STATUS_TRANSFER_ACTIVE
;
2183 dwc3_trace(trace_dwc3_gadget
, "%s: reason %s",
2184 dep
->name
, active
? "Transfer Active"
2185 : "Transfer Not Active");
2187 ret
= __dwc3_gadget_kick_transfer(dep
, 0);
2188 if (!ret
|| ret
== -EBUSY
)
2191 dwc3_trace(trace_dwc3_gadget
,
2192 "%s: failed to kick transfers\n",
2197 case DWC3_DEPEVT_STREAMEVT
:
2198 if (!usb_endpoint_xfer_bulk(dep
->endpoint
.desc
)) {
2199 dev_err(dwc
->dev
, "Stream event for non-Bulk %s\n",
2204 switch (event
->status
) {
2205 case DEPEVT_STREAMEVT_FOUND
:
2206 dwc3_trace(trace_dwc3_gadget
,
2207 "Stream %d found and started",
2211 case DEPEVT_STREAMEVT_NOTFOUND
:
2214 dwc3_trace(trace_dwc3_gadget
,
2215 "unable to find suitable stream\n");
2218 case DWC3_DEPEVT_RXTXFIFOEVT
:
2219 dwc3_trace(trace_dwc3_gadget
, "%s FIFO Overrun\n", dep
->name
);
2221 case DWC3_DEPEVT_EPCMDCMPLT
:
2222 dwc3_trace(trace_dwc3_gadget
, "Endpoint Command Complete");
2227 static void dwc3_disconnect_gadget(struct dwc3
*dwc
)
2229 if (dwc
->gadget_driver
&& dwc
->gadget_driver
->disconnect
) {
2230 spin_unlock(&dwc
->lock
);
2231 dwc
->gadget_driver
->disconnect(&dwc
->gadget
);
2232 spin_lock(&dwc
->lock
);
2236 static void dwc3_suspend_gadget(struct dwc3
*dwc
)
2238 if (dwc
->gadget_driver
&& dwc
->gadget_driver
->suspend
) {
2239 spin_unlock(&dwc
->lock
);
2240 dwc
->gadget_driver
->suspend(&dwc
->gadget
);
2241 spin_lock(&dwc
->lock
);
2245 static void dwc3_resume_gadget(struct dwc3
*dwc
)
2247 if (dwc
->gadget_driver
&& dwc
->gadget_driver
->resume
) {
2248 spin_unlock(&dwc
->lock
);
2249 dwc
->gadget_driver
->resume(&dwc
->gadget
);
2250 spin_lock(&dwc
->lock
);
2254 static void dwc3_reset_gadget(struct dwc3
*dwc
)
2256 if (!dwc
->gadget_driver
)
2259 if (dwc
->gadget
.speed
!= USB_SPEED_UNKNOWN
) {
2260 spin_unlock(&dwc
->lock
);
2261 usb_gadget_udc_reset(&dwc
->gadget
, dwc
->gadget_driver
);
2262 spin_lock(&dwc
->lock
);
2266 static void dwc3_stop_active_transfer(struct dwc3
*dwc
, u32 epnum
, bool force
)
2268 struct dwc3_ep
*dep
;
2269 struct dwc3_gadget_ep_cmd_params params
;
2273 dep
= dwc
->eps
[epnum
];
2275 if (!dep
->resource_index
)
2279 * NOTICE: We are violating what the Databook says about the
2280 * EndTransfer command. Ideally we would _always_ wait for the
2281 * EndTransfer Command Completion IRQ, but that's causing too
2282 * much trouble synchronizing between us and gadget driver.
2284 * We have discussed this with the IP Provider and it was
2285 * suggested to giveback all requests here, but give HW some
2286 * extra time to synchronize with the interconnect. We're using
2287 * an arbitrary 100us delay for that.
2289 * Note also that a similar handling was tested by Synopsys
2290 * (thanks a lot Paul) and nothing bad has come out of it.
2291 * In short, what we're doing is:
2293 * - Issue EndTransfer WITH CMDIOC bit set
2297 cmd
= DWC3_DEPCMD_ENDTRANSFER
;
2298 cmd
|= force
? DWC3_DEPCMD_HIPRI_FORCERM
: 0;
2299 cmd
|= DWC3_DEPCMD_CMDIOC
;
2300 cmd
|= DWC3_DEPCMD_PARAM(dep
->resource_index
);
2301 memset(¶ms
, 0, sizeof(params
));
2302 ret
= dwc3_send_gadget_ep_cmd(dep
, cmd
, ¶ms
);
2304 dep
->resource_index
= 0;
2305 dep
->flags
&= ~DWC3_EP_BUSY
;
2309 static void dwc3_stop_active_transfers(struct dwc3
*dwc
)
2313 for (epnum
= 2; epnum
< DWC3_ENDPOINTS_NUM
; epnum
++) {
2314 struct dwc3_ep
*dep
;
2316 dep
= dwc
->eps
[epnum
];
2320 if (!(dep
->flags
& DWC3_EP_ENABLED
))
2323 dwc3_remove_requests(dwc
, dep
);
2327 static void dwc3_clear_stall_all_ep(struct dwc3
*dwc
)
2331 for (epnum
= 1; epnum
< DWC3_ENDPOINTS_NUM
; epnum
++) {
2332 struct dwc3_ep
*dep
;
2335 dep
= dwc
->eps
[epnum
];
2339 if (!(dep
->flags
& DWC3_EP_STALL
))
2342 dep
->flags
&= ~DWC3_EP_STALL
;
2344 ret
= dwc3_send_clear_stall_ep_cmd(dep
);
2349 static void dwc3_gadget_disconnect_interrupt(struct dwc3
*dwc
)
2353 reg
= dwc3_readl(dwc
->regs
, DWC3_DCTL
);
2354 reg
&= ~DWC3_DCTL_INITU1ENA
;
2355 dwc3_writel(dwc
->regs
, DWC3_DCTL
, reg
);
2357 reg
&= ~DWC3_DCTL_INITU2ENA
;
2358 dwc3_writel(dwc
->regs
, DWC3_DCTL
, reg
);
2360 dwc3_disconnect_gadget(dwc
);
2362 dwc
->gadget
.speed
= USB_SPEED_UNKNOWN
;
2363 dwc
->setup_packet_pending
= false;
2364 usb_gadget_set_state(&dwc
->gadget
, USB_STATE_NOTATTACHED
);
2366 dwc
->connected
= false;
2369 static void dwc3_gadget_reset_interrupt(struct dwc3
*dwc
)
2373 dwc
->connected
= true;
2376 * WORKAROUND: DWC3 revisions <1.88a have an issue which
2377 * would cause a missing Disconnect Event if there's a
2378 * pending Setup Packet in the FIFO.
2380 * There's no suggested workaround on the official Bug
2381 * report, which states that "unless the driver/application
2382 * is doing any special handling of a disconnect event,
2383 * there is no functional issue".
2385 * Unfortunately, it turns out that we _do_ some special
2386 * handling of a disconnect event, namely complete all
2387 * pending transfers, notify gadget driver of the
2388 * disconnection, and so on.
2390 * Our suggested workaround is to follow the Disconnect
2391 * Event steps here, instead, based on a setup_packet_pending
2392 * flag. Such flag gets set whenever we have a SETUP_PENDING
2393 * status for EP0 TRBs and gets cleared on XferComplete for the
2398 * STAR#9000466709: RTL: Device : Disconnect event not
2399 * generated if setup packet pending in FIFO
2401 if (dwc
->revision
< DWC3_REVISION_188A
) {
2402 if (dwc
->setup_packet_pending
)
2403 dwc3_gadget_disconnect_interrupt(dwc
);
2406 dwc3_reset_gadget(dwc
);
2408 reg
= dwc3_readl(dwc
->regs
, DWC3_DCTL
);
2409 reg
&= ~DWC3_DCTL_TSTCTRL_MASK
;
2410 dwc3_writel(dwc
->regs
, DWC3_DCTL
, reg
);
2411 dwc
->test_mode
= false;
2413 dwc3_stop_active_transfers(dwc
);
2414 dwc3_clear_stall_all_ep(dwc
);
2416 /* Reset device address to zero */
2417 reg
= dwc3_readl(dwc
->regs
, DWC3_DCFG
);
2418 reg
&= ~(DWC3_DCFG_DEVADDR_MASK
);
2419 dwc3_writel(dwc
->regs
, DWC3_DCFG
, reg
);
2422 static void dwc3_update_ram_clk_sel(struct dwc3
*dwc
, u32 speed
)
2425 u32 usb30_clock
= DWC3_GCTL_CLK_BUS
;
2428 * We change the clock only at SS but I dunno why I would want to do
2429 * this. Maybe it becomes part of the power saving plan.
2432 if ((speed
!= DWC3_DSTS_SUPERSPEED
) &&
2433 (speed
!= DWC3_DSTS_SUPERSPEED_PLUS
))
2437 * RAMClkSel is reset to 0 after USB reset, so it must be reprogrammed
2438 * each time on Connect Done.
2443 reg
= dwc3_readl(dwc
->regs
, DWC3_GCTL
);
2444 reg
|= DWC3_GCTL_RAMCLKSEL(usb30_clock
);
2445 dwc3_writel(dwc
->regs
, DWC3_GCTL
, reg
);
2448 static void dwc3_gadget_conndone_interrupt(struct dwc3
*dwc
)
2450 struct dwc3_ep
*dep
;
2455 reg
= dwc3_readl(dwc
->regs
, DWC3_DSTS
);
2456 speed
= reg
& DWC3_DSTS_CONNECTSPD
;
2459 dwc3_update_ram_clk_sel(dwc
, speed
);
2462 case DWC3_DSTS_SUPERSPEED_PLUS
:
2463 dwc3_gadget_ep0_desc
.wMaxPacketSize
= cpu_to_le16(512);
2464 dwc
->gadget
.ep0
->maxpacket
= 512;
2465 dwc
->gadget
.speed
= USB_SPEED_SUPER_PLUS
;
2467 case DWC3_DSTS_SUPERSPEED
:
2469 * WORKAROUND: DWC3 revisions <1.90a have an issue which
2470 * would cause a missing USB3 Reset event.
2472 * In such situations, we should force a USB3 Reset
2473 * event by calling our dwc3_gadget_reset_interrupt()
2478 * STAR#9000483510: RTL: SS : USB3 reset event may
2479 * not be generated always when the link enters poll
2481 if (dwc
->revision
< DWC3_REVISION_190A
)
2482 dwc3_gadget_reset_interrupt(dwc
);
2484 dwc3_gadget_ep0_desc
.wMaxPacketSize
= cpu_to_le16(512);
2485 dwc
->gadget
.ep0
->maxpacket
= 512;
2486 dwc
->gadget
.speed
= USB_SPEED_SUPER
;
2488 case DWC3_DSTS_HIGHSPEED
:
2489 dwc3_gadget_ep0_desc
.wMaxPacketSize
= cpu_to_le16(64);
2490 dwc
->gadget
.ep0
->maxpacket
= 64;
2491 dwc
->gadget
.speed
= USB_SPEED_HIGH
;
2493 case DWC3_DSTS_FULLSPEED2
:
2494 case DWC3_DSTS_FULLSPEED1
:
2495 dwc3_gadget_ep0_desc
.wMaxPacketSize
= cpu_to_le16(64);
2496 dwc
->gadget
.ep0
->maxpacket
= 64;
2497 dwc
->gadget
.speed
= USB_SPEED_FULL
;
2499 case DWC3_DSTS_LOWSPEED
:
2500 dwc3_gadget_ep0_desc
.wMaxPacketSize
= cpu_to_le16(8);
2501 dwc
->gadget
.ep0
->maxpacket
= 8;
2502 dwc
->gadget
.speed
= USB_SPEED_LOW
;
2506 /* Enable USB2 LPM Capability */
2508 if ((dwc
->revision
> DWC3_REVISION_194A
) &&
2509 (speed
!= DWC3_DSTS_SUPERSPEED
) &&
2510 (speed
!= DWC3_DSTS_SUPERSPEED_PLUS
)) {
2511 reg
= dwc3_readl(dwc
->regs
, DWC3_DCFG
);
2512 reg
|= DWC3_DCFG_LPM_CAP
;
2513 dwc3_writel(dwc
->regs
, DWC3_DCFG
, reg
);
2515 reg
= dwc3_readl(dwc
->regs
, DWC3_DCTL
);
2516 reg
&= ~(DWC3_DCTL_HIRD_THRES_MASK
| DWC3_DCTL_L1_HIBER_EN
);
2518 reg
|= DWC3_DCTL_HIRD_THRES(dwc
->hird_threshold
);
2521 * When dwc3 revisions >= 2.40a, LPM Erratum is enabled and
2522 * DCFG.LPMCap is set, core responses with an ACK and the
2523 * BESL value in the LPM token is less than or equal to LPM
2526 WARN_ONCE(dwc
->revision
< DWC3_REVISION_240A
2527 && dwc
->has_lpm_erratum
,
2528 "LPM Erratum not available on dwc3 revisisions < 2.40a\n");
2530 if (dwc
->has_lpm_erratum
&& dwc
->revision
>= DWC3_REVISION_240A
)
2531 reg
|= DWC3_DCTL_LPM_ERRATA(dwc
->lpm_nyet_threshold
);
2533 dwc3_writel(dwc
->regs
, DWC3_DCTL
, reg
);
2535 reg
= dwc3_readl(dwc
->regs
, DWC3_DCTL
);
2536 reg
&= ~DWC3_DCTL_HIRD_THRES_MASK
;
2537 dwc3_writel(dwc
->regs
, DWC3_DCTL
, reg
);
2541 ret
= __dwc3_gadget_ep_enable(dep
, &dwc3_gadget_ep0_desc
, NULL
, true,
2544 dev_err(dwc
->dev
, "failed to enable %s\n", dep
->name
);
2549 ret
= __dwc3_gadget_ep_enable(dep
, &dwc3_gadget_ep0_desc
, NULL
, true,
2552 dev_err(dwc
->dev
, "failed to enable %s\n", dep
->name
);
2557 * Configure PHY via GUSB3PIPECTLn if required.
2559 * Update GTXFIFOSIZn
2561 * In both cases reset values should be sufficient.
2565 static void dwc3_gadget_wakeup_interrupt(struct dwc3
*dwc
)
2568 * TODO take core out of low power mode when that's
2572 if (dwc
->gadget_driver
&& dwc
->gadget_driver
->resume
) {
2573 spin_unlock(&dwc
->lock
);
2574 dwc
->gadget_driver
->resume(&dwc
->gadget
);
2575 spin_lock(&dwc
->lock
);
2579 static void dwc3_gadget_linksts_change_interrupt(struct dwc3
*dwc
,
2580 unsigned int evtinfo
)
2582 enum dwc3_link_state next
= evtinfo
& DWC3_LINK_STATE_MASK
;
2583 unsigned int pwropt
;
2586 * WORKAROUND: DWC3 < 2.50a have an issue when configured without
2587 * Hibernation mode enabled which would show up when device detects
2588 * host-initiated U3 exit.
2590 * In that case, device will generate a Link State Change Interrupt
2591 * from U3 to RESUME which is only necessary if Hibernation is
2594 * There are no functional changes due to such spurious event and we
2595 * just need to ignore it.
2599 * STAR#9000570034 RTL: SS Resume event generated in non-Hibernation
2602 pwropt
= DWC3_GHWPARAMS1_EN_PWROPT(dwc
->hwparams
.hwparams1
);
2603 if ((dwc
->revision
< DWC3_REVISION_250A
) &&
2604 (pwropt
!= DWC3_GHWPARAMS1_EN_PWROPT_HIB
)) {
2605 if ((dwc
->link_state
== DWC3_LINK_STATE_U3
) &&
2606 (next
== DWC3_LINK_STATE_RESUME
)) {
2607 dwc3_trace(trace_dwc3_gadget
,
2608 "ignoring transition U3 -> Resume");
2614 * WORKAROUND: DWC3 Revisions <1.83a have an issue which, depending
2615 * on the link partner, the USB session might do multiple entry/exit
2616 * of low power states before a transfer takes place.
2618 * Due to this problem, we might experience lower throughput. The
2619 * suggested workaround is to disable DCTL[12:9] bits if we're
2620 * transitioning from U1/U2 to U0 and enable those bits again
2621 * after a transfer completes and there are no pending transfers
2622 * on any of the enabled endpoints.
2624 * This is the first half of that workaround.
2628 * STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us
2629 * core send LGO_Ux entering U0
2631 if (dwc
->revision
< DWC3_REVISION_183A
) {
2632 if (next
== DWC3_LINK_STATE_U0
) {
2636 switch (dwc
->link_state
) {
2637 case DWC3_LINK_STATE_U1
:
2638 case DWC3_LINK_STATE_U2
:
2639 reg
= dwc3_readl(dwc
->regs
, DWC3_DCTL
);
2640 u1u2
= reg
& (DWC3_DCTL_INITU2ENA
2641 | DWC3_DCTL_ACCEPTU2ENA
2642 | DWC3_DCTL_INITU1ENA
2643 | DWC3_DCTL_ACCEPTU1ENA
);
2646 dwc
->u1u2
= reg
& u1u2
;
2650 dwc3_writel(dwc
->regs
, DWC3_DCTL
, reg
);
2660 case DWC3_LINK_STATE_U1
:
2661 if (dwc
->speed
== USB_SPEED_SUPER
)
2662 dwc3_suspend_gadget(dwc
);
2664 case DWC3_LINK_STATE_U2
:
2665 case DWC3_LINK_STATE_U3
:
2666 dwc3_suspend_gadget(dwc
);
2668 case DWC3_LINK_STATE_RESUME
:
2669 dwc3_resume_gadget(dwc
);
2676 dwc
->link_state
= next
;
2679 static void dwc3_gadget_hibernation_interrupt(struct dwc3
*dwc
,
2680 unsigned int evtinfo
)
2682 unsigned int is_ss
= evtinfo
& BIT(4);
2685 * WORKAROUND: DWC3 revison 2.20a with hibernation support
2686 * have a known issue which can cause USB CV TD.9.23 to fail
2689 * Because of this issue, core could generate bogus hibernation
2690 * events which SW needs to ignore.
2694 * STAR#9000546576: Device Mode Hibernation: Issue in USB 2.0
2695 * Device Fallback from SuperSpeed
2697 if (is_ss
^ (dwc
->speed
== USB_SPEED_SUPER
))
2700 /* enter hibernation here */
2703 static void dwc3_gadget_interrupt(struct dwc3
*dwc
,
2704 const struct dwc3_event_devt
*event
)
2706 switch (event
->type
) {
2707 case DWC3_DEVICE_EVENT_DISCONNECT
:
2708 dwc3_gadget_disconnect_interrupt(dwc
);
2710 case DWC3_DEVICE_EVENT_RESET
:
2711 dwc3_gadget_reset_interrupt(dwc
);
2713 case DWC3_DEVICE_EVENT_CONNECT_DONE
:
2714 dwc3_gadget_conndone_interrupt(dwc
);
2716 case DWC3_DEVICE_EVENT_WAKEUP
:
2717 dwc3_gadget_wakeup_interrupt(dwc
);
2719 case DWC3_DEVICE_EVENT_HIBER_REQ
:
2720 if (dev_WARN_ONCE(dwc
->dev
, !dwc
->has_hibernation
,
2721 "unexpected hibernation event\n"))
2724 dwc3_gadget_hibernation_interrupt(dwc
, event
->event_info
);
2726 case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE
:
2727 dwc3_gadget_linksts_change_interrupt(dwc
, event
->event_info
);
2729 case DWC3_DEVICE_EVENT_EOPF
:
2730 dwc3_trace(trace_dwc3_gadget
, "End of Periodic Frame");
2732 case DWC3_DEVICE_EVENT_SOF
:
2733 dwc3_trace(trace_dwc3_gadget
, "Start of Periodic Frame");
2735 case DWC3_DEVICE_EVENT_ERRATIC_ERROR
:
2736 dwc3_trace(trace_dwc3_gadget
, "Erratic Error");
2738 case DWC3_DEVICE_EVENT_CMD_CMPL
:
2739 dwc3_trace(trace_dwc3_gadget
, "Command Complete");
2741 case DWC3_DEVICE_EVENT_OVERFLOW
:
2742 dwc3_trace(trace_dwc3_gadget
, "Overflow");
2745 dev_WARN(dwc
->dev
, "UNKNOWN IRQ %d\n", event
->type
);
2749 static void dwc3_process_event_entry(struct dwc3
*dwc
,
2750 const union dwc3_event
*event
)
2752 trace_dwc3_event(event
->raw
);
2754 /* Endpoint IRQ, handle it and return early */
2755 if (event
->type
.is_devspec
== 0) {
2757 return dwc3_endpoint_interrupt(dwc
, &event
->depevt
);
2760 switch (event
->type
.type
) {
2761 case DWC3_EVENT_TYPE_DEV
:
2762 dwc3_gadget_interrupt(dwc
, &event
->devt
);
2764 /* REVISIT what to do with Carkit and I2C events ? */
2766 dev_err(dwc
->dev
, "UNKNOWN IRQ type %d\n", event
->raw
);
2770 static irqreturn_t
dwc3_process_event_buf(struct dwc3_event_buffer
*evt
)
2772 struct dwc3
*dwc
= evt
->dwc
;
2773 irqreturn_t ret
= IRQ_NONE
;
2779 if (!(evt
->flags
& DWC3_EVENT_PENDING
))
2783 union dwc3_event event
;
2785 event
.raw
= *(u32
*) (evt
->buf
+ evt
->lpos
);
2787 dwc3_process_event_entry(dwc
, &event
);
2790 * FIXME we wrap around correctly to the next entry as
2791 * almost all entries are 4 bytes in size. There is one
2792 * entry which has 12 bytes which is a regular entry
2793 * followed by 8 bytes data. ATM I don't know how
2794 * things are organized if we get next to the a
2795 * boundary so I worry about that once we try to handle
2798 evt
->lpos
= (evt
->lpos
+ 4) % DWC3_EVENT_BUFFERS_SIZE
;
2801 dwc3_writel(dwc
->regs
, DWC3_GEVNTCOUNT(0), 4);
2805 evt
->flags
&= ~DWC3_EVENT_PENDING
;
2808 /* Unmask interrupt */
2809 reg
= dwc3_readl(dwc
->regs
, DWC3_GEVNTSIZ(0));
2810 reg
&= ~DWC3_GEVNTSIZ_INTMASK
;
2811 dwc3_writel(dwc
->regs
, DWC3_GEVNTSIZ(0), reg
);
2816 static irqreturn_t
dwc3_thread_interrupt(int irq
, void *_evt
)
2818 struct dwc3_event_buffer
*evt
= _evt
;
2819 struct dwc3
*dwc
= evt
->dwc
;
2820 unsigned long flags
;
2821 irqreturn_t ret
= IRQ_NONE
;
2823 spin_lock_irqsave(&dwc
->lock
, flags
);
2824 ret
= dwc3_process_event_buf(evt
);
2825 spin_unlock_irqrestore(&dwc
->lock
, flags
);
2830 static irqreturn_t
dwc3_check_event_buf(struct dwc3_event_buffer
*evt
)
2832 struct dwc3
*dwc
= evt
->dwc
;
2836 if (pm_runtime_suspended(dwc
->dev
)) {
2837 pm_runtime_get(dwc
->dev
);
2838 disable_irq_nosync(dwc
->irq_gadget
);
2839 dwc
->pending_events
= true;
2843 count
= dwc3_readl(dwc
->regs
, DWC3_GEVNTCOUNT(0));
2844 count
&= DWC3_GEVNTCOUNT_MASK
;
2849 evt
->flags
|= DWC3_EVENT_PENDING
;
2851 /* Mask interrupt */
2852 reg
= dwc3_readl(dwc
->regs
, DWC3_GEVNTSIZ(0));
2853 reg
|= DWC3_GEVNTSIZ_INTMASK
;
2854 dwc3_writel(dwc
->regs
, DWC3_GEVNTSIZ(0), reg
);
2856 return IRQ_WAKE_THREAD
;
2859 static irqreturn_t
dwc3_interrupt(int irq
, void *_evt
)
2861 struct dwc3_event_buffer
*evt
= _evt
;
2863 return dwc3_check_event_buf(evt
);
2867 * dwc3_gadget_init - Initializes gadget related registers
2868 * @dwc: pointer to our controller context structure
2870 * Returns 0 on success otherwise negative errno.
2872 int dwc3_gadget_init(struct dwc3
*dwc
)
2876 dwc
->ctrl_req
= dma_alloc_coherent(dwc
->dev
, sizeof(*dwc
->ctrl_req
),
2877 &dwc
->ctrl_req_addr
, GFP_KERNEL
);
2878 if (!dwc
->ctrl_req
) {
2879 dev_err(dwc
->dev
, "failed to allocate ctrl request\n");
2884 dwc
->ep0_trb
= dma_alloc_coherent(dwc
->dev
, sizeof(*dwc
->ep0_trb
) * 2,
2885 &dwc
->ep0_trb_addr
, GFP_KERNEL
);
2886 if (!dwc
->ep0_trb
) {
2887 dev_err(dwc
->dev
, "failed to allocate ep0 trb\n");
2892 dwc
->setup_buf
= kzalloc(DWC3_EP0_BOUNCE_SIZE
, GFP_KERNEL
);
2893 if (!dwc
->setup_buf
) {
2898 dwc
->ep0_bounce
= dma_alloc_coherent(dwc
->dev
,
2899 DWC3_EP0_BOUNCE_SIZE
, &dwc
->ep0_bounce_addr
,
2901 if (!dwc
->ep0_bounce
) {
2902 dev_err(dwc
->dev
, "failed to allocate ep0 bounce buffer\n");
2907 dwc
->zlp_buf
= kzalloc(DWC3_ZLP_BUF_SIZE
, GFP_KERNEL
);
2908 if (!dwc
->zlp_buf
) {
2913 dwc
->gadget
.ops
= &dwc3_gadget_ops
;
2914 dwc
->gadget
.speed
= USB_SPEED_UNKNOWN
;
2915 dwc
->gadget
.sg_supported
= true;
2916 dwc
->gadget
.name
= "dwc3-gadget";
2917 dwc
->gadget
.is_otg
= dwc
->dr_mode
== USB_DR_MODE_OTG
;
2920 * FIXME We might be setting max_speed to <SUPER, however versions
2921 * <2.20a of dwc3 have an issue with metastability (documented
2922 * elsewhere in this driver) which tells us we can't set max speed to
2923 * anything lower than SUPER.
2925 * Because gadget.max_speed is only used by composite.c and function
2926 * drivers (i.e. it won't go into dwc3's registers) we are allowing this
2927 * to happen so we avoid sending SuperSpeed Capability descriptor
2928 * together with our BOS descriptor as that could confuse host into
2929 * thinking we can handle super speed.
2931 * Note that, in fact, we won't even support GetBOS requests when speed
2932 * is less than super speed because we don't have means, yet, to tell
2933 * composite.c that we are USB 2.0 + LPM ECN.
2935 if (dwc
->revision
< DWC3_REVISION_220A
)
2936 dwc3_trace(trace_dwc3_gadget
,
2937 "Changing max_speed on rev %08x\n",
2940 dwc
->gadget
.max_speed
= dwc
->maximum_speed
;
2943 * Per databook, DWC3 needs buffer size to be aligned to MaxPacketSize
2946 dwc
->gadget
.quirk_ep_out_aligned_size
= true;
2949 * REVISIT: Here we should clear all pending IRQs to be
2950 * sure we're starting from a well known location.
2953 ret
= dwc3_gadget_init_endpoints(dwc
);
2957 ret
= usb_add_gadget_udc(dwc
->dev
, &dwc
->gadget
);
2959 dev_err(dwc
->dev
, "failed to register udc\n");
2966 kfree(dwc
->zlp_buf
);
2969 dwc3_gadget_free_endpoints(dwc
);
2970 dma_free_coherent(dwc
->dev
, DWC3_EP0_BOUNCE_SIZE
,
2971 dwc
->ep0_bounce
, dwc
->ep0_bounce_addr
);
2974 kfree(dwc
->setup_buf
);
2977 dma_free_coherent(dwc
->dev
, sizeof(*dwc
->ep0_trb
),
2978 dwc
->ep0_trb
, dwc
->ep0_trb_addr
);
2981 dma_free_coherent(dwc
->dev
, sizeof(*dwc
->ctrl_req
),
2982 dwc
->ctrl_req
, dwc
->ctrl_req_addr
);
2988 /* -------------------------------------------------------------------------- */
2990 void dwc3_gadget_exit(struct dwc3
*dwc
)
2992 usb_del_gadget_udc(&dwc
->gadget
);
2994 dwc3_gadget_free_endpoints(dwc
);
2996 dma_free_coherent(dwc
->dev
, DWC3_EP0_BOUNCE_SIZE
,
2997 dwc
->ep0_bounce
, dwc
->ep0_bounce_addr
);
2999 kfree(dwc
->setup_buf
);
3000 kfree(dwc
->zlp_buf
);
3002 dma_free_coherent(dwc
->dev
, sizeof(*dwc
->ep0_trb
),
3003 dwc
->ep0_trb
, dwc
->ep0_trb_addr
);
3005 dma_free_coherent(dwc
->dev
, sizeof(*dwc
->ctrl_req
),
3006 dwc
->ctrl_req
, dwc
->ctrl_req_addr
);
3009 int dwc3_gadget_suspend(struct dwc3
*dwc
)
3013 if (!dwc
->gadget_driver
)
3016 ret
= dwc3_gadget_run_stop(dwc
, false, false);
3020 dwc3_disconnect_gadget(dwc
);
3021 __dwc3_gadget_stop(dwc
);
3026 int dwc3_gadget_resume(struct dwc3
*dwc
)
3030 if (!dwc
->gadget_driver
)
3033 ret
= __dwc3_gadget_start(dwc
);
3037 ret
= dwc3_gadget_run_stop(dwc
, true, false);
3044 __dwc3_gadget_stop(dwc
);
3050 void dwc3_gadget_process_pending_events(struct dwc3
*dwc
)
3052 if (dwc
->pending_events
) {
3053 dwc3_interrupt(dwc
->irq_gadget
, dwc
->ev_buf
);
3054 dwc
->pending_events
= false;
3055 enable_irq(dwc
->irq_gadget
);