static void __dwc3_ep0_do_control_data(struct dwc3 *dwc,
struct dwc3_ep *dep, struct dwc3_request *req)
{
int ret;
req->direction = !!dep->number;
if (req->request.length == 0) {
ret = dwc3_ep0_start_trans(dwc, dep->number,
dwc->ctrl_req_addr, 0,
DWC3_TRBCTL_CONTROL_DATA, false);
} else if (!IS_ALIGNED(req->request.length, dep->endpoint.maxpacket)
&& (dep->number == 0)) {
u32 transfer_size = 0;
u32 maxpacket;
ret = usb_gadget_map_request(&dwc->gadget, &req->request,
dep->number);
if (ret) {
dwc3_trace(trace_dwc3_ep0, "failed to map request\n");
return;
}
maxpacket = dep->endpoint.maxpacket;
if (req->request.length > DWC3_EP0_BOUNCE_SIZE) {
transfer_size = ALIGN(req->request.length - maxpacket,
maxpacket);
ret = dwc3_ep0_start_trans(dwc, dep->number,
req->request.dma,
transfer_size,
DWC3_TRBCTL_CONTROL_DATA,
true);
}
transfer_size = roundup((req->request.length - transfer_size),
maxpacket);
dwc->ep0_bounced = true;
ret = dwc3_ep0_start_trans(dwc, dep->number,
dwc->ep0_bounce_addr, transfer_size,
DWC3_TRBCTL_CONTROL_DATA, false);
} else {
ret = usb_gadget_map_request(&dwc->gadget, &req->request,
dep->number);
if (ret) {
dwc3_trace(trace_dwc3_ep0, "failed to map request\n");
return;
}
ret = dwc3_ep0_start_trans(dwc, dep->number, req->request.dma,
req->request.length, DWC3_TRBCTL_CONTROL_DATA,
false);
}
WARN_ON(ret < 0);
}
static void dwc3_ep0_do_control_data(struct dwc3 *dwc,
const struct dwc3_event_depevt *event)
{
struct dwc3_ep *dep;
struct dwc3_request *req;
int ret;
dep = dwc->eps[0];
dwc->ep0state = EP0_DATA_PHASE;
if (dwc->ep0_status_pending) {
dwc3_ep0_send_status_response(dwc);
return;
}
if (list_empty(&dep->request_list)) {
dev_vdbg(dwc->dev, "pending request for EP0 Data phase\n");
dep->flags |= DWC3_EP_PENDING_REQUEST;
if (event->endpoint_number)
dep->flags |= DWC3_EP0_DIR_IN;
return;
}
req = next_request(&dep->request_list);
req->direction = !!event->endpoint_number;
dwc->ep0state = EP0_DATA_PHASE;
if (req->request.length == 0) {
ret = dwc3_ep0_start_trans(dwc, event->endpoint_number,
dwc->ctrl_req_addr, 0,
DWC3_TRBCTL_CONTROL_DATA);
} else if ((req->request.length % dep->endpoint.maxpacket)
&& (event->endpoint_number == 0)) {
dwc3_map_buffer_to_dma(req);
WARN_ON(req->request.length > dep->endpoint.maxpacket);
dwc->ep0_bounced = true;
/*
* REVISIT in case request length is bigger than EP0
* wMaxPacketSize, we will need two chained TRBs to handle
* the transfer.
*/
ret = dwc3_ep0_start_trans(dwc, event->endpoint_number,
dwc->ep0_bounce_addr, dep->endpoint.maxpacket,
DWC3_TRBCTL_CONTROL_DATA);
} else {
dwc3_map_buffer_to_dma(req);
ret = dwc3_ep0_start_trans(dwc, event->endpoint_number,
req->request.dma, req->request.length,
DWC3_TRBCTL_CONTROL_DATA);
}
WARN_ON(ret < 0);
}
static void __dwc3_ep0_do_control_data(struct dwc3 *dwc,
struct dwc3_ep *dep, struct dwc3_request *req)
{
int ret;
req->direction = !!dep->number;
if (req->request.length == 0) {
ret = dwc3_ep0_start_trans(dwc, dep->number,
dwc->ctrl_req_addr, 0,
DWC3_TRBCTL_CONTROL_DATA);
} else if (!IS_ALIGNED(req->request.length, dep->endpoint.maxpacket)
&& (dep->number == 0)) {
u32 transfer_size;
u32 maxpacket;
ret = usb_gadget_map_request(&dwc->gadget, &req->request,
dep->number);
if (ret) {
dev_dbg(dwc->dev, "failed to map request\n");
return;
}
WARN_ON(req->request.length > DWC3_EP0_BOUNCE_SIZE);
maxpacket = dep->endpoint.maxpacket;
transfer_size = roundup(req->request.length, maxpacket);
dwc->ep0_bounced = true;
/*
* REVISIT in case request length is bigger than
* DWC3_EP0_BOUNCE_SIZE we will need two chained
* TRBs to handle the transfer.
*/
ret = dwc3_ep0_start_trans(dwc, dep->number,
dwc->ep0_bounce_addr, transfer_size,
DWC3_TRBCTL_CONTROL_DATA);
} else {
ret = usb_gadget_map_request(&dwc->gadget, &req->request,
dep->number);
if (ret) {
dev_dbg(dwc->dev, "failed to map request\n");
return;
}
if (dep->number &&
!(req->request.length % dwc->gadget.ep0->maxpacket))
req->request.zero = true;
ret = dwc3_ep0_start_trans(dwc, dep->number, req->request.dma,
req->request.length, DWC3_TRBCTL_CONTROL_DATA);
}
dbg_queue(dep->number, &req->request, ret);
WARN_ON(ret < 0);
}
/*
* Send a zero length packet for the status phase of the control transfer
*/
static void dwc3_ep0_do_setup_status(struct dwc3 *dwc,
const struct dwc3_event_depevt *event)
{
struct dwc3_ep *dep;
int ret;
u32 epnum;
epnum = event->endpoint_number;
dep = dwc->eps[epnum];
if (epnum)
dwc->ep0state = EP0_IN_STATUS_PHASE;
else
dwc->ep0state = EP0_OUT_STATUS_PHASE;
/*
* Not sure Why I need a buffer for a zero transfer. Maybe the
* HW reacts strange on a NULL pointer
*/
ret = dwc3_ep0_start_trans(dwc, epnum, dwc->ctrl_req_addr, 0);
if (ret) {
dev_dbg(dwc->dev, "failed to start transfer, stalling\n");
dwc3_ep0_stall_and_restart(dwc);
}
}
void dwc3_ep0_out_start(struct dwc3 *dwc)
{
int ret;
ret = dwc3_ep0_start_trans(dwc, 0, dwc->ctrl_req_addr, 8,
DWC3_TRBCTL_CONTROL_SETUP);
WARN_ON(ret < 0);
}
void dwc3_ep0_out_start(struct dwc3 *dwc)
{
struct dwc3_ep *dep;
int ret;
dep = dwc->eps[0];
ret = dwc3_ep0_start_trans(dwc, 0, dwc->ctrl_req_addr, 8);
WARN_ON(ret < 0);
}
static int dwc3_ep0_start_control_status(struct dwc3_ep *dep)
{
struct dwc3 *dwc = dep->dwc;
u32 type;
type = dwc->three_stage_setup ? DWC3_TRBCTL_CONTROL_STATUS3
: DWC3_TRBCTL_CONTROL_STATUS2;
dwc3_ep0_prepare_one_trb(dep, dwc->ep0_trb_addr, 0, type, false);
return dwc3_ep0_start_trans(dep);
}
static int dwc3_ep0_start_control_status(struct dwc3_ep *dep)
{
struct dwc3 *dwc = dep->dwc;
u32 type;
type = dwc->three_stage_setup ? DWC3_TRBCTL_CONTROL_STATUS3
: DWC3_TRBCTL_CONTROL_STATUS2;
return dwc3_ep0_start_trans(dwc, dep->number,
dwc->ctrl_req_addr, 0, type);
}
void dwc3_ep0_out_start(struct dwc3 *dwc)
{
struct dwc3_ep *dep;
int ret;
complete(&dwc->ep0_in_setup);
dep = dwc->eps[0];
dwc3_ep0_prepare_one_trb(dep, dwc->ep0_trb_addr, 8,
DWC3_TRBCTL_CONTROL_SETUP, false);
ret = dwc3_ep0_start_trans(dep);
WARN_ON(ret < 0);
}
static void dwc3_ep0_send_status_response(struct dwc3 *dwc)
{
u32 epnum;
if (dwc->ep0state == EP0_IN_DATA_PHASE)
epnum = 1;
else
epnum = 0;
dwc3_ep0_start_trans(dwc, epnum, dwc->ctrl_req_addr,
dwc->ep0_usb_req.length);
dwc->ep0_status_pending = 1;
}
static int __dwc3_gadget_ep0_queue(struct dwc3_ep *dep,
struct dwc3_request *req)
{
struct dwc3 *dwc = dep->dwc;
u32 type;
int ret = 0;
req->request.actual = 0;
req->request.status = -EINPROGRESS;
req->epnum = dep->number;
list_add_tail(&req->list, &dep->request_list);
/*
* Gadget driver might not be quick enough to queue a request
* before we get a Transfer Not Ready event on this endpoint.
*
* In that case, we will set DWC3_EP_PENDING_REQUEST. When that
* flag is set, it's telling us that as soon as Gadget queues the
* required request, we should kick the transfer here because the
* IRQ we were waiting for is long gone.
*/
if (dep->flags & DWC3_EP_PENDING_REQUEST) {
unsigned direction;
direction = !!(dep->flags & DWC3_EP0_DIR_IN);
if (dwc->ep0state == EP0_STATUS_PHASE) {
type = dwc->three_stage_setup
? DWC3_TRBCTL_CONTROL_STATUS3
: DWC3_TRBCTL_CONTROL_STATUS2;
} else if (dwc->ep0state == EP0_DATA_PHASE) {
type = DWC3_TRBCTL_CONTROL_DATA;
} else {
/* should never happen */
WARN_ON(1);
return 0;
}
ret = dwc3_ep0_start_trans(dwc, direction,
req->request.dma, req->request.length, type);
dep->flags &= ~(DWC3_EP_PENDING_REQUEST |
DWC3_EP0_DIR_IN);
} else if (dwc->delayed_status && (dwc->ep0state == EP0_STATUS_PHASE)) {
dwc->delayed_status = false;
dwc3_ep0_do_control_status(dwc, 1);
}
return ret;
}
static void dwc3_ep0_do_control_status(struct dwc3 *dwc,
const struct dwc3_event_depevt *event)
{
u32 type;
int ret;
dwc->ep0state = EP0_STATUS_PHASE;
type = dwc->three_stage_setup ? DWC3_TRBCTL_CONTROL_STATUS3
: DWC3_TRBCTL_CONTROL_STATUS2;
ret = dwc3_ep0_start_trans(dwc, event->endpoint_number,
dwc->ctrl_req_addr, 0, type);
WARN_ON(ret < 0);
}
static int __dwc3_gadget_ep0_queue(struct dwc3_ep *dep,
struct dwc3_request *req)
{
int ret = 0;
req->request.actual = 0;
req->request.status = -EINPROGRESS;
req->epnum = dep->number;
list_add_tail(&req->list, &dep->request_list);
/*
* Gadget driver might not be quick enough to queue a request
* before we get a Transfer Not Ready event on this endpoint.
*
* In that case, we will set DWC3_EP_PENDING_REQUEST. When that
* flag is set, it's telling us that as soon as Gadget queues the
* required request, we should kick the transfer here because the
* IRQ we were waiting for is long gone.
*/
if (dep->flags & DWC3_EP_PENDING_REQUEST) {
struct dwc3 *dwc = dep->dwc;
unsigned direction;
u32 type;
direction = !!(dep->flags & DWC3_EP0_DIR_IN);
if (dwc->ep0state != EP0_DATA_PHASE) {
dev_WARN(dwc->dev, "Unexpected pending request\n");
return 0;
}
ret = dwc3_ep0_start_trans(dwc, direction,
req->request.dma, req->request.length,
DWC3_TRBCTL_CONTROL_DATA);
dep->flags &= ~(DWC3_EP_PENDING_REQUEST |
DWC3_EP0_DIR_IN);
}
return ret;
}
static int __dwc3_gadget_ep0_queue(struct dwc3_ep *dep,
struct dwc3_request *req)
{
struct dwc3 *dwc = dep->dwc;
int ret;
req->request.actual = 0;
req->request.status = -EINPROGRESS;
req->direction = dep->direction;
req->epnum = dep->number;
list_add_tail(&req->list, &dep->request_list);
dwc3_map_buffer_to_dma(req);
ret = dwc3_ep0_start_trans(dwc, dep->number, req->request.dma,
req->request.length);
if (ret < 0) {
list_del(&req->list);
dwc3_unmap_buffer_from_dma(req);
}
return ret;
}
static void __dwc3_ep0_do_control_data(struct dwc3 *dwc,
struct dwc3_ep *dep, struct dwc3_request *req)
{
int ret;
req->direction = !!dep->number;
if (req->request.length == 0) {
dwc3_ep0_prepare_one_trb(dep, dwc->ep0_trb_addr, 0,
DWC3_TRBCTL_CONTROL_DATA, false);
ret = dwc3_ep0_start_trans(dep);
} else if (!IS_ALIGNED(req->request.length, dep->endpoint.maxpacket)
&& (dep->number == 0)) {
u32 maxpacket;
u32 rem;
ret = usb_gadget_map_request_by_dev(dwc->sysdev,
&req->request, dep->number);
if (ret)
return;
maxpacket = dep->endpoint.maxpacket;
rem = req->request.length % maxpacket;
dwc->ep0_bounced = true;
/* prepare normal TRB */
dwc3_ep0_prepare_one_trb(dep, req->request.dma,
req->request.length,
DWC3_TRBCTL_CONTROL_DATA,
true);
req->trb = &dwc->ep0_trb[dep->trb_enqueue - 1];
/* Now prepare one extra TRB to align transfer size */
dwc3_ep0_prepare_one_trb(dep, dwc->bounce_addr,
maxpacket - rem,
DWC3_TRBCTL_CONTROL_DATA,
false);
ret = dwc3_ep0_start_trans(dep);
} else if (IS_ALIGNED(req->request.length, dep->endpoint.maxpacket) &&
req->request.length && req->request.zero) {
u32 maxpacket;
u32 rem;
ret = usb_gadget_map_request_by_dev(dwc->sysdev,
&req->request, dep->number);
if (ret)
return;
maxpacket = dep->endpoint.maxpacket;
rem = req->request.length % maxpacket;
/* prepare normal TRB */
dwc3_ep0_prepare_one_trb(dep, req->request.dma,
req->request.length,
DWC3_TRBCTL_CONTROL_DATA,
true);
req->trb = &dwc->ep0_trb[dep->trb_enqueue - 1];
/* Now prepare one extra TRB to align transfer size */
dwc3_ep0_prepare_one_trb(dep, dwc->bounce_addr,
0, DWC3_TRBCTL_CONTROL_DATA,
false);
ret = dwc3_ep0_start_trans(dep);
} else {
ret = usb_gadget_map_request_by_dev(dwc->sysdev,
&req->request, dep->number);
if (ret)
return;
dwc3_ep0_prepare_one_trb(dep, req->request.dma,
req->request.length, DWC3_TRBCTL_CONTROL_DATA,
false);
req->trb = &dwc->ep0_trb[dep->trb_enqueue];
ret = dwc3_ep0_start_trans(dep);
}
WARN_ON(ret < 0);
}
static void dwc3_ep0_complete_data(struct dwc3 *dwc,
const struct dwc3_event_depevt *event)
{
struct dwc3_request *r = NULL;
struct usb_request *ur;
struct dwc3_trb *trb;
struct dwc3_ep *ep0;
u32 transferred;
u32 status;
u32 length;
u8 epnum;
epnum = event->endpoint_number;
ep0 = dwc->eps[0];
dwc->ep0_next_event = DWC3_EP0_NRDY_STATUS;
trb = dwc->ep0_trb;
r = next_request(&ep0->request_list);
if (!r)
return;
status = DWC3_TRB_SIZE_TRBSTS(trb->size);
if (status == DWC3_TRBSTS_SETUP_PENDING) {
dev_dbg(dwc->dev, "Setup Pending received\n");
if (r)
dwc3_gadget_giveback(ep0, r, -ECONNRESET);
return;
}
if (dwc->ep0_zlp_sent)
goto finish_zlp;
ur = &r->request;
length = trb->size & DWC3_TRB_SIZE_MASK;
if (dwc->ep0_bounced) {
unsigned transfer_size = ur->length;
unsigned maxp = ep0->endpoint.maxpacket;
transfer_size += (maxp - (transfer_size % maxp));
transferred = min_t(u32, ur->length,
transfer_size - length);
memcpy(ur->buf, dwc->ep0_bounce, transferred);
} else {
transferred = ur->length - length;
}
ur->actual += transferred;
if ((epnum & 1) && ur->actual < ur->length) {
/* for some reason we did not get everything out */
dwc3_ep0_stall_and_restart(dwc);
return;
}
/* handle the case where we have to send a zero packet */
if ((epnum & 1) && ur->zero &&
(ur->length % ep0->endpoint.maxpacket == 0)) {
int ret;
ret = dwc3_ep0_start_trans(dwc, epnum, dwc->ctrl_req_addr, 0,
DWC3_TRBCTL_CONTROL_DATA);
WARN_ON(ret < 0);
dwc->ep0_zlp_sent = 1;
return;
}
finish_zlp:
dwc3_gadget_giveback(ep0, r, 0);
}
static void dwc3_ep0_send_status_response(struct dwc3 *dwc)
{
dwc3_ep0_start_trans(dwc, 1, dwc->setup_buf_addr,
dwc->ep0_usb_req.length,
DWC3_TRBCTL_CONTROL_DATA);
}
static void dwc3_ep0_xfernotready(struct dwc3 *dwc,
const struct dwc3_event_depevt *event)
{
u8 epnum;
int ret;
dwc->setup_packet_pending = true;
epnum = event->endpoint_number;
switch (event->status) {
case DEPEVT_STATUS_CONTROL_DATA:
dev_vdbg(dwc->dev, "Control Data\n");
/*
* We already have a DATA transfer in the controller's cache,
* if we receive a XferNotReady(DATA) we will ignore it, unless
* it's for the wrong direction.
*
* In that case, we must issue END_TRANSFER command to the Data
* Phase we already have started and issue SetStall on the
* control endpoint.
*/
if (dwc->ep0_expect_in != event->endpoint_number) {
struct dwc3_ep *dep = dwc->eps[dwc->ep0_expect_in];
dev_vdbg(dwc->dev, "Wrong direction for Data phase\n");
dwc3_ep0_end_control_data(dwc, dep);
dwc3_ep0_stall_and_restart(dwc);
return;
}
if (zlp_required) {
zlp_required = false;
ret = dwc3_ep0_start_trans(dwc, epnum,
dwc->ctrl_req_addr, 0,
DWC3_TRBCTL_CONTROL_DATA);
dbg_event(epnum, "ZLP", ret);
WARN_ON(ret < 0);
}
break;
case DEPEVT_STATUS_CONTROL_STATUS:
if (dwc->ep0_next_event != DWC3_EP0_NRDY_STATUS)
return;
dev_vdbg(dwc->dev, "Control Status\n");
zlp_required = false;
dwc->ep0state = EP0_STATUS_PHASE;
if (dwc->delayed_status &&
list_empty(&dwc->eps[0]->request_list)) {
WARN_ON_ONCE(event->endpoint_number != 1);
dev_vdbg(dwc->dev, "Mass Storage delayed status\n");
return;
}
dwc->delayed_status = false;
dwc3_ep0_do_control_status(dwc, event);
}
}
static void dwc3_ep0_xfernotready(struct dwc3 *dwc,
const struct dwc3_event_depevt *event)
{
dwc->setup_packet_pending = true;
switch (event->status) {
case DEPEVT_STATUS_CONTROL_DATA:
dev_vdbg(dwc->dev, "Control Data\n");
/*
* We already have a DATA transfer in the controller's cache,
* if we receive a XferNotReady(DATA) we will ignore it, unless
* it's for the wrong direction.
*
* In that case, we must issue END_TRANSFER command to the Data
* Phase we already have started and issue SetStall on the
* control endpoint.
*/
if (dwc->ep0_expect_in != event->endpoint_number) {
struct dwc3_ep *dep = dwc->eps[dwc->ep0_expect_in];
dev_vdbg(dwc->dev, "Wrong direction for Data phase\n");
dwc3_ep0_end_control_data(dwc, dep);
dwc3_ep0_stall_and_restart(dwc);
return;
}
/*
* Per databook, if an XferNotready(Data) is received after
* XferComplete(Data), one possible reason is host is trying
* to complete data stage by moving a 0-length packet.
*
* REVISIT in case of other cases
*/
if (dwc->ep0_next_event == DWC3_EP0_NRDY_STATUS) {
u32 size = 0;
struct dwc3_ep *dep = dwc->eps[event->endpoint_number];
if (dep->number == 0)
size = dep->endpoint.maxpacket;
dwc3_ep0_start_trans(dwc, dep->number,
dwc->ctrl_req_addr, size,
DWC3_TRBCTL_CONTROL_DATA);
}
break;
case DEPEVT_STATUS_CONTROL_STATUS:
if (dwc->ep0_next_event != DWC3_EP0_NRDY_STATUS)
return;
dev_vdbg(dwc->dev, "Control Status\n");
dwc->ep0state = EP0_STATUS_PHASE;
if (dwc->delayed_status) {
WARN_ON_ONCE(event->endpoint_number != 1);
dev_vdbg(dwc->dev, "Mass Storage delayed status\n");
if (list_empty(&dwc->eps[0]->request_list))
return;
else
dwc->delayed_status = false;
}
dwc3_ep0_do_control_status(dwc, event);
}
}
static void dwc3_ep0_complete_data(struct dwc3 *dwc,
const struct dwc3_event_depevt *event)
{
struct dwc3_request *r = NULL;
struct usb_request *ur;
struct dwc3_trb *trb;
struct dwc3_ep *ep0;
unsigned transfer_size = 0;
unsigned maxp;
unsigned remaining_ur_length;
void *buf;
u32 transferred = 0;
u32 status;
u32 length;
u8 epnum;
epnum = event->endpoint_number;
ep0 = dwc->eps[0];
dwc->ep0_next_event = DWC3_EP0_NRDY_STATUS;
trb = dwc->ep0_trb;
trace_dwc3_complete_trb(ep0, trb);
r = next_request(&ep0->request_list);
if (!r)
return;
status = DWC3_TRB_SIZE_TRBSTS(trb->size);
if (status == DWC3_TRBSTS_SETUP_PENDING) {
dwc->setup_packet_pending = true;
dwc3_trace(trace_dwc3_ep0, "Setup Pending received");
if (r)
dwc3_gadget_giveback(ep0, r, -ECONNRESET);
return;
}
ur = &r->request;
buf = ur->buf;
remaining_ur_length = ur->length;
length = trb->size & DWC3_TRB_SIZE_MASK;
maxp = ep0->endpoint.maxpacket;
if (dwc->ep0_bounced) {
/*
* Handle the first TRB before handling the bounce buffer if
* the request length is greater than the bounce buffer size
*/
if (ur->length > DWC3_EP0_BOUNCE_SIZE) {
transfer_size = ALIGN(ur->length - maxp, maxp);
transferred = transfer_size - length;
buf = (u8 *)buf + transferred;
ur->actual += transferred;
remaining_ur_length -= transferred;
trb++;
length = trb->size & DWC3_TRB_SIZE_MASK;
ep0->free_slot = 0;
}
transfer_size = roundup((ur->length - transfer_size),
maxp);
transferred = min_t(u32, remaining_ur_length,
transfer_size - length);
memcpy(buf, dwc->ep0_bounce, transferred);
} else {
transferred = ur->length - length;
}
ur->actual += transferred;
if ((epnum & 1) && ur->actual < ur->length) {
/* for some reason we did not get everything out */
dwc3_ep0_stall_and_restart(dwc);
} else {
dwc3_gadget_giveback(ep0, r, 0);
if (IS_ALIGNED(ur->length, ep0->endpoint.maxpacket) &&
ur->length && ur->zero) {
int ret;
dwc->ep0_next_event = DWC3_EP0_COMPLETE;
ret = dwc3_ep0_start_trans(dwc, epnum,
dwc->ctrl_req_addr, 0,
DWC3_TRBCTL_CONTROL_DATA, false);
WARN_ON(ret < 0);
}
}
}
