static void xhci_enqueue_td(transfer_ring_t *const tr, const int ep, const size_t mps, const int dalen, void *const data, const int dir) { trb_t *trb = NULL; /* cur TRB */ u8 *cur_start = data; /* cur data pointer */ size_t length = dalen; /* remaining bytes */ size_t packets = (length + mps - 1) / mps; /* remaining packets */ size_t residue = 0; /* residue from last TRB */ size_t trb_count = 0; /* TRBs added so far */ while (length || !trb_count /* enqueue at least one */) { const size_t cur_end = ((size_t)cur_start + 0x10000) & ~0xffff; size_t cur_length = cur_end - (size_t)cur_start; if (length < cur_length) { cur_length = length; packets = 0; length = 0; } else { packets -= (residue + cur_length) / mps; residue = (residue + cur_length) % mps; length -= cur_length; } trb = tr->cur; xhci_clear_trb(trb, tr->pcs); trb->ptr_low = virt_to_phys(cur_start); TRB_SET(TL, trb, cur_length); TRB_SET(TDS, trb, packets); TRB_SET(CH, trb, 1); /* Check for first, data stage TRB */ if (!trb_count && ep == 1) { TRB_SET(DIR, trb, dir); TRB_SET(TT, trb, TRB_DATA_STAGE); } else { TRB_SET(TT, trb, TRB_NORMAL); } xhci_enqueue_trb(tr); cur_start += cur_length; ++trb_count; } trb = tr->cur; xhci_clear_trb(trb, tr->pcs); trb->ptr_low = virt_to_phys(trb); /* for easier debugging only */ TRB_SET(TT, trb, TRB_EVENT_DATA); TRB_SET(IOC, trb, 1); xhci_enqueue_trb(tr); }
/* read one intr-packet from queue, if available. extend the queue for new input. return NULL if nothing new available. Recommended use: while (data=poll_intr_queue(q)) process(data); */ static u8 * xhci_poll_intr_queue(void *const q) { if (!q) return NULL; intrq_t *const intrq = (intrq_t *)q; endpoint_t *const ep = intrq->ep; xhci_t *const xhci = XHCI_INST(ep->dev->controller); /* TODO: Reset interrupt queue if it gets halted? */ xhci_handle_events(xhci); u8 *reqdata = NULL; while (!reqdata && intrq->ready) { const int ep_id = xhci_ep_id(ep); transfer_ring_t *const tr = xhci->dev[ep->dev->address].transfer_rings[ep_id]; /* Fetch the request's buffer */ reqdata = phys_to_virt(intrq->next->ptr_low); /* Enqueue the last (spare) TRB and ring doorbell */ xhci_enqueue_trb(tr); xhci->dbreg[ep->dev->address] = ep_id; /* Reuse the current buffer for the next spare TRB */ xhci_clear_trb(tr->cur, tr->pcs); tr->cur->ptr_low = virt_to_phys(reqdata); tr->cur->ptr_high = 0; TRB_SET(TL, tr->cur, intrq->size); TRB_SET(TT, tr->cur, TRB_NORMAL); TRB_SET(ISP, tr->cur, 1); TRB_SET(IOC, tr->cur, 1); /* Check if anything was transferred */ const size_t read = TRB_GET(TL, intrq->next); if (!read) reqdata = NULL; else if (read < intrq->size) /* At least zero it, poll interface is rather limited */ memset(reqdata + read, 0x00, intrq->size - read); /* Advance the interrupt queue */ if (intrq->ready == intrq->next) /* This was last TRB being ready */ intrq->ready = NULL; intrq->next = xhci_next_trb(intrq->next, NULL); } return reqdata; }
/* create and hook-up an intr queue into device schedule */ static void * xhci_create_intr_queue(endpoint_t *const ep, const int reqsize, const int reqcount, const int reqtiming) { /* reqtiming: We ignore it and use the interval from the endpoint descriptor configured earlier. */ xhci_t *const xhci = XHCI_INST(ep->dev->controller); const int slot_id = ep->dev->address; const int ep_id = xhci_ep_id(ep); transfer_ring_t *const tr = xhci->dev[slot_id].transfer_rings[ep_id]; if (reqcount > (TRANSFER_RING_SIZE - 2)) { xhci_debug("reqcount is too high, at most %d supported\n", TRANSFER_RING_SIZE - 2); return NULL; } if (reqsize > 0x10000) { xhci_debug("reqsize is too large, at most 64KiB supported\n"); return NULL; } if (xhci->dev[slot_id].interrupt_queues[ep_id]) { xhci_debug("Only one interrupt queue per endpoint supported\n"); return NULL; } /* Allocate intrq structure and reqdata chunks */ intrq_t *const intrq = malloc(sizeof(*intrq)); if (!intrq) { xhci_debug("Out of memory\n"); return NULL; } int i; int pcs = tr->pcs; trb_t *cur = tr->cur; for (i = 0; i < reqcount; ++i) { if (TRB_GET(C, cur) == pcs) { xhci_debug("Not enough empty TRBs\n"); goto _free_return; } void *const reqdata = xhci_align(1, reqsize); if (!reqdata) { xhci_debug("Out of memory\n"); goto _free_return; } xhci_clear_trb(cur, pcs); cur->ptr_low = virt_to_phys(reqdata); cur->ptr_high = 0; TRB_SET(TL, cur, reqsize); TRB_SET(TT, cur, TRB_NORMAL); TRB_SET(ISP, cur, 1); TRB_SET(IOC, cur, 1); cur = xhci_next_trb(cur, &pcs); } intrq->size = reqsize; intrq->count = reqcount; intrq->next = tr->cur; intrq->ready = NULL; intrq->ep = ep; xhci->dev[slot_id].interrupt_queues[ep_id] = intrq; /* Now enqueue all the prepared TRBs but the last and ring the doorbell. */ for (i = 0; i < (reqcount - 1); ++i) xhci_enqueue_trb(tr); xhci->dbreg[slot_id] = ep_id; return intrq; _free_return: cur = tr->cur; for (--i; i >= 0; --i) { free(phys_to_virt(cur->ptr_low)); cur = xhci_next_trb(cur, NULL); } free(intrq); return NULL; }
static int xhci_control(usbdev_t *const dev, const direction_t dir, const int drlen, void *const devreq, const int dalen, unsigned char *const src) { unsigned char *data = src; xhci_t *const xhci = XHCI_INST(dev->controller); epctx_t *const epctx = xhci->dev[dev->address].ctx.ep0; transfer_ring_t *const tr = xhci->dev[dev->address].transfer_rings[1]; const size_t off = (size_t)data & 0xffff; if ((off + dalen) > ((TRANSFER_RING_SIZE - 4) << 16)) { xhci_debug("Unsupported transfer size\n"); return -1; } /* Reset endpoint if it's halted */ const unsigned ep_state = EC_GET(STATE, epctx); if (ep_state == 2 || ep_state == 4) { if (xhci_reset_endpoint(dev, NULL, 0)) return -1; } if (dalen && !dma_coherent(src)) { data = xhci->dma_buffer; if (dalen > DMA_SIZE) { xhci_debug("Control transfer too large: %d\n", dalen); return -1; } if (dir == OUT) memcpy(data, src, dalen); } /* Fill and enqueue setup TRB */ trb_t *const setup = tr->cur; xhci_clear_trb(setup, tr->pcs); setup->ptr_low = ((u32 *)devreq)[0]; setup->ptr_high = ((u32 *)devreq)[1]; TRB_SET(TL, setup, 8); TRB_SET(TRT, setup, (dalen) ? ((dir == OUT) ? TRB_TRT_OUT_DATA : TRB_TRT_IN_DATA) : TRB_TRT_NO_DATA); TRB_SET(TT, setup, TRB_SETUP_STAGE); TRB_SET(IDT, setup, 1); TRB_SET(IOC, setup, 1); xhci_enqueue_trb(tr); /* Fill and enqueue data TRBs (if any) */ if (dalen) { const unsigned mps = EC_GET(MPS, epctx); const unsigned dt_dir = (dir == OUT) ? TRB_DIR_OUT : TRB_DIR_IN; xhci_enqueue_td(tr, 1, mps, dalen, data, dt_dir); } /* Fill status TRB */ trb_t *const status = tr->cur; xhci_clear_trb(status, tr->pcs); TRB_SET(DIR, status, (dir == OUT) ? TRB_DIR_IN : TRB_DIR_OUT); TRB_SET(TT, status, TRB_STATUS_STAGE); TRB_SET(IOC, status, 1); xhci_enqueue_trb(tr); /* Ring doorbell for EP0 */ xhci->dbreg[dev->address] = 1; /* Wait for transfer events */ int i, transferred = 0; const int n_stages = 2 + !!dalen; for (i = 0; i < n_stages; ++i) { const int ret = xhci_wait_for_transfer(xhci, dev->address, 1); transferred += ret; if (ret < 0) { if (ret == TIMEOUT) { xhci_debug("Stopping ID %d EP 1\n", dev->address); xhci_cmd_stop_endpoint(xhci, dev->address, 1); } xhci_debug("Stage %d/%d failed: %d\n" " trb ring: @%p\n" " setup trb: @%p\n" " status trb: @%p\n" " ep state: %d -> %d\n" " usbsts: 0x%08"PRIx32"\n", i, n_stages, ret, tr->ring, setup, status, ep_state, EC_GET(STATE, epctx), xhci->opreg->usbsts); return ret; } } if (dir == IN && data != src) memcpy(src, data, transferred); return transferred; }
trb_t * xhci_next_command_trb(xhci_t *const xhci) { xhci_clear_trb(xhci->cr.cur, xhci->cr.pcs); return xhci->cr.cur; }