static void
xhci_enqueue_td(transfer_ring_t *const tr, const int ep, const size_t mps,
const int dalen, void *const data, const int dir)
{
trb_t *trb = NULL; /* cur TRB */
u8 *cur_start = data; /* cur data pointer */
size_t length = dalen; /* remaining bytes */
size_t packets = (length + mps - 1) / mps; /* remaining packets */
size_t residue = 0; /* residue from last TRB */
size_t trb_count = 0; /* TRBs added so far */
while (length || !trb_count /* enqueue at least one */) {
const size_t cur_end = ((size_t)cur_start + 0x10000) & ~0xffff;
size_t cur_length = cur_end - (size_t)cur_start;
if (length < cur_length) {
cur_length = length;
packets = 0;
length = 0;
} else {
packets -= (residue + cur_length) / mps;
residue = (residue + cur_length) % mps;
length -= cur_length;
}
trb = tr->cur;
xhci_clear_trb(trb, tr->pcs);
trb->ptr_low = virt_to_phys(cur_start);
TRB_SET(TL, trb, cur_length);
TRB_SET(TDS, trb, packets);
TRB_SET(CH, trb, 1);
/* Check for first, data stage TRB */
if (!trb_count && ep == 1) {
TRB_SET(DIR, trb, dir);
TRB_SET(TT, trb, TRB_DATA_STAGE);
} else {
TRB_SET(TT, trb, TRB_NORMAL);
}
xhci_enqueue_trb(tr);
cur_start += cur_length;
++trb_count;
}
trb = tr->cur;
xhci_clear_trb(trb, tr->pcs);
trb->ptr_low = virt_to_phys(trb); /* for easier debugging only */
TRB_SET(TT, trb, TRB_EVENT_DATA);
TRB_SET(IOC, trb, 1);
xhci_enqueue_trb(tr);
}
/* read one intr-packet from queue, if available. extend the queue for new input.
return NULL if nothing new available.
Recommended use: while (data=poll_intr_queue(q)) process(data);
*/
static u8 *
xhci_poll_intr_queue(void *const q)
{
if (!q)
return NULL;
intrq_t *const intrq = (intrq_t *)q;
endpoint_t *const ep = intrq->ep;
xhci_t *const xhci = XHCI_INST(ep->dev->controller);
/* TODO: Reset interrupt queue if it gets halted? */
xhci_handle_events(xhci);
u8 *reqdata = NULL;
while (!reqdata && intrq->ready) {
const int ep_id = xhci_ep_id(ep);
transfer_ring_t *const tr =
xhci->dev[ep->dev->address].transfer_rings[ep_id];
/* Fetch the request's buffer */
reqdata = phys_to_virt(intrq->next->ptr_low);
/* Enqueue the last (spare) TRB and ring doorbell */
xhci_enqueue_trb(tr);
xhci->dbreg[ep->dev->address] = ep_id;
/* Reuse the current buffer for the next spare TRB */
xhci_clear_trb(tr->cur, tr->pcs);
tr->cur->ptr_low = virt_to_phys(reqdata);
tr->cur->ptr_high = 0;
TRB_SET(TL, tr->cur, intrq->size);
TRB_SET(TT, tr->cur, TRB_NORMAL);
TRB_SET(ISP, tr->cur, 1);
TRB_SET(IOC, tr->cur, 1);
/* Check if anything was transferred */
const size_t read = TRB_GET(TL, intrq->next);
if (!read)
reqdata = NULL;
else if (read < intrq->size)
/* At least zero it, poll interface is rather limited */
memset(reqdata + read, 0x00, intrq->size - read);
/* Advance the interrupt queue */
if (intrq->ready == intrq->next)
/* This was last TRB being ready */
intrq->ready = NULL;
intrq->next = xhci_next_trb(intrq->next, NULL);
}
return reqdata;
}
/* create and hook-up an intr queue into device schedule */
static void *
xhci_create_intr_queue(endpoint_t *const ep,
const int reqsize, const int reqcount,
const int reqtiming)
{
/* reqtiming: We ignore it and use the interval from the
endpoint descriptor configured earlier. */
xhci_t *const xhci = XHCI_INST(ep->dev->controller);
const int slot_id = ep->dev->address;
const int ep_id = xhci_ep_id(ep);
transfer_ring_t *const tr = xhci->dev[slot_id].transfer_rings[ep_id];
if (reqcount > (TRANSFER_RING_SIZE - 2)) {
xhci_debug("reqcount is too high, at most %d supported\n",
TRANSFER_RING_SIZE - 2);
return NULL;
}
if (reqsize > 0x10000) {
xhci_debug("reqsize is too large, at most 64KiB supported\n");
return NULL;
}
if (xhci->dev[slot_id].interrupt_queues[ep_id]) {
xhci_debug("Only one interrupt queue per endpoint supported\n");
return NULL;
}
/* Allocate intrq structure and reqdata chunks */
intrq_t *const intrq = malloc(sizeof(*intrq));
if (!intrq) {
xhci_debug("Out of memory\n");
return NULL;
}
int i;
int pcs = tr->pcs;
trb_t *cur = tr->cur;
for (i = 0; i < reqcount; ++i) {
if (TRB_GET(C, cur) == pcs) {
xhci_debug("Not enough empty TRBs\n");
goto _free_return;
}
void *const reqdata = xhci_align(1, reqsize);
if (!reqdata) {
xhci_debug("Out of memory\n");
goto _free_return;
}
xhci_clear_trb(cur, pcs);
cur->ptr_low = virt_to_phys(reqdata);
cur->ptr_high = 0;
TRB_SET(TL, cur, reqsize);
TRB_SET(TT, cur, TRB_NORMAL);
TRB_SET(ISP, cur, 1);
TRB_SET(IOC, cur, 1);
cur = xhci_next_trb(cur, &pcs);
}
intrq->size = reqsize;
intrq->count = reqcount;
intrq->next = tr->cur;
intrq->ready = NULL;
intrq->ep = ep;
xhci->dev[slot_id].interrupt_queues[ep_id] = intrq;
/* Now enqueue all the prepared TRBs but the last
and ring the doorbell. */
for (i = 0; i < (reqcount - 1); ++i)
xhci_enqueue_trb(tr);
xhci->dbreg[slot_id] = ep_id;
return intrq;
_free_return:
cur = tr->cur;
for (--i; i >= 0; --i) {
free(phys_to_virt(cur->ptr_low));
cur = xhci_next_trb(cur, NULL);
}
free(intrq);
return NULL;
}
static int
xhci_control(usbdev_t *const dev, const direction_t dir,
const int drlen, void *const devreq,
const int dalen, unsigned char *const src)
{
unsigned char *data = src;
xhci_t *const xhci = XHCI_INST(dev->controller);
epctx_t *const epctx = xhci->dev[dev->address].ctx.ep0;
transfer_ring_t *const tr = xhci->dev[dev->address].transfer_rings[1];
const size_t off = (size_t)data & 0xffff;
if ((off + dalen) > ((TRANSFER_RING_SIZE - 4) << 16)) {
xhci_debug("Unsupported transfer size\n");
return -1;
}
/* Reset endpoint if it's halted */
const unsigned ep_state = EC_GET(STATE, epctx);
if (ep_state == 2 || ep_state == 4) {
if (xhci_reset_endpoint(dev, NULL, 0))
return -1;
}
if (dalen && !dma_coherent(src)) {
data = xhci->dma_buffer;
if (dalen > DMA_SIZE) {
xhci_debug("Control transfer too large: %d\n", dalen);
return -1;
}
if (dir == OUT)
memcpy(data, src, dalen);
}
/* Fill and enqueue setup TRB */
trb_t *const setup = tr->cur;
xhci_clear_trb(setup, tr->pcs);
setup->ptr_low = ((u32 *)devreq)[0];
setup->ptr_high = ((u32 *)devreq)[1];
TRB_SET(TL, setup, 8);
TRB_SET(TRT, setup, (dalen)
? ((dir == OUT) ? TRB_TRT_OUT_DATA : TRB_TRT_IN_DATA)
: TRB_TRT_NO_DATA);
TRB_SET(TT, setup, TRB_SETUP_STAGE);
TRB_SET(IDT, setup, 1);
TRB_SET(IOC, setup, 1);
xhci_enqueue_trb(tr);
/* Fill and enqueue data TRBs (if any) */
if (dalen) {
const unsigned mps = EC_GET(MPS, epctx);
const unsigned dt_dir = (dir == OUT) ? TRB_DIR_OUT : TRB_DIR_IN;
xhci_enqueue_td(tr, 1, mps, dalen, data, dt_dir);
}
/* Fill status TRB */
trb_t *const status = tr->cur;
xhci_clear_trb(status, tr->pcs);
TRB_SET(DIR, status, (dir == OUT) ? TRB_DIR_IN : TRB_DIR_OUT);
TRB_SET(TT, status, TRB_STATUS_STAGE);
TRB_SET(IOC, status, 1);
xhci_enqueue_trb(tr);
/* Ring doorbell for EP0 */
xhci->dbreg[dev->address] = 1;
/* Wait for transfer events */
int i, transferred = 0;
const int n_stages = 2 + !!dalen;
for (i = 0; i < n_stages; ++i) {
const int ret = xhci_wait_for_transfer(xhci, dev->address, 1);
transferred += ret;
if (ret < 0) {
if (ret == TIMEOUT) {
xhci_debug("Stopping ID %d EP 1\n",
dev->address);
xhci_cmd_stop_endpoint(xhci, dev->address, 1);
}
xhci_debug("Stage %d/%d failed: %d\n"
" trb ring: @%p\n"
" setup trb: @%p\n"
" status trb: @%p\n"
" ep state: %d -> %d\n"
" usbsts: 0x%08"PRIx32"\n",
i, n_stages, ret,
tr->ring, setup, status,
ep_state, EC_GET(STATE, epctx),
xhci->opreg->usbsts);
return ret;
}
}
if (dir == IN && data != src)
memcpy(src, data, transferred);
return transferred;
}
trb_t *
xhci_next_command_trb(xhci_t *const xhci)
{
xhci_clear_trb(xhci->cr.cur, xhci->cr.pcs);
return xhci->cr.cur;
}
