static void cppi_dump_rxq(int level, const char *tag, struct cppi_channel *rx) { struct cppi_descriptor *bd; cppi_dump_rx(level, rx, tag); if (rx->last_processed) cppi_dump_rxbd("last", rx->last_processed); for (bd = rx->head; bd; bd = bd->next) cppi_dump_rxbd("active", bd); }
static void cppi_dump_rxq(int level, const char *tag, struct cppi_channel *rx) { #ifdef CONFIG_USB_MUSB_DEBUG struct cppi_descriptor *bd; if (!_dbg_level(level)) return; cppi_dump_rx(level, rx, tag); if (rx->last_processed) cppi_dump_rxbd("last", rx->last_processed); for (bd = rx->head; bd; bd = bd->next) cppi_dump_rxbd("active", bd); #endif }
/** * cppi_next_rx_segment - dma read for the next chunk of a buffer * @musb: the controller * @rx: dma channel * @onepacket: true unless caller treats short reads as errors, and * performs fault recovery above usbcore. * Context: controller irqlocked * * See above notes about why we can't use multi-BD RX queues except in * rare cases (mass storage class), and can never use the hardware "rndis" * mode (since it's not a "true" RNDIS mode) with complete safety.. * * It's ESSENTIAL that callers specify "onepacket" mode unless they kick in * code to recover from corrupted datastreams after each short transfer. */ static void cppi_next_rx_segment(struct musb *musb, struct cppi_channel *rx, int onepacket) { unsigned maxpacket = rx->maxpacket; dma_addr_t addr = rx->buf_dma + rx->offset; size_t length = rx->buf_len - rx->offset; struct cppi_descriptor *bd, *tail; unsigned n_bds; unsigned i; void __iomem *tibase = musb->ctrl_base; int is_rndis = 0; struct cppi_rx_stateram __iomem *rx_ram = rx->state_ram; struct cppi_descriptor *d; if (onepacket) { /* almost every USB driver, host or peripheral side */ n_bds = 1; /* maybe apply the heuristic above */ if (cppi_rx_rndis && is_peripheral_active(musb) && length > maxpacket && (length & ~0xffff) == 0 && (length & 0x0fff) != 0 && (length & (maxpacket - 1)) == 0) { maxpacket = length; is_rndis = 1; } } else { /* virtually nothing except mass storage class */ if (length > 0xffff) { n_bds = 0xffff / maxpacket; length = n_bds * maxpacket; } else { n_bds = DIV_ROUND_UP(length, maxpacket); } if (n_bds == 1) onepacket = 1; else n_bds = min(n_bds, (unsigned) NUM_RXCHAN_BD); } /* In host mode, autorequest logic can generate some IN tokens; it's * tricky since we can't leave REQPKT set in RXCSR after the transfer * finishes. So: multipacket transfers involve two or more segments. * And always at least two IRQs ... RNDIS mode is not an option. */ if (is_host_active(musb)) n_bds = cppi_autoreq_update(rx, tibase, onepacket, n_bds); cppi_rndis_update(rx, 1, musb->ctrl_base, is_rndis); length = min(n_bds * maxpacket, length); musb_dbg(musb, "RX DMA%d seg, maxp %d %s bds %d (cnt %d) " "dma 0x%llx len %u %u/%u", rx->index, maxpacket, onepacket ? (is_rndis ? "rndis" : "onepacket") : "multipacket", n_bds, musb_readl(tibase, DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4)) & 0xffff, (unsigned long long)addr, length, rx->channel.actual_len, rx->buf_len); /* only queue one segment at a time, since the hardware prevents * correct queue shutdown after unexpected short packets */ bd = cppi_bd_alloc(rx); rx->head = bd; /* Build BDs for all packets in this segment */ for (i = 0, tail = NULL; bd && i < n_bds; i++, tail = bd) { u32 bd_len; if (i) { bd = cppi_bd_alloc(rx); if (!bd) break; tail->next = bd; tail->hw_next = bd->dma; } bd->hw_next = 0; /* all but the last packet will be maxpacket size */ if (maxpacket < length) bd_len = maxpacket; else bd_len = length; bd->hw_bufp = addr; addr += bd_len; rx->offset += bd_len; bd->hw_off_len = (0 /*offset*/ << 16) + bd_len; bd->buflen = bd_len; bd->hw_options = CPPI_OWN_SET | (i == 0 ? length : 0); length -= bd_len; } /* we always expect at least one reusable BD! */ if (!tail) { WARNING("rx dma%d -- no BDs? need %d\n", rx->index, n_bds); return; } else if (i < n_bds) WARNING("rx dma%d -- only %d of %d BDs\n", rx->index, i, n_bds); tail->next = NULL; tail->hw_next = 0; bd = rx->head; rx->tail = tail; /* short reads and other faults should terminate this entire * dma segment. we want one "dma packet" per dma segment, not * one per USB packet, terminating the whole queue at once... * NOTE that current hardware seems to ignore SOP and EOP. */ bd->hw_options |= CPPI_SOP_SET; tail->hw_options |= CPPI_EOP_SET; for (d = rx->head; d; d = d->next) cppi_dump_rxbd("S", d); /* in case the preceding transfer left some state... */ tail = rx->last_processed; if (tail) { tail->next = bd; tail->hw_next = bd->dma; } core_rxirq_enable(tibase, rx->index + 1); /* BDs live in DMA-coherent memory, but writes might be pending */ cpu_drain_writebuffer(); /* REVISIT specs say to write this AFTER the BUFCNT register * below ... but that loses badly. */ musb_writel(&rx_ram->rx_head, 0, bd->dma); /* bufferCount must be at least 3, and zeroes on completion * unless it underflows below zero, or stops at two, or keeps * growing ... grr. */ i = musb_readl(tibase, DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4)) & 0xffff; if (!i) musb_writel(tibase, DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4), n_bds + 2); else if (n_bds > (i - 3)) musb_writel(tibase, DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4), n_bds - (i - 3)); i = musb_readl(tibase, DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4)) & 0xffff; if (i < (2 + n_bds)) { musb_dbg(musb, "bufcnt%d underrun - %d (for %d)", rx->index, i, n_bds); musb_writel(tibase, DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4), n_bds + 2); } cppi_dump_rx(4, rx, "/S"); }
/* * Context: controller irqlocked, endpoint selected */ static int cppi_channel_abort(struct dma_channel *channel) { struct cppi_channel *cppi_ch; struct cppi *controller; void __iomem *mbase; void __iomem *tibase; void __iomem *regs; u32 value; struct cppi_descriptor *queue; cppi_ch = container_of(channel, struct cppi_channel, channel); controller = cppi_ch->controller; switch (channel->status) { case MUSB_DMA_STATUS_BUS_ABORT: case MUSB_DMA_STATUS_CORE_ABORT: /* from RX or TX fault irq handler */ case MUSB_DMA_STATUS_BUSY: /* the hardware needs shutting down */ regs = cppi_ch->hw_ep->regs; break; case MUSB_DMA_STATUS_UNKNOWN: case MUSB_DMA_STATUS_FREE: return 0; default: return -EINVAL; } if (!cppi_ch->transmit && cppi_ch->head) cppi_dump_rxq(3, "/abort", cppi_ch); mbase = controller->mregs; tibase = controller->tibase; queue = cppi_ch->head; cppi_ch->head = NULL; cppi_ch->tail = NULL; /* REVISIT should rely on caller having done this, * and caller should rely on us not changing it. * peripheral code is safe ... check host too. */ musb_ep_select(mbase, cppi_ch->index + 1); if (cppi_ch->transmit) { struct cppi_tx_stateram __iomem *tx_ram; /* REVISIT put timeouts on these controller handshakes */ cppi_dump_tx(6, cppi_ch, " (teardown)"); /* teardown DMA engine then usb core */ do { value = musb_readl(tibase, DAVINCI_TXCPPI_TEAR_REG); } while (!(value & CPPI_TEAR_READY)); musb_writel(tibase, DAVINCI_TXCPPI_TEAR_REG, cppi_ch->index); tx_ram = cppi_ch->state_ram; do { value = musb_readl(&tx_ram->tx_complete, 0); } while (0xFFFFFFFC != value); /* FIXME clean up the transfer state ... here? * the completion routine should get called with * an appropriate status code. */ value = musb_readw(regs, MUSB_TXCSR); value &= ~MUSB_TXCSR_DMAENAB; value |= MUSB_TXCSR_FLUSHFIFO; musb_writew(regs, MUSB_TXCSR, value); musb_writew(regs, MUSB_TXCSR, value); /* * 1. Write to completion Ptr value 0x1(bit 0 set) * (write back mode) * 2. Wait for abort interrupt and then put the channel in * compare mode by writing 1 to the tx_complete register. */ cppi_reset_tx(tx_ram, 1); cppi_ch->head = NULL; musb_writel(&tx_ram->tx_complete, 0, 1); cppi_dump_tx(5, cppi_ch, " (done teardown)"); /* REVISIT tx side _should_ clean up the same way * as the RX side ... this does no cleanup at all! */ } else /* RX */ { u16 csr; /* NOTE: docs don't guarantee any of this works ... we * expect that if the usb core stops telling the cppi core * to pull more data from it, then it'll be safe to flush * current RX DMA state iff any pending fifo transfer is done. */ core_rxirq_disable(tibase, cppi_ch->index + 1); /* for host, ensure ReqPkt is never set again */ if (is_host_active(cppi_ch->controller->controller.musb)) { value = musb_readl(tibase, DAVINCI_AUTOREQ_REG); value &= ~((0x3) << (cppi_ch->index * 2)); musb_writel(tibase, DAVINCI_AUTOREQ_REG, value); } csr = musb_readw(regs, MUSB_RXCSR); /* for host, clear (just) ReqPkt at end of current packet(s) */ if (is_host_active(cppi_ch->controller->controller.musb)) { csr |= MUSB_RXCSR_H_WZC_BITS; csr &= ~MUSB_RXCSR_H_REQPKT; } else csr |= MUSB_RXCSR_P_WZC_BITS; /* clear dma enable */ csr &= ~(MUSB_RXCSR_DMAENAB); musb_writew(regs, MUSB_RXCSR, csr); csr = musb_readw(regs, MUSB_RXCSR); /* Quiesce: wait for current dma to finish (if not cleanup). * We can't use bit zero of stateram->rx_sop, since that * refers to an entire "DMA packet" not just emptying the * current fifo. Most segments need multiple usb packets. */ if (channel->status == MUSB_DMA_STATUS_BUSY) udelay(50); /* scan the current list, reporting any data that was * transferred and acking any IRQ */ cppi_rx_scan(controller, cppi_ch->index); /* clobber the existing state once it's idle * * NOTE: arguably, we should also wait for all the other * RX channels to quiesce (how??) and then temporarily * disable RXCPPI_CTRL_REG ... but it seems that we can * rely on the controller restarting from state ram, with * only RXCPPI_BUFCNT state being bogus. BUFCNT will * correct itself after the next DMA transfer though. * * REVISIT does using rndis mode change that? */ cppi_reset_rx(cppi_ch->state_ram); /* next DMA request _should_ load cppi head ptr */ /* ... we don't "free" that list, only mutate it in place. */ cppi_dump_rx(5, cppi_ch, " (done abort)"); /* clean up previously pending bds */ cppi_bd_free(cppi_ch, cppi_ch->last_processed); cppi_ch->last_processed = NULL; while (queue) { struct cppi_descriptor *tmp = queue->next; cppi_bd_free(cppi_ch, queue); queue = tmp; } } channel->status = MUSB_DMA_STATUS_FREE; cppi_ch->buf_dma = 0; cppi_ch->offset = 0; cppi_ch->buf_len = 0; cppi_ch->maxpacket = 0; return 0; }
static bool cppi_rx_scan(struct cppi *cppi, unsigned ch) { struct cppi_channel *rx = &cppi->rx[ch]; struct cppi_rx_stateram __iomem *state = rx->state_ram; struct cppi_descriptor *bd; struct cppi_descriptor *last = rx->last_processed; bool completed = false; bool acked = false; int i; dma_addr_t safe2ack; void __iomem *regs = rx->hw_ep->regs; struct musb *musb = cppi->controller.musb; cppi_dump_rx(6, rx, "/K"); bd = last ? last->next : rx->head; if (!bd) return false; /* run through all completed BDs */ for (i = 0, safe2ack = musb_readl(&state->rx_complete, 0); (safe2ack || completed) && bd && i < NUM_RXCHAN_BD; i++, bd = bd->next) { u16 len; /* catch latest BD writes from CPPI */ rmb(); if (!completed && (bd->hw_options & CPPI_OWN_SET)) break; musb_dbg(musb, "C/RXBD %llx: nxt %08x buf %08x " "off.len %08x opt.len %08x (%d)", (unsigned long long)bd->dma, bd->hw_next, bd->hw_bufp, bd->hw_off_len, bd->hw_options, rx->channel.actual_len); /* actual packet received length */ if ((bd->hw_options & CPPI_SOP_SET) && !completed) len = bd->hw_off_len & CPPI_RECV_PKTLEN_MASK; else len = 0; if (bd->hw_options & CPPI_EOQ_MASK) completed = true; if (!completed && len < bd->buflen) { /* NOTE: when we get a short packet, RXCSR_H_REQPKT * must have been cleared, and no more DMA packets may * active be in the queue... TI docs didn't say, but * CPPI ignores those BDs even though OWN is still set. */ completed = true; musb_dbg(musb, "rx short %d/%d (%d)", len, bd->buflen, rx->channel.actual_len); } /* If we got here, we expect to ack at least one BD; meanwhile * CPPI may completing other BDs while we scan this list... * * RACE: we can notice OWN cleared before CPPI raises the * matching irq by writing that BD as the completion pointer. * In such cases, stop scanning and wait for the irq, avoiding * lost acks and states where BD ownership is unclear. */ if (bd->dma == safe2ack) { musb_writel(&state->rx_complete, 0, safe2ack); safe2ack = musb_readl(&state->rx_complete, 0); acked = true; if (bd->dma == safe2ack) safe2ack = 0; } rx->channel.actual_len += len; cppi_bd_free(rx, last); last = bd; /* stop scanning on end-of-segment */ if (bd->hw_next == 0) completed = true; } rx->last_processed = last; /* dma abort, lost ack, or ... */ if (!acked && last) { int csr; if (safe2ack == 0 || safe2ack == rx->last_processed->dma) musb_writel(&state->rx_complete, 0, safe2ack); if (safe2ack == 0) { cppi_bd_free(rx, last); rx->last_processed = NULL; /* if we land here on the host side, H_REQPKT will * be clear and we need to restart the queue... */ WARN_ON(rx->head); } musb_ep_select(cppi->mregs, rx->index + 1); csr = musb_readw(regs, MUSB_RXCSR); if (csr & MUSB_RXCSR_DMAENAB) { musb_dbg(musb, "list%d %p/%p, last %llx%s, csr %04x", rx->index, rx->head, rx->tail, rx->last_processed ? (unsigned long long) rx->last_processed->dma : 0, completed ? ", completed" : "", csr); cppi_dump_rxq(4, "/what?", rx); } } if (!completed) { int csr; rx->head = bd; /* REVISIT seems like "autoreq all but EOP" doesn't... * setting it here "should" be racey, but seems to work */ csr = musb_readw(rx->hw_ep->regs, MUSB_RXCSR); if (is_host_active(cppi->controller.musb) && bd && !(csr & MUSB_RXCSR_H_REQPKT)) { csr |= MUSB_RXCSR_H_REQPKT; musb_writew(regs, MUSB_RXCSR, MUSB_RXCSR_H_WZC_BITS | csr); csr = musb_readw(rx->hw_ep->regs, MUSB_RXCSR); } } else { rx->head = NULL; rx->tail = NULL; } cppi_dump_rx(6, rx, completed ? "/completed" : "/cleaned"); return completed; }