static void cppi_dump_rxq(int level, const char *tag, struct cppi_channel *rx)
{
struct cppi_descriptor *bd;
cppi_dump_rx(level, rx, tag);
if (rx->last_processed)
cppi_dump_rxbd("last", rx->last_processed);
for (bd = rx->head; bd; bd = bd->next)
cppi_dump_rxbd("active", bd);
}
static void cppi_dump_rxq(int level, const char *tag, struct cppi_channel *rx)
{
#ifdef CONFIG_USB_MUSB_DEBUG
struct cppi_descriptor *bd;
if (!_dbg_level(level))
return;
cppi_dump_rx(level, rx, tag);
if (rx->last_processed)
cppi_dump_rxbd("last", rx->last_processed);
for (bd = rx->head; bd; bd = bd->next)
cppi_dump_rxbd("active", bd);
#endif
}
/**
* cppi_next_rx_segment - dma read for the next chunk of a buffer
* @musb: the controller
* @rx: dma channel
* @onepacket: true unless caller treats short reads as errors, and
* performs fault recovery above usbcore.
* Context: controller irqlocked
*
* See above notes about why we can't use multi-BD RX queues except in
* rare cases (mass storage class), and can never use the hardware "rndis"
* mode (since it's not a "true" RNDIS mode) with complete safety..
*
* It's ESSENTIAL that callers specify "onepacket" mode unless they kick in
* code to recover from corrupted datastreams after each short transfer.
*/
static void
cppi_next_rx_segment(struct musb *musb, struct cppi_channel *rx, int onepacket)
{
unsigned maxpacket = rx->maxpacket;
dma_addr_t addr = rx->buf_dma + rx->offset;
size_t length = rx->buf_len - rx->offset;
struct cppi_descriptor *bd, *tail;
unsigned n_bds;
unsigned i;
void __iomem *tibase = musb->ctrl_base;
int is_rndis = 0;
struct cppi_rx_stateram __iomem *rx_ram = rx->state_ram;
struct cppi_descriptor *d;
if (onepacket) {
/* almost every USB driver, host or peripheral side */
n_bds = 1;
/* maybe apply the heuristic above */
if (cppi_rx_rndis
&& is_peripheral_active(musb)
&& length > maxpacket
&& (length & ~0xffff) == 0
&& (length & 0x0fff) != 0
&& (length & (maxpacket - 1)) == 0) {
maxpacket = length;
is_rndis = 1;
}
} else {
/* virtually nothing except mass storage class */
if (length > 0xffff) {
n_bds = 0xffff / maxpacket;
length = n_bds * maxpacket;
} else {
n_bds = DIV_ROUND_UP(length, maxpacket);
}
if (n_bds == 1)
onepacket = 1;
else
n_bds = min(n_bds, (unsigned) NUM_RXCHAN_BD);
}
/* In host mode, autorequest logic can generate some IN tokens; it's
* tricky since we can't leave REQPKT set in RXCSR after the transfer
* finishes. So: multipacket transfers involve two or more segments.
* And always at least two IRQs ... RNDIS mode is not an option.
*/
if (is_host_active(musb))
n_bds = cppi_autoreq_update(rx, tibase, onepacket, n_bds);
cppi_rndis_update(rx, 1, musb->ctrl_base, is_rndis);
length = min(n_bds * maxpacket, length);
musb_dbg(musb, "RX DMA%d seg, maxp %d %s bds %d (cnt %d) "
"dma 0x%llx len %u %u/%u",
rx->index, maxpacket,
onepacket
? (is_rndis ? "rndis" : "onepacket")
: "multipacket",
n_bds,
musb_readl(tibase,
DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4))
& 0xffff,
(unsigned long long)addr, length,
rx->channel.actual_len, rx->buf_len);
/* only queue one segment at a time, since the hardware prevents
* correct queue shutdown after unexpected short packets
*/
bd = cppi_bd_alloc(rx);
rx->head = bd;
/* Build BDs for all packets in this segment */
for (i = 0, tail = NULL; bd && i < n_bds; i++, tail = bd) {
u32 bd_len;
if (i) {
bd = cppi_bd_alloc(rx);
if (!bd)
break;
tail->next = bd;
tail->hw_next = bd->dma;
}
bd->hw_next = 0;
/* all but the last packet will be maxpacket size */
if (maxpacket < length)
bd_len = maxpacket;
else
bd_len = length;
bd->hw_bufp = addr;
addr += bd_len;
rx->offset += bd_len;
bd->hw_off_len = (0 /*offset*/ << 16) + bd_len;
bd->buflen = bd_len;
bd->hw_options = CPPI_OWN_SET | (i == 0 ? length : 0);
length -= bd_len;
}
/* we always expect at least one reusable BD! */
if (!tail) {
WARNING("rx dma%d -- no BDs? need %d\n", rx->index, n_bds);
return;
} else if (i < n_bds)
WARNING("rx dma%d -- only %d of %d BDs\n", rx->index, i, n_bds);
tail->next = NULL;
tail->hw_next = 0;
bd = rx->head;
rx->tail = tail;
/* short reads and other faults should terminate this entire
* dma segment. we want one "dma packet" per dma segment, not
* one per USB packet, terminating the whole queue at once...
* NOTE that current hardware seems to ignore SOP and EOP.
*/
bd->hw_options |= CPPI_SOP_SET;
tail->hw_options |= CPPI_EOP_SET;
for (d = rx->head; d; d = d->next)
cppi_dump_rxbd("S", d);
/* in case the preceding transfer left some state... */
tail = rx->last_processed;
if (tail) {
tail->next = bd;
tail->hw_next = bd->dma;
}
core_rxirq_enable(tibase, rx->index + 1);
/* BDs live in DMA-coherent memory, but writes might be pending */
cpu_drain_writebuffer();
/* REVISIT specs say to write this AFTER the BUFCNT register
* below ... but that loses badly.
*/
musb_writel(&rx_ram->rx_head, 0, bd->dma);
/* bufferCount must be at least 3, and zeroes on completion
* unless it underflows below zero, or stops at two, or keeps
* growing ... grr.
*/
i = musb_readl(tibase,
DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4))
& 0xffff;
if (!i)
musb_writel(tibase,
DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4),
n_bds + 2);
else if (n_bds > (i - 3))
musb_writel(tibase,
DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4),
n_bds - (i - 3));
i = musb_readl(tibase,
DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4))
& 0xffff;
if (i < (2 + n_bds)) {
musb_dbg(musb, "bufcnt%d underrun - %d (for %d)",
rx->index, i, n_bds);
musb_writel(tibase,
DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4),
n_bds + 2);
}
cppi_dump_rx(4, rx, "/S");
}
/*
* Context: controller irqlocked, endpoint selected
*/
static int cppi_channel_abort(struct dma_channel *channel)
{
struct cppi_channel *cppi_ch;
struct cppi *controller;
void __iomem *mbase;
void __iomem *tibase;
void __iomem *regs;
u32 value;
struct cppi_descriptor *queue;
cppi_ch = container_of(channel, struct cppi_channel, channel);
controller = cppi_ch->controller;
switch (channel->status) {
case MUSB_DMA_STATUS_BUS_ABORT:
case MUSB_DMA_STATUS_CORE_ABORT:
/* from RX or TX fault irq handler */
case MUSB_DMA_STATUS_BUSY:
/* the hardware needs shutting down */
regs = cppi_ch->hw_ep->regs;
break;
case MUSB_DMA_STATUS_UNKNOWN:
case MUSB_DMA_STATUS_FREE:
return 0;
default:
return -EINVAL;
}
if (!cppi_ch->transmit && cppi_ch->head)
cppi_dump_rxq(3, "/abort", cppi_ch);
mbase = controller->mregs;
tibase = controller->tibase;
queue = cppi_ch->head;
cppi_ch->head = NULL;
cppi_ch->tail = NULL;
/* REVISIT should rely on caller having done this,
* and caller should rely on us not changing it.
* peripheral code is safe ... check host too.
*/
musb_ep_select(mbase, cppi_ch->index + 1);
if (cppi_ch->transmit) {
struct cppi_tx_stateram __iomem *tx_ram;
/* REVISIT put timeouts on these controller handshakes */
cppi_dump_tx(6, cppi_ch, " (teardown)");
/* teardown DMA engine then usb core */
do {
value = musb_readl(tibase, DAVINCI_TXCPPI_TEAR_REG);
} while (!(value & CPPI_TEAR_READY));
musb_writel(tibase, DAVINCI_TXCPPI_TEAR_REG, cppi_ch->index);
tx_ram = cppi_ch->state_ram;
do {
value = musb_readl(&tx_ram->tx_complete, 0);
} while (0xFFFFFFFC != value);
/* FIXME clean up the transfer state ... here?
* the completion routine should get called with
* an appropriate status code.
*/
value = musb_readw(regs, MUSB_TXCSR);
value &= ~MUSB_TXCSR_DMAENAB;
value |= MUSB_TXCSR_FLUSHFIFO;
musb_writew(regs, MUSB_TXCSR, value);
musb_writew(regs, MUSB_TXCSR, value);
/*
* 1. Write to completion Ptr value 0x1(bit 0 set)
* (write back mode)
* 2. Wait for abort interrupt and then put the channel in
* compare mode by writing 1 to the tx_complete register.
*/
cppi_reset_tx(tx_ram, 1);
cppi_ch->head = NULL;
musb_writel(&tx_ram->tx_complete, 0, 1);
cppi_dump_tx(5, cppi_ch, " (done teardown)");
/* REVISIT tx side _should_ clean up the same way
* as the RX side ... this does no cleanup at all!
*/
} else /* RX */ {
u16 csr;
/* NOTE: docs don't guarantee any of this works ... we
* expect that if the usb core stops telling the cppi core
* to pull more data from it, then it'll be safe to flush
* current RX DMA state iff any pending fifo transfer is done.
*/
core_rxirq_disable(tibase, cppi_ch->index + 1);
/* for host, ensure ReqPkt is never set again */
if (is_host_active(cppi_ch->controller->controller.musb)) {
value = musb_readl(tibase, DAVINCI_AUTOREQ_REG);
value &= ~((0x3) << (cppi_ch->index * 2));
musb_writel(tibase, DAVINCI_AUTOREQ_REG, value);
}
csr = musb_readw(regs, MUSB_RXCSR);
/* for host, clear (just) ReqPkt at end of current packet(s) */
if (is_host_active(cppi_ch->controller->controller.musb)) {
csr |= MUSB_RXCSR_H_WZC_BITS;
csr &= ~MUSB_RXCSR_H_REQPKT;
} else
csr |= MUSB_RXCSR_P_WZC_BITS;
/* clear dma enable */
csr &= ~(MUSB_RXCSR_DMAENAB);
musb_writew(regs, MUSB_RXCSR, csr);
csr = musb_readw(regs, MUSB_RXCSR);
/* Quiesce: wait for current dma to finish (if not cleanup).
* We can't use bit zero of stateram->rx_sop, since that
* refers to an entire "DMA packet" not just emptying the
* current fifo. Most segments need multiple usb packets.
*/
if (channel->status == MUSB_DMA_STATUS_BUSY)
udelay(50);
/* scan the current list, reporting any data that was
* transferred and acking any IRQ
*/
cppi_rx_scan(controller, cppi_ch->index);
/* clobber the existing state once it's idle
*
* NOTE: arguably, we should also wait for all the other
* RX channels to quiesce (how??) and then temporarily
* disable RXCPPI_CTRL_REG ... but it seems that we can
* rely on the controller restarting from state ram, with
* only RXCPPI_BUFCNT state being bogus. BUFCNT will
* correct itself after the next DMA transfer though.
*
* REVISIT does using rndis mode change that?
*/
cppi_reset_rx(cppi_ch->state_ram);
/* next DMA request _should_ load cppi head ptr */
/* ... we don't "free" that list, only mutate it in place. */
cppi_dump_rx(5, cppi_ch, " (done abort)");
/* clean up previously pending bds */
cppi_bd_free(cppi_ch, cppi_ch->last_processed);
cppi_ch->last_processed = NULL;
while (queue) {
struct cppi_descriptor *tmp = queue->next;
cppi_bd_free(cppi_ch, queue);
queue = tmp;
}
}
channel->status = MUSB_DMA_STATUS_FREE;
cppi_ch->buf_dma = 0;
cppi_ch->offset = 0;
cppi_ch->buf_len = 0;
cppi_ch->maxpacket = 0;
return 0;
}
static bool cppi_rx_scan(struct cppi *cppi, unsigned ch)
{
struct cppi_channel *rx = &cppi->rx[ch];
struct cppi_rx_stateram __iomem *state = rx->state_ram;
struct cppi_descriptor *bd;
struct cppi_descriptor *last = rx->last_processed;
bool completed = false;
bool acked = false;
int i;
dma_addr_t safe2ack;
void __iomem *regs = rx->hw_ep->regs;
struct musb *musb = cppi->controller.musb;
cppi_dump_rx(6, rx, "/K");
bd = last ? last->next : rx->head;
if (!bd)
return false;
/* run through all completed BDs */
for (i = 0, safe2ack = musb_readl(&state->rx_complete, 0);
(safe2ack || completed) && bd && i < NUM_RXCHAN_BD;
i++, bd = bd->next) {
u16 len;
/* catch latest BD writes from CPPI */
rmb();
if (!completed && (bd->hw_options & CPPI_OWN_SET))
break;
musb_dbg(musb, "C/RXBD %llx: nxt %08x buf %08x "
"off.len %08x opt.len %08x (%d)",
(unsigned long long)bd->dma, bd->hw_next, bd->hw_bufp,
bd->hw_off_len, bd->hw_options,
rx->channel.actual_len);
/* actual packet received length */
if ((bd->hw_options & CPPI_SOP_SET) && !completed)
len = bd->hw_off_len & CPPI_RECV_PKTLEN_MASK;
else
len = 0;
if (bd->hw_options & CPPI_EOQ_MASK)
completed = true;
if (!completed && len < bd->buflen) {
/* NOTE: when we get a short packet, RXCSR_H_REQPKT
* must have been cleared, and no more DMA packets may
* active be in the queue... TI docs didn't say, but
* CPPI ignores those BDs even though OWN is still set.
*/
completed = true;
musb_dbg(musb, "rx short %d/%d (%d)",
len, bd->buflen,
rx->channel.actual_len);
}
/* If we got here, we expect to ack at least one BD; meanwhile
* CPPI may completing other BDs while we scan this list...
*
* RACE: we can notice OWN cleared before CPPI raises the
* matching irq by writing that BD as the completion pointer.
* In such cases, stop scanning and wait for the irq, avoiding
* lost acks and states where BD ownership is unclear.
*/
if (bd->dma == safe2ack) {
musb_writel(&state->rx_complete, 0, safe2ack);
safe2ack = musb_readl(&state->rx_complete, 0);
acked = true;
if (bd->dma == safe2ack)
safe2ack = 0;
}
rx->channel.actual_len += len;
cppi_bd_free(rx, last);
last = bd;
/* stop scanning on end-of-segment */
if (bd->hw_next == 0)
completed = true;
}
rx->last_processed = last;
/* dma abort, lost ack, or ... */
if (!acked && last) {
int csr;
if (safe2ack == 0 || safe2ack == rx->last_processed->dma)
musb_writel(&state->rx_complete, 0, safe2ack);
if (safe2ack == 0) {
cppi_bd_free(rx, last);
rx->last_processed = NULL;
/* if we land here on the host side, H_REQPKT will
* be clear and we need to restart the queue...
*/
WARN_ON(rx->head);
}
musb_ep_select(cppi->mregs, rx->index + 1);
csr = musb_readw(regs, MUSB_RXCSR);
if (csr & MUSB_RXCSR_DMAENAB) {
musb_dbg(musb, "list%d %p/%p, last %llx%s, csr %04x",
rx->index,
rx->head, rx->tail,
rx->last_processed
? (unsigned long long)
rx->last_processed->dma
: 0,
completed ? ", completed" : "",
csr);
cppi_dump_rxq(4, "/what?", rx);
}
}
if (!completed) {
int csr;
rx->head = bd;
/* REVISIT seems like "autoreq all but EOP" doesn't...
* setting it here "should" be racey, but seems to work
*/
csr = musb_readw(rx->hw_ep->regs, MUSB_RXCSR);
if (is_host_active(cppi->controller.musb)
&& bd
&& !(csr & MUSB_RXCSR_H_REQPKT)) {
csr |= MUSB_RXCSR_H_REQPKT;
musb_writew(regs, MUSB_RXCSR,
MUSB_RXCSR_H_WZC_BITS | csr);
csr = musb_readw(rx->hw_ep->regs, MUSB_RXCSR);
}
} else {
rx->head = NULL;
rx->tail = NULL;
}
cppi_dump_rx(6, rx, completed ? "/completed" : "/cleaned");
return completed;
}
