static void omap2_mcspi_tx_dma(struct spi_device *spi,
struct spi_transfer *xfer,
struct dma_slave_config cfg)
{
struct omap2_mcspi *mcspi;
struct omap2_mcspi_dma *mcspi_dma;
unsigned int count;
mcspi = spi_master_get_devdata(spi->master);
mcspi_dma = &mcspi->dma_channels[spi->chip_select];
count = xfer->len;
if (mcspi_dma->dma_tx) {
struct dma_async_tx_descriptor *tx;
struct scatterlist sg;
dmaengine_slave_config(mcspi_dma->dma_tx, &cfg);
sg_init_table(&sg, 1);
sg_dma_address(&sg) = xfer->tx_dma;
sg_dma_len(&sg) = xfer->len;
tx = dmaengine_prep_slave_sg(mcspi_dma->dma_tx, &sg, 1,
DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
if (tx) {
tx->callback = omap2_mcspi_tx_callback;
tx->callback_param = spi;
dmaengine_submit(tx);
} else {
/* FIXME: fall back to PIO? */
}
}
dma_async_issue_pending(mcspi_dma->dma_tx);
omap2_mcspi_set_dma_req(spi, 0, 1);
}
struct siu_stream *siu_stream = &port_info->playback;
struct snd_pcm_substream *substream = siu_stream->substream;
struct device *dev = substream->pcm->card->dev;
struct dma_async_tx_descriptor *desc;
dma_cookie_t cookie;
struct scatterlist sg;
u32 stfifo;
sg_init_table(&sg, 1);
sg_set_page(&sg, pfn_to_page(PFN_DOWN(buff)),
size, offset_in_page(buff));
sg_dma_len(&sg) = size;
sg_dma_address(&sg) = buff;
<<<<<<< HEAD
desc = dmaengine_prep_slave_sg(siu_stream->chan,
&sg, 1, DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
=======
<<<<<<< HEAD
desc = dmaengine_prep_slave_sg(siu_stream->chan,
&sg, 1, DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
=======
desc = siu_stream->chan->device->device_prep_slave_sg(siu_stream->chan,
&sg, 1, DMA_TO_DEVICE, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
>>>>>>> 58a75b6a81be54a8b491263ca1af243e9d8617b9
>>>>>>> ae1773bb70f3d7cf73324ce8fba787e01d8fa9f2
if (!desc) {
dev_err(dev, "Failed to allocate a dma descriptor\n");
return -ENOMEM;
}
desc->callback = siu_dma_tx_complete;
static void rockchip_spi_prepare_dma(struct rockchip_spi *rs)
{
unsigned long flags;
struct dma_slave_config rxconf, txconf;
struct dma_async_tx_descriptor *rxdesc, *txdesc;
spin_lock_irqsave(&rs->lock, flags);
rs->state &= ~RXBUSY;
rs->state &= ~TXBUSY;
spin_unlock_irqrestore(&rs->lock, flags);
rxdesc = NULL;
if (rs->rx) {
rxconf.direction = rs->dma_rx.direction;
rxconf.src_addr = rs->dma_rx.addr;
rxconf.src_addr_width = rs->n_bytes;
rxconf.src_maxburst = rs->n_bytes;
dmaengine_slave_config(rs->dma_rx.ch, &rxconf);
rxdesc = dmaengine_prep_slave_sg(
rs->dma_rx.ch,
rs->rx_sg.sgl, rs->rx_sg.nents,
rs->dma_rx.direction, DMA_PREP_INTERRUPT);
rxdesc->callback = rockchip_spi_dma_rxcb;
rxdesc->callback_param = rs;
}
txdesc = NULL;
if (rs->tx) {
txconf.direction = rs->dma_tx.direction;
txconf.dst_addr = rs->dma_tx.addr;
txconf.dst_addr_width = rs->n_bytes;
txconf.dst_maxburst = rs->n_bytes;
dmaengine_slave_config(rs->dma_tx.ch, &txconf);
txdesc = dmaengine_prep_slave_sg(
rs->dma_tx.ch,
rs->tx_sg.sgl, rs->tx_sg.nents,
rs->dma_tx.direction, DMA_PREP_INTERRUPT);
txdesc->callback = rockchip_spi_dma_txcb;
txdesc->callback_param = rs;
}
/* rx must be started before tx due to spi instinct */
if (rxdesc) {
spin_lock_irqsave(&rs->lock, flags);
rs->state |= RXBUSY;
spin_unlock_irqrestore(&rs->lock, flags);
dmaengine_submit(rxdesc);
dma_async_issue_pending(rs->dma_rx.ch);
}
if (txdesc) {
spin_lock_irqsave(&rs->lock, flags);
rs->state |= TXBUSY;
spin_unlock_irqrestore(&rs->lock, flags);
dmaengine_submit(txdesc);
dma_async_issue_pending(rs->dma_tx.ch);
}
}
static int img_spfi_start_dma(struct spi_master *master,
struct spi_device *spi,
struct spi_transfer *xfer)
{
struct img_spfi *spfi = spi_master_get_devdata(spi->master);
struct dma_async_tx_descriptor *rxdesc = NULL, *txdesc = NULL;
struct dma_slave_config rxconf, txconf;
spfi->rx_dma_busy = false;
spfi->tx_dma_busy = false;
if (xfer->rx_buf) {
rxconf.direction = DMA_DEV_TO_MEM;
if (xfer->len % 4 == 0) {
rxconf.src_addr = spfi->phys + SPFI_RX_32BIT_VALID_DATA;
rxconf.src_addr_width = 4;
rxconf.src_maxburst = 4;
} else {
rxconf.src_addr = spfi->phys + SPFI_RX_8BIT_VALID_DATA;
rxconf.src_addr_width = 1;
rxconf.src_maxburst = 4;
}
dmaengine_slave_config(spfi->rx_ch, &rxconf);
rxdesc = dmaengine_prep_slave_sg(spfi->rx_ch, xfer->rx_sg.sgl,
xfer->rx_sg.nents,
DMA_DEV_TO_MEM,
DMA_PREP_INTERRUPT);
if (!rxdesc)
goto stop_dma;
rxdesc->callback = img_spfi_dma_rx_cb;
rxdesc->callback_param = spfi;
}
if (xfer->tx_buf) {
txconf.direction = DMA_MEM_TO_DEV;
if (xfer->len % 4 == 0) {
txconf.dst_addr = spfi->phys + SPFI_TX_32BIT_VALID_DATA;
txconf.dst_addr_width = 4;
txconf.dst_maxburst = 4;
} else {
txconf.dst_addr = spfi->phys + SPFI_TX_8BIT_VALID_DATA;
txconf.dst_addr_width = 1;
txconf.dst_maxburst = 4;
}
dmaengine_slave_config(spfi->tx_ch, &txconf);
txdesc = dmaengine_prep_slave_sg(spfi->tx_ch, xfer->tx_sg.sgl,
xfer->tx_sg.nents,
DMA_MEM_TO_DEV,
DMA_PREP_INTERRUPT);
if (!txdesc)
goto stop_dma;
txdesc->callback = img_spfi_dma_tx_cb;
txdesc->callback_param = spfi;
}
if (xfer->rx_buf) {
spfi->rx_dma_busy = true;
dmaengine_submit(rxdesc);
dma_async_issue_pending(spfi->rx_ch);
}
spfi_start(spfi);
if (xfer->tx_buf) {
spfi->tx_dma_busy = true;
dmaengine_submit(txdesc);
dma_async_issue_pending(spfi->tx_ch);
}
return 1;
stop_dma:
dmaengine_terminate_all(spfi->rx_ch);
dmaengine_terminate_all(spfi->tx_ch);
return -EIO;
}
/* prepares DMA channel and DMA descriptor, returns non-zero on failure */
static int __mmci_dma_prep_data(struct mmci_host *host, struct mmc_data *data,
struct dma_chan **dma_chan,
struct dma_async_tx_descriptor **dma_desc)
{
struct variant_data *variant = host->variant;
struct dma_slave_config conf = {
.src_addr = host->phybase + MMCIFIFO,
.dst_addr = host->phybase + MMCIFIFO,
.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES,
.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES,
.src_maxburst = variant->fifohalfsize >> 2, /* # of words */
.dst_maxburst = variant->fifohalfsize >> 2, /* # of words */
.device_fc = false,
};
struct dma_chan *chan;
struct dma_device *device;
struct dma_async_tx_descriptor *desc;
int nr_sg;
unsigned long flags = DMA_CTRL_ACK;
if (data->flags & MMC_DATA_READ) {
conf.direction = DMA_DEV_TO_MEM;
chan = host->dma_rx_channel;
} else {
conf.direction = DMA_MEM_TO_DEV;
chan = host->dma_tx_channel;
}
/* If there's no DMA channel, fall back to PIO */
if (!chan)
return -EINVAL;
/* If less than or equal to the fifo size, don't bother with DMA */
if (data->blksz * data->blocks <= variant->fifosize)
return -EINVAL;
device = chan->device;
nr_sg = dma_map_sg(device->dev, data->sg, data->sg_len,
mmc_get_dma_dir(data));
if (nr_sg == 0)
return -EINVAL;
if (host->variant->qcom_dml)
flags |= DMA_PREP_INTERRUPT;
dmaengine_slave_config(chan, &conf);
desc = dmaengine_prep_slave_sg(chan, data->sg, nr_sg,
conf.direction, flags);
if (!desc)
goto unmap_exit;
*dma_chan = chan;
*dma_desc = desc;
return 0;
unmap_exit:
dma_unmap_sg(device->dev, data->sg, data->sg_len,
mmc_get_dma_dir(data));
return -ENOMEM;
}
static inline int mmci_dma_prep_data(struct mmci_host *host,
struct mmc_data *data)
{
/* Check if next job is already prepared. */
if (host->dma_current && host->dma_desc_current)
return 0;
/* No job were prepared thus do it now. */
return __mmci_dma_prep_data(host, data, &host->dma_current,
&host->dma_desc_current);
}
static inline int mmci_dma_prep_next(struct mmci_host *host,
struct mmc_data *data)
{
struct mmci_host_next *nd = &host->next_data;
return __mmci_dma_prep_data(host, data, &nd->dma_chan, &nd->dma_desc);
}
static int mmci_dma_start_data(struct mmci_host *host, unsigned int datactrl)
{
int ret;
struct mmc_data *data = host->data;
ret = mmci_dma_prep_data(host, host->data);
if (ret)
return ret;
/* Okay, go for it. */
dev_vdbg(mmc_dev(host->mmc),
"Submit MMCI DMA job, sglen %d blksz %04x blks %04x flags %08x\n",
data->sg_len, data->blksz, data->blocks, data->flags);
host->dma_in_progress = true;
dmaengine_submit(host->dma_desc_current);
dma_async_issue_pending(host->dma_current);
if (host->variant->qcom_dml)
dml_start_xfer(host, data);
datactrl |= MCI_DPSM_DMAENABLE;
/* Trigger the DMA transfer */
mmci_write_datactrlreg(host, datactrl);
/*
* Let the MMCI say when the data is ended and it's time
* to fire next DMA request. When that happens, MMCI will
* call mmci_data_end()
*/
writel(readl(host->base + MMCIMASK0) | MCI_DATAENDMASK,
host->base + MMCIMASK0);
return 0;
}
static int mxs_i2c_dma_setup_xfer(struct i2c_adapter *adap,
struct i2c_msg *msg, uint32_t flags)
{
struct dma_async_tx_descriptor *desc;
struct mxs_i2c_dev *i2c = i2c_get_adapdata(adap);
if (msg->flags & I2C_M_RD) {
i2c->dma_read = 1;
i2c->addr_data = (msg->addr << 1) | I2C_SMBUS_READ;
/*
* SELECT command.
*/
/* Queue the PIO register write transfer. */
i2c->pio_data[0] = MXS_CMD_I2C_SELECT;
desc = dmaengine_prep_slave_sg(i2c->dmach,
(struct scatterlist *)&i2c->pio_data[0],
1, DMA_TRANS_NONE, 0);
if (!desc) {
dev_err(i2c->dev,
"Failed to get PIO reg. write descriptor.\n");
goto select_init_pio_fail;
}
/* Queue the DMA data transfer. */
sg_init_one(&i2c->sg_io[0], &i2c->addr_data, 1);
dma_map_sg(i2c->dev, &i2c->sg_io[0], 1, DMA_TO_DEVICE);
desc = dmaengine_prep_slave_sg(i2c->dmach, &i2c->sg_io[0], 1,
DMA_MEM_TO_DEV,
DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
if (!desc) {
dev_err(i2c->dev,
"Failed to get DMA data write descriptor.\n");
goto select_init_dma_fail;
}
/*
* READ command.
*/
/* Queue the PIO register write transfer. */
i2c->pio_data[1] = flags | MXS_CMD_I2C_READ |
MXS_I2C_CTRL0_XFER_COUNT(msg->len);
desc = dmaengine_prep_slave_sg(i2c->dmach,
(struct scatterlist *)&i2c->pio_data[1],
1, DMA_TRANS_NONE, DMA_PREP_INTERRUPT);
if (!desc) {
dev_err(i2c->dev,
"Failed to get PIO reg. write descriptor.\n");
goto select_init_dma_fail;
}
/* Queue the DMA data transfer. */
sg_init_one(&i2c->sg_io[1], msg->buf, msg->len);
dma_map_sg(i2c->dev, &i2c->sg_io[1], 1, DMA_FROM_DEVICE);
desc = dmaengine_prep_slave_sg(i2c->dmach, &i2c->sg_io[1], 1,
DMA_DEV_TO_MEM,
DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
if (!desc) {
dev_err(i2c->dev,
"Failed to get DMA data write descriptor.\n");
goto read_init_dma_fail;
}
} else {
i2c->dma_read = 0;
i2c->addr_data = (msg->addr << 1) | I2C_SMBUS_WRITE;
/*
* WRITE command.
*/
/* Queue the PIO register write transfer. */
i2c->pio_data[0] = flags | MXS_CMD_I2C_WRITE |
MXS_I2C_CTRL0_XFER_COUNT(msg->len + 1);
desc = dmaengine_prep_slave_sg(i2c->dmach,
(struct scatterlist *)&i2c->pio_data[0],
1, DMA_TRANS_NONE, 0);
if (!desc) {
dev_err(i2c->dev,
"Failed to get PIO reg. write descriptor.\n");
goto write_init_pio_fail;
}
/* Queue the DMA data transfer. */
sg_init_table(i2c->sg_io, 2);
sg_set_buf(&i2c->sg_io[0], &i2c->addr_data, 1);
sg_set_buf(&i2c->sg_io[1], msg->buf, msg->len);
dma_map_sg(i2c->dev, i2c->sg_io, 2, DMA_TO_DEVICE);
desc = dmaengine_prep_slave_sg(i2c->dmach, i2c->sg_io, 2,
DMA_MEM_TO_DEV,
DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
if (!desc) {
dev_err(i2c->dev,
"Failed to get DMA data write descriptor.\n");
goto write_init_dma_fail;
}
}
/*
* The last descriptor must have this callback,
* to finish the DMA transaction.
*/
desc->callback = mxs_i2c_dma_irq_callback;
desc->callback_param = i2c;
/* Start the transfer. */
dmaengine_submit(desc);
dma_async_issue_pending(i2c->dmach);
return 0;
/* Read failpath. */
read_init_dma_fail:
dma_unmap_sg(i2c->dev, &i2c->sg_io[1], 1, DMA_FROM_DEVICE);
select_init_dma_fail:
dma_unmap_sg(i2c->dev, &i2c->sg_io[0], 1, DMA_TO_DEVICE);
select_init_pio_fail:
dmaengine_terminate_all(i2c->dmach);
return -EINVAL;
/* Write failpath. */
write_init_dma_fail:
dma_unmap_sg(i2c->dev, i2c->sg_io, 2, DMA_TO_DEVICE);
write_init_pio_fail:
dmaengine_terminate_all(i2c->dmach);
return -EINVAL;
}
static int spi_imx_dma_transfer(struct spi_imx_data *spi_imx,
struct spi_transfer *transfer)
{
struct dma_async_tx_descriptor *desc_tx = NULL, *desc_rx = NULL;
int ret;
u32 dma;
int left;
struct spi_master *master = spi_imx->bitbang.master;
struct sg_table *tx = &transfer->tx_sg, *rx = &transfer->rx_sg;
if (tx) {
desc_tx = dmaengine_prep_slave_sg(master->dma_tx,
tx->sgl, tx->nents, DMA_TO_DEVICE,
DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
if (!desc_tx)
goto no_dma;
desc_tx->callback = spi_imx_dma_tx_callback;
desc_tx->callback_param = (void *)spi_imx;
dmaengine_submit(desc_tx);
}
if (rx) {
desc_rx = dmaengine_prep_slave_sg(master->dma_rx,
rx->sgl, rx->nents, DMA_FROM_DEVICE,
DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
if (!desc_rx)
goto no_dma;
desc_rx->callback = spi_imx_dma_rx_callback;
desc_rx->callback_param = (void *)spi_imx;
dmaengine_submit(desc_rx);
}
reinit_completion(&spi_imx->dma_rx_completion);
reinit_completion(&spi_imx->dma_tx_completion);
/* Trigger the cspi module. */
spi_imx->dma_finished = 0;
dma = readl(spi_imx->base + MX51_ECSPI_DMA);
dma = dma & (~MX51_ECSPI_DMA_RXT_WML_MASK);
/* Change RX_DMA_LENGTH trigger dma fetch tail data */
left = transfer->len % spi_imx->rxt_wml;
if (left)
writel(dma | (left << MX51_ECSPI_DMA_RXT_WML_OFFSET),
spi_imx->base + MX51_ECSPI_DMA);
spi_imx->devtype_data->trigger(spi_imx);
dma_async_issue_pending(master->dma_tx);
dma_async_issue_pending(master->dma_rx);
/* Wait SDMA to finish the data transfer.*/
ret = wait_for_completion_timeout(&spi_imx->dma_tx_completion,
IMX_DMA_TIMEOUT);
if (!ret) {
pr_warn("%s %s: I/O Error in DMA TX\n",
dev_driver_string(&master->dev),
dev_name(&master->dev));
dmaengine_terminate_all(master->dma_tx);
} else {
ret = wait_for_completion_timeout(&spi_imx->dma_rx_completion,
IMX_DMA_TIMEOUT);
if (!ret) {
pr_warn("%s %s: I/O Error in DMA RX\n",
dev_driver_string(&master->dev),
dev_name(&master->dev));
spi_imx->devtype_data->reset(spi_imx);
dmaengine_terminate_all(master->dma_rx);
}
writel(dma |
spi_imx->rxt_wml << MX51_ECSPI_DMA_RXT_WML_OFFSET,
spi_imx->base + MX51_ECSPI_DMA);
}
spi_imx->dma_finished = 1;
spi_imx->devtype_data->trigger(spi_imx);
if (!ret)
ret = -ETIMEDOUT;
else if (ret > 0)
ret = transfer->len;
return ret;
no_dma:
pr_warn_once("%s %s: DMA not available, falling back to PIO\n",
dev_driver_string(&master->dev),
dev_name(&master->dev));
return -EAGAIN;
}
static void at91_twi_write_data_dma(struct at91_twi_dev *dev)
{
dma_addr_t dma_addr;
struct dma_async_tx_descriptor *txdesc;
struct at91_twi_dma *dma = &dev->dma;
struct dma_chan *chan_tx = dma->chan_tx;
unsigned int sg_len = 1;
if (!dev->buf_len)
return;
dma->direction = DMA_TO_DEVICE;
at91_twi_irq_save(dev);
dma_addr = dma_map_single(dev->dev, dev->buf, dev->buf_len,
DMA_TO_DEVICE);
if (dma_mapping_error(dev->dev, dma_addr)) {
dev_err(dev->dev, "dma map failed\n");
return;
}
dma->buf_mapped = true;
at91_twi_irq_restore(dev);
if (dev->fifo_size) {
size_t part1_len, part2_len;
struct scatterlist *sg;
unsigned fifo_mr;
sg_len = 0;
part1_len = dev->buf_len & ~0x3;
if (part1_len) {
sg = &dma->sg[sg_len++];
sg_dma_len(sg) = part1_len;
sg_dma_address(sg) = dma_addr;
}
part2_len = dev->buf_len & 0x3;
if (part2_len) {
sg = &dma->sg[sg_len++];
sg_dma_len(sg) = part2_len;
sg_dma_address(sg) = dma_addr + part1_len;
}
/*
* DMA controller is triggered when at least 4 data can be
* written into the TX FIFO
*/
fifo_mr = at91_twi_read(dev, AT91_TWI_FMR);
fifo_mr &= ~AT91_TWI_FMR_TXRDYM_MASK;
fifo_mr |= AT91_TWI_FMR_TXRDYM(AT91_TWI_FOUR_DATA);
at91_twi_write(dev, AT91_TWI_FMR, fifo_mr);
} else {
sg_dma_len(&dma->sg[0]) = dev->buf_len;
sg_dma_address(&dma->sg[0]) = dma_addr;
}
txdesc = dmaengine_prep_slave_sg(chan_tx, dma->sg, sg_len,
DMA_MEM_TO_DEV,
DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
if (!txdesc) {
dev_err(dev->dev, "dma prep slave sg failed\n");
goto error;
}
txdesc->callback = at91_twi_write_data_dma_callback;
txdesc->callback_param = dev;
dma->xfer_in_progress = true;
dmaengine_submit(txdesc);
dma_async_issue_pending(chan_tx);
return;
error:
at91_twi_dma_cleanup(dev);
}
netdev_tx_t mpodp_start_xmit(struct sk_buff *skb,
struct net_device *netdev)
{
struct mpodp_if_priv *priv = netdev_priv(netdev);
struct mpodp_tx *tx;
struct dma_async_tx_descriptor *dma_txd;
struct mpodp_cache_entry *entry;
int ret;
uint8_t fifo_mode;
int16_t requested_engine;
struct mpodp_pkt_hdr *hdr;
uint32_t tx_autoloop_next;
uint32_t tx_submitted, tx_next, tx_done;
uint32_t tx_mppa_idx;
int qidx;
unsigned long flags = 0;
struct mpodp_txq *txq;
/* Fetch HW queue selected by the kernel */
qidx = skb_get_queue_mapping(skb);
txq = &priv->txqs[qidx];
if (atomic_read(&priv->reset) == 1) {
mpodp_clean_tx_unlocked(priv, txq, -1);
goto addr_error;
}
tx_submitted = atomic_read(&txq->submitted);
/* Compute txd id */
tx_next = (tx_submitted + 1);
if (tx_next == txq->size)
tx_next = 0;
/* MPPA H2C Entry to use */
tx_mppa_idx = atomic_read(&txq->autoloop_cur);
tx_done = atomic_read(&txq->done);
if (tx_done != tx_submitted &&
((txq->ring[tx_done].jiffies + msecs_to_jiffies(5) >= jiffies) ||
(tx_submitted < tx_done && tx_submitted + txq->size - tx_done >= TX_POLL_THRESHOLD) ||
(tx_submitted >= tx_done && tx_submitted - tx_done >= TX_POLL_THRESHOLD))) {
mpodp_clean_tx_unlocked(priv, txq, -1);
}
/* Check if there are txd available */
if (tx_next == atomic_read(&txq->done)) {
/* Ring is full */
if (netif_msg_tx_err(priv))
netdev_err(netdev, "txq[%d]: ring full \n", txq->id);
netif_tx_stop_queue(txq->txq);
return NETDEV_TX_BUSY;
}
tx = &(txq->ring[tx_submitted]);
entry = &(txq->cache[tx_mppa_idx]);
/* take the time */
mppa_pcie_time_get(priv->tx_time, &tx->time);
/* configure channel */
tx->dst_addr = entry->addr;
/* Check the provided address */
ret =
mppa_pcie_dma_check_addr(priv->pdata, tx->dst_addr, &fifo_mode,
&requested_engine);
if (ret) {
if (netif_msg_tx_err(priv))
netdev_err(netdev, "txq[%d] tx[%d]: invalid send address %llx\n",
txq->id, tx_submitted, tx->dst_addr);
goto addr_error;
}
if (!fifo_mode) {
if (netif_msg_tx_err(priv))
netdev_err(netdev, "txq[%d] tx[%d]: %llx is not a PCI2Noc addres\n",
txq->id, tx_submitted, tx->dst_addr);
goto addr_error;
}
if (requested_engine >= MPODP_NOC_CHAN_COUNT) {
if (netif_msg_tx_err(priv))
netdev_err(netdev,
"txq[%d] tx[%d]: address %llx using NoC engine out of range (%d >= %d)\n",
txq->id, tx_submitted, tx->dst_addr,
requested_engine, MPODP_NOC_CHAN_COUNT);
goto addr_error;
}
tx->chanidx = requested_engine;
/* The packet needs a header to determine size,timestamp, etc.
* Add it */
if (skb_headroom(skb) < sizeof(struct mpodp_pkt_hdr)) {
struct sk_buff *skb_new;
skb_new =
skb_realloc_headroom(skb, sizeof(struct mpodp_pkt_hdr));
if (!skb_new) {
netdev->stats.tx_errors++;
kfree_skb(skb);
return NETDEV_TX_OK;
}
kfree_skb(skb);
skb = skb_new;
}
hdr = (struct mpodp_pkt_hdr *)
skb_push(skb, sizeof(struct mpodp_pkt_hdr));
hdr->timestamp = priv->packet_id;
hdr->info._.pkt_id = priv->packet_id;
hdr->info.dword = 0ULL;
hdr->info._.pkt_size = skb->len; /* Also count the header size */
hdr->info._.pkt_id = priv->packet_id;
priv->packet_id++;
/* save skb to free it later */
tx->skb = skb;
tx->len = skb->len;
/* prepare sg */
if (map_skb(&priv->pdev->dev, skb, tx)){
if (netif_msg_tx_err(priv))
netdev_err(netdev, "tx %d: failed to map skb to dma\n",
tx_submitted);
goto busy;
}
if (priv->n_txqs > MPODP_NOC_CHAN_COUNT)
spin_lock_irqsave(&priv->tx_lock[requested_engine], flags);
/* Prepare slave args */
priv->tx_config[requested_engine].cfg.dst_addr = tx->dst_addr;
priv->tx_config[requested_engine].requested_engine = requested_engine;
/* FIFO mode, direction, latency were filled at setup */
if (dmaengine_slave_config(priv->tx_chan[requested_engine],
&priv->tx_config[requested_engine].cfg)) {
/* board has reset, wait for reset of netdev */
netif_tx_stop_queue(txq->txq);
netif_carrier_off(netdev);
if (netif_msg_tx_err(priv))
netdev_err(netdev, "txq[%d] tx[%d]: cannot configure channel\n",
txq->id, tx_submitted);
goto busy;
}
/* get transfer descriptor */
dma_txd =
dmaengine_prep_slave_sg(priv->tx_chan[requested_engine], tx->sg,
tx->sg_len, DMA_MEM_TO_DEV, 0);
if (dma_txd == NULL) {
/* dmaengine_prep_slave_sg failed, retry */
if (netif_msg_tx_err(priv))
netdev_err(netdev, "txq[%d] tx[%d]: cannot get dma descriptor\n",
txq->id, tx_submitted);
goto busy;
}
if (netif_msg_tx_queued(priv))
netdev_info(netdev,
"txq[%d] tx[%d]: transfer start (submitted: %d done: %d) len=%d, sg_len=%d\n",
txq->id, tx_submitted, tx_next, atomic_read(&txq->done),
tx->len, tx->sg_len);
skb_orphan(skb);
/* submit and issue descriptor */
tx->jiffies = jiffies;
tx->cookie = dmaengine_submit(dma_txd);
dma_async_issue_pending(priv->tx_chan[requested_engine]);
if (priv->n_txqs > MPODP_NOC_CHAN_COUNT)
spin_unlock_irqrestore(&priv->tx_lock[requested_engine], flags);
/* Count number of bytes on the fly for DQL */
netdev_tx_sent_queue(txq->txq, skb->len);
if (test_bit(__QUEUE_STATE_STACK_XOFF, &txq->txq->state)){
/* We reached over the limit of DQL. Try to clean some
* tx so we are rescheduled right now */
mpodp_clean_tx_unlocked(priv, txq, -1);
}
/* Increment tail pointer locally */
atomic_set(&txq->submitted, tx_next);
/* Update H2C entry offset */
tx_autoloop_next = tx_mppa_idx + 1;
if (tx_autoloop_next == txq->cached_head)
tx_autoloop_next = 0;
atomic_set(&txq->autoloop_cur, tx_autoloop_next);
skb_tx_timestamp(skb);
/* Check if there is room for another txd
* or stop the queue if there is not */
tx_next = (tx_next + 1);
if (tx_next == txq->size)
tx_next = 0;
if (tx_next == atomic_read(&txq->done)) {
if (netif_msg_tx_queued(priv))
netdev_info(netdev, "txq[%d]: ring full \n", txq->id);
netif_tx_stop_queue(txq->txq);
}
return NETDEV_TX_OK;
busy:
unmap_skb(&priv->pdev->dev, skb, tx);
return NETDEV_TX_BUSY;
addr_error:
netdev->stats.tx_dropped++;
dev_kfree_skb(skb);
/* We can't do anything, just stop the queue artificially */
netif_tx_stop_queue(txq->txq);
return NETDEV_TX_OK;
}
static void
mmc_omap_prepare_data(struct mmc_omap_host *host, struct mmc_request *req)
{
struct mmc_data *data = req->data;
int i, use_dma, block_size;
unsigned sg_len;
host->data = data;
if (data == NULL) {
OMAP_MMC_WRITE(host, BLEN, 0);
OMAP_MMC_WRITE(host, NBLK, 0);
OMAP_MMC_WRITE(host, BUF, 0);
host->dma_in_use = 0;
set_cmd_timeout(host, req);
return;
}
block_size = data->blksz;
OMAP_MMC_WRITE(host, NBLK, data->blocks - 1);
OMAP_MMC_WRITE(host, BLEN, block_size - 1);
set_data_timeout(host, req);
/* cope with calling layer confusion; it issues "single
* block" writes using multi-block scatterlists.
*/
sg_len = (data->blocks == 1) ? 1 : data->sg_len;
/* Only do DMA for entire blocks */
use_dma = host->use_dma;
if (use_dma) {
for (i = 0; i < sg_len; i++) {
if ((data->sg[i].length % block_size) != 0) {
use_dma = 0;
break;
}
}
}
host->sg_idx = 0;
if (use_dma) {
enum dma_data_direction dma_data_dir;
struct dma_async_tx_descriptor *tx;
struct dma_chan *c;
u32 burst, *bp;
u16 buf;
/*
* FIFO is 16x2 bytes on 15xx, and 32x2 bytes on 16xx
* and 24xx. Use 16 or 32 word frames when the
* blocksize is at least that large. Blocksize is
* usually 512 bytes; but not for some SD reads.
*/
burst = cpu_is_omap15xx() ? 32 : 64;
if (burst > data->blksz)
burst = data->blksz;
burst >>= 1;
if (data->flags & MMC_DATA_WRITE) {
c = host->dma_tx;
bp = &host->dma_tx_burst;
buf = 0x0f80 | (burst - 1) << 0;
dma_data_dir = DMA_TO_DEVICE;
} else {
c = host->dma_rx;
bp = &host->dma_rx_burst;
buf = 0x800f | (burst - 1) << 8;
dma_data_dir = DMA_FROM_DEVICE;
}
if (!c)
goto use_pio;
/* Only reconfigure if we have a different burst size */
if (*bp != burst) {
struct dma_slave_config cfg;
cfg.src_addr = host->phys_base + OMAP_MMC_REG(host, DATA);
cfg.dst_addr = host->phys_base + OMAP_MMC_REG(host, DATA);
cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES;
cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES;
cfg.src_maxburst = burst;
cfg.dst_maxburst = burst;
if (dmaengine_slave_config(c, &cfg))
goto use_pio;
*bp = burst;
}
host->sg_len = dma_map_sg(c->device->dev, data->sg, sg_len,
dma_data_dir);
if (host->sg_len == 0)
goto use_pio;
tx = dmaengine_prep_slave_sg(c, data->sg, host->sg_len,
data->flags & MMC_DATA_WRITE ? DMA_MEM_TO_DEV : DMA_DEV_TO_MEM,
DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
if (!tx)
goto use_pio;
OMAP_MMC_WRITE(host, BUF, buf);
tx->callback = mmc_omap_dma_callback;
tx->callback_param = host;
dmaengine_submit(tx);
host->brs_received = 0;
host->dma_done = 0;
host->dma_in_use = 1;
return;
}
use_pio:
/* Revert to PIO? */
OMAP_MMC_WRITE(host, BUF, 0x1f1f);
host->total_bytes_left = data->blocks * block_size;
host->sg_len = sg_len;
mmc_omap_sg_to_buf(host);
host->dma_in_use = 0;
}
static int spi_imx_dma_transfer(struct spi_imx_data *spi_imx,
struct spi_transfer *transfer)
{
struct dma_async_tx_descriptor *desc_tx = NULL, *desc_rx = NULL;
int ret;
int left = 0;
struct spi_master *master = spi_imx->bitbang.master;
struct sg_table *tx = &transfer->tx_sg, *rx = &transfer->rx_sg;
if (tx) {
desc_tx = dmaengine_prep_slave_sg(master->dma_tx,
tx->sgl, tx->nents, DMA_MEM_TO_DEV,
DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
if (!desc_tx)
goto tx_nodma;
desc_tx->callback = spi_imx_dma_tx_callback;
desc_tx->callback_param = (void *)spi_imx;
dmaengine_submit(desc_tx);
}
if (rx) {
struct scatterlist *sgl_last = &rx->sgl[rx->nents - 1];
unsigned int orig_length = sgl_last->length;
int wml_mask = ~(spi_imx->rx_wml - 1);
/*
* Adjust the transfer lenth of the last scattlist if there are
* some tail data, use PIO read to get the tail data since DMA
* sometimes miss the last tail interrupt.
*/
left = transfer->len % spi_imx->rx_wml;
if (left)
sgl_last->length = orig_length & wml_mask;
desc_rx = dmaengine_prep_slave_sg(master->dma_rx,
rx->sgl, rx->nents, DMA_DEV_TO_MEM,
DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
if (!desc_rx)
goto rx_nodma;
desc_rx->callback = spi_imx_dma_rx_callback;
desc_rx->callback_param = (void *)spi_imx;
dmaengine_submit(desc_rx);
}
reinit_completion(&spi_imx->dma_rx_completion);
reinit_completion(&spi_imx->dma_tx_completion);
/* Trigger the cspi module. */
spi_imx->dma_finished = 0;
spi_imx->devtype_data->trigger(spi_imx);
dma_async_issue_pending(master->dma_tx);
dma_async_issue_pending(master->dma_rx);
/* Wait SDMA to finish the data transfer.*/
ret = wait_for_completion_timeout(&spi_imx->dma_tx_completion,
IMX_DMA_TIMEOUT(transfer->len));
if (!ret) {
pr_warn("%s %s: I/O Error in DMA TX:%x\n",
dev_driver_string(&master->dev),
dev_name(&master->dev), transfer->len);
dmaengine_terminate_all(master->dma_tx);
} else {
ret = wait_for_completion_timeout(&spi_imx->dma_rx_completion,
IMX_DMA_TIMEOUT(transfer->len));
if (!ret) {
pr_warn("%s %s: I/O Error in DMA RX:%x\n",
dev_driver_string(&master->dev),
dev_name(&master->dev), transfer->len);
spi_imx->devtype_data->reset(spi_imx);
dmaengine_terminate_all(master->dma_rx);
} else if (left) {
/* read the tail data by PIO */
dma_sync_sg_for_cpu(master->dma_rx->device->dev,
&rx->sgl[rx->nents - 1], 1,
DMA_FROM_DEVICE);
spi_imx->rx_buf = transfer->rx_buf
+ (transfer->len - left);
spi_imx_tail_pio_set(spi_imx, left);
reinit_completion(&spi_imx->xfer_done);
spi_imx->devtype_data->intctrl(spi_imx, MXC_INT_TCEN);
ret = wait_for_completion_timeout(&spi_imx->xfer_done,
IMX_DMA_TIMEOUT(transfer->len));
if (!ret) {
pr_warn("%s %s: I/O Error in RX tail\n",
dev_driver_string(&master->dev),
dev_name(&master->dev));
}
}
}
spi_imx->dma_finished = 1;
if (spi_imx->devtype_data->devtype == IMX6UL_ECSPI)
spi_imx->devtype_data->trigger(spi_imx);
if (!ret)
ret = -ETIMEDOUT;
else if (ret > 0)
ret = transfer->len;
return ret;
rx_nodma:
dmaengine_terminate_all(master->dma_tx);
tx_nodma:
pr_warn_once("%s %s: DMA not available, falling back to PIO\n",
dev_driver_string(&master->dev),
dev_name(&master->dev));
return -EAGAIN;
}
static int imx_asrc_dma_config(struct asrc_pair_params *params,
struct dma_chan *chan, u32 dma_addr,
void *buf_addr, u32 buf_len, bool in,
enum asrc_word_width word_width)
{
enum asrc_pair_index index = params->index;
struct dma_async_tx_descriptor *desc;
struct dma_slave_config slave_config;
enum dma_slave_buswidth buswidth;
struct scatterlist *sg;
unsigned int sg_nent, i;
int ret;
if (in) {
sg = params->input_sg;
sg_nent = params->input_sg_nodes;
desc = params->desc_in;
} else {
sg = params->output_sg;
sg_nent = params->output_sg_nodes;
desc = params->desc_out;
}
switch (word_width) {
case ASRC_WIDTH_16_BIT:
buswidth = DMA_SLAVE_BUSWIDTH_2_BYTES;
break;
case ASRC_WIDTH_24_BIT:
buswidth = DMA_SLAVE_BUSWIDTH_4_BYTES;
break;
default:
pair_err("invalid word width\n");
return -EINVAL;
}
slave_config.dma_request0 = 0;
slave_config.dma_request1 = 0;
if (in) {
slave_config.direction = DMA_MEM_TO_DEV;
slave_config.dst_addr = dma_addr;
slave_config.dst_addr_width = buswidth;
slave_config.dst_maxburst =
params->input_wm * params->channel_nums / buswidth;
} else {
slave_config.direction = DMA_DEV_TO_MEM;
slave_config.src_addr = dma_addr;
slave_config.src_addr_width = buswidth;
slave_config.src_maxburst =
params->output_wm * params->channel_nums / buswidth;
}
ret = dmaengine_slave_config(chan, &slave_config);
if (ret) {
pair_err("failed to config dmaengine for %sput task: %d\n",
in ? "in" : "out", ret);
return -EINVAL;
}
sg_init_table(sg, sg_nent);
switch (sg_nent) {
case 1:
sg_init_one(sg, buf_addr, buf_len);
break;
case 2:
case 3:
case 4:
for (i = 0; i < (sg_nent - 1); i++)
sg_set_buf(&sg[i], buf_addr + i * ASRC_MAX_BUFFER_SIZE,
ASRC_MAX_BUFFER_SIZE);
sg_set_buf(&sg[i], buf_addr + i * ASRC_MAX_BUFFER_SIZE,
buf_len - ASRC_MAX_BUFFER_SIZE * i);
break;
default:
pair_err("invalid input DMA nodes number: %d\n", sg_nent);
return -EINVAL;
}
ret = dma_map_sg(NULL, sg, sg_nent, slave_config.direction);
if (ret != sg_nent) {
pair_err("failed to map dma sg for %sput task\n",
in ? "in" : "out");
return -EINVAL;
}
desc = dmaengine_prep_slave_sg(chan, sg, sg_nent,
slave_config.direction, DMA_PREP_INTERRUPT);
if (!desc) {
pair_err("failed to prepare slave sg for %sput task\n",
in ? "in" : "out");
return -EINVAL;
}
if (in) {
params->desc_in = desc;
params->desc_in->callback = asrc_input_dma_callback;
} else {
params->desc_out = desc;
params->desc_out->callback = asrc_output_dma_callback;
}
desc->callback = ASRC_xPUT_DMA_CALLBACK(in);
desc->callback_param = params;
return 0;
}
static int spi_imx_dma_transfer(struct spi_imx_data *spi_imx,
struct spi_transfer *transfer)
{
struct dma_async_tx_descriptor *desc_tx, *desc_rx;
unsigned long transfer_timeout;
unsigned long timeout;
struct spi_master *master = spi_imx->bitbang.master;
struct sg_table *tx = &transfer->tx_sg, *rx = &transfer->rx_sg;
/*
* The TX DMA setup starts the transfer, so make sure RX is configured
* before TX.
*/
desc_rx = dmaengine_prep_slave_sg(master->dma_rx,
rx->sgl, rx->nents, DMA_DEV_TO_MEM,
DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
if (!desc_rx)
return -EINVAL;
desc_rx->callback = spi_imx_dma_rx_callback;
desc_rx->callback_param = (void *)spi_imx;
dmaengine_submit(desc_rx);
reinit_completion(&spi_imx->dma_rx_completion);
dma_async_issue_pending(master->dma_rx);
desc_tx = dmaengine_prep_slave_sg(master->dma_tx,
tx->sgl, tx->nents, DMA_MEM_TO_DEV,
DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
if (!desc_tx) {
dmaengine_terminate_all(master->dma_tx);
return -EINVAL;
}
desc_tx->callback = spi_imx_dma_tx_callback;
desc_tx->callback_param = (void *)spi_imx;
dmaengine_submit(desc_tx);
reinit_completion(&spi_imx->dma_tx_completion);
dma_async_issue_pending(master->dma_tx);
transfer_timeout = spi_imx_calculate_timeout(spi_imx, transfer->len);
/* Wait SDMA to finish the data transfer.*/
timeout = wait_for_completion_timeout(&spi_imx->dma_tx_completion,
transfer_timeout);
if (!timeout) {
dev_err(spi_imx->dev, "I/O Error in DMA TX\n");
dmaengine_terminate_all(master->dma_tx);
dmaengine_terminate_all(master->dma_rx);
return -ETIMEDOUT;
}
timeout = wait_for_completion_timeout(&spi_imx->dma_rx_completion,
transfer_timeout);
if (!timeout) {
dev_err(&master->dev, "I/O Error in DMA RX\n");
spi_imx->devtype_data->reset(spi_imx);
dmaengine_terminate_all(master->dma_rx);
return -ETIMEDOUT;
}
return transfer->len;
}
