static struct dma_async_tx_descriptor *fsl_edma_prep_slave_sg(
struct dma_chan *chan, struct scatterlist *sgl,
unsigned int sg_len, enum dma_transfer_direction direction,
unsigned long flags, void *context)
{
struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan);
struct fsl_edma_desc *fsl_desc;
struct scatterlist *sg;
u32 src_addr, dst_addr, last_sg, nbytes;
u16 soff, doff, iter;
int i;
if (!is_slave_direction(fsl_chan->fsc.dir))
return NULL;
fsl_desc = fsl_edma_alloc_desc(fsl_chan, sg_len);
if (!fsl_desc)
return NULL;
fsl_desc->iscyclic = false;
nbytes = fsl_chan->fsc.addr_width * fsl_chan->fsc.burst;
for_each_sg(sgl, sg, sg_len, i) {
/* get next sg's physical address */
last_sg = fsl_desc->tcd[(i + 1) % sg_len].ptcd;
if (fsl_chan->fsc.dir == DMA_MEM_TO_DEV) {
src_addr = sg_dma_address(sg);
dst_addr = fsl_chan->fsc.dev_addr;
soff = fsl_chan->fsc.addr_width;
doff = 0;
} else {
src_addr = fsl_chan->fsc.dev_addr;
dst_addr = sg_dma_address(sg);
soff = 0;
doff = fsl_chan->fsc.addr_width;
}
iter = sg_dma_len(sg) / nbytes;
if (i < sg_len - 1) {
last_sg = fsl_desc->tcd[(i + 1)].ptcd;
fill_tcd_params(fsl_chan->edma, fsl_desc->tcd[i].vtcd,
src_addr, dst_addr, fsl_chan->fsc.attr,
soff, nbytes, 0, iter, iter, doff, last_sg,
false, false, true);
} else {
last_sg = 0;
fill_tcd_params(fsl_chan->edma, fsl_desc->tcd[i].vtcd,
src_addr, dst_addr, fsl_chan->fsc.attr,
soff, nbytes, 0, iter, iter, doff, last_sg,
true, true, false);
}
}
static struct dma_async_tx_descriptor *fsl_edma_prep_dma_cyclic(
struct dma_chan *chan, dma_addr_t dma_addr, size_t buf_len,
size_t period_len, enum dma_transfer_direction direction,
unsigned long flags)
{
struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan);
struct fsl_edma_desc *fsl_desc;
dma_addr_t dma_buf_next;
int sg_len, i;
u32 src_addr, dst_addr, last_sg, nbytes;
u16 soff, doff, iter;
if (!is_slave_direction(fsl_chan->fsc.dir))
return NULL;
sg_len = buf_len / period_len;
fsl_desc = fsl_edma_alloc_desc(fsl_chan, sg_len);
if (!fsl_desc)
return NULL;
fsl_desc->iscyclic = true;
dma_buf_next = dma_addr;
nbytes = fsl_chan->fsc.addr_width * fsl_chan->fsc.burst;
iter = period_len / nbytes;
for (i = 0; i < sg_len; i++) {
if (dma_buf_next >= dma_addr + buf_len)
dma_buf_next = dma_addr;
/* get next sg's physical address */
last_sg = fsl_desc->tcd[(i + 1) % sg_len].ptcd;
if (fsl_chan->fsc.dir == DMA_MEM_TO_DEV) {
src_addr = dma_buf_next;
dst_addr = fsl_chan->fsc.dev_addr;
soff = fsl_chan->fsc.addr_width;
doff = 0;
} else {
src_addr = fsl_chan->fsc.dev_addr;
dst_addr = dma_buf_next;
soff = 0;
doff = fsl_chan->fsc.addr_width;
}
fill_tcd_params(fsl_chan->edma, fsl_desc->tcd[i].vtcd, src_addr,
dst_addr, fsl_chan->fsc.attr, soff, nbytes, 0,
iter, iter, doff, last_sg, true, false, true);
dma_buf_next += period_len;
}
return vchan_tx_prep(&fsl_chan->vchan, &fsl_desc->vdesc, flags);
}
static inline struct st_fdma_chan *st_fdma_prep_common(struct dma_chan *chan,
size_t len, enum dma_transfer_direction direction)
{
struct st_fdma_chan *fchan;
if (!chan || !len)
return NULL;
fchan = to_st_fdma_chan(chan);
if (!is_slave_direction(direction)) {
dev_err(fchan->fdev->dev, "bad direction?\n");
return NULL;
}
return fchan;
}
static struct dma_async_tx_descriptor *mdc_prep_slave_sg(
struct dma_chan *chan, struct scatterlist *sgl,
unsigned int sg_len, enum dma_transfer_direction dir,
unsigned long flags, void *context)
{
struct mdc_chan *mchan = to_mdc_chan(chan);
struct mdc_dma *mdma = mchan->mdma;
struct mdc_tx_desc *mdesc;
struct scatterlist *sg;
struct mdc_hw_list_desc *curr, *prev = NULL;
dma_addr_t curr_phys, prev_phys;
unsigned int i;
if (!sgl)
return NULL;
if (!is_slave_direction(dir))
return NULL;
if (mdc_check_slave_width(mchan, dir) < 0)
return NULL;
mdesc = kzalloc(sizeof(*mdesc), GFP_NOWAIT);
if (!mdesc)
return NULL;
mdesc->chan = mchan;
for_each_sg(sgl, sg, sg_len, i) {
dma_addr_t buf = sg_dma_address(sg);
size_t buf_len = sg_dma_len(sg);
while (buf_len > 0) {
size_t xfer_size;
curr = dma_pool_alloc(mdma->desc_pool, GFP_NOWAIT,
&curr_phys);
if (!curr)
goto free_desc;
if (!prev) {
mdesc->list_phys = curr_phys;
mdesc->list = curr;
} else {
prev->node_addr = curr_phys;
prev->next_desc = curr;
}
xfer_size = min_t(size_t, mdma->max_xfer_size,
buf_len);
if (dir == DMA_MEM_TO_DEV) {
mdc_list_desc_config(mchan, curr, dir, buf,
mchan->config.dst_addr,
xfer_size);
} else {
mdc_list_desc_config(mchan, curr, dir,
mchan->config.src_addr,
buf, xfer_size);
}
prev = curr;
prev_phys = curr_phys;
mdesc->list_len++;
mdesc->list_xfer_size += xfer_size;
buf += xfer_size;
buf_len -= xfer_size;
}
}
static struct dma_async_tx_descriptor *mdc_prep_dma_cyclic(
struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
size_t period_len, enum dma_transfer_direction dir,
unsigned long flags)
{
struct mdc_chan *mchan = to_mdc_chan(chan);
struct mdc_dma *mdma = mchan->mdma;
struct mdc_tx_desc *mdesc;
struct mdc_hw_list_desc *curr, *prev = NULL;
dma_addr_t curr_phys, prev_phys;
if (!buf_len && !period_len)
return NULL;
if (!is_slave_direction(dir))
return NULL;
if (mdc_check_slave_width(mchan, dir) < 0)
return NULL;
mdesc = kzalloc(sizeof(*mdesc), GFP_NOWAIT);
if (!mdesc)
return NULL;
mdesc->chan = mchan;
mdesc->cyclic = true;
mdesc->list_xfer_size = buf_len;
mdesc->list_period_len = DIV_ROUND_UP(period_len,
mdma->max_xfer_size);
while (buf_len > 0) {
size_t remainder = min(period_len, buf_len);
while (remainder > 0) {
size_t xfer_size;
curr = dma_pool_alloc(mdma->desc_pool, GFP_NOWAIT,
&curr_phys);
if (!curr)
goto free_desc;
if (!prev) {
mdesc->list_phys = curr_phys;
mdesc->list = curr;
} else {
prev->node_addr = curr_phys;
prev->next_desc = curr;
}
xfer_size = min_t(size_t, mdma->max_xfer_size,
remainder);
if (dir == DMA_MEM_TO_DEV) {
mdc_list_desc_config(mchan, curr, dir,
buf_addr,
mchan->config.dst_addr,
xfer_size);
} else {
mdc_list_desc_config(mchan, curr, dir,
mchan->config.src_addr,
buf_addr,
xfer_size);
}
prev = curr;
prev_phys = curr_phys;
mdesc->list_len++;
buf_addr += xfer_size;
buf_len -= xfer_size;
remainder -= xfer_size;
}
}
prev->node_addr = mdesc->list_phys;
return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags);
free_desc:
mdc_desc_free(&mdesc->vd);
return NULL;
}
static struct dma_async_tx_descriptor *
at_xdmac_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr,
size_t buf_len, size_t period_len,
enum dma_transfer_direction direction,
unsigned long flags)
{
struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan);
struct at_xdmac_desc *first = NULL, *prev = NULL;
unsigned int periods = buf_len / period_len;
int i;
unsigned long irqflags;
dev_dbg(chan2dev(chan), "%s: buf_addr=%pad, buf_len=%zd, period_len=%zd, dir=%s, flags=0x%lx\n",
__func__, &buf_addr, buf_len, period_len,
direction == DMA_MEM_TO_DEV ? "mem2per" : "per2mem", flags);
if (!is_slave_direction(direction)) {
dev_err(chan2dev(chan), "invalid DMA direction\n");
return NULL;
}
if (test_and_set_bit(AT_XDMAC_CHAN_IS_CYCLIC, &atchan->status)) {
dev_err(chan2dev(chan), "channel currently used\n");
return NULL;
}
if (at_xdmac_compute_chan_conf(chan, direction))
return NULL;
for (i = 0; i < periods; i++) {
struct at_xdmac_desc *desc = NULL;
spin_lock_irqsave(&atchan->lock, irqflags);
desc = at_xdmac_get_desc(atchan);
if (!desc) {
dev_err(chan2dev(chan), "can't get descriptor\n");
if (first)
list_splice_init(&first->descs_list, &atchan->free_descs_list);
spin_unlock_irqrestore(&atchan->lock, irqflags);
return NULL;
}
spin_unlock_irqrestore(&atchan->lock, irqflags);
dev_dbg(chan2dev(chan),
"%s: desc=0x%p, tx_dma_desc.phys=%pad\n",
__func__, desc, &desc->tx_dma_desc.phys);
if (direction == DMA_DEV_TO_MEM) {
desc->lld.mbr_sa = atchan->sconfig.src_addr;
desc->lld.mbr_da = buf_addr + i * period_len;
} else {
desc->lld.mbr_sa = buf_addr + i * period_len;
desc->lld.mbr_da = atchan->sconfig.dst_addr;
}
desc->lld.mbr_cfg = atchan->cfg;
desc->lld.mbr_ubc = AT_XDMAC_MBR_UBC_NDV1
| AT_XDMAC_MBR_UBC_NDEN
| AT_XDMAC_MBR_UBC_NSEN
| period_len >> at_xdmac_get_dwidth(desc->lld.mbr_cfg);
dev_dbg(chan2dev(chan),
"%s: lld: mbr_sa=%pad, mbr_da=%pad, mbr_ubc=0x%08x\n",
__func__, &desc->lld.mbr_sa, &desc->lld.mbr_da, desc->lld.mbr_ubc);
/* Chain lld. */
if (prev)
at_xdmac_queue_desc(chan, prev, desc);
prev = desc;
if (!first)
first = desc;
dev_dbg(chan2dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n",
__func__, desc, first);
list_add_tail(&desc->desc_node, &first->descs_list);
}
prev->lld.mbr_nda = first->tx_dma_desc.phys;
dev_dbg(chan2dev(chan),
"%s: chain lld: prev=0x%p, mbr_nda=%pad\n",
__func__, prev, &prev->lld.mbr_nda);
first->tx_dma_desc.flags = flags;
first->xfer_size = buf_len;
first->direction = direction;
return &first->tx_dma_desc;
}
static struct dma_async_tx_descriptor *
at_xdmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
unsigned int sg_len, enum dma_transfer_direction direction,
unsigned long flags, void *context)
{
struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan);
struct at_xdmac_desc *first = NULL, *prev = NULL;
struct scatterlist *sg;
int i;
unsigned int xfer_size = 0;
unsigned long irqflags;
struct dma_async_tx_descriptor *ret = NULL;
if (!sgl)
return NULL;
if (!is_slave_direction(direction)) {
dev_err(chan2dev(chan), "invalid DMA direction\n");
return NULL;
}
dev_dbg(chan2dev(chan), "%s: sg_len=%d, dir=%s, flags=0x%lx\n",
__func__, sg_len,
direction == DMA_MEM_TO_DEV ? "to device" : "from device",
flags);
/* Protect dma_sconfig field that can be modified by set_slave_conf. */
spin_lock_irqsave(&atchan->lock, irqflags);
if (at_xdmac_compute_chan_conf(chan, direction))
goto spin_unlock;
/* Prepare descriptors. */
for_each_sg(sgl, sg, sg_len, i) {
struct at_xdmac_desc *desc = NULL;
u32 len, mem, dwidth, fixed_dwidth;
len = sg_dma_len(sg);
mem = sg_dma_address(sg);
if (unlikely(!len)) {
dev_err(chan2dev(chan), "sg data length is zero\n");
goto spin_unlock;
}
dev_dbg(chan2dev(chan), "%s: * sg%d len=%u, mem=0x%08x\n",
__func__, i, len, mem);
desc = at_xdmac_get_desc(atchan);
if (!desc) {
dev_err(chan2dev(chan), "can't get descriptor\n");
if (first)
list_splice_init(&first->descs_list, &atchan->free_descs_list);
goto spin_unlock;
}
/* Linked list descriptor setup. */
if (direction == DMA_DEV_TO_MEM) {
desc->lld.mbr_sa = atchan->sconfig.src_addr;
desc->lld.mbr_da = mem;
} else {
desc->lld.mbr_sa = mem;
desc->lld.mbr_da = atchan->sconfig.dst_addr;
}
dwidth = at_xdmac_get_dwidth(atchan->cfg);
fixed_dwidth = IS_ALIGNED(len, 1 << dwidth)
? dwidth
: AT_XDMAC_CC_DWIDTH_BYTE;
desc->lld.mbr_ubc = AT_XDMAC_MBR_UBC_NDV2 /* next descriptor view */
| AT_XDMAC_MBR_UBC_NDEN /* next descriptor dst parameter update */
| AT_XDMAC_MBR_UBC_NSEN /* next descriptor src parameter update */
| (len >> fixed_dwidth); /* microblock length */
desc->lld.mbr_cfg = (atchan->cfg & ~AT_XDMAC_CC_DWIDTH_MASK) |
AT_XDMAC_CC_DWIDTH(fixed_dwidth);
dev_dbg(chan2dev(chan),
"%s: lld: mbr_sa=%pad, mbr_da=%pad, mbr_ubc=0x%08x\n",
__func__, &desc->lld.mbr_sa, &desc->lld.mbr_da, desc->lld.mbr_ubc);
/* Chain lld. */
if (prev)
at_xdmac_queue_desc(chan, prev, desc);
prev = desc;
if (!first)
first = desc;
dev_dbg(chan2dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n",
__func__, desc, first);
list_add_tail(&desc->desc_node, &first->descs_list);
xfer_size += len;
}
first->tx_dma_desc.flags = flags;
first->xfer_size = xfer_size;
first->direction = direction;
ret = &first->tx_dma_desc;
spin_unlock:
spin_unlock_irqrestore(&atchan->lock, irqflags);
return ret;
}
