static inline u32 at_xdmac_align_width(struct dma_chan *chan, dma_addr_t addr)
{
u32 width;
/*
* Check address alignment to select the greater data width we
* can use.
*
* Some XDMAC implementations don't provide dword transfer, in
* this case selecting dword has the same behavior as
* selecting word transfers.
*/
if (!(addr & 7)) {
width = AT_XDMAC_CC_DWIDTH_DWORD;
dev_dbg(chan2dev(chan), "%s: dwidth: double word\n", __func__);
} else if (!(addr & 3)) {
width = AT_XDMAC_CC_DWIDTH_WORD;
dev_dbg(chan2dev(chan), "%s: dwidth: word\n", __func__);
} else if (!(addr & 1)) {
width = AT_XDMAC_CC_DWIDTH_HALFWORD;
dev_dbg(chan2dev(chan), "%s: dwidth: half word\n", __func__);
} else {
width = AT_XDMAC_CC_DWIDTH_BYTE;
dev_dbg(chan2dev(chan), "%s: dwidth: byte\n", __func__);
}
return width;
}
static int at_xdmac_compute_chan_conf(struct dma_chan *chan,
enum dma_transfer_direction direction)
{
struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan);
int csize, dwidth;
if (direction == DMA_DEV_TO_MEM) {
atchan->cfg =
AT91_XDMAC_DT_PERID(atchan->perid)
| AT_XDMAC_CC_DAM_INCREMENTED_AM
| AT_XDMAC_CC_SAM_FIXED_AM
| AT_XDMAC_CC_DIF(atchan->memif)
| AT_XDMAC_CC_SIF(atchan->perif)
| AT_XDMAC_CC_SWREQ_HWR_CONNECTED
| AT_XDMAC_CC_DSYNC_PER2MEM
| AT_XDMAC_CC_MBSIZE_SIXTEEN
| AT_XDMAC_CC_TYPE_PER_TRAN;
csize = ffs(atchan->sconfig.src_maxburst) - 1;
if (csize < 0) {
dev_err(chan2dev(chan), "invalid src maxburst value\n");
return -EINVAL;
}
atchan->cfg |= AT_XDMAC_CC_CSIZE(csize);
dwidth = ffs(atchan->sconfig.src_addr_width) - 1;
if (dwidth < 0) {
dev_err(chan2dev(chan), "invalid src addr width value\n");
return -EINVAL;
}
atchan->cfg |= AT_XDMAC_CC_DWIDTH(dwidth);
} else if (direction == DMA_MEM_TO_DEV) {
atchan->cfg =
AT91_XDMAC_DT_PERID(atchan->perid)
| AT_XDMAC_CC_DAM_FIXED_AM
| AT_XDMAC_CC_SAM_INCREMENTED_AM
| AT_XDMAC_CC_DIF(atchan->perif)
| AT_XDMAC_CC_SIF(atchan->memif)
| AT_XDMAC_CC_SWREQ_HWR_CONNECTED
| AT_XDMAC_CC_DSYNC_MEM2PER
| AT_XDMAC_CC_MBSIZE_SIXTEEN
| AT_XDMAC_CC_TYPE_PER_TRAN;
csize = ffs(atchan->sconfig.dst_maxburst) - 1;
if (csize < 0) {
dev_err(chan2dev(chan), "invalid src maxburst value\n");
return -EINVAL;
}
atchan->cfg |= AT_XDMAC_CC_CSIZE(csize);
dwidth = ffs(atchan->sconfig.dst_addr_width) - 1;
if (dwidth < 0) {
dev_err(chan2dev(chan), "invalid dst addr width value\n");
return -EINVAL;
}
atchan->cfg |= AT_XDMAC_CC_DWIDTH(dwidth);
}
dev_dbg(chan2dev(chan), "%s: cfg=0x%08x\n", __func__, atchan->cfg);
return 0;
}
static inline void at_xdmac_increment_block_count(struct dma_chan *chan,
struct at_xdmac_desc *desc)
{
if (!desc)
return;
desc->lld.mbr_bc++;
dev_dbg(chan2dev(chan),
"%s: incrementing the block count of the desc 0x%p\n",
__func__, desc);
}
static void at_xdmac_queue_desc(struct dma_chan *chan,
struct at_xdmac_desc *prev,
struct at_xdmac_desc *desc)
{
if (!prev || !desc)
return;
prev->lld.mbr_nda = desc->tx_dma_desc.phys;
prev->lld.mbr_ubc |= AT_XDMAC_MBR_UBC_NDE;
dev_dbg(chan2dev(chan), "%s: chain lld: prev=0x%p, mbr_nda=%pad\n",
__func__, prev, &prev->lld.mbr_nda);
}
static int at_xdmac_set_slave_config(struct dma_chan *chan,
struct dma_slave_config *sconfig)
{
struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan);
if (at_xdmac_check_slave_config(sconfig)) {
dev_err(chan2dev(chan), "invalid slave configuration\n");
return -EINVAL;
}
memcpy(&atchan->sconfig, sconfig, sizeof(atchan->sconfig));
return 0;
}
static void pcbit_fsm_timer(unsigned long data)
{
struct pcbit_dev *dev;
struct pcbit_chan *chan;
chan = (struct pcbit_chan *) data;
del_timer(&chan->fsm_timer);
chan->fsm_timer.function = NULL;
dev = chan2dev(chan);
if (dev == NULL) {
printk(KERN_WARNING "pcbit: timer for unknown device\n");
return;
}
pcbit_fsm_event(dev, chan, EV_TIMER, NULL);
}
static dma_cookie_t at_xdmac_tx_submit(struct dma_async_tx_descriptor *tx)
{
struct at_xdmac_desc *desc = txd_to_at_desc(tx);
struct at_xdmac_chan *atchan = to_at_xdmac_chan(tx->chan);
dma_cookie_t cookie;
unsigned long irqflags;
spin_lock_irqsave(&atchan->lock, irqflags);
cookie = dma_cookie_assign(tx);
dev_vdbg(chan2dev(tx->chan), "%s: atchan 0x%p, add desc 0x%p to xfers_list\n",
__func__, atchan, desc);
list_add_tail(&desc->xfer_node, &atchan->xfers_list);
if (list_is_singular(&atchan->xfers_list))
at_xdmac_start_xfer(atchan, desc);
spin_unlock_irqrestore(&atchan->lock, irqflags);
return cookie;
}
struct dma_async_tx_descriptor *
at_xdmac_prep_dma_memset(struct dma_chan *chan, dma_addr_t dest, int value,
size_t len, unsigned long flags)
{
struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan);
struct at_xdmac_desc *desc;
dev_dbg(chan2dev(chan), "%s: dest=0x%08x, len=%d, pattern=0x%x, flags=0x%lx\n",
__func__, dest, len, value, flags);
if (unlikely(!len))
return NULL;
desc = at_xdmac_memset_create_desc(chan, atchan, dest, len, value);
list_add_tail(&desc->desc_node, &desc->descs_list);
desc->tx_dma_desc.cookie = -EBUSY;
desc->tx_dma_desc.flags = flags;
desc->xfer_size = len;
return &desc->tx_dma_desc;
}
static struct dma_async_tx_descriptor *
at_xdmac_prep_interleaved(struct dma_chan *chan,
struct dma_interleaved_template *xt,
unsigned long flags)
{
struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan);
struct at_xdmac_desc *prev = NULL, *first = NULL;
struct data_chunk *chunk, *prev_chunk = NULL;
dma_addr_t dst_addr, src_addr;
size_t dst_skip, src_skip, len = 0;
size_t prev_dst_icg = 0, prev_src_icg = 0;
int i;
if (!xt || (xt->numf != 1) || (xt->dir != DMA_MEM_TO_MEM))
return NULL;
dev_dbg(chan2dev(chan), "%s: src=0x%08x, dest=0x%08x, numf=%d, frame_size=%d, flags=0x%lx\n",
__func__, xt->src_start, xt->dst_start, xt->numf,
xt->frame_size, flags);
src_addr = xt->src_start;
dst_addr = xt->dst_start;
for (i = 0; i < xt->frame_size; i++) {
struct at_xdmac_desc *desc;
size_t src_icg, dst_icg;
chunk = xt->sgl + i;
dst_icg = dmaengine_get_dst_icg(xt, chunk);
src_icg = dmaengine_get_src_icg(xt, chunk);
src_skip = chunk->size + src_icg;
dst_skip = chunk->size + dst_icg;
dev_dbg(chan2dev(chan),
"%s: chunk size=%d, src icg=%d, dst icg=%d\n",
__func__, chunk->size, src_icg, dst_icg);
/*
* Handle the case where we just have the same
* transfer to setup, we can just increase the
* block number and reuse the same descriptor.
*/
if (prev_chunk && prev &&
(prev_chunk->size == chunk->size) &&
(prev_src_icg == src_icg) &&
(prev_dst_icg == dst_icg)) {
dev_dbg(chan2dev(chan),
"%s: same configuration that the previous chunk, merging the descriptors...\n",
__func__);
at_xdmac_increment_block_count(chan, prev);
continue;
}
desc = at_xdmac_interleaved_queue_desc(chan, atchan,
prev,
src_addr, dst_addr,
xt, chunk);
if (!desc) {
list_splice_init(&first->descs_list,
&atchan->free_descs_list);
return NULL;
}
if (!first)
first = desc;
dev_dbg(chan2dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n",
__func__, desc, first);
list_add_tail(&desc->desc_node, &first->descs_list);
if (xt->src_sgl)
src_addr += src_skip;
if (xt->dst_sgl)
dst_addr += dst_skip;
len += chunk->size;
prev_chunk = chunk;
prev_dst_icg = dst_icg;
prev_src_icg = src_icg;
prev = desc;
}
first->tx_dma_desc.cookie = -EBUSY;
first->tx_dma_desc.flags = flags;
first->xfer_size = len;
return &first->tx_dma_desc;
}
static struct at_xdmac_desc *
at_xdmac_interleaved_queue_desc(struct dma_chan *chan,
struct at_xdmac_chan *atchan,
struct at_xdmac_desc *prev,
dma_addr_t src, dma_addr_t dst,
struct dma_interleaved_template *xt,
struct data_chunk *chunk)
{
struct at_xdmac_desc *desc;
u32 dwidth;
unsigned long flags;
size_t ublen;
/*
* WARNING: The channel configuration is set here since there is no
* dmaengine_slave_config call in this case. Moreover we don't know the
* direction, it involves we can't dynamically set the source and dest
* interface so we have to use the same one. Only interface 0 allows EBI
* access. Hopefully we can access DDR through both ports (at least on
* SAMA5D4x), so we can use the same interface for source and dest,
* that solves the fact we don't know the direction.
*/
u32 chan_cc = AT_XDMAC_CC_DIF(0)
| AT_XDMAC_CC_SIF(0)
| AT_XDMAC_CC_MBSIZE_SIXTEEN
| AT_XDMAC_CC_TYPE_MEM_TRAN;
dwidth = at_xdmac_align_width(chan, src | dst | chunk->size);
if (chunk->size >= (AT_XDMAC_MBR_UBC_UBLEN_MAX << dwidth)) {
dev_dbg(chan2dev(chan),
"%s: chunk too big (%d, max size %lu)...\n",
__func__, chunk->size,
AT_XDMAC_MBR_UBC_UBLEN_MAX << dwidth);
return NULL;
}
if (prev)
dev_dbg(chan2dev(chan),
"Adding items at the end of desc 0x%p\n", prev);
if (xt->src_inc) {
if (xt->src_sgl)
chan_cc |= AT_XDMAC_CC_SAM_UBS_DS_AM;
else
chan_cc |= AT_XDMAC_CC_SAM_INCREMENTED_AM;
}
if (xt->dst_inc) {
if (xt->dst_sgl)
chan_cc |= AT_XDMAC_CC_DAM_UBS_DS_AM;
else
chan_cc |= AT_XDMAC_CC_DAM_INCREMENTED_AM;
}
spin_lock_irqsave(&atchan->lock, flags);
desc = at_xdmac_get_desc(atchan);
spin_unlock_irqrestore(&atchan->lock, flags);
if (!desc) {
dev_err(chan2dev(chan), "can't get descriptor\n");
return NULL;
}
chan_cc |= AT_XDMAC_CC_DWIDTH(dwidth);
ublen = chunk->size >> dwidth;
desc->lld.mbr_sa = src;
desc->lld.mbr_da = dst;
desc->lld.mbr_sus = dmaengine_get_src_icg(xt, chunk);
desc->lld.mbr_dus = dmaengine_get_dst_icg(xt, chunk);
desc->lld.mbr_ubc = AT_XDMAC_MBR_UBC_NDV3
| AT_XDMAC_MBR_UBC_NDEN
| AT_XDMAC_MBR_UBC_NSEN
| ublen;
desc->lld.mbr_cfg = chan_cc;
dev_dbg(chan2dev(chan),
"%s: lld: mbr_sa=0x%08x, mbr_da=0x%08x, mbr_ubc=0x%08x, mbr_cfg=0x%08x\n",
__func__, desc->lld.mbr_sa, desc->lld.mbr_da,
desc->lld.mbr_ubc, desc->lld.mbr_cfg);
/* Chain lld. */
if (prev)
at_xdmac_queue_desc(chan, prev, desc);
return desc;
}
static struct dma_async_tx_descriptor *
at_xdmac_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr,
size_t buf_len, size_t period_len,
enum dma_transfer_direction direction,
unsigned long flags)
{
struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan);
struct at_xdmac_desc *first = NULL, *prev = NULL;
unsigned int periods = buf_len / period_len;
int i;
unsigned long irqflags;
dev_dbg(chan2dev(chan), "%s: buf_addr=%pad, buf_len=%zd, period_len=%zd, dir=%s, flags=0x%lx\n",
__func__, &buf_addr, buf_len, period_len,
direction == DMA_MEM_TO_DEV ? "mem2per" : "per2mem", flags);
if (!is_slave_direction(direction)) {
dev_err(chan2dev(chan), "invalid DMA direction\n");
return NULL;
}
if (test_and_set_bit(AT_XDMAC_CHAN_IS_CYCLIC, &atchan->status)) {
dev_err(chan2dev(chan), "channel currently used\n");
return NULL;
}
if (at_xdmac_compute_chan_conf(chan, direction))
return NULL;
for (i = 0; i < periods; i++) {
struct at_xdmac_desc *desc = NULL;
spin_lock_irqsave(&atchan->lock, irqflags);
desc = at_xdmac_get_desc(atchan);
if (!desc) {
dev_err(chan2dev(chan), "can't get descriptor\n");
if (first)
list_splice_init(&first->descs_list, &atchan->free_descs_list);
spin_unlock_irqrestore(&atchan->lock, irqflags);
return NULL;
}
spin_unlock_irqrestore(&atchan->lock, irqflags);
dev_dbg(chan2dev(chan),
"%s: desc=0x%p, tx_dma_desc.phys=%pad\n",
__func__, desc, &desc->tx_dma_desc.phys);
if (direction == DMA_DEV_TO_MEM) {
desc->lld.mbr_sa = atchan->sconfig.src_addr;
desc->lld.mbr_da = buf_addr + i * period_len;
} else {
desc->lld.mbr_sa = buf_addr + i * period_len;
desc->lld.mbr_da = atchan->sconfig.dst_addr;
}
desc->lld.mbr_cfg = atchan->cfg;
desc->lld.mbr_ubc = AT_XDMAC_MBR_UBC_NDV1
| AT_XDMAC_MBR_UBC_NDEN
| AT_XDMAC_MBR_UBC_NSEN
| period_len >> at_xdmac_get_dwidth(desc->lld.mbr_cfg);
dev_dbg(chan2dev(chan),
"%s: lld: mbr_sa=%pad, mbr_da=%pad, mbr_ubc=0x%08x\n",
__func__, &desc->lld.mbr_sa, &desc->lld.mbr_da, desc->lld.mbr_ubc);
/* Chain lld. */
if (prev)
at_xdmac_queue_desc(chan, prev, desc);
prev = desc;
if (!first)
first = desc;
dev_dbg(chan2dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n",
__func__, desc, first);
list_add_tail(&desc->desc_node, &first->descs_list);
}
prev->lld.mbr_nda = first->tx_dma_desc.phys;
dev_dbg(chan2dev(chan),
"%s: chain lld: prev=0x%p, mbr_nda=%pad\n",
__func__, prev, &prev->lld.mbr_nda);
first->tx_dma_desc.flags = flags;
first->xfer_size = buf_len;
first->direction = direction;
return &first->tx_dma_desc;
}
static struct dma_async_tx_descriptor *
at_xdmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
unsigned int sg_len, enum dma_transfer_direction direction,
unsigned long flags, void *context)
{
struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan);
struct at_xdmac_desc *first = NULL, *prev = NULL;
struct scatterlist *sg;
int i;
unsigned int xfer_size = 0;
unsigned long irqflags;
struct dma_async_tx_descriptor *ret = NULL;
if (!sgl)
return NULL;
if (!is_slave_direction(direction)) {
dev_err(chan2dev(chan), "invalid DMA direction\n");
return NULL;
}
dev_dbg(chan2dev(chan), "%s: sg_len=%d, dir=%s, flags=0x%lx\n",
__func__, sg_len,
direction == DMA_MEM_TO_DEV ? "to device" : "from device",
flags);
/* Protect dma_sconfig field that can be modified by set_slave_conf. */
spin_lock_irqsave(&atchan->lock, irqflags);
if (at_xdmac_compute_chan_conf(chan, direction))
goto spin_unlock;
/* Prepare descriptors. */
for_each_sg(sgl, sg, sg_len, i) {
struct at_xdmac_desc *desc = NULL;
u32 len, mem, dwidth, fixed_dwidth;
len = sg_dma_len(sg);
mem = sg_dma_address(sg);
if (unlikely(!len)) {
dev_err(chan2dev(chan), "sg data length is zero\n");
goto spin_unlock;
}
dev_dbg(chan2dev(chan), "%s: * sg%d len=%u, mem=0x%08x\n",
__func__, i, len, mem);
desc = at_xdmac_get_desc(atchan);
if (!desc) {
dev_err(chan2dev(chan), "can't get descriptor\n");
if (first)
list_splice_init(&first->descs_list, &atchan->free_descs_list);
goto spin_unlock;
}
/* Linked list descriptor setup. */
if (direction == DMA_DEV_TO_MEM) {
desc->lld.mbr_sa = atchan->sconfig.src_addr;
desc->lld.mbr_da = mem;
} else {
desc->lld.mbr_sa = mem;
desc->lld.mbr_da = atchan->sconfig.dst_addr;
}
dwidth = at_xdmac_get_dwidth(atchan->cfg);
fixed_dwidth = IS_ALIGNED(len, 1 << dwidth)
? dwidth
: AT_XDMAC_CC_DWIDTH_BYTE;
desc->lld.mbr_ubc = AT_XDMAC_MBR_UBC_NDV2 /* next descriptor view */
| AT_XDMAC_MBR_UBC_NDEN /* next descriptor dst parameter update */
| AT_XDMAC_MBR_UBC_NSEN /* next descriptor src parameter update */
| (len >> fixed_dwidth); /* microblock length */
desc->lld.mbr_cfg = (atchan->cfg & ~AT_XDMAC_CC_DWIDTH_MASK) |
AT_XDMAC_CC_DWIDTH(fixed_dwidth);
dev_dbg(chan2dev(chan),
"%s: lld: mbr_sa=%pad, mbr_da=%pad, mbr_ubc=0x%08x\n",
__func__, &desc->lld.mbr_sa, &desc->lld.mbr_da, desc->lld.mbr_ubc);
/* Chain lld. */
if (prev)
at_xdmac_queue_desc(chan, prev, desc);
prev = desc;
if (!first)
first = desc;
dev_dbg(chan2dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n",
__func__, desc, first);
list_add_tail(&desc->desc_node, &first->descs_list);
xfer_size += len;
}
first->tx_dma_desc.flags = flags;
first->xfer_size = xfer_size;
first->direction = direction;
ret = &first->tx_dma_desc;
spin_unlock:
spin_unlock_irqrestore(&atchan->lock, irqflags);
return ret;
}
/* Call with lock hold. */
static void at_xdmac_start_xfer(struct at_xdmac_chan *atchan,
struct at_xdmac_desc *first)
{
struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device);
u32 reg;
dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, first);
if (at_xdmac_chan_is_enabled(atchan))
return;
/* Set transfer as active to not try to start it again. */
first->active_xfer = true;
/* Tell xdmac where to get the first descriptor. */
reg = AT_XDMAC_CNDA_NDA(first->tx_dma_desc.phys)
| AT_XDMAC_CNDA_NDAIF(atchan->memif);
at_xdmac_chan_write(atchan, AT_XDMAC_CNDA, reg);
/*
* When doing non cyclic transfer we need to use the next
* descriptor view 2 since some fields of the configuration register
* depend on transfer size and src/dest addresses.
*/
if (at_xdmac_chan_is_cyclic(atchan))
reg = AT_XDMAC_CNDC_NDVIEW_NDV1;
else if (first->lld.mbr_ubc & AT_XDMAC_MBR_UBC_NDV3)
reg = AT_XDMAC_CNDC_NDVIEW_NDV3;
else
reg = AT_XDMAC_CNDC_NDVIEW_NDV2;
/*
* Even if the register will be updated from the configuration in the
* descriptor when using view 2 or higher, the PROT bit won't be set
* properly. This bit can be modified only by using the channel
* configuration register.
*/
at_xdmac_chan_write(atchan, AT_XDMAC_CC, first->lld.mbr_cfg);
reg |= AT_XDMAC_CNDC_NDDUP
| AT_XDMAC_CNDC_NDSUP
| AT_XDMAC_CNDC_NDE;
at_xdmac_chan_write(atchan, AT_XDMAC_CNDC, reg);
dev_vdbg(chan2dev(&atchan->chan),
"%s: CC=0x%08x CNDA=0x%08x, CNDC=0x%08x, CSA=0x%08x, CDA=0x%08x, CUBC=0x%08x\n",
__func__, at_xdmac_chan_read(atchan, AT_XDMAC_CC),
at_xdmac_chan_read(atchan, AT_XDMAC_CNDA),
at_xdmac_chan_read(atchan, AT_XDMAC_CNDC),
at_xdmac_chan_read(atchan, AT_XDMAC_CSA),
at_xdmac_chan_read(atchan, AT_XDMAC_CDA),
at_xdmac_chan_read(atchan, AT_XDMAC_CUBC));
at_xdmac_chan_write(atchan, AT_XDMAC_CID, 0xffffffff);
reg = AT_XDMAC_CIE_RBEIE | AT_XDMAC_CIE_WBEIE | AT_XDMAC_CIE_ROIE;
/*
* There is no end of list when doing cyclic dma, we need to get
* an interrupt after each periods.
*/
if (at_xdmac_chan_is_cyclic(atchan))
at_xdmac_chan_write(atchan, AT_XDMAC_CIE,
reg | AT_XDMAC_CIE_BIE);
else
at_xdmac_chan_write(atchan, AT_XDMAC_CIE,
reg | AT_XDMAC_CIE_LIE);
at_xdmac_write(atxdmac, AT_XDMAC_GIE, atchan->mask);
dev_vdbg(chan2dev(&atchan->chan),
"%s: enable channel (0x%08x)\n", __func__, atchan->mask);
wmb();
at_xdmac_write(atxdmac, AT_XDMAC_GE, atchan->mask);
dev_vdbg(chan2dev(&atchan->chan),
"%s: CC=0x%08x CNDA=0x%08x, CNDC=0x%08x, CSA=0x%08x, CDA=0x%08x, CUBC=0x%08x\n",
__func__, at_xdmac_chan_read(atchan, AT_XDMAC_CC),
at_xdmac_chan_read(atchan, AT_XDMAC_CNDA),
at_xdmac_chan_read(atchan, AT_XDMAC_CNDC),
at_xdmac_chan_read(atchan, AT_XDMAC_CSA),
at_xdmac_chan_read(atchan, AT_XDMAC_CDA),
at_xdmac_chan_read(atchan, AT_XDMAC_CUBC));
}
static struct at_xdmac_desc *at_xdmac_memset_create_desc(struct dma_chan *chan,
struct at_xdmac_chan *atchan,
dma_addr_t dst_addr,
size_t len,
int value)
{
struct at_xdmac_desc *desc;
unsigned long flags;
size_t ublen;
u32 dwidth;
/*
* WARNING: The channel configuration is set here since there is no
* dmaengine_slave_config call in this case. Moreover we don't know the
* direction, it involves we can't dynamically set the source and dest
* interface so we have to use the same one. Only interface 0 allows EBI
* access. Hopefully we can access DDR through both ports (at least on
* SAMA5D4x), so we can use the same interface for source and dest,
* that solves the fact we don't know the direction.
*/
u32 chan_cc = AT_XDMAC_CC_DAM_INCREMENTED_AM
| AT_XDMAC_CC_SAM_INCREMENTED_AM
| AT_XDMAC_CC_DIF(0)
| AT_XDMAC_CC_SIF(0)
| AT_XDMAC_CC_MBSIZE_SIXTEEN
| AT_XDMAC_CC_MEMSET_HW_MODE
| AT_XDMAC_CC_TYPE_MEM_TRAN;
dwidth = at_xdmac_align_width(chan, dst_addr);
if (len >= (AT_XDMAC_MBR_UBC_UBLEN_MAX << dwidth)) {
dev_err(chan2dev(chan),
"%s: Transfer too large, aborting...\n",
__func__);
return NULL;
}
spin_lock_irqsave(&atchan->lock, flags);
desc = at_xdmac_get_desc(atchan);
spin_unlock_irqrestore(&atchan->lock, flags);
if (!desc) {
dev_err(chan2dev(chan), "can't get descriptor\n");
return NULL;
}
chan_cc |= AT_XDMAC_CC_DWIDTH(dwidth);
ublen = len >> dwidth;
desc->lld.mbr_da = dst_addr;
desc->lld.mbr_ds = value;
desc->lld.mbr_ubc = AT_XDMAC_MBR_UBC_NDV3
| AT_XDMAC_MBR_UBC_NDEN
| AT_XDMAC_MBR_UBC_NSEN
| ublen;
desc->lld.mbr_cfg = chan_cc;
dev_dbg(chan2dev(chan),
"%s: lld: mbr_da=0x%08x, mbr_ds=0x%08x, mbr_ubc=0x%08x, mbr_cfg=0x%08x\n",
__func__, desc->lld.mbr_da, desc->lld.mbr_ds, desc->lld.mbr_ubc,
desc->lld.mbr_cfg);
return desc;
}
static struct dma_async_tx_descriptor *
at_xdmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
size_t len, unsigned long flags)
{
struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan);
struct at_xdmac_desc *first = NULL, *prev = NULL;
size_t remaining_size = len, xfer_size = 0, ublen;
dma_addr_t src_addr = src, dst_addr = dest;
u32 dwidth;
/*
* WARNING: We don't know the direction, it involves we can't
* dynamically set the source and dest interface so we have to use the
* same one. Only interface 0 allows EBI access. Hopefully we can
* access DDR through both ports (at least on SAMA5D4x), so we can use
* the same interface for source and dest, that solves the fact we
* don't know the direction.
*/
u32 chan_cc = AT_XDMAC_CC_DAM_INCREMENTED_AM
| AT_XDMAC_CC_SAM_INCREMENTED_AM
| AT_XDMAC_CC_DIF(0)
| AT_XDMAC_CC_SIF(0)
| AT_XDMAC_CC_MBSIZE_SIXTEEN
| AT_XDMAC_CC_TYPE_MEM_TRAN;
unsigned long irqflags;
dev_dbg(chan2dev(chan), "%s: src=%pad, dest=%pad, len=%zd, flags=0x%lx\n",
__func__, &src, &dest, len, flags);
if (unlikely(!len))
return NULL;
dwidth = at_xdmac_align_width(chan, src_addr | dst_addr);
/* Prepare descriptors. */
while (remaining_size) {
struct at_xdmac_desc *desc = NULL;
dev_dbg(chan2dev(chan), "%s: remaining_size=%zu\n", __func__, remaining_size);
spin_lock_irqsave(&atchan->lock, irqflags);
desc = at_xdmac_get_desc(atchan);
spin_unlock_irqrestore(&atchan->lock, irqflags);
if (!desc) {
dev_err(chan2dev(chan), "can't get descriptor\n");
if (first)
list_splice_init(&first->descs_list, &atchan->free_descs_list);
return NULL;
}
/* Update src and dest addresses. */
src_addr += xfer_size;
dst_addr += xfer_size;
if (remaining_size >= AT_XDMAC_MBR_UBC_UBLEN_MAX << dwidth)
xfer_size = AT_XDMAC_MBR_UBC_UBLEN_MAX << dwidth;
else
xfer_size = remaining_size;
dev_dbg(chan2dev(chan), "%s: xfer_size=%zu\n", __func__, xfer_size);
/* Check remaining length and change data width if needed. */
dwidth = at_xdmac_align_width(chan,
src_addr | dst_addr | xfer_size);
chan_cc |= AT_XDMAC_CC_DWIDTH(dwidth);
ublen = xfer_size >> dwidth;
remaining_size -= xfer_size;
desc->lld.mbr_sa = src_addr;
desc->lld.mbr_da = dst_addr;
desc->lld.mbr_ubc = AT_XDMAC_MBR_UBC_NDV2
| AT_XDMAC_MBR_UBC_NDEN
| AT_XDMAC_MBR_UBC_NSEN
| ublen;
desc->lld.mbr_cfg = chan_cc;
dev_dbg(chan2dev(chan),
"%s: lld: mbr_sa=%pad, mbr_da=%pad, mbr_ubc=0x%08x, mbr_cfg=0x%08x\n",
__func__, &desc->lld.mbr_sa, &desc->lld.mbr_da, desc->lld.mbr_ubc, desc->lld.mbr_cfg);
/* Chain lld. */
if (prev)
at_xdmac_queue_desc(chan, prev, desc);
prev = desc;
if (!first)
first = desc;
dev_dbg(chan2dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n",
__func__, desc, first);
list_add_tail(&desc->desc_node, &first->descs_list);
}
first->tx_dma_desc.flags = flags;
first->xfer_size = len;
return &first->tx_dma_desc;
}
