static void ak98_sdio_stop_data(struct ak98_mci_host *host)
{
u32 masks;
PK1("%s\n", __func__);
writel(0, host->base + AK98MCIDMACTRL);
writel(0, host->base + AK98MCIDATACTRL);
masks = readl(host->base + AK98MCIMASK);
masks &= ~(MCI_DATAIRQMASKS|MCI_FIFOFULLMASK|MCI_FIFOEMPTYMASK);
writel(masks, host->base + AK98MCIMASK);
PK("DISABLE DATA IRQ\n");
#ifdef MCI_USE_L2FIFO_DMA
if (host->data->flags & MMC_DATA_WRITE) {
dma_sync_sg_for_cpu(mmc_dev(host->mmc), host->data->sg, host->data->sg_len, DMA_TO_DEVICE);
dma_unmap_sg(mmc_dev(host->mmc), host->data->sg, host->data->sg_len, DMA_TO_DEVICE);
} else {
dma_sync_sg_for_cpu(mmc_dev(host->mmc), host->data->sg, host->data->sg_len, DMA_FROM_DEVICE);
dma_unmap_sg(mmc_dev(host->mmc), host->data->sg, host->data->sg_len, DMA_FROM_DEVICE);
}
#endif
host->data = NULL;
}
void s5p_mfc_cache_inv(void *alloc_ctx)
{
struct vb2_ion_buf *buf = (struct vb2_ion_buf *)alloc_ctx;
dma_sync_sg_for_cpu(buf->conf->dev, buf->sg, buf->nents,
DMA_FROM_DEVICE);
}
int ion_cma_cache_ops(struct ion_heap *heap,
struct ion_buffer *buffer, void *vaddr,
unsigned int offset, unsigned int length,
unsigned int cmd)
{
void (*outer_cache_op)(phys_addr_t, phys_addr_t);
switch (cmd) {
case ION_IOC_CLEAN_CACHES:
if (!vaddr)
dma_sync_sg_for_device(NULL, buffer->sg_table->sgl,
buffer->sg_table->nents, DMA_TO_DEVICE);
else
dmac_clean_range(vaddr, vaddr + length);
outer_cache_op = outer_clean_range;
break;
case ION_IOC_INV_CACHES:
if (!vaddr)
dma_sync_sg_for_cpu(NULL, buffer->sg_table->sgl,
buffer->sg_table->nents, DMA_FROM_DEVICE);
else
dmac_inv_range(vaddr, vaddr + length);
outer_cache_op = outer_inv_range;
break;
case ION_IOC_CLEAN_INV_CACHES:
if (!vaddr) {
dma_sync_sg_for_device(NULL, buffer->sg_table->sgl,
buffer->sg_table->nents, DMA_TO_DEVICE);
dma_sync_sg_for_cpu(NULL, buffer->sg_table->sgl,
buffer->sg_table->nents, DMA_FROM_DEVICE);
} else {
dmac_flush_range(vaddr, vaddr + length);
}
outer_cache_op = outer_flush_range;
break;
default:
return -EINVAL;
}
if (cma_heap_has_outer_cache) {
struct ion_cma_buffer_info *info = buffer->priv_virt;
outer_cache_op(info->handle, info->handle + length);
}
return 0;
}
int videobuf_dma_sync(struct videobuf_queue *q, struct videobuf_dmabuf *dma)
{
MAGIC_CHECK(dma->magic, MAGIC_DMABUF);
BUG_ON(!dma->sglen);
dma_sync_sg_for_cpu(q->dev, dma->sglist, dma->nr_pages, dma->direction);
return 0;
}
static void isp_stat_buf_sync_for_cpu(struct ispstat *stat,
struct ispstat_buffer *buf)
{
if (IS_COHERENT_BUF(stat))
return;
dma_sync_sg_for_cpu(stat->isp->dev, buf->iovm->sgt->sgl,
buf->iovm->sgt->nents, DMA_FROM_DEVICE);
}
static void isp_stat_buf_sync_for_cpu(struct ispstat *stat,
struct ispstat_buffer *buf)
{
if (ISP_STAT_USES_DMAENGINE(stat))
return;
dma_sync_sg_for_cpu(stat->isp->dev, buf->sgt.sgl,
buf->sgt.nents, DMA_FROM_DEVICE);
}
static void vb2_dma_sg_finish(void *buf_priv)
{
struct vb2_dma_sg_buf *buf = buf_priv;
struct sg_table *sgt = buf->dma_sgt;
/* DMABUF exporter will flush the cache for us */
if (buf->db_attach)
return;
dma_sync_sg_for_cpu(buf->dev, sgt->sgl, sgt->nents, buf->dma_dir);
}
/**
* usb_buffer_dmasync_sg - synchronize DMA and CPU view of scatterlist buffer(s)
* @dev: device to which the scatterlist will be mapped
* @is_in: mapping transfer direction
* @sg: the scatterlist to synchronize
* @n_hw_ents: the positive return value from usb_buffer_map_sg
*
* Use this when you are re-using a scatterlist's data buffers for
* another USB request.
*/
void usb_buffer_dmasync_sg(const struct usb_device *dev, int is_in,
struct scatterlist *sg, int n_hw_ents)
{
struct usb_bus *bus;
struct device *controller;
if (!dev
|| !(bus = dev->bus)
|| !(controller = bus->controller)
|| !controller->dma_mask)
return;
dma_sync_sg_for_cpu(controller, sg, n_hw_ents,
is_in ? DMA_FROM_DEVICE : DMA_TO_DEVICE);
}
void kbase_sync_to_cpu(phys_addr_t paddr, void *vaddr, size_t sz)
{
#ifdef CONFIG_ARM
__cpuc_flush_dcache_area(vaddr, sz);
outer_flush_range(paddr, paddr + sz);
#elif defined(CONFIG_ARM64)
/* FIXME (MID64-46): There's no other suitable cache flush function for ARM64 */
flush_cache_all();
#elif defined(CONFIG_X86)
struct scatterlist scl = { 0, };
sg_set_page(&scl, pfn_to_page(PFN_DOWN(paddr)), sz, paddr & (PAGE_SIZE - 1));
dma_sync_sg_for_cpu(NULL, &scl, 1, DMA_FROM_DEVICE);
#else
#error Implement cache maintenance for your architecture here
#endif
}
long fops_dma_l2l(struct pci_dev *dev, struct si_dma *dma)
{
struct vm_area_struct *src_vma;
struct vm_area_struct *dst_vma;
struct vma_private_data *src_vma_data;
struct vma_private_data *dst_vma_data;
u64 src_aperture_gpu_addr;
u64 dst_aperture_gpu_addr;
u64 src_gpu_addr;
u64 dst_gpu_addr;
struct si_dma_l2l *l2l;
struct dmas_timeouts_info t_info;
long r;
r = 0;
if (dma->type == SI_DMA_TO_DEVICE || dma->type == SI_DMA_FROM_DEVICE
|| dma->type == SI_DMA_ON_HOST)
down_read(¤t->mm->mmap_sem);
l2l = &dma->params.l2l;
/* if the src addr is a cpu aperture addr... */
if (dma->type == SI_DMA_TO_DEVICE || dma->type == SI_DMA_ON_HOST) {
r = cpu_addr_to_aperture_gpu_addr(dev, &src_aperture_gpu_addr,
(void __iomem *)l2l->src_addr, &src_vma);
if (r != 0) {
if (r == NO_VMA_FOUND)
r = -EINVAL;
/*
* XXX:a vma can be around without a gpu aperture,
* for instance after a suspend.
*/
else if (r == NO_BA_MAP_FOUND)
r = 0; /* blank dma operation */
goto unlock_mmap_sem;
}
}
/* if the dst addr is a cpu aperture addr... */
if (dma->type == SI_DMA_FROM_DEVICE || dma->type == SI_DMA_ON_HOST) {
r = cpu_addr_to_aperture_gpu_addr(dev, &dst_aperture_gpu_addr,
(void __iomem *)l2l->dst_addr, &dst_vma);
if (r != 0) {
if (r == NO_VMA_FOUND)
r = -EINVAL;
/*
* XXX:a vma can be around without a gpu aperture,
* for instance after a suspend.
*/
else if (r == NO_BA_MAP_FOUND)
r = 0; /* blank dma operation */
goto unlock_mmap_sem;
}
}
switch (dma->dir) {
case SI_DMA_FROM_DEVICE:
dst_gpu_addr = dst_aperture_gpu_addr;
src_gpu_addr = l2l->src_addr;
break;
case SI_DMA_TO_DEVICE:
dst_gpu_addr = l2l->dst_addr;
src_gpu_addr = src_aperture_gpu_addr;
break;
case SI_DMA_ON_DEVICE:
dst_gpu_addr = l2l->dst_addr;
src_gpu_addr = l2l->src_addr;
break;
case SI_DMA_ON_HOST:
dst_gpu_addr = dst_aperture_gpu_addr;
src_gpu_addr = src_aperture_gpu_addr;
break;
default:
r = -EINVAL;
goto unlock_mmap_sem;
}
if (dma->type == SI_DMA_TO_DEVICE || dma->type == SI_DMA_ON_HOST) {
src_vma_data = src_vma->vm_private_data;
dma_sync_sg_for_device(&dev->dev, src_vma_data->sg_tbl.sgl,
src_vma_data->sg_tbl.nents, DMA_BIDIRECTIONAL);
}
memcpy(&t_info, &dma->t_info, sizeof(t_info));
r = dmas_cpy(dev, dst_gpu_addr, src_gpu_addr, l2l->sz, t_info);
if (r < 0) {
if (r == -DMAS_RING_TIMEOUT)
r = SI_RING_TIMEOUT;
else if (r == -DMAS_FENCE_TIMEOUT)
r = SI_FENCE_TIMEOUT;
goto unlock_mmap_sem;
}
if (dma->type == SI_DMA_FROM_DEVICE || dma->type == SI_DMA_ON_HOST) {
dst_vma_data = dst_vma->vm_private_data;
dma_sync_sg_for_cpu(&dev->dev, dst_vma_data->sg_tbl.sgl,
dst_vma_data->sg_tbl.nents, DMA_BIDIRECTIONAL);
}
unlock_mmap_sem:
if (dma->type == SI_DMA_TO_DEVICE || dma->type == SI_DMA_FROM_DEVICE
|| dma->type == SI_DMA_ON_HOST)
up_read(¤t->mm->mmap_sem);
return r;
}
static int spi_imx_dma_transfer(struct spi_imx_data *spi_imx,
struct spi_transfer *transfer)
{
struct dma_async_tx_descriptor *desc_tx = NULL, *desc_rx = NULL;
int ret;
int left = 0;
struct spi_master *master = spi_imx->bitbang.master;
struct sg_table *tx = &transfer->tx_sg, *rx = &transfer->rx_sg;
if (tx) {
desc_tx = dmaengine_prep_slave_sg(master->dma_tx,
tx->sgl, tx->nents, DMA_MEM_TO_DEV,
DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
if (!desc_tx)
goto tx_nodma;
desc_tx->callback = spi_imx_dma_tx_callback;
desc_tx->callback_param = (void *)spi_imx;
dmaengine_submit(desc_tx);
}
if (rx) {
struct scatterlist *sgl_last = &rx->sgl[rx->nents - 1];
unsigned int orig_length = sgl_last->length;
int wml_mask = ~(spi_imx->rx_wml - 1);
/*
* Adjust the transfer lenth of the last scattlist if there are
* some tail data, use PIO read to get the tail data since DMA
* sometimes miss the last tail interrupt.
*/
left = transfer->len % spi_imx->rx_wml;
if (left)
sgl_last->length = orig_length & wml_mask;
desc_rx = dmaengine_prep_slave_sg(master->dma_rx,
rx->sgl, rx->nents, DMA_DEV_TO_MEM,
DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
if (!desc_rx)
goto rx_nodma;
desc_rx->callback = spi_imx_dma_rx_callback;
desc_rx->callback_param = (void *)spi_imx;
dmaengine_submit(desc_rx);
}
reinit_completion(&spi_imx->dma_rx_completion);
reinit_completion(&spi_imx->dma_tx_completion);
/* Trigger the cspi module. */
spi_imx->dma_finished = 0;
spi_imx->devtype_data->trigger(spi_imx);
dma_async_issue_pending(master->dma_tx);
dma_async_issue_pending(master->dma_rx);
/* Wait SDMA to finish the data transfer.*/
ret = wait_for_completion_timeout(&spi_imx->dma_tx_completion,
IMX_DMA_TIMEOUT(transfer->len));
if (!ret) {
pr_warn("%s %s: I/O Error in DMA TX:%x\n",
dev_driver_string(&master->dev),
dev_name(&master->dev), transfer->len);
dmaengine_terminate_all(master->dma_tx);
} else {
ret = wait_for_completion_timeout(&spi_imx->dma_rx_completion,
IMX_DMA_TIMEOUT(transfer->len));
if (!ret) {
pr_warn("%s %s: I/O Error in DMA RX:%x\n",
dev_driver_string(&master->dev),
dev_name(&master->dev), transfer->len);
spi_imx->devtype_data->reset(spi_imx);
dmaengine_terminate_all(master->dma_rx);
} else if (left) {
/* read the tail data by PIO */
dma_sync_sg_for_cpu(master->dma_rx->device->dev,
&rx->sgl[rx->nents - 1], 1,
DMA_FROM_DEVICE);
spi_imx->rx_buf = transfer->rx_buf
+ (transfer->len - left);
spi_imx_tail_pio_set(spi_imx, left);
reinit_completion(&spi_imx->xfer_done);
spi_imx->devtype_data->intctrl(spi_imx, MXC_INT_TCEN);
ret = wait_for_completion_timeout(&spi_imx->xfer_done,
IMX_DMA_TIMEOUT(transfer->len));
if (!ret) {
pr_warn("%s %s: I/O Error in RX tail\n",
dev_driver_string(&master->dev),
dev_name(&master->dev));
}
}
}
spi_imx->dma_finished = 1;
if (spi_imx->devtype_data->devtype == IMX6UL_ECSPI)
spi_imx->devtype_data->trigger(spi_imx);
if (!ret)
ret = -ETIMEDOUT;
else if (ret > 0)
ret = transfer->len;
return ret;
rx_nodma:
dmaengine_terminate_all(master->dma_tx);
tx_nodma:
pr_warn_once("%s %s: DMA not available, falling back to PIO\n",
dev_driver_string(&master->dev),
dev_name(&master->dev));
return -EAGAIN;
}
int ion_system_heap_cache_ops(struct ion_heap *heap, struct ion_buffer *buffer,
void *vaddr, unsigned int offset, unsigned int length,
unsigned int cmd)
{
void (*outer_cache_op)(phys_addr_t, phys_addr_t);
switch (cmd) {
case ION_IOC_CLEAN_CACHES:
if (!vaddr)
dma_sync_sg_for_device(NULL, buffer->sglist, 1, DMA_TO_DEVICE);
else
dmac_clean_range(vaddr, vaddr + length);
outer_cache_op = outer_clean_range;
break;
case ION_IOC_INV_CACHES:
if (!vaddr)
dma_sync_sg_for_cpu(NULL, buffer->sglist, 1, DMA_FROM_DEVICE);
else
dmac_inv_range(vaddr, vaddr + length);
outer_cache_op = outer_inv_range;
break;
case ION_IOC_CLEAN_INV_CACHES:
if (!vaddr) {
dma_sync_sg_for_device(NULL, buffer->sglist, 1, DMA_TO_DEVICE);
dma_sync_sg_for_cpu(NULL, buffer->sglist, 1, DMA_FROM_DEVICE);
} else {
dmac_flush_range(vaddr, vaddr + length);
}
outer_cache_op = outer_flush_range;
break;
default:
return -EINVAL;
}
if (system_heap_has_outer_cache) {
unsigned long pstart;
void *vend;
void *vtemp;
unsigned long ln = 0;
vend = buffer->priv_virt + buffer->size;
vtemp = buffer->priv_virt + offset;
if ((vtemp+length) > vend) {
pr_err("Trying to flush outside of mapped range.\n");
pr_err("End of mapped range: %p, trying to flush to "
"address %p\n", vend, vtemp+length);
WARN(1, "%s: called with heap name %s, buffer size 0x%x, "
"vaddr 0x%p, offset 0x%x, length: 0x%x\n",
__func__, heap->name, buffer->size, vaddr,
offset, length);
return -EINVAL;
}
for (; ln < length && vtemp < vend;
vtemp += PAGE_SIZE, ln += PAGE_SIZE) {
struct page *page = vmalloc_to_page(vtemp);
if (!page) {
WARN(1, "Could not find page for virt. address %p\n",
vtemp);
return -EINVAL;
}
pstart = page_to_phys(page);
/*
* If page -> phys is returning NULL, something
* has really gone wrong...
*/
if (!pstart) {
WARN(1, "Could not translate %p to physical address\n",
vtemp);
return -EINVAL;
}
outer_cache_op(pstart, pstart + PAGE_SIZE);
}
}
return 0;
}
