/** * async_memcpy - attempt to copy memory with a dma engine. * @dest: destination page * @src: src page * @dest_offset: offset into 'dest' to start transaction * @src_offset: offset into 'src' to start transaction * @len: length in bytes * @submit: submission / completion modifiers * * honored flags: ASYNC_TX_ACK */ struct dma_async_tx_descriptor * async_memcpy(struct page *dest, struct page *src, unsigned int dest_offset, unsigned int src_offset, size_t len, struct async_submit_ctl *submit) { struct dma_chan *chan = async_tx_find_channel(submit, DMA_MEMCPY, &dest, 1, &src, 1, len); struct dma_device *device = chan ? chan->device : NULL; struct dma_async_tx_descriptor *tx = NULL; if (device && is_dma_copy_aligned(device, src_offset, dest_offset, len)) { dma_addr_t dma_dest, dma_src; unsigned long dma_prep_flags = 0; if (submit->cb_fn) dma_prep_flags |= DMA_PREP_INTERRUPT; if (submit->flags & ASYNC_TX_FENCE) dma_prep_flags |= DMA_PREP_FENCE; dma_dest = dma_map_page(device->dev, dest, dest_offset, len, DMA_FROM_DEVICE); dma_src = dma_map_page(device->dev, src, src_offset, len, DMA_TO_DEVICE); tx = device->device_prep_dma_memcpy(chan, dma_dest, dma_src, len, dma_prep_flags); if (!tx) { dma_unmap_page(device->dev, dma_dest, len, DMA_FROM_DEVICE); dma_unmap_page(device->dev, dma_src, len, DMA_TO_DEVICE); } } if (tx) { pr_debug("%s: (async) len: %zu\n", __func__, len); async_tx_submit(chan, tx, submit); } else { void *dest_buf, *src_buf; pr_debug("%s: (sync) len: %zu\n", __func__, len); /* wait for any prerequisite operations */ async_tx_quiesce(&submit->depend_tx); dest_buf = kmap_atomic(dest, KM_USER0) + dest_offset; src_buf = kmap_atomic(src, KM_USER1) + src_offset; memcpy(dest_buf, src_buf, len); kunmap_atomic(src_buf, KM_USER1); kunmap_atomic(dest_buf, KM_USER0); async_tx_sync_epilog(submit); } return tx; }
/** * async_memcpy - attempt to copy memory with a dma engine. * @dest: destination page * @src: src page * @dest_offset: offset into 'dest' to start transaction * @src_offset: offset into 'src' to start transaction * @len: length in bytes * @submit: submission / completion modifiers * * honored flags: ASYNC_TX_ACK */ struct dma_async_tx_descriptor * async_memcpy(struct page *dest, struct page *src, unsigned int dest_offset, unsigned int src_offset, size_t len, struct async_submit_ctl *submit) { struct dma_chan *chan = async_tx_find_channel(submit, DMA_MEMCPY, &dest, 1, &src, 1, len); struct dma_device *device = chan ? chan->device : NULL; struct dma_async_tx_descriptor *tx = NULL; struct dmaengine_unmap_data *unmap = NULL; if (device) unmap = dmaengine_get_unmap_data(device->dev, 2, GFP_NOWAIT); if (unmap && is_dma_copy_aligned(device, src_offset, dest_offset, len)) { unsigned long dma_prep_flags = 0; if (submit->cb_fn) dma_prep_flags |= DMA_PREP_INTERRUPT; if (submit->flags & ASYNC_TX_FENCE) dma_prep_flags |= DMA_PREP_FENCE; unmap->to_cnt = 1; unmap->addr[0] = dma_map_page(device->dev, src, src_offset, len, DMA_TO_DEVICE); unmap->from_cnt = 1; unmap->addr[1] = dma_map_page(device->dev, dest, dest_offset, len, DMA_FROM_DEVICE); unmap->len = len; tx = device->device_prep_dma_memcpy(chan, unmap->addr[1], unmap->addr[0], len, dma_prep_flags); } if (tx) { pr_debug("%s: (async) len: %zu\n", __func__, len); dma_set_unmap(tx, unmap); async_tx_submit(chan, tx, submit); } else { void *dest_buf, *src_buf; pr_debug("%s: (sync) len: %zu\n", __func__, len); /* wait for any prerequisite operations */ async_tx_quiesce(&submit->depend_tx); dest_buf = kmap_atomic(dest) + dest_offset; src_buf = kmap_atomic(src) + src_offset; memcpy(dest_buf, src_buf, len); kunmap_atomic(src_buf); kunmap_atomic(dest_buf); async_tx_sync_epilog(submit); } dmaengine_unmap_put(unmap); return tx; }
static ssize_t perf_copy(struct pthr_ctx *pctx, char __iomem *dst, char *src, size_t size) { struct perf_ctx *perf = pctx->perf; struct dma_async_tx_descriptor *txd; struct dma_chan *chan = pctx->dma_chan; struct dma_device *device; struct dmaengine_unmap_data *unmap; dma_cookie_t cookie; size_t src_off, dst_off; struct perf_mw *mw = &perf->mw; void __iomem *vbase; void __iomem *dst_vaddr; dma_addr_t dst_phys; int retries = 0; if (!use_dma) { memcpy_toio(dst, src, size); return size; } if (!chan) { dev_err(&perf->ntb->dev, "DMA engine does not exist\n"); return -EINVAL; } device = chan->device; src_off = (uintptr_t)src & ~PAGE_MASK; dst_off = (uintptr_t __force)dst & ~PAGE_MASK; if (!is_dma_copy_aligned(device, src_off, dst_off, size)) return -ENODEV; vbase = mw->vbase; dst_vaddr = dst; dst_phys = mw->phys_addr + (dst_vaddr - vbase); unmap = dmaengine_get_unmap_data(device->dev, 1, GFP_NOWAIT); if (!unmap) return -ENOMEM; unmap->len = size; unmap->addr[0] = dma_map_page(device->dev, virt_to_page(src), src_off, size, DMA_TO_DEVICE); if (dma_mapping_error(device->dev, unmap->addr[0])) goto err_get_unmap; unmap->to_cnt = 1; do { txd = device->device_prep_dma_memcpy(chan, dst_phys, unmap->addr[0], size, DMA_PREP_INTERRUPT); if (!txd) { set_current_state(TASK_INTERRUPTIBLE); schedule_timeout(DMA_OUT_RESOURCE_TO); } } while (!txd && (++retries < DMA_RETRIES)); if (!txd) { pctx->dma_prep_err++; goto err_get_unmap; } txd->callback = perf_copy_callback; txd->callback_param = pctx; dma_set_unmap(txd, unmap); cookie = dmaengine_submit(txd); if (dma_submit_error(cookie)) goto err_set_unmap; dmaengine_unmap_put(unmap); atomic_inc(&pctx->dma_sync); dma_async_issue_pending(chan); return size; err_set_unmap: dmaengine_unmap_put(unmap); err_get_unmap: dmaengine_unmap_put(unmap); return 0; }