/** * async_memcpy - attempt to copy memory with a dma engine. * @dest: destination page * @src: src page * @dest_offset: offset into 'dest' to start transaction * @src_offset: offset into 'src' to start transaction * @len: length in bytes * @submit: submission / completion modifiers * * honored flags: ASYNC_TX_ACK */ struct dma_async_tx_descriptor * async_memcpy(struct page *dest, struct page *src, unsigned int dest_offset, unsigned int src_offset, size_t len, struct async_submit_ctl *submit) { struct dma_chan *chan = async_tx_find_channel(submit, DMA_MEMCPY, &dest, 1, &src, 1, len); struct dma_device *device = chan ? chan->device : NULL; struct dma_async_tx_descriptor *tx = NULL; struct dmaengine_unmap_data *unmap = NULL; if (device) unmap = dmaengine_get_unmap_data(device->dev, 2, GFP_NOWAIT); if (unmap && is_dma_copy_aligned(device, src_offset, dest_offset, len)) { unsigned long dma_prep_flags = 0; if (submit->cb_fn) dma_prep_flags |= DMA_PREP_INTERRUPT; if (submit->flags & ASYNC_TX_FENCE) dma_prep_flags |= DMA_PREP_FENCE; unmap->to_cnt = 1; unmap->addr[0] = dma_map_page(device->dev, src, src_offset, len, DMA_TO_DEVICE); unmap->from_cnt = 1; unmap->addr[1] = dma_map_page(device->dev, dest, dest_offset, len, DMA_FROM_DEVICE); unmap->len = len; tx = device->device_prep_dma_memcpy(chan, unmap->addr[1], unmap->addr[0], len, dma_prep_flags); } if (tx) { pr_debug("%s: (async) len: %zu\n", __func__, len); dma_set_unmap(tx, unmap); async_tx_submit(chan, tx, submit); } else { void *dest_buf, *src_buf; pr_debug("%s: (sync) len: %zu\n", __func__, len); /* wait for any prerequisite operations */ async_tx_quiesce(&submit->depend_tx); dest_buf = kmap_atomic(dest) + dest_offset; src_buf = kmap_atomic(src) + src_offset; memcpy(dest_buf, src_buf, len); kunmap_atomic(src_buf); kunmap_atomic(dest_buf); async_tx_sync_epilog(submit); } dmaengine_unmap_put(unmap); return tx; }
static ssize_t perf_copy(struct pthr_ctx *pctx, char __iomem *dst, char *src, size_t size) { struct perf_ctx *perf = pctx->perf; struct dma_async_tx_descriptor *txd; struct dma_chan *chan = pctx->dma_chan; struct dma_device *device; struct dmaengine_unmap_data *unmap; dma_cookie_t cookie; size_t src_off, dst_off; struct perf_mw *mw = &perf->mw; void __iomem *vbase; void __iomem *dst_vaddr; dma_addr_t dst_phys; int retries = 0; if (!use_dma) { memcpy_toio(dst, src, size); return size; } if (!chan) { dev_err(&perf->ntb->dev, "DMA engine does not exist\n"); return -EINVAL; } device = chan->device; src_off = (uintptr_t)src & ~PAGE_MASK; dst_off = (uintptr_t __force)dst & ~PAGE_MASK; if (!is_dma_copy_aligned(device, src_off, dst_off, size)) return -ENODEV; vbase = mw->vbase; dst_vaddr = dst; dst_phys = mw->phys_addr + (dst_vaddr - vbase); unmap = dmaengine_get_unmap_data(device->dev, 1, GFP_NOWAIT); if (!unmap) return -ENOMEM; unmap->len = size; unmap->addr[0] = dma_map_page(device->dev, virt_to_page(src), src_off, size, DMA_TO_DEVICE); if (dma_mapping_error(device->dev, unmap->addr[0])) goto err_get_unmap; unmap->to_cnt = 1; do { txd = device->device_prep_dma_memcpy(chan, dst_phys, unmap->addr[0], size, DMA_PREP_INTERRUPT); if (!txd) { set_current_state(TASK_INTERRUPTIBLE); schedule_timeout(DMA_OUT_RESOURCE_TO); } } while (!txd && (++retries < DMA_RETRIES)); if (!txd) { pctx->dma_prep_err++; goto err_get_unmap; } txd->callback = perf_copy_callback; txd->callback_param = pctx; dma_set_unmap(txd, unmap); cookie = dmaengine_submit(txd); if (dma_submit_error(cookie)) goto err_set_unmap; dmaengine_unmap_put(unmap); atomic_inc(&pctx->dma_sync); dma_async_issue_pending(chan); return size; err_set_unmap: dmaengine_unmap_put(unmap); err_get_unmap: dmaengine_unmap_put(unmap); return 0; }
/* do_async_xor - dma map the pages and perform the xor with an engine */ static __async_inline struct dma_async_tx_descriptor * do_async_xor(struct dma_chan *chan, struct dmaengine_unmap_data *unmap, struct async_submit_ctl *submit) { struct dma_device *dma = chan->device; struct dma_async_tx_descriptor *tx = NULL; dma_async_tx_callback cb_fn_orig = submit->cb_fn; void *cb_param_orig = submit->cb_param; enum async_tx_flags flags_orig = submit->flags; enum dma_ctrl_flags dma_flags = 0; int src_cnt = unmap->to_cnt; int xor_src_cnt; dma_addr_t dma_dest = unmap->addr[unmap->to_cnt]; dma_addr_t *src_list = unmap->addr; while (src_cnt) { dma_addr_t tmp; submit->flags = flags_orig; xor_src_cnt = min(src_cnt, (int)dma->max_xor); /* if we are submitting additional xors, leave the chain open * and clear the callback parameters */ if (src_cnt > xor_src_cnt) { submit->flags &= ~ASYNC_TX_ACK; submit->flags |= ASYNC_TX_FENCE; submit->cb_fn = NULL; submit->cb_param = NULL; } else { submit->cb_fn = cb_fn_orig; submit->cb_param = cb_param_orig; } if (submit->cb_fn) dma_flags |= DMA_PREP_INTERRUPT; if (submit->flags & ASYNC_TX_FENCE) dma_flags |= DMA_PREP_FENCE; /* Drivers force forward progress in case they can not provide a * descriptor */ tmp = src_list[0]; if (src_list > unmap->addr) src_list[0] = dma_dest; tx = dma->device_prep_dma_xor(chan, dma_dest, src_list, xor_src_cnt, unmap->len, dma_flags); if (unlikely(!tx)) async_tx_quiesce(&submit->depend_tx); /* spin wait for the preceding transactions to complete */ while (unlikely(!tx)) { dma_async_issue_pending(chan); tx = dma->device_prep_dma_xor(chan, dma_dest, src_list, xor_src_cnt, unmap->len, dma_flags); } src_list[0] = tmp; dma_set_unmap(tx, unmap); async_tx_submit(chan, tx, submit); submit->depend_tx = tx; if (src_cnt > xor_src_cnt) { /* drop completed sources */ src_cnt -= xor_src_cnt; /* use the intermediate result a source */ src_cnt++; src_list += xor_src_cnt - 1; } else break; } return tx; }
/** * do_async_gen_syndrome - asynchronously calculate P and/or Q */ static __async_inline struct dma_async_tx_descriptor * do_async_gen_syndrome(struct dma_chan *chan, const unsigned char *scfs, int disks, struct dmaengine_unmap_data *unmap, enum dma_ctrl_flags dma_flags, struct async_submit_ctl *submit) { struct dma_async_tx_descriptor *tx = NULL; struct dma_device *dma = chan->device; enum async_tx_flags flags_orig = submit->flags; dma_async_tx_callback cb_fn_orig = submit->cb_fn; dma_async_tx_callback cb_param_orig = submit->cb_param; int src_cnt = disks - 2; unsigned short pq_src_cnt; dma_addr_t dma_dest[2]; int src_off = 0; if (submit->flags & ASYNC_TX_FENCE) dma_flags |= DMA_PREP_FENCE; while (src_cnt > 0) { submit->flags = flags_orig; pq_src_cnt = min(src_cnt, dma_maxpq(dma, dma_flags)); /* if we are submitting additional pqs, leave the chain open, * clear the callback parameters, and leave the destination * buffers mapped */ if (src_cnt > pq_src_cnt) { submit->flags &= ~ASYNC_TX_ACK; submit->flags |= ASYNC_TX_FENCE; submit->cb_fn = NULL; submit->cb_param = NULL; } else { submit->cb_fn = cb_fn_orig; submit->cb_param = cb_param_orig; if (cb_fn_orig) dma_flags |= DMA_PREP_INTERRUPT; } /* Drivers force forward progress in case they can not provide * a descriptor */ for (;;) { dma_dest[0] = unmap->addr[disks - 2]; dma_dest[1] = unmap->addr[disks - 1]; tx = dma->device_prep_dma_pq(chan, dma_dest, &unmap->addr[src_off], pq_src_cnt, &scfs[src_off], unmap->len, dma_flags); if (likely(tx)) break; async_tx_quiesce(&submit->depend_tx); dma_async_issue_pending(chan); } dma_set_unmap(tx, unmap); async_tx_submit(chan, tx, submit); submit->depend_tx = tx; /* drop completed sources */ src_cnt -= pq_src_cnt; src_off += pq_src_cnt; dma_flags |= DMA_PREP_CONTINUE; } return tx; }
/** * async_syndrome_val - asynchronously validate a raid6 syndrome * @blocks: source blocks from idx 0..disks-3, P @ disks-2 and Q @ disks-1 * @offset: common offset into each block (src and dest) to start transaction * @disks: number of blocks (including missing P or Q, see below) * @len: length of operation in bytes * @pqres: on val failure SUM_CHECK_P_RESULT and/or SUM_CHECK_Q_RESULT are set * @spare: temporary result buffer for the synchronous case * @submit: submission / completion modifiers * * The same notes from async_gen_syndrome apply to the 'blocks', * and 'disks' parameters of this routine. The synchronous path * requires a temporary result buffer and submit->scribble to be * specified. */ struct dma_async_tx_descriptor * async_syndrome_val(struct page **blocks, unsigned int offset, int disks, size_t len, enum sum_check_flags *pqres, struct page *spare, struct async_submit_ctl *submit) { struct dma_chan *chan = pq_val_chan(submit, blocks, disks, len); struct dma_device *device = chan ? chan->device : NULL; struct dma_async_tx_descriptor *tx; unsigned char coefs[disks-2]; enum dma_ctrl_flags dma_flags = submit->cb_fn ? DMA_PREP_INTERRUPT : 0; struct dmaengine_unmap_data *unmap = NULL; BUG_ON(disks < 4); if (device) unmap = dmaengine_get_unmap_data(device->dev, disks, GFP_NOWAIT); if (unmap && disks <= dma_maxpq(device, 0) && is_dma_pq_aligned(device, offset, 0, len)) { struct device *dev = device->dev; dma_addr_t pq[2]; int i, j = 0, src_cnt = 0; pr_debug("%s: (async) disks: %d len: %zu\n", __func__, disks, len); unmap->len = len; for (i = 0; i < disks-2; i++) if (likely(blocks[i])) { unmap->addr[j] = dma_map_page(dev, blocks[i], offset, len, DMA_TO_DEVICE); coefs[j] = raid6_gfexp[i]; unmap->to_cnt++; src_cnt++; j++; } if (!P(blocks, disks)) { pq[0] = 0; dma_flags |= DMA_PREP_PQ_DISABLE_P; } else { pq[0] = dma_map_page(dev, P(blocks, disks), offset, len, DMA_TO_DEVICE); unmap->addr[j++] = pq[0]; unmap->to_cnt++; } if (!Q(blocks, disks)) { pq[1] = 0; dma_flags |= DMA_PREP_PQ_DISABLE_Q; } else { pq[1] = dma_map_page(dev, Q(blocks, disks), offset, len, DMA_TO_DEVICE); unmap->addr[j++] = pq[1]; unmap->to_cnt++; } if (submit->flags & ASYNC_TX_FENCE) dma_flags |= DMA_PREP_FENCE; for (;;) { tx = device->device_prep_dma_pq_val(chan, pq, unmap->addr, src_cnt, coefs, len, pqres, dma_flags); if (likely(tx)) break; async_tx_quiesce(&submit->depend_tx); dma_async_issue_pending(chan); } dma_set_unmap(tx, unmap); async_tx_submit(chan, tx, submit); return tx; } else { struct page *p_src = P(blocks, disks); struct page *q_src = Q(blocks, disks); enum async_tx_flags flags_orig = submit->flags; dma_async_tx_callback cb_fn_orig = submit->cb_fn; void *scribble = submit->scribble; void *cb_param_orig = submit->cb_param; void *p, *q, *s; pr_debug("%s: (sync) disks: %d len: %zu\n", __func__, disks, len); /* caller must provide a temporary result buffer and * allow the input parameters to be preserved */ BUG_ON(!spare || !scribble); /* wait for any prerequisite operations */ async_tx_quiesce(&submit->depend_tx); /* recompute p and/or q into the temporary buffer and then * check to see the result matches the current value */ tx = NULL; *pqres = 0; if (p_src) { init_async_submit(submit, ASYNC_TX_XOR_ZERO_DST, NULL, NULL, NULL, scribble); tx = async_xor(spare, blocks, offset, disks-2, len, submit); async_tx_quiesce(&tx); p = page_address(p_src) + offset; s = page_address(spare) + offset; *pqres |= !!memcmp(p, s, len) << SUM_CHECK_P; } if (q_src) { P(blocks, disks) = NULL; Q(blocks, disks) = spare; init_async_submit(submit, 0, NULL, NULL, NULL, scribble); tx = async_gen_syndrome(blocks, offset, disks, len, submit); async_tx_quiesce(&tx); q = page_address(q_src) + offset; s = page_address(spare) + offset; *pqres |= !!memcmp(q, s, len) << SUM_CHECK_Q; } /* restore P, Q and submit */ P(blocks, disks) = p_src; Q(blocks, disks) = q_src; submit->cb_fn = cb_fn_orig; submit->cb_param = cb_param_orig; submit->flags = flags_orig; async_tx_sync_epilog(submit); return NULL; } }
/** * async_xor_val - attempt a xor parity check with a dma engine. * @dest: destination page used if the xor is performed synchronously * @src_list: array of source pages * @offset: offset in pages to start transaction * @src_cnt: number of source pages * @len: length in bytes * @result: 0 if sum == 0 else non-zero * @submit: submission / completion modifiers * * honored flags: ASYNC_TX_ACK * * src_list note: if the dest is also a source it must be at index zero. * The contents of this array will be overwritten if a scribble region * is not specified. */ struct dma_async_tx_descriptor * async_xor_val(struct page *dest, struct page **src_list, unsigned int offset, int src_cnt, size_t len, enum sum_check_flags *result, struct async_submit_ctl *submit) { struct dma_chan *chan = xor_val_chan(submit, dest, src_list, src_cnt, len); struct dma_device *device = chan ? chan->device : NULL; struct dma_async_tx_descriptor *tx = NULL; struct dmaengine_unmap_data *unmap = NULL; BUG_ON(src_cnt <= 1); if (device) unmap = dmaengine_get_unmap_data(device->dev, src_cnt, GFP_NOWAIT); if (unmap && src_cnt <= device->max_xor && is_dma_xor_aligned(device, offset, 0, len)) { unsigned long dma_prep_flags = 0; int i; pr_debug("%s: (async) len: %zu\n", __func__, len); if (submit->cb_fn) dma_prep_flags |= DMA_PREP_INTERRUPT; if (submit->flags & ASYNC_TX_FENCE) dma_prep_flags |= DMA_PREP_FENCE; for (i = 0; i < src_cnt; i++) { unmap->addr[i] = dma_map_page(device->dev, src_list[i], offset, len, DMA_TO_DEVICE); unmap->to_cnt++; } unmap->len = len; tx = device->device_prep_dma_xor_val(chan, unmap->addr, src_cnt, len, result, dma_prep_flags); if (unlikely(!tx)) { async_tx_quiesce(&submit->depend_tx); while (!tx) { dma_async_issue_pending(chan); tx = device->device_prep_dma_xor_val(chan, unmap->addr, src_cnt, len, result, dma_prep_flags); } } dma_set_unmap(tx, unmap); async_tx_submit(chan, tx, submit); } else { enum async_tx_flags flags_orig = submit->flags; pr_debug("%s: (sync) len: %zu\n", __func__, len); WARN_ONCE(device && src_cnt <= device->max_xor, "%s: no space for dma address conversion\n", __func__); submit->flags |= ASYNC_TX_XOR_DROP_DST; submit->flags &= ~ASYNC_TX_ACK; tx = async_xor(dest, src_list, offset, src_cnt, len, submit); async_tx_quiesce(&tx); *result = !page_is_zero(dest, offset, len) << SUM_CHECK_P; async_tx_sync_epilog(submit); submit->flags = flags_orig; } dmaengine_unmap_put(unmap); return tx; }