struct dma_chan * __async_tx_find_channel(struct async_submit_ctl *submit, enum dma_transaction_type tx_type) { struct dma_async_tx_descriptor *depend_tx = submit->depend_tx; if (depend_tx && dma_has_cap(tx_type, depend_tx->chan->device->cap_mask)) return depend_tx->chan; return async_dma_find_channel(tx_type); }
struct dma_async_tx_descriptor * async_raid6_datap_recov(int disks, size_t bytes, int faila, struct page **blocks, struct async_submit_ctl *submit) { struct dma_async_tx_descriptor *tx = NULL; struct page *p, *q, *dq; u8 coef; enum async_tx_flags flags = submit->flags; dma_async_tx_callback cb_fn = submit->cb_fn; void *cb_param = submit->cb_param; void *scribble = submit->scribble; int good_srcs, good, i; struct page *srcs[2]; pr_debug("%s: disks: %d len: %zu\n", __func__, disks, bytes); /* if a dma resource is not available or a scribble buffer is not * available punt to the synchronous path. In the 'dma not * available' case be sure to use the scribble buffer to * preserve the content of 'blocks' as the caller intended. */ if (!async_dma_find_channel(DMA_PQ) || !scribble) { void **ptrs = scribble ? scribble : (void **) blocks; async_tx_quiesce(&submit->depend_tx); for (i = 0; i < disks; i++) if (blocks[i] == NULL) ptrs[i] = (void*)raid6_empty_zero_page; else ptrs[i] = page_address(blocks[i]); raid6_datap_recov(disks, bytes, faila, ptrs); async_tx_sync_epilog(submit); return NULL; } good_srcs = 0; good = -1; for (i = 0; i < disks-2; i++) { if (i == faila) continue; if (blocks[i]) { good = i; good_srcs++; if (good_srcs > 1) break; } } BUG_ON(good_srcs == 0); p = blocks[disks-2]; q = blocks[disks-1]; /* Compute syndrome with zero for the missing data page * Use the dead data page as temporary storage for delta q */ dq = blocks[faila]; blocks[faila] = NULL; blocks[disks-1] = dq; /* in the 4-disk case we only need to perform a single source * multiplication with the one good data block. */ if (good_srcs == 1) { struct page *g = blocks[good]; init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble); tx = async_memcpy(p, g, 0, 0, bytes, submit); init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble); tx = async_mult(dq, g, raid6_gfexp[good], bytes, submit); } else { init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble); tx = async_gen_syndrome(blocks, 0, disks, bytes, submit); } /* Restore pointer table */ blocks[faila] = dq; blocks[disks-1] = q; /* calculate g^{-faila} */ coef = raid6_gfinv[raid6_gfexp[faila]]; srcs[0] = dq; srcs[1] = q; init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx, NULL, NULL, scribble); tx = async_xor(dq, srcs, 0, 2, bytes, submit); init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble); tx = async_mult(dq, dq, coef, bytes, submit); srcs[0] = p; srcs[1] = dq; init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn, cb_param, scribble); tx = async_xor(p, srcs, 0, 2, bytes, submit); return tx; }
struct dma_async_tx_descriptor * async_raid6_2data_recov(int disks, size_t bytes, int faila, int failb, struct page **blocks, struct async_submit_ctl *submit) { void *scribble = submit->scribble; int non_zero_srcs, i; BUG_ON(faila == failb); if (failb < faila) swap(faila, failb); pr_debug("%s: disks: %d len: %zu\n", __func__, disks, bytes); /* if a dma resource is not available or a scribble buffer is not * available punt to the synchronous path. In the 'dma not * available' case be sure to use the scribble buffer to * preserve the content of 'blocks' as the caller intended. */ if (!async_dma_find_channel(DMA_PQ) || !scribble) { void **ptrs = scribble ? scribble : (void **) blocks; async_tx_quiesce(&submit->depend_tx); for (i = 0; i < disks; i++) if (blocks[i] == NULL) ptrs[i] = (void *) raid6_empty_zero_page; else ptrs[i] = page_address(blocks[i]); raid6_2data_recov(disks, bytes, faila, failb, ptrs); async_tx_sync_epilog(submit); return NULL; } non_zero_srcs = 0; for (i = 0; i < disks-2 && non_zero_srcs < 4; i++) if (blocks[i]) non_zero_srcs++; switch (non_zero_srcs) { case 0: case 1: /* There must be at least 2 sources - the failed devices. */ BUG(); case 2: /* dma devices do not uniformly understand a zero source pq * operation (in contrast to the synchronous case), so * explicitly handle the special case of a 4 disk array with * both data disks missing. */ return __2data_recov_4(disks, bytes, faila, failb, blocks, submit); case 3: /* dma devices do not uniformly understand a single * source pq operation (in contrast to the synchronous * case), so explicitly handle the special case of a 5 disk * array with 2 of 3 data disks missing. */ return __2data_recov_5(disks, bytes, faila, failb, blocks, submit); default: return __2data_recov_n(disks, bytes, faila, failb, blocks, submit); } }