static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes) { const unsigned int bsize = TF_BLOCK_SIZE; struct crypt_priv *ctx = priv; int i; ctx->fpu_enabled = twofish_fpu_begin(ctx->fpu_enabled, nbytes); while (nbytes >= TF_AVX2_PARALLEL_BLOCKS * bsize) { twofish_ecb_dec_16way(ctx->ctx, srcdst, srcdst); srcdst += bsize * TF_AVX2_PARALLEL_BLOCKS; nbytes -= bsize * TF_AVX2_PARALLEL_BLOCKS; } while (nbytes >= 8 * bsize) { twofish_ecb_dec_8way(ctx->ctx, srcdst, srcdst); srcdst += bsize * 8; nbytes -= bsize * 8; } while (nbytes >= 3 * bsize) { twofish_dec_blk_3way(ctx->ctx, srcdst, srcdst); srcdst += bsize * 3; nbytes -= bsize * 3; } for (i = 0; i < nbytes / bsize; i++, srcdst += bsize) twofish_dec_blk(ctx->ctx, srcdst, srcdst); }
static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes) { const unsigned int bsize = TF_BLOCK_SIZE; struct twofish_ctx *ctx = priv; int i; if (nbytes == 3 * bsize) { twofish_dec_blk_3way(ctx, srcdst, srcdst); return; } for (i = 0; i < nbytes / bsize; i++, srcdst += bsize) twofish_dec_blk(ctx, srcdst, srcdst); }
static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes) { const unsigned int bsize = TF_BLOCK_SIZE; struct crypt_priv *ctx = priv; int i; ctx->fpu_enabled = twofish_fpu_begin(ctx->fpu_enabled, nbytes); if (nbytes == bsize * TWOFISH_PARALLEL_BLOCKS) { twofish_dec_blk_xway(ctx->ctx, srcdst, srcdst); return; } for (i = 0; i < nbytes / (bsize * 3); i++, srcdst += bsize * 3) twofish_dec_blk_3way(ctx->ctx, srcdst, srcdst); nbytes %= bsize * 3; for (i = 0; i < nbytes / bsize; i++, srcdst += bsize) twofish_dec_blk(ctx->ctx, srcdst, srcdst); }
static void twofish_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) { twofish_dec_blk(tfm, dst, src); }
static unsigned int __cbc_decrypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk) { struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); unsigned int bsize = TF_BLOCK_SIZE; unsigned int nbytes = walk->nbytes; u128 *src = (u128 *)walk->src.virt.addr; u128 *dst = (u128 *)walk->dst.virt.addr; u128 ivs[3 - 1]; u128 last_iv; /* Start of the last block. */ src += nbytes / bsize - 1; dst += nbytes / bsize - 1; last_iv = *src; /* Process three block batch */ if (nbytes >= bsize * 3) { do { nbytes -= bsize * (3 - 1); src -= 3 - 1; dst -= 3 - 1; ivs[0] = src[0]; ivs[1] = src[1]; twofish_dec_blk_3way(ctx, (u8 *)dst, (u8 *)src); u128_xor(dst + 1, dst + 1, ivs + 0); u128_xor(dst + 2, dst + 2, ivs + 1); nbytes -= bsize; if (nbytes < bsize) goto done; u128_xor(dst, dst, src - 1); src -= 1; dst -= 1; } while (nbytes >= bsize * 3); if (nbytes < bsize) goto done; } /* Handle leftovers */ for (;;) { twofish_dec_blk(ctx, (u8 *)dst, (u8 *)src); nbytes -= bsize; if (nbytes < bsize) break; u128_xor(dst, dst, src - 1); src -= 1; dst -= 1; } done: u128_xor(dst, dst, (u128 *)walk->iv); *(u128 *)walk->iv = last_iv; return nbytes; }