static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes) { const unsigned int bsize = TF_BLOCK_SIZE; struct crypt_priv *ctx = priv; int i; ctx->fpu_enabled = twofish_fpu_begin(ctx->fpu_enabled, nbytes); while (nbytes >= TF_AVX2_PARALLEL_BLOCKS * bsize) { twofish_ecb_enc_16way(ctx->ctx, srcdst, srcdst); srcdst += bsize * TF_AVX2_PARALLEL_BLOCKS; nbytes -= bsize * TF_AVX2_PARALLEL_BLOCKS; } while (nbytes >= 8 * bsize) { twofish_ecb_enc_8way(ctx->ctx, srcdst, srcdst); srcdst += bsize * 8; nbytes -= bsize * 8; } while (nbytes >= 3 * bsize) { twofish_enc_blk_3way(ctx->ctx, srcdst, srcdst); srcdst += bsize * 3; nbytes -= bsize * 3; } for (i = 0; i < nbytes / bsize; i++, srcdst += bsize) twofish_enc_blk(ctx->ctx, srcdst, srcdst); }
void twofish_enc_blk_ctr(void *ctx, u128 *dst, const u128 *src, u128 *iv) { be128 ctrblk; if (dst != src) *dst = *src; u128_to_be128(&ctrblk, iv); u128_inc(iv); twofish_enc_blk(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk); u128_xor(dst, dst, (u128 *)&ctrblk); }
static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes) { const unsigned int bsize = TF_BLOCK_SIZE; struct twofish_ctx *ctx = priv; int i; if (nbytes == 3 * bsize) { twofish_enc_blk_3way(ctx, srcdst, srcdst); return; } for (i = 0; i < nbytes / bsize; i++, srcdst += bsize) twofish_enc_blk(ctx, srcdst, srcdst); }
static void ctr_crypt_final(struct blkcipher_desc *desc, struct blkcipher_walk *walk) { struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); u8 *ctrblk = walk->iv; u8 keystream[TF_BLOCK_SIZE]; u8 *src = walk->src.virt.addr; u8 *dst = walk->dst.virt.addr; unsigned int nbytes = walk->nbytes; twofish_enc_blk(ctx, keystream, ctrblk); crypto_xor(keystream, src, nbytes); memcpy(dst, keystream, nbytes); crypto_inc(ctrblk, TF_BLOCK_SIZE); }
static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes) { const unsigned int bsize = TF_BLOCK_SIZE; struct crypt_priv *ctx = priv; int i; ctx->fpu_enabled = twofish_fpu_begin(ctx->fpu_enabled, nbytes); if (nbytes == bsize * TWOFISH_PARALLEL_BLOCKS) { twofish_enc_blk_xway(ctx->ctx, srcdst, srcdst); return; } for (i = 0; i < nbytes / (bsize * 3); i++, srcdst += bsize * 3) twofish_enc_blk_3way(ctx->ctx, srcdst, srcdst); nbytes %= bsize * 3; for (i = 0; i < nbytes / bsize; i++, srcdst += bsize) twofish_enc_blk(ctx->ctx, srcdst, srcdst); }
static unsigned int __cbc_encrypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk) { struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); unsigned int bsize = TF_BLOCK_SIZE; unsigned int nbytes = walk->nbytes; u128 *src = (u128 *)walk->src.virt.addr; u128 *dst = (u128 *)walk->dst.virt.addr; u128 *iv = (u128 *)walk->iv; do { u128_xor(dst, src, iv); twofish_enc_blk(ctx, (u8 *)dst, (u8 *)dst); iv = dst; src += 1; dst += 1; nbytes -= bsize; } while (nbytes >= bsize); u128_xor((u128 *)walk->iv, (u128 *)walk->iv, iv); return nbytes; }
static void twofish_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) { twofish_enc_blk(tfm, dst, src); }
static unsigned int __ctr_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk) { struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); unsigned int bsize = TF_BLOCK_SIZE; unsigned int nbytes = walk->nbytes; u128 *src = (u128 *)walk->src.virt.addr; u128 *dst = (u128 *)walk->dst.virt.addr; u128 ctrblk; be128 ctrblocks[3]; be128_to_u128(&ctrblk, (be128 *)walk->iv); /* Process three block batch */ if (nbytes >= bsize * 3) { do { if (dst != src) { dst[0] = src[0]; dst[1] = src[1]; dst[2] = src[2]; } /* create ctrblks for parallel encrypt */ u128_to_be128(&ctrblocks[0], &ctrblk); u128_inc(&ctrblk); u128_to_be128(&ctrblocks[1], &ctrblk); u128_inc(&ctrblk); u128_to_be128(&ctrblocks[2], &ctrblk); u128_inc(&ctrblk); twofish_enc_blk_xor_3way(ctx, (u8 *)dst, (u8 *)ctrblocks); src += 3; dst += 3; nbytes -= bsize * 3; } while (nbytes >= bsize * 3); if (nbytes < bsize) goto done; } /* Handle leftovers */ do { if (dst != src) *dst = *src; u128_to_be128(&ctrblocks[0], &ctrblk); u128_inc(&ctrblk); twofish_enc_blk(ctx, (u8 *)ctrblocks, (u8 *)ctrblocks); u128_xor(dst, dst, (u128 *)ctrblocks); src += 1; dst += 1; nbytes -= bsize; } while (nbytes >= bsize); done: u128_to_be128((be128 *)walk->iv, &ctrblk); return nbytes; }