static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct cast6_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); be128 buf[CAST6_PARALLEL_BLOCKS]; struct crypt_priv crypt_ctx = { .ctx = &ctx->cast6_ctx, .fpu_enabled = false, }; struct lrw_crypt_req req = { .tbuf = buf, .tbuflen = sizeof(buf), .table_ctx = &ctx->lrw_table, .crypt_ctx = &crypt_ctx, .crypt_fn = encrypt_callback, }; int ret; desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; ret = lrw_crypt(desc, dst, src, nbytes, &req); cast6_fpu_end(crypt_ctx.fpu_enabled); return ret; } static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct cast6_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); be128 buf[CAST6_PARALLEL_BLOCKS]; struct crypt_priv crypt_ctx = { .ctx = &ctx->cast6_ctx, .fpu_enabled = false, }; struct lrw_crypt_req req = { .tbuf = buf, .tbuflen = sizeof(buf), .table_ctx = &ctx->lrw_table, .crypt_ctx = &crypt_ctx, .crypt_fn = decrypt_callback, }; int ret; desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; ret = lrw_crypt(desc, dst, src, nbytes, &req); cast6_fpu_end(crypt_ctx.fpu_enabled); return ret; } static void lrw_exit_tfm(struct crypto_tfm *tfm) { struct cast6_lrw_ctx *ctx = crypto_tfm_ctx(tfm); lrw_free_table(&ctx->lrw_table); } struct cast6_xts_ctx { struct cast6_ctx tweak_ctx; struct cast6_ctx crypt_ctx; }; static int xts_cast6_setkey(struct crypto_tfm *tfm, const u8 *key, unsigned int keylen) { struct cast6_xts_ctx *ctx = crypto_tfm_ctx(tfm); u32 *flags = &tfm->crt_flags; int err; err = xts_check_key(tfm, key, keylen); if (err) return err; /* first half of xts-key is for crypt */ err = __cast6_setkey(&ctx->crypt_ctx, key, keylen / 2, flags); if (err) return err; /* second half of xts-key is for tweak */ return __cast6_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2, flags); } static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct cast6_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); return glue_xts_crypt_128bit(&cast6_enc_xts, desc, dst, src, nbytes, XTS_TWEAK_CAST(__cast6_encrypt), &ctx->tweak_ctx, &ctx->crypt_ctx); } static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct cast6_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); return glue_xts_crypt_128bit(&cast6_dec_xts, desc, dst, src, nbytes, XTS_TWEAK_CAST(__cast6_encrypt), &ctx->tweak_ctx, &ctx->crypt_ctx); } static struct crypto_alg cast6_algs[10] = { { .cra_name = "__ecb-cast6-avx", .cra_driver_name = "__driver-ecb-cast6-avx", .cra_priority = 0, .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, .cra_blocksize = CAST6_BLOCK_SIZE, .cra_ctxsize = sizeof(struct cast6_ctx), .cra_alignmask = 0, .cra_type = &crypto_blkcipher_type, .cra_module = THIS_MODULE, .cra_u = { .blkcipher = { .min_keysize = CAST6_MIN_KEY_SIZE, .max_keysize = CAST6_MAX_KEY_SIZE, .setkey = cast6_setkey, .encrypt = ecb_encrypt, .decrypt = ecb_decrypt, }, }, }, { .cra_name = "__cbc-cast6-avx", .cra_driver_name = "__driver-cbc-cast6-avx", .cra_priority = 0, .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, .cra_blocksize = CAST6_BLOCK_SIZE, .cra_ctxsize = sizeof(struct cast6_ctx), .cra_alignmask = 0, .cra_type = &crypto_blkcipher_type, .cra_module = THIS_MODULE, .cra_u = { .blkcipher = { .min_keysize = CAST6_MIN_KEY_SIZE, .max_keysize = CAST6_MAX_KEY_SIZE, .setkey = cast6_setkey, .encrypt = cbc_encrypt, .decrypt = cbc_decrypt, }, }, }, { .cra_name = "__ctr-cast6-avx", .cra_driver_name = "__driver-ctr-cast6-avx", .cra_priority = 0, .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, .cra_blocksize = 1, .cra_ctxsize = sizeof(struct cast6_ctx), .cra_alignmask = 0, .cra_type = &crypto_blkcipher_type, .cra_module = THIS_MODULE, .cra_u = { .blkcipher = { .min_keysize = CAST6_MIN_KEY_SIZE, .max_keysize = CAST6_MAX_KEY_SIZE, .ivsize = CAST6_BLOCK_SIZE, .setkey = cast6_setkey, .encrypt = ctr_crypt, .decrypt = ctr_crypt, }, }, }, { .cra_name = "__lrw-cast6-avx", .cra_driver_name = "__driver-lrw-cast6-avx", .cra_priority = 0, .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, .cra_blocksize = CAST6_BLOCK_SIZE, .cra_ctxsize = sizeof(struct cast6_lrw_ctx), .cra_alignmask = 0, .cra_type = &crypto_blkcipher_type, .cra_module = THIS_MODULE, .cra_exit = lrw_exit_tfm, .cra_u = { .blkcipher = { .min_keysize = CAST6_MIN_KEY_SIZE + CAST6_BLOCK_SIZE, .max_keysize = CAST6_MAX_KEY_SIZE + CAST6_BLOCK_SIZE, .ivsize = CAST6_BLOCK_SIZE, .setkey = lrw_cast6_setkey, .encrypt = lrw_encrypt, .decrypt = lrw_decrypt, }, }, }, { .cra_name = "__xts-cast6-avx",
static int crypt(struct blkcipher_desc *d, struct blkcipher_walk *w, struct priv *ctx, void (*fn)(struct crypto_tfm *, u8 *, const u8 *)) { int err; unsigned int avail; const int bs = LRW_BLOCK_SIZE; struct sinfo s = { .tfm = crypto_cipher_tfm(ctx->child), .fn = fn }; be128 *iv; u8 *wsrc; u8 *wdst; err = blkcipher_walk_virt(d, w); if (!(avail = w->nbytes)) return err; wsrc = w->src.virt.addr; wdst = w->dst.virt.addr; /* calculate first value of T */ iv = (be128 *)w->iv; s.t = *iv; /* T <- I*Key2 */ gf128mul_64k_bbe(&s.t, ctx->table.table); goto first; for (;;) { do { /* T <- I*Key2, using the optimization * discussed in the specification */ be128_xor(&s.t, &s.t, &ctx->table.mulinc[get_index128(iv)]); inc(iv); first: lrw_round(&s, wdst, wsrc); wsrc += bs; wdst += bs; } while ((avail -= bs) >= bs); err = blkcipher_walk_done(d, w, avail); if (!(avail = w->nbytes)) break; wsrc = w->src.virt.addr; wdst = w->dst.virt.addr; } return err; } static int encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct priv *ctx = crypto_blkcipher_ctx(desc->tfm); struct blkcipher_walk w; blkcipher_walk_init(&w, dst, src, nbytes); return crypt(desc, &w, ctx, crypto_cipher_alg(ctx->child)->cia_encrypt); } static int decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct priv *ctx = crypto_blkcipher_ctx(desc->tfm); struct blkcipher_walk w; blkcipher_walk_init(&w, dst, src, nbytes); return crypt(desc, &w, ctx, crypto_cipher_alg(ctx->child)->cia_decrypt); } int lrw_crypt(struct blkcipher_desc *desc, struct scatterlist *sdst, struct scatterlist *ssrc, unsigned int nbytes, struct lrw_crypt_req *req) { const unsigned int bsize = LRW_BLOCK_SIZE; const unsigned int max_blks = req->tbuflen / bsize; struct lrw_table_ctx *ctx = req->table_ctx; struct blkcipher_walk walk; unsigned int nblocks; be128 *iv, *src, *dst, *t; be128 *t_buf = req->tbuf; int err, i; BUG_ON(max_blks < 1); blkcipher_walk_init(&walk, sdst, ssrc, nbytes); err = blkcipher_walk_virt(desc, &walk); nbytes = walk.nbytes; if (!nbytes) return err; nblocks = min(walk.nbytes / bsize, max_blks); src = (be128 *)walk.src.virt.addr; dst = (be128 *)walk.dst.virt.addr; /* calculate first value of T */ iv = (be128 *)walk.iv; t_buf[0] = *iv; /* T <- I*Key2 */ gf128mul_64k_bbe(&t_buf[0], ctx->table); i = 0; goto first; for (;;) { do { for (i = 0; i < nblocks; i++) { /* T <- I*Key2, using the optimization * discussed in the specification */ be128_xor(&t_buf[i], t, &ctx->mulinc[get_index128(iv)]); inc(iv); first: t = &t_buf[i]; /* PP <- T xor P */ be128_xor(dst + i, t, src + i); } /* CC <- E(Key2,PP) */ req->crypt_fn(req->crypt_ctx, (u8 *)dst, nblocks * bsize); /* C <- T xor CC */ for (i = 0; i < nblocks; i++) be128_xor(dst + i, dst + i, &t_buf[i]); src += nblocks; dst += nblocks; nbytes -= nblocks * bsize; nblocks = min(nbytes / bsize, max_blks); } while (nblocks > 0); err = blkcipher_walk_done(desc, &walk, nbytes); nbytes = walk.nbytes; if (!nbytes) break; nblocks = min(nbytes / bsize, max_blks); src = (be128 *)walk.src.virt.addr; dst = (be128 *)walk.dst.virt.addr; } return err; } EXPORT_SYMBOL_GPL(lrw_crypt); static int init_tfm(struct crypto_tfm *tfm) { struct crypto_cipher *cipher; struct crypto_instance *inst = (void *)tfm->__crt_alg; struct crypto_spawn *spawn = crypto_instance_ctx(inst); struct priv *ctx = crypto_tfm_ctx(tfm); u32 *flags = &tfm->crt_flags; cipher = crypto_spawn_cipher(spawn); if (IS_ERR(cipher)) return PTR_ERR(cipher); if (crypto_cipher_blocksize(cipher) != LRW_BLOCK_SIZE) { *flags |= CRYPTO_TFM_RES_BAD_BLOCK_LEN; crypto_free_cipher(cipher); return -EINVAL; } ctx->child = cipher; return 0; } static void exit_tfm(struct crypto_tfm *tfm) { struct priv *ctx = crypto_tfm_ctx(tfm); lrw_free_table(&ctx->table); crypto_free_cipher(ctx->child); } static struct crypto_instance *alloc(struct rtattr **tb) { struct crypto_instance *inst; struct crypto_alg *alg; int err; err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_BLKCIPHER); if (err) return ERR_PTR(err); alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_CIPHER, CRYPTO_ALG_TYPE_MASK); if (IS_ERR(alg)) return ERR_CAST(alg); inst = crypto_alloc_instance("lrw", alg); if (IS_ERR(inst)) goto out_put_alg; inst->alg.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER; inst->alg.cra_priority = alg->cra_priority; inst->alg.cra_blocksize = alg->cra_blocksize; if (alg->cra_alignmask < 7) inst->alg.cra_alignmask = 7; else inst->alg.cra_alignmask = alg->cra_alignmask; inst->alg.cra_type = &crypto_blkcipher_type; if (!(alg->cra_blocksize % 4)) inst->alg.cra_alignmask |= 3; inst->alg.cra_blkcipher.ivsize = alg->cra_blocksize; inst->alg.cra_blkcipher.min_keysize = alg->cra_cipher.cia_min_keysize + alg->cra_blocksize; inst->alg.cra_blkcipher.max_keysize = alg->cra_cipher.cia_max_keysize + alg->cra_blocksize; inst->alg.cra_ctxsize = sizeof(struct priv); inst->alg.cra_init = init_tfm; inst->alg.cra_exit = exit_tfm; inst->alg.cra_blkcipher.setkey = setkey; inst->alg.cra_blkcipher.encrypt = encrypt; inst->alg.cra_blkcipher.decrypt = decrypt; out_put_alg: crypto_mod_put(alg); return inst; } static void free(struct crypto_instance *inst) { crypto_drop_spawn(crypto_instance_ctx(inst)); kfree(inst); }
static void lrw_aesni_exit_tfm(struct crypto_tfm *tfm) { struct aesni_lrw_ctx *ctx = crypto_tfm_ctx(tfm); lrw_free_table(&ctx->lrw_table); }
static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); be128 buf[3]; struct lrw_crypt_req req = { .tbuf = buf, .tbuflen = sizeof(buf), .table_ctx = &ctx->lrw_table, .crypt_ctx = &ctx->twofish_ctx, .crypt_fn = encrypt_callback, }; return lrw_crypt(desc, dst, src, nbytes, &req); } static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); be128 buf[3]; struct lrw_crypt_req req = { .tbuf = buf, .tbuflen = sizeof(buf), .table_ctx = &ctx->lrw_table, .crypt_ctx = &ctx->twofish_ctx, .crypt_fn = decrypt_callback, }; return lrw_crypt(desc, dst, src, nbytes, &req); } void lrw_twofish_exit_tfm(struct crypto_tfm *tfm) { struct twofish_lrw_ctx *ctx = crypto_tfm_ctx(tfm); lrw_free_table(&ctx->lrw_table); } EXPORT_SYMBOL_GPL(lrw_twofish_exit_tfm); int xts_twofish_setkey(struct crypto_tfm *tfm, const u8 *key, unsigned int keylen) { struct twofish_xts_ctx *ctx = crypto_tfm_ctx(tfm); u32 *flags = &tfm->crt_flags; int err; /* key consists of keys of equal size concatenated, therefore * the length must be even */ if (keylen % 2) { *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; return -EINVAL; } /* first half of xts-key is for crypt */ err = __twofish_setkey(&ctx->crypt_ctx, key, keylen / 2, flags); if (err) return err; /* second half of xts-key is for tweak */ return __twofish_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2, flags); } EXPORT_SYMBOL_GPL(xts_twofish_setkey); static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); be128 buf[3]; struct xts_crypt_req req = { .tbuf = buf, .tbuflen = sizeof(buf), .tweak_ctx = &ctx->tweak_ctx, .tweak_fn = XTS_TWEAK_CAST(twofish_enc_blk), .crypt_ctx = &ctx->crypt_ctx, .crypt_fn = encrypt_callback, }; return xts_crypt(desc, dst, src, nbytes, &req); } static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); be128 buf[3]; struct xts_crypt_req req = { .tbuf = buf, .tbuflen = sizeof(buf), .tweak_ctx = &ctx->tweak_ctx, .tweak_fn = XTS_TWEAK_CAST(twofish_enc_blk), .crypt_ctx = &ctx->crypt_ctx, .crypt_fn = decrypt_callback, }; return xts_crypt(desc, dst, src, nbytes, &req); } static struct crypto_alg tf_algs[5] = { { .cra_name = "ecb(twofish)", .cra_driver_name = "ecb-twofish-3way", .cra_priority = 300, .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, .cra_blocksize = TF_BLOCK_SIZE, .cra_ctxsize = sizeof(struct twofish_ctx), .cra_alignmask = 0, .cra_type = &crypto_blkcipher_type, .cra_module = THIS_MODULE, .cra_list = LIST_HEAD_INIT(tf_algs[0].cra_list), .cra_u = { .blkcipher = { .min_keysize = TF_MIN_KEY_SIZE, .max_keysize = TF_MAX_KEY_SIZE, .setkey = twofish_setkey, .encrypt = ecb_encrypt, .decrypt = ecb_decrypt, }, }, }, { .cra_name = "cbc(twofish)", .cra_driver_name = "cbc-twofish-3way", .cra_priority = 300, .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, .cra_blocksize = TF_BLOCK_SIZE, .cra_ctxsize = sizeof(struct twofish_ctx), .cra_alignmask = 0, .cra_type = &crypto_blkcipher_type, .cra_module = THIS_MODULE, .cra_list = LIST_HEAD_INIT(tf_algs[1].cra_list), .cra_u = { .blkcipher = { .min_keysize = TF_MIN_KEY_SIZE, .max_keysize = TF_MAX_KEY_SIZE, .ivsize = TF_BLOCK_SIZE, .setkey = twofish_setkey, .encrypt = cbc_encrypt, .decrypt = cbc_decrypt, }, }, }, { .cra_name = "ctr(twofish)", .cra_driver_name = "ctr-twofish-3way", .cra_priority = 300, .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, .cra_blocksize = 1, .cra_ctxsize = sizeof(struct twofish_ctx), .cra_alignmask = 0, .cra_type = &crypto_blkcipher_type, .cra_module = THIS_MODULE, .cra_list = LIST_HEAD_INIT(tf_algs[2].cra_list), .cra_u = { .blkcipher = { .min_keysize = TF_MIN_KEY_SIZE, .max_keysize = TF_MAX_KEY_SIZE, .ivsize = TF_BLOCK_SIZE, .setkey = twofish_setkey, .encrypt = ctr_crypt, .decrypt = ctr_crypt, }, }, }, { .cra_name = "lrw(twofish)", .cra_driver_name = "lrw-twofish-3way", .cra_priority = 300, .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, .cra_blocksize = TF_BLOCK_SIZE, .cra_ctxsize = sizeof(struct twofish_lrw_ctx), .cra_alignmask = 0, .cra_type = &crypto_blkcipher_type, .cra_module = THIS_MODULE, .cra_list = LIST_HEAD_INIT(tf_algs[3].cra_list), .cra_exit = lrw_twofish_exit_tfm, .cra_u = { .blkcipher = { .min_keysize = TF_MIN_KEY_SIZE + TF_BLOCK_SIZE, .max_keysize = TF_MAX_KEY_SIZE + TF_BLOCK_SIZE, .ivsize = TF_BLOCK_SIZE, .setkey = lrw_twofish_setkey, .encrypt = lrw_encrypt, .decrypt = lrw_decrypt, }, }, }, { .cra_name = "xts(twofish)",
static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct serpent_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); be128 buf[SERPENT_PARALLEL_BLOCKS]; struct crypt_priv crypt_ctx = { .ctx = &ctx->serpent_ctx, .fpu_enabled = false, }; struct lrw_crypt_req req = { .tbuf = buf, .tbuflen = sizeof(buf), .table_ctx = &ctx->lrw_table, .crypt_ctx = &crypt_ctx, .crypt_fn = encrypt_callback, }; int ret; desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; ret = lrw_crypt(desc, dst, src, nbytes, &req); serpent_fpu_end(crypt_ctx.fpu_enabled); return ret; } static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct serpent_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); be128 buf[SERPENT_PARALLEL_BLOCKS]; struct crypt_priv crypt_ctx = { .ctx = &ctx->serpent_ctx, .fpu_enabled = false, }; struct lrw_crypt_req req = { .tbuf = buf, .tbuflen = sizeof(buf), .table_ctx = &ctx->lrw_table, .crypt_ctx = &crypt_ctx, .crypt_fn = decrypt_callback, }; int ret; desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; ret = lrw_crypt(desc, dst, src, nbytes, &req); serpent_fpu_end(crypt_ctx.fpu_enabled); return ret; } void lrw_serpent_exit_tfm(struct crypto_tfm *tfm) { struct serpent_lrw_ctx *ctx = crypto_tfm_ctx(tfm); lrw_free_table(&ctx->lrw_table); } EXPORT_SYMBOL_GPL(lrw_serpent_exit_tfm); int xts_serpent_setkey(struct crypto_tfm *tfm, const u8 *key, unsigned int keylen) { struct serpent_xts_ctx *ctx = crypto_tfm_ctx(tfm); u32 *flags = &tfm->crt_flags; int err; /* key consists of keys of equal size concatenated, therefore * the length must be even */ if (keylen % 2) { *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; return -EINVAL; } /* first half of xts-key is for crypt */ err = __serpent_setkey(&ctx->crypt_ctx, key, keylen / 2); if (err) return err; /* second half of xts-key is for tweak */ return __serpent_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2); } EXPORT_SYMBOL_GPL(xts_serpent_setkey); static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); return glue_xts_crypt_128bit(&serpent_enc_xts, desc, dst, src, nbytes, XTS_TWEAK_CAST(__serpent_encrypt), &ctx->tweak_ctx, &ctx->crypt_ctx); } static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); return glue_xts_crypt_128bit(&serpent_dec_xts, desc, dst, src, nbytes, XTS_TWEAK_CAST(__serpent_encrypt), &ctx->tweak_ctx, &ctx->crypt_ctx); } static struct crypto_alg serpent_algs[10] = { { .cra_name = "__ecb-serpent-avx", .cra_driver_name = "__driver-ecb-serpent-avx", .cra_priority = 0, .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | CRYPTO_ALG_INTERNAL, .cra_blocksize = SERPENT_BLOCK_SIZE, .cra_ctxsize = sizeof(struct serpent_ctx), .cra_alignmask = 0, .cra_type = &crypto_blkcipher_type, .cra_module = THIS_MODULE, .cra_u = { .blkcipher = { .min_keysize = SERPENT_MIN_KEY_SIZE, .max_keysize = SERPENT_MAX_KEY_SIZE, .setkey = serpent_setkey, .encrypt = ecb_encrypt, .decrypt = ecb_decrypt, }, }, }, { .cra_name = "__cbc-serpent-avx", .cra_driver_name = "__driver-cbc-serpent-avx", .cra_priority = 0, .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | CRYPTO_ALG_INTERNAL, .cra_blocksize = SERPENT_BLOCK_SIZE, .cra_ctxsize = sizeof(struct serpent_ctx), .cra_alignmask = 0, .cra_type = &crypto_blkcipher_type, .cra_module = THIS_MODULE, .cra_u = { .blkcipher = { .min_keysize = SERPENT_MIN_KEY_SIZE, .max_keysize = SERPENT_MAX_KEY_SIZE, .setkey = serpent_setkey, .encrypt = cbc_encrypt, .decrypt = cbc_decrypt, }, }, }, { .cra_name = "__ctr-serpent-avx", .cra_driver_name = "__driver-ctr-serpent-avx", .cra_priority = 0, .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | CRYPTO_ALG_INTERNAL, .cra_blocksize = 1, .cra_ctxsize = sizeof(struct serpent_ctx), .cra_alignmask = 0, .cra_type = &crypto_blkcipher_type, .cra_module = THIS_MODULE, .cra_u = { .blkcipher = { .min_keysize = SERPENT_MIN_KEY_SIZE, .max_keysize = SERPENT_MAX_KEY_SIZE, .ivsize = SERPENT_BLOCK_SIZE, .setkey = serpent_setkey, .encrypt = ctr_crypt, .decrypt = ctr_crypt, }, }, }, { .cra_name = "__lrw-serpent-avx", .cra_driver_name = "__driver-lrw-serpent-avx", .cra_priority = 0, .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | CRYPTO_ALG_INTERNAL, .cra_blocksize = SERPENT_BLOCK_SIZE, .cra_ctxsize = sizeof(struct serpent_lrw_ctx), .cra_alignmask = 0, .cra_type = &crypto_blkcipher_type, .cra_module = THIS_MODULE, .cra_exit = lrw_serpent_exit_tfm, .cra_u = { .blkcipher = { .min_keysize = SERPENT_MIN_KEY_SIZE + SERPENT_BLOCK_SIZE, .max_keysize = SERPENT_MAX_KEY_SIZE + SERPENT_BLOCK_SIZE, .ivsize = SERPENT_BLOCK_SIZE, .setkey = lrw_serpent_setkey, .encrypt = lrw_encrypt, .decrypt = lrw_decrypt, }, }, }, { .cra_name = "__xts-serpent-avx",
static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct cast6_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); be128 buf[CAST6_PARALLEL_BLOCKS]; struct crypt_priv crypt_ctx = { .ctx = &ctx->cast6_ctx, .fpu_enabled = false, }; struct lrw_crypt_req req = { .tbuf = buf, .tbuflen = sizeof(buf), .table_ctx = &ctx->lrw_table, .crypt_ctx = &crypt_ctx, .crypt_fn = encrypt_callback, }; int ret; desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; ret = lrw_crypt(desc, dst, src, nbytes, &req); cast6_fpu_end(crypt_ctx.fpu_enabled); return ret; } static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct cast6_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); be128 buf[CAST6_PARALLEL_BLOCKS]; struct crypt_priv crypt_ctx = { .ctx = &ctx->cast6_ctx, .fpu_enabled = false, }; struct lrw_crypt_req req = { .tbuf = buf, .tbuflen = sizeof(buf), .table_ctx = &ctx->lrw_table, .crypt_ctx = &crypt_ctx, .crypt_fn = decrypt_callback, }; int ret; desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; ret = lrw_crypt(desc, dst, src, nbytes, &req); cast6_fpu_end(crypt_ctx.fpu_enabled); return ret; } static void lrw_exit_tfm(struct crypto_tfm *tfm) { struct cast6_lrw_ctx *ctx = crypto_tfm_ctx(tfm); lrw_free_table(&ctx->lrw_table); } struct cast6_xts_ctx { struct cast6_ctx tweak_ctx; struct cast6_ctx crypt_ctx; }; static int xts_cast6_setkey(struct crypto_tfm *tfm, const u8 *key, unsigned int keylen) { struct cast6_xts_ctx *ctx = crypto_tfm_ctx(tfm); u32 *flags = &tfm->crt_flags; int err; /* key consists of keys of equal size concatenated, therefore * the length must be even */ if (keylen % 2) { *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; return -EINVAL; } /* first half of xts-key is for crypt */ err = __cast6_setkey(&ctx->crypt_ctx, key, keylen / 2, flags); if (err) return err; /* second half of xts-key is for tweak */ return __cast6_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2, flags); } static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct cast6_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); be128 buf[CAST6_PARALLEL_BLOCKS]; struct crypt_priv crypt_ctx = { .ctx = &ctx->crypt_ctx, .fpu_enabled = false, }; struct xts_crypt_req req = { .tbuf = buf, .tbuflen = sizeof(buf), .tweak_ctx = &ctx->tweak_ctx, .tweak_fn = XTS_TWEAK_CAST(__cast6_encrypt), .crypt_ctx = &crypt_ctx, .crypt_fn = encrypt_callback, }; int ret; desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; ret = xts_crypt(desc, dst, src, nbytes, &req); cast6_fpu_end(crypt_ctx.fpu_enabled); return ret; } static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct cast6_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); be128 buf[CAST6_PARALLEL_BLOCKS]; struct crypt_priv crypt_ctx = { .ctx = &ctx->crypt_ctx, .fpu_enabled = false, }; struct xts_crypt_req req = { .tbuf = buf, .tbuflen = sizeof(buf), .tweak_ctx = &ctx->tweak_ctx, .tweak_fn = XTS_TWEAK_CAST(__cast6_encrypt), .crypt_ctx = &crypt_ctx, .crypt_fn = decrypt_callback, }; int ret; desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; ret = xts_crypt(desc, dst, src, nbytes, &req); cast6_fpu_end(crypt_ctx.fpu_enabled); return ret; } static struct crypto_alg cast6_algs[10] = { { .cra_name = "__ecb-cast6-avx", .cra_driver_name = "__driver-ecb-cast6-avx", .cra_priority = 0, .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, .cra_blocksize = CAST6_BLOCK_SIZE, .cra_ctxsize = sizeof(struct cast6_ctx), .cra_alignmask = 0, .cra_type = &crypto_blkcipher_type, .cra_module = THIS_MODULE, .cra_u = { .blkcipher = { .min_keysize = CAST6_MIN_KEY_SIZE, .max_keysize = CAST6_MAX_KEY_SIZE, .setkey = cast6_setkey, .encrypt = ecb_encrypt, .decrypt = ecb_decrypt, }, }, }, { .cra_name = "__cbc-cast6-avx", .cra_driver_name = "__driver-cbc-cast6-avx", .cra_priority = 0, .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, .cra_blocksize = CAST6_BLOCK_SIZE, .cra_ctxsize = sizeof(struct cast6_ctx), .cra_alignmask = 0, .cra_type = &crypto_blkcipher_type, .cra_module = THIS_MODULE, .cra_u = { .blkcipher = { .min_keysize = CAST6_MIN_KEY_SIZE, .max_keysize = CAST6_MAX_KEY_SIZE, .setkey = cast6_setkey, .encrypt = cbc_encrypt, .decrypt = cbc_decrypt, }, }, }, { .cra_name = "__ctr-cast6-avx", .cra_driver_name = "__driver-ctr-cast6-avx", .cra_priority = 0, .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, .cra_blocksize = 1, .cra_ctxsize = sizeof(struct cast6_ctx), .cra_alignmask = 0, .cra_type = &crypto_blkcipher_type, .cra_module = THIS_MODULE, .cra_u = { .blkcipher = { .min_keysize = CAST6_MIN_KEY_SIZE, .max_keysize = CAST6_MAX_KEY_SIZE, .ivsize = CAST6_BLOCK_SIZE, .setkey = cast6_setkey, .encrypt = ctr_crypt, .decrypt = ctr_crypt, }, }, }, { .cra_name = "__lrw-cast6-avx", .cra_driver_name = "__driver-lrw-cast6-avx", .cra_priority = 0, .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, .cra_blocksize = CAST6_BLOCK_SIZE, .cra_ctxsize = sizeof(struct cast6_lrw_ctx), .cra_alignmask = 0, .cra_type = &crypto_blkcipher_type, .cra_module = THIS_MODULE, .cra_exit = lrw_exit_tfm, .cra_u = { .blkcipher = { .min_keysize = CAST6_MIN_KEY_SIZE + CAST6_BLOCK_SIZE, .max_keysize = CAST6_MAX_KEY_SIZE + CAST6_BLOCK_SIZE, .ivsize = CAST6_BLOCK_SIZE, .setkey = lrw_cast6_setkey, .encrypt = lrw_encrypt, .decrypt = lrw_decrypt, }, }, }, { .cra_name = "__xts-cast6-avx", .cra_driver_name = "__driver-xts-cast6-avx", .cra_priority = 0, .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, .cra_blocksize = CAST6_BLOCK_SIZE, .cra_ctxsize = sizeof(struct cast6_xts_ctx), .cra_alignmask = 0, .cra_type = &crypto_blkcipher_type, .cra_module = THIS_MODULE, .cra_u = { .blkcipher = { .min_keysize = CAST6_MIN_KEY_SIZE * 2, .max_keysize = CAST6_MAX_KEY_SIZE * 2, .ivsize = CAST6_BLOCK_SIZE, .setkey = xts_cast6_setkey, .encrypt = xts_encrypt, .decrypt = xts_decrypt, }, }, }, { .cra_name = "ecb(cast6)", .cra_driver_name = "ecb-cast6-avx", .cra_priority = 200, .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = CAST6_BLOCK_SIZE, .cra_ctxsize = sizeof(struct async_helper_ctx), .cra_alignmask = 0, .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = ablk_init, .cra_exit = ablk_exit, .cra_u = { .ablkcipher = { .min_keysize = CAST6_MIN_KEY_SIZE, .max_keysize = CAST6_MAX_KEY_SIZE, .setkey = ablk_set_key, .encrypt = ablk_encrypt, .decrypt = ablk_decrypt, }, }, }, { .cra_name = "cbc(cast6)", .cra_driver_name = "cbc-cast6-avx", .cra_priority = 200, .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = CAST6_BLOCK_SIZE, .cra_ctxsize = sizeof(struct async_helper_ctx), .cra_alignmask = 0, .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = ablk_init, .cra_exit = ablk_exit, .cra_u = { .ablkcipher = { .min_keysize = CAST6_MIN_KEY_SIZE, .max_keysize = CAST6_MAX_KEY_SIZE, .ivsize = CAST6_BLOCK_SIZE, .setkey = ablk_set_key, .encrypt = __ablk_encrypt, .decrypt = ablk_decrypt, }, }, }, { .cra_name = "ctr(cast6)", .cra_driver_name = "ctr-cast6-avx", .cra_priority = 200, .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = 1, .cra_ctxsize = sizeof(struct async_helper_ctx), .cra_alignmask = 0, .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = ablk_init, .cra_exit = ablk_exit, .cra_u = { .ablkcipher = { .min_keysize = CAST6_MIN_KEY_SIZE, .max_keysize = CAST6_MAX_KEY_SIZE, .ivsize = CAST6_BLOCK_SIZE, .setkey = ablk_set_key, .encrypt = ablk_encrypt, .decrypt = ablk_encrypt, .geniv = "chainiv", }, }, }, { .cra_name = "lrw(cast6)",