static int ccm_encrypt(struct aead_request *req) { struct crypto_aead *aead = crypto_aead_reqtfm(req); struct crypto_aes_ctx *ctx = crypto_aead_ctx(aead); struct blkcipher_desc desc = { .info = req->iv }; struct blkcipher_walk walk; struct scatterlist srcbuf[2]; struct scatterlist dstbuf[2]; struct scatterlist *src; struct scatterlist *dst; u8 __aligned(8) mac[AES_BLOCK_SIZE]; u8 buf[AES_BLOCK_SIZE]; u32 len = req->cryptlen; int err; err = ccm_init_mac(req, mac, len); if (err) return err; kernel_neon_begin_partial(6); if (req->assoclen) ccm_calculate_auth_mac(req, mac); /* preserve the original iv for the final round */ memcpy(buf, req->iv, AES_BLOCK_SIZE); src = scatterwalk_ffwd(srcbuf, req->src, req->assoclen); dst = src; if (req->src != req->dst) dst = scatterwalk_ffwd(dstbuf, req->dst, req->assoclen); blkcipher_walk_init(&walk, dst, src, len); err = blkcipher_aead_walk_virt_block(&desc, &walk, aead, AES_BLOCK_SIZE); while (walk.nbytes) { u32 tail = walk.nbytes % AES_BLOCK_SIZE; if (walk.nbytes == len) tail = 0; ce_aes_ccm_encrypt(walk.dst.virt.addr, walk.src.virt.addr, walk.nbytes - tail, ctx->key_enc, num_rounds(ctx), mac, walk.iv); len -= walk.nbytes - tail; err = blkcipher_walk_done(&desc, &walk, tail); } if (!err) ce_aes_ccm_final(mac, buf, ctx->key_enc, num_rounds(ctx)); kernel_neon_end(); if (err) return err; /* copy authtag to end of dst */ scatterwalk_map_and_copy(mac, dst, req->cryptlen, crypto_aead_authsize(aead), 1); return 0; } static int ccm_decrypt(struct aead_request *req) { struct crypto_aead *aead = crypto_aead_reqtfm(req); struct crypto_aes_ctx *ctx = crypto_aead_ctx(aead); unsigned int authsize = crypto_aead_authsize(aead); struct blkcipher_desc desc = { .info = req->iv }; struct blkcipher_walk walk; struct scatterlist srcbuf[2]; struct scatterlist dstbuf[2]; struct scatterlist *src; struct scatterlist *dst; u8 __aligned(8) mac[AES_BLOCK_SIZE]; u8 buf[AES_BLOCK_SIZE]; u32 len = req->cryptlen - authsize; int err; err = ccm_init_mac(req, mac, len); if (err) return err; kernel_neon_begin_partial(6); if (req->assoclen) ccm_calculate_auth_mac(req, mac); /* preserve the original iv for the final round */ memcpy(buf, req->iv, AES_BLOCK_SIZE); src = scatterwalk_ffwd(srcbuf, req->src, req->assoclen); dst = src; if (req->src != req->dst) dst = scatterwalk_ffwd(dstbuf, req->dst, req->assoclen); blkcipher_walk_init(&walk, dst, src, len); err = blkcipher_aead_walk_virt_block(&desc, &walk, aead, AES_BLOCK_SIZE); while (walk.nbytes) { u32 tail = walk.nbytes % AES_BLOCK_SIZE; if (walk.nbytes == len) tail = 0; ce_aes_ccm_decrypt(walk.dst.virt.addr, walk.src.virt.addr, walk.nbytes - tail, ctx->key_enc, num_rounds(ctx), mac, walk.iv); len -= walk.nbytes - tail; err = blkcipher_walk_done(&desc, &walk, tail); } if (!err) ce_aes_ccm_final(mac, buf, ctx->key_enc, num_rounds(ctx)); kernel_neon_end(); if (err) return err; /* compare calculated auth tag with the stored one */ scatterwalk_map_and_copy(buf, src, req->cryptlen - authsize, authsize, 0); if (crypto_memneq(mac, buf, authsize)) return -EBADMSG; return 0; } static struct aead_alg ccm_aes_alg = { .base = { .cra_name = "ccm(aes)", .cra_driver_name = "ccm-aes-ce", .cra_flags = CRYPTO_ALG_AEAD_NEW, .cra_priority = 300, .cra_blocksize = 1, .cra_ctxsize = sizeof(struct crypto_aes_ctx), .cra_alignmask = 7, .cra_module = THIS_MODULE, }, .ivsize = AES_BLOCK_SIZE, .maxauthsize = AES_BLOCK_SIZE, .setkey = ccm_setkey, .setauthsize = ccm_setauthsize, .encrypt = ccm_encrypt, .decrypt = ccm_decrypt, }; static int __init aes_mod_init(void) { if (!(elf_hwcap & HWCAP_AES)) return -ENODEV; return crypto_register_aead(&ccm_aes_alg); } static void __exit aes_mod_exit(void) { crypto_unregister_aead(&ccm_aes_alg); }
struct simd_aead_alg *simd_aead_create_compat(const char *algname, const char *drvname, const char *basename) { struct simd_aead_alg *salg; struct crypto_aead *tfm; struct aead_alg *ialg; struct aead_alg *alg; int err; tfm = crypto_alloc_aead(basename, CRYPTO_ALG_INTERNAL, CRYPTO_ALG_INTERNAL | CRYPTO_ALG_ASYNC); if (IS_ERR(tfm)) return ERR_CAST(tfm); ialg = crypto_aead_alg(tfm); salg = kzalloc(sizeof(*salg), GFP_KERNEL); if (!salg) { salg = ERR_PTR(-ENOMEM); goto out_put_tfm; } salg->ialg_name = basename; alg = &salg->alg; err = -ENAMETOOLONG; if (snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", algname) >= CRYPTO_MAX_ALG_NAME) goto out_free_salg; if (snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", drvname) >= CRYPTO_MAX_ALG_NAME) goto out_free_salg; alg->base.cra_flags = CRYPTO_ALG_ASYNC; alg->base.cra_priority = ialg->base.cra_priority; alg->base.cra_blocksize = ialg->base.cra_blocksize; alg->base.cra_alignmask = ialg->base.cra_alignmask; alg->base.cra_module = ialg->base.cra_module; alg->base.cra_ctxsize = sizeof(struct simd_aead_ctx); alg->ivsize = ialg->ivsize; alg->maxauthsize = ialg->maxauthsize; alg->chunksize = ialg->chunksize; alg->init = simd_aead_init; alg->exit = simd_aead_exit; alg->setkey = simd_aead_setkey; alg->setauthsize = simd_aead_setauthsize; alg->encrypt = simd_aead_encrypt; alg->decrypt = simd_aead_decrypt; err = crypto_register_aead(alg); if (err) goto out_free_salg; out_put_tfm: crypto_free_aead(tfm); return salg; out_free_salg: kfree(salg); salg = ERR_PTR(err); goto out_put_tfm; }
static int __init aes_mod_init(void) { if (!cpu_have_named_feature(AES)) return -ENODEV; return crypto_register_aead(&ccm_aes_alg); }