static int crypto_rfc4543_copy_src_to_dst(struct aead_request *req, bool enc)
{
struct crypto_aead *aead = crypto_aead_reqtfm(req);
struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(aead);
unsigned int authsize = crypto_aead_authsize(aead);
unsigned int nbytes = req->cryptlen - (enc ? 0 : authsize);
struct blkcipher_desc desc = {
.tfm = ctx->null,
};
return crypto_blkcipher_encrypt(&desc, req->dst, req->src, nbytes);
}
static int crypto_rfc4543_encrypt(struct aead_request *req)
{
struct crypto_aead *aead = crypto_aead_reqtfm(req);
struct crypto_rfc4543_req_ctx *rctx = crypto_rfc4543_reqctx(req);
struct aead_request *subreq;
int err;
if (req->src != req->dst) {
err = crypto_rfc4543_copy_src_to_dst(req, true);
if (err)
return err;
}
subreq = crypto_rfc4543_crypt(req, true);
err = crypto_aead_encrypt(subreq);
if (err)
return err;
scatterwalk_map_and_copy(rctx->auth_tag, req->dst, req->cryptlen,
crypto_aead_authsize(aead), 1);
return 0;
}
static int crypto_rfc4543_decrypt(struct aead_request *req)
{
int err;
if (req->src != req->dst) {
err = crypto_rfc4543_copy_src_to_dst(req, false);
if (err)
return err;
}
req = crypto_rfc4543_crypt(req, false);
return crypto_aead_decrypt(req);
}
static int crypto_gcm_decrypt(struct aead_request *req)
{
struct crypto_aead *aead = crypto_aead_reqtfm(req);
struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
struct ablkcipher_request *abreq = &pctx->abreq;
struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash;
unsigned int cryptlen = req->cryptlen;
unsigned int authsize = crypto_aead_authsize(aead);
int err;
if (cryptlen < authsize)
return -EINVAL;
cryptlen -= authsize;
crypto_gcm_init_crypt(abreq, req, cryptlen);
ablkcipher_request_set_callback(abreq, aead_request_flags(req),
crypto_gcm_decrypt_done, req);
crypto_gcm_ghash_update_sg(ghash, req->src, cryptlen);
err = crypto_ablkcipher_decrypt(abreq);
if (err)
return err;
return crypto_gcm_verify(req);
}
static int crypto4xx_crypt_aes_ccm(struct aead_request *req, bool decrypt)
{
struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
struct crypto_aead *aead = crypto_aead_reqtfm(req);
unsigned int len = req->cryptlen;
__le32 iv[16];
u32 tmp_sa[ctx->sa_len * 4];
struct dynamic_sa_ctl *sa = (struct dynamic_sa_ctl *)tmp_sa;
if (crypto4xx_aead_need_fallback(req, true, decrypt))
return crypto4xx_aead_fallback(req, ctx, decrypt);
if (decrypt)
len -= crypto_aead_authsize(aead);
memcpy(tmp_sa, decrypt ? ctx->sa_in : ctx->sa_out, sizeof(tmp_sa));
sa->sa_command_0.bf.digest_len = crypto_aead_authsize(aead) >> 2;
if (req->iv[0] == 1) {
/* CRYPTO_MODE_AES_ICM */
sa->sa_command_1.bf.crypto_mode9_8 = 1;
}
iv[3] = cpu_to_le32(0);
crypto4xx_memcpy_to_le32(iv, req->iv, 16 - (req->iv[0] + 1));
return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst,
len, iv, sizeof(iv),
sa, ctx->sa_len, req->assoclen);
}
static int crypto_gcm_decrypt(struct aead_request *req)
{
struct crypto_aead *aead = crypto_aead_reqtfm(req);
struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
struct ablkcipher_request *abreq = &pctx->u.abreq;
struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
unsigned int authsize = crypto_aead_authsize(aead);
unsigned int cryptlen = req->cryptlen;
int err;
if (cryptlen < authsize)
return -EINVAL;
cryptlen -= authsize;
gctx->src = req->src;
gctx->cryptlen = cryptlen;
gctx->complete = gcm_dec_hash_done;
err = gcm_hash(req, pctx);
if (err)
return err;
ablkcipher_request_set_callback(abreq, aead_request_flags(req),
gcm_decrypt_done, req);
crypto_gcm_init_crypt(abreq, req, cryptlen);
err = crypto_ablkcipher_decrypt(abreq);
if (err)
return err;
return crypto_gcm_verify(req, pctx);
}
static int seqiv_aead_decrypt(struct aead_request *req)
{
struct crypto_aead *geniv = crypto_aead_reqtfm(req);
struct aead_geniv_ctx *ctx = crypto_aead_ctx(geniv);
struct aead_request *subreq = aead_request_ctx(req);
crypto_completion_t compl;
void *data;
unsigned int ivsize = 8;
if (req->cryptlen < ivsize + crypto_aead_authsize(geniv))
return -EINVAL;
aead_request_set_tfm(subreq, ctx->child);
compl = req->base.complete;
data = req->base.data;
aead_request_set_callback(subreq, req->base.flags, compl, data);
aead_request_set_crypt(subreq, req->src, req->dst,
req->cryptlen - ivsize, req->iv);
aead_request_set_ad(subreq, req->assoclen + ivsize);
scatterwalk_map_and_copy(req->iv, req->src, req->assoclen, ivsize, 0);
return crypto_aead_decrypt(subreq);
}
static int crypto_gcm_verify(struct aead_request *req)
{
struct crypto_aead *aead = crypto_aead_reqtfm(req);
struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash;
u8 *auth_tag = pctx->auth_tag;
u8 *iauth_tag = pctx->iauth_tag;
unsigned int authsize = crypto_aead_authsize(aead);
unsigned int cryptlen = req->cryptlen - authsize;
crypto_gcm_ghash_final_xor(ghash, req->assoclen, cryptlen, auth_tag);
authsize = crypto_aead_authsize(aead);
scatterwalk_map_and_copy(iauth_tag, req->src, cryptlen, authsize, 0);
return memcmp(iauth_tag, auth_tag, authsize) ? -EBADMSG : 0;
}
static int omap_aes_gcm_crypt(struct aead_request *req, unsigned long mode)
{
struct omap_aes_reqctx *rctx = aead_request_ctx(req);
struct crypto_aead *aead = crypto_aead_reqtfm(req);
unsigned int authlen = crypto_aead_authsize(aead);
struct omap_aes_dev *dd;
__be32 counter = cpu_to_be32(1);
int err, assoclen;
memset(rctx->auth_tag, 0, sizeof(rctx->auth_tag));
memcpy(rctx->iv + GCM_AES_IV_SIZE, &counter, 4);
err = do_encrypt_iv(req, (u32 *)rctx->auth_tag, (u32 *)rctx->iv);
if (err)
return err;
if (mode & FLAGS_RFC4106_GCM)
assoclen = req->assoclen - 8;
else
assoclen = req->assoclen;
if (assoclen + req->cryptlen == 0) {
scatterwalk_map_and_copy(rctx->auth_tag, req->dst, 0, authlen,
1);
return 0;
}
dd = omap_aes_find_dev(rctx);
if (!dd)
return -ENODEV;
rctx->mode = mode;
return omap_aes_gcm_handle_queue(dd, req);
}
static int crypto_ccm_decrypt(struct aead_request *req)
{
struct crypto_aead *aead = crypto_aead_reqtfm(req);
struct crypto_ccm_ctx *ctx = crypto_aead_ctx(aead);
struct crypto_ccm_req_priv_ctx *pctx = crypto_ccm_reqctx(req);
struct ablkcipher_request *abreq = &pctx->abreq;
struct scatterlist *dst;
unsigned int authsize = crypto_aead_authsize(aead);
unsigned int cryptlen = req->cryptlen;
u8 *authtag = pctx->auth_tag;
u8 *odata = pctx->odata;
u8 *iv = req->iv;
int err;
if (cryptlen < authsize)
return -EINVAL;
cryptlen -= authsize;
err = crypto_ccm_check_iv(iv);
if (err)
return err;
pctx->flags = aead_request_flags(req);
scatterwalk_map_and_copy(authtag, req->src, cryptlen, authsize, 0);
memset(iv + 15 - iv[0], 0, iv[0] + 1);
sg_init_table(pctx->src, 2);
sg_set_buf(pctx->src, authtag, 16);
scatterwalk_sg_chain(pctx->src, 2, req->src);
dst = pctx->src;
if (req->src != req->dst) {
sg_init_table(pctx->dst, 2);
sg_set_buf(pctx->dst, authtag, 16);
scatterwalk_sg_chain(pctx->dst, 2, req->dst);
dst = pctx->dst;
}
ablkcipher_request_set_tfm(abreq, ctx->ctr);
ablkcipher_request_set_callback(abreq, pctx->flags,
crypto_ccm_decrypt_done, req);
ablkcipher_request_set_crypt(abreq, pctx->src, dst, cryptlen + 16, iv);
err = crypto_ablkcipher_decrypt(abreq);
if (err)
return err;
err = crypto_ccm_auth(req, req->dst, cryptlen);
if (err)
return err;
/* verify */
if (crypto_memneq(authtag, odata, authsize))
return -EBADMSG;
return err;
}
static void gcm_enc_copy_hash(struct aead_request *req,
struct crypto_gcm_req_priv_ctx *pctx)
{
struct crypto_aead *aead = crypto_aead_reqtfm(req);
u8 *auth_tag = pctx->auth_tag;
scatterwalk_map_and_copy(auth_tag, req->dst, req->cryptlen,
crypto_aead_authsize(aead), 1);
}
static int ccm_encrypt(struct aead_request *req)
{
struct crypto_aead *aead = crypto_aead_reqtfm(req);
struct crypto_aes_ctx *ctx = crypto_aead_ctx(aead);
struct skcipher_walk walk;
u8 __aligned(8) mac[AES_BLOCK_SIZE];
u8 buf[AES_BLOCK_SIZE];
u32 len = req->cryptlen;
int err;
err = ccm_init_mac(req, mac, len);
if (err)
return err;
if (req->assoclen)
ccm_calculate_auth_mac(req, mac);
/* preserve the original iv for the final round */
memcpy(buf, req->iv, AES_BLOCK_SIZE);
err = skcipher_walk_aead_encrypt(&walk, req, false);
if (crypto_simd_usable()) {
while (walk.nbytes) {
u32 tail = walk.nbytes % AES_BLOCK_SIZE;
if (walk.nbytes == walk.total)
tail = 0;
kernel_neon_begin();
ce_aes_ccm_encrypt(walk.dst.virt.addr,
walk.src.virt.addr,
walk.nbytes - tail, ctx->key_enc,
num_rounds(ctx), mac, walk.iv);
kernel_neon_end();
err = skcipher_walk_done(&walk, tail);
}
if (!err) {
kernel_neon_begin();
ce_aes_ccm_final(mac, buf, ctx->key_enc,
num_rounds(ctx));
kernel_neon_end();
}
} else {
err = ccm_crypt_fallback(&walk, mac, buf, ctx, true);
}
if (err)
return err;
/* copy authtag to end of dst */
scatterwalk_map_and_copy(mac, req->dst, req->assoclen + req->cryptlen,
crypto_aead_authsize(aead), 1);
return 0;
}
static void crypto_ccm_encrypt_done(struct crypto_async_request *areq, int err)
{
struct aead_request *req = areq->data;
struct crypto_aead *aead = crypto_aead_reqtfm(req);
struct crypto_ccm_req_priv_ctx *pctx = crypto_ccm_reqctx(req);
u8 *odata = pctx->odata;
if (!err)
scatterwalk_map_and_copy(odata, req->dst, req->cryptlen,
crypto_aead_authsize(aead), 1);
aead_request_complete(req, err);
}
static struct aead_request *crypto_rfc4543_crypt(struct aead_request *req,
int enc)
{
struct crypto_aead *aead = crypto_aead_reqtfm(req);
struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(aead);
struct crypto_rfc4543_req_ctx *rctx = crypto_rfc4543_reqctx(req);
struct aead_request *subreq = &rctx->subreq;
struct scatterlist *dst = req->dst;
struct scatterlist *cipher = rctx->cipher;
struct scatterlist *payload = rctx->payload;
struct scatterlist *assoc = rctx->assoc;
unsigned int authsize = crypto_aead_authsize(aead);
unsigned int assoclen = req->assoclen;
struct page *dstp;
u8 *vdst;
u8 *iv = PTR_ALIGN((u8 *)(rctx + 1) + crypto_aead_reqsize(ctx->child),
crypto_aead_alignmask(ctx->child) + 1);
memcpy(iv, ctx->nonce, 4);
memcpy(iv + 4, req->iv, 8);
/* construct cipher/plaintext */
if (enc)
memset(rctx->auth_tag, 0, authsize);
else
scatterwalk_map_and_copy(rctx->auth_tag, dst,
req->cryptlen - authsize,
authsize, 0);
sg_init_one(cipher, rctx->auth_tag, authsize);
/* construct the aad */
dstp = sg_page(dst);
vdst = PageHighMem(dstp) ? NULL : page_address(dstp) + dst->offset;
sg_init_table(payload, 2);
sg_set_buf(payload, req->iv, 8);
scatterwalk_crypto_chain(payload, dst, vdst == req->iv + 8, 2);
assoclen += 8 + req->cryptlen - (enc ? 0 : authsize);
sg_init_table(assoc, 2);
sg_set_page(assoc, sg_page(req->assoc), req->assoc->length,
req->assoc->offset);
scatterwalk_crypto_chain(assoc, payload, 0, 2);
aead_request_set_tfm(subreq, ctx->child);
aead_request_set_callback(subreq, req->base.flags, req->base.complete,
req->base.data);
aead_request_set_crypt(subreq, cipher, cipher, enc ? 0 : authsize, iv);
aead_request_set_assoc(subreq, assoc, assoclen);
return subreq;
}
static int crypto_ccm_encrypt(struct aead_request *req)
{
struct crypto_aead *aead = crypto_aead_reqtfm(req);
struct crypto_ccm_ctx *ctx = crypto_aead_ctx(aead);
struct crypto_ccm_req_priv_ctx *pctx = crypto_ccm_reqctx(req);
struct ablkcipher_request *abreq = &pctx->abreq;
struct scatterlist *dst;
unsigned int cryptlen = req->cryptlen;
u8 *odata = pctx->odata;
u8 *iv = req->iv;
int err;
err = crypto_ccm_check_iv(iv);
if (err)
return err;
pctx->flags = aead_request_flags(req);
err = crypto_ccm_auth(req, req->src, cryptlen);
if (err)
return err;
/* Note: rfc 3610 and NIST 800-38C require counter of
* zero to encrypt auth tag.
*/
memset(iv + 15 - iv[0], 0, iv[0] + 1);
sg_init_table(pctx->src, 2);
sg_set_buf(pctx->src, odata, 16);
scatterwalk_sg_chain(pctx->src, 2, req->src);
dst = pctx->src;
if (req->src != req->dst) {
sg_init_table(pctx->dst, 2);
sg_set_buf(pctx->dst, odata, 16);
scatterwalk_sg_chain(pctx->dst, 2, req->dst);
dst = pctx->dst;
}
ablkcipher_request_set_tfm(abreq, ctx->ctr);
ablkcipher_request_set_callback(abreq, pctx->flags,
crypto_ccm_encrypt_done, req);
ablkcipher_request_set_crypt(abreq, pctx->src, dst, cryptlen + 16, iv);
err = crypto_ablkcipher_encrypt(abreq);
if (err)
return err;
/* copy authtag to end of dst */
scatterwalk_map_and_copy(odata, req->dst, cryptlen,
crypto_aead_authsize(aead), 1);
return err;
}
static int crypto_gcm_verify(struct aead_request *req,
struct crypto_gcm_req_priv_ctx *pctx)
{
struct crypto_aead *aead = crypto_aead_reqtfm(req);
u8 *auth_tag = pctx->auth_tag;
u8 *iauth_tag = pctx->iauth_tag;
unsigned int authsize = crypto_aead_authsize(aead);
unsigned int cryptlen = req->cryptlen - authsize;
crypto_xor(auth_tag, iauth_tag, 16);
scatterwalk_map_and_copy(iauth_tag, req->src, cryptlen, authsize, 0);
return crypto_memneq(iauth_tag, auth_tag, authsize) ? -EBADMSG : 0;
}
static void crypto_rfc4543_done(struct crypto_async_request *areq, int err)
{
struct aead_request *req = areq->data;
struct crypto_aead *aead = crypto_aead_reqtfm(req);
struct crypto_rfc4543_req_ctx *rctx = crypto_rfc4543_reqctx(req);
if (!err) {
scatterwalk_map_and_copy(rctx->auth_tag, req->dst,
req->cryptlen,
crypto_aead_authsize(aead), 1);
}
aead_request_complete(req, err);
}
static int crypto_gcm_hash(struct aead_request *req)
{
struct crypto_aead *aead = crypto_aead_reqtfm(req);
struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
u8 *auth_tag = pctx->auth_tag;
struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash;
crypto_gcm_ghash_update_sg(ghash, req->dst, req->cryptlen);
crypto_gcm_ghash_final_xor(ghash, req->assoclen, req->cryptlen,
auth_tag);
scatterwalk_map_and_copy(auth_tag, req->dst, req->cryptlen,
crypto_aead_authsize(aead), 1);
return 0;
}
static void crypto_ccm_decrypt_done(struct crypto_async_request *areq,
int err)
{
struct aead_request *req = areq->data;
struct crypto_ccm_req_priv_ctx *pctx = crypto_ccm_reqctx(req);
struct crypto_aead *aead = crypto_aead_reqtfm(req);
unsigned int authsize = crypto_aead_authsize(aead);
unsigned int cryptlen = req->cryptlen - authsize;
if (!err) {
err = crypto_ccm_auth(req, req->dst, cryptlen);
if (!err && crypto_memneq(pctx->auth_tag, pctx->odata, authsize))
err = -EBADMSG;
}
aead_request_complete(req, err);
}
static int crypto_rfc4543_encrypt(struct aead_request *req)
{
struct crypto_aead *aead = crypto_aead_reqtfm(req);
struct crypto_rfc4543_req_ctx *rctx = crypto_rfc4543_reqctx(req);
struct aead_request *subreq;
int err;
subreq = crypto_rfc4543_crypt(req, 1);
err = crypto_aead_encrypt(subreq);
if (err)
return err;
scatterwalk_map_and_copy(rctx->auth_tag, req->dst, req->cryptlen,
crypto_aead_authsize(aead), 1);
return 0;
}
static int crypto_ccm_decrypt(struct aead_request *req)
{
struct crypto_aead *aead = crypto_aead_reqtfm(req);
struct crypto_ccm_ctx *ctx = crypto_aead_ctx(aead);
struct crypto_ccm_req_priv_ctx *pctx = crypto_ccm_reqctx(req);
struct ablkcipher_request *abreq = &pctx->abreq;
struct scatterlist *dst;
unsigned int authsize = crypto_aead_authsize(aead);
unsigned int cryptlen = req->cryptlen;
u8 *authtag = pctx->auth_tag;
u8 *odata = pctx->odata;
u8 *iv = req->iv;
int err;
cryptlen -= authsize;
err = crypto_ccm_init_crypt(req, authtag);
if (err)
return err;
scatterwalk_map_and_copy(authtag, sg_next(pctx->src), cryptlen,
authsize, 0);
dst = pctx->src;
if (req->src != req->dst)
dst = pctx->dst;
ablkcipher_request_set_tfm(abreq, ctx->ctr);
ablkcipher_request_set_callback(abreq, pctx->flags,
crypto_ccm_decrypt_done, req);
ablkcipher_request_set_crypt(abreq, pctx->src, dst, cryptlen + 16, iv);
err = crypto_ablkcipher_decrypt(abreq);
if (err)
return err;
err = crypto_ccm_auth(req, sg_next(dst), cryptlen);
if (err)
return err;
/* verify */
if (crypto_memneq(authtag, odata, authsize))
return -EBADMSG;
return err;
}
static int crypto_aegis256_aesni_encrypt(struct aead_request *req)
{
static const struct aegis_crypt_ops OPS = {
.skcipher_walk_init = skcipher_walk_aead_encrypt,
.crypt_blocks = crypto_aegis256_aesni_enc,
.crypt_tail = crypto_aegis256_aesni_enc_tail,
};
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct aegis_block tag = {};
unsigned int authsize = crypto_aead_authsize(tfm);
unsigned int cryptlen = req->cryptlen;
crypto_aegis256_aesni_crypt(req, &tag, cryptlen, &OPS);
scatterwalk_map_and_copy(tag.bytes, req->dst,
req->assoclen + cryptlen, authsize, 1);
return 0;
}
static int format_input(u8 *info, struct aead_request *req,
unsigned int cryptlen)
{
struct crypto_aead *aead = crypto_aead_reqtfm(req);
unsigned int lp = req->iv[0];
unsigned int l = lp + 1;
unsigned int m;
m = crypto_aead_authsize(aead);
memcpy(info, req->iv, 16);
/* format control info per RFC 3610 and
* NIST Special Publication 800-38C
*/
*info |= (8 * ((m - 2) / 2));
if (req->assoclen)
*info |= 64;
return set_msg_len(info + 16 - l, cryptlen, l);
}
static inline int crypto4xx_crypt_aes_gcm(struct aead_request *req,
bool decrypt)
{
struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
unsigned int len = req->cryptlen;
__le32 iv[4];
if (crypto4xx_aead_need_fallback(req, false, decrypt))
return crypto4xx_aead_fallback(req, ctx, decrypt);
crypto4xx_memcpy_to_le32(iv, req->iv, GCM_AES_IV_SIZE);
iv[3] = cpu_to_le32(1);
if (decrypt)
len -= crypto_aead_authsize(crypto_aead_reqtfm(req));
return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst,
len, iv, sizeof(iv),
decrypt ? ctx->sa_in : ctx->sa_out,
ctx->sa_len, req->assoclen);
}
static int crypto_aegis256_aesni_decrypt(struct aead_request *req)
{
static const struct aegis_block zeros = {};
static const struct aegis_crypt_ops OPS = {
.skcipher_walk_init = skcipher_walk_aead_decrypt,
.crypt_blocks = crypto_aegis256_aesni_dec,
.crypt_tail = crypto_aegis256_aesni_dec_tail,
};
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct aegis_block tag;
unsigned int authsize = crypto_aead_authsize(tfm);
unsigned int cryptlen = req->cryptlen - authsize;
scatterwalk_map_and_copy(tag.bytes, req->src,
req->assoclen + cryptlen, authsize, 0);
crypto_aegis256_aesni_crypt(req, &tag, cryptlen, &OPS);
return crypto_memneq(tag.bytes, zeros.bytes, authsize) ? -EBADMSG : 0;
}
static int ccm_init_mac(struct aead_request *req, u8 maciv[], u32 msglen)
{
struct crypto_aead *aead = crypto_aead_reqtfm(req);
__be32 *n = (__be32 *)&maciv[AES_BLOCK_SIZE - 8];
u32 l = req->iv[0] + 1;
/* verify that CCM dimension 'L' is set correctly in the IV */
if (l < 2 || l > 8)
return -EINVAL;
/* verify that msglen can in fact be represented in L bytes */
if (l < 4 && msglen >> (8 * l))
return -EOVERFLOW;
/*
* Even if the CCM spec allows L values of up to 8, the Linux cryptoapi
* uses a u32 type to represent msglen so the top 4 bytes are always 0.
*/
n[0] = 0;
n[1] = cpu_to_be32(msglen);
memcpy(maciv, req->iv, AES_BLOCK_SIZE - l);
/*
* Meaning of byte 0 according to CCM spec (RFC 3610/NIST 800-38C)
* - bits 0..2 : max # of bytes required to represent msglen, minus 1
* (already set by caller)
* - bits 3..5 : size of auth tag (1 => 4 bytes, 2 => 6 bytes, etc)
* - bit 6 : indicates presence of authenticate-only data
*/
maciv[0] |= (crypto_aead_authsize(aead) - 2) << 2;
if (req->assoclen)
maciv[0] |= 0x40;
memset(&req->iv[AES_BLOCK_SIZE - l], 0, l);
return 0;
}
static int crypto_ccm_encrypt(struct aead_request *req)
{
struct crypto_aead *aead = crypto_aead_reqtfm(req);
struct crypto_ccm_ctx *ctx = crypto_aead_ctx(aead);
struct crypto_ccm_req_priv_ctx *pctx = crypto_ccm_reqctx(req);
struct ablkcipher_request *abreq = &pctx->abreq;
struct scatterlist *dst;
unsigned int cryptlen = req->cryptlen;
u8 *odata = pctx->odata;
u8 *iv = req->iv;
int err;
err = crypto_ccm_init_crypt(req, odata);
if (err)
return err;
err = crypto_ccm_auth(req, sg_next(pctx->src), cryptlen);
if (err)
return err;
dst = pctx->src;
if (req->src != req->dst)
dst = pctx->dst;
ablkcipher_request_set_tfm(abreq, ctx->ctr);
ablkcipher_request_set_callback(abreq, pctx->flags,
crypto_ccm_encrypt_done, req);
ablkcipher_request_set_crypt(abreq, pctx->src, dst, cryptlen + 16, iv);
err = crypto_ablkcipher_encrypt(abreq);
if (err)
return err;
/* copy authtag to end of dst */
scatterwalk_map_and_copy(odata, sg_next(dst), cryptlen,
crypto_aead_authsize(aead), 1);
return err;
}
static struct aead_request *crypto_rfc4543_crypt(struct aead_request *req,
bool enc)
{
struct crypto_aead *aead = crypto_aead_reqtfm(req);
struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(aead);
struct crypto_rfc4543_req_ctx *rctx = crypto_rfc4543_reqctx(req);
struct aead_request *subreq = &rctx->subreq;
struct scatterlist *src = req->src;
struct scatterlist *cipher = rctx->cipher;
struct scatterlist *payload = rctx->payload;
struct scatterlist *assoc = rctx->assoc;
unsigned int authsize = crypto_aead_authsize(aead);
unsigned int assoclen = req->assoclen;
struct page *srcp;
u8 *vsrc;
u8 *iv = PTR_ALIGN((u8 *)(rctx + 1) + crypto_aead_reqsize(ctx->child),
crypto_aead_alignmask(ctx->child) + 1);
memcpy(iv, ctx->nonce, 4);
memcpy(iv + 4, req->iv, 8);
/* construct cipher/plaintext */
if (enc)
memset(rctx->auth_tag, 0, authsize);
else
scatterwalk_map_and_copy(rctx->auth_tag, src,
req->cryptlen - authsize,
authsize, 0);
sg_init_one(cipher, rctx->auth_tag, authsize);
/* construct the aad */
srcp = sg_page(src);
vsrc = PageHighMem(srcp) ? NULL : page_address(srcp) + src->offset;
sg_init_table(payload, 2);
sg_set_buf(payload, req->iv, 8);
scatterwalk_crypto_chain(payload, src, vsrc == req->iv + 8, 2);
assoclen += 8 + req->cryptlen - (enc ? 0 : authsize);
if (req->assoc->length == req->assoclen) {
sg_init_table(assoc, 2);
sg_set_page(assoc, sg_page(req->assoc), req->assoc->length,
req->assoc->offset);
} else {
BUG_ON(req->assoclen > sizeof(rctx->assocbuf));
scatterwalk_map_and_copy(rctx->assocbuf, req->assoc, 0,
req->assoclen, 0);
sg_init_table(assoc, 2);
sg_set_buf(assoc, rctx->assocbuf, req->assoclen);
}
scatterwalk_crypto_chain(assoc, payload, 0, 2);
aead_request_set_tfm(subreq, ctx->child);
aead_request_set_callback(subreq, req->base.flags, crypto_rfc4543_done,
req);
aead_request_set_crypt(subreq, cipher, cipher, enc ? 0 : authsize, iv);
aead_request_set_assoc(subreq, assoc, assoclen);
return subreq;
}
static int __driver_rfc4106_encrypt(struct aead_request *req)
{
u8 one_entry_in_sg = 0;
u8 *src, *dst, *assoc;
__be32 counter = cpu_to_be32(1);
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
void *aes_ctx = &(ctx->aes_key_expanded);
unsigned long auth_tag_len = crypto_aead_authsize(tfm);
u8 iv_tab[16+AESNI_ALIGN];
u8* iv = (u8 *) PTR_ALIGN((u8 *)iv_tab, AESNI_ALIGN);
struct scatter_walk src_sg_walk;
struct scatter_walk assoc_sg_walk;
struct scatter_walk dst_sg_walk;
unsigned int i;
/* Assuming we are supporting rfc4106 64-bit extended */
/* sequence numbers We need to have the AAD length equal */
/* to 8 or 12 bytes */
if (unlikely(req->assoclen != 8 && req->assoclen != 12))
return -EINVAL;
/* IV below built */
for (i = 0; i < 4; i++)
*(iv+i) = ctx->nonce[i];
for (i = 0; i < 8; i++)
*(iv+4+i) = req->iv[i];
*((__be32 *)(iv+12)) = counter;
if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) {
one_entry_in_sg = 1;
scatterwalk_start(&src_sg_walk, req->src);
scatterwalk_start(&assoc_sg_walk, req->assoc);
src = scatterwalk_map(&src_sg_walk);
assoc = scatterwalk_map(&assoc_sg_walk);
dst = src;
if (unlikely(req->src != req->dst)) {
scatterwalk_start(&dst_sg_walk, req->dst);
dst = scatterwalk_map(&dst_sg_walk);
}
} else {
/* Allocate memory for src, dst, assoc */
src = kmalloc(req->cryptlen + auth_tag_len + req->assoclen,
GFP_ATOMIC);
if (unlikely(!src))
return -ENOMEM;
assoc = (src + req->cryptlen + auth_tag_len);
scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0);
scatterwalk_map_and_copy(assoc, req->assoc, 0,
req->assoclen, 0);
dst = src;
}
aesni_gcm_enc(aes_ctx, dst, src, (unsigned long)req->cryptlen, iv,
ctx->hash_subkey, assoc, (unsigned long)req->assoclen, dst
+ ((unsigned long)req->cryptlen), auth_tag_len);
/* The authTag (aka the Integrity Check Value) needs to be written
* back to the packet. */
if (one_entry_in_sg) {
if (unlikely(req->src != req->dst)) {
scatterwalk_unmap(dst);
scatterwalk_done(&dst_sg_walk, 0, 0);
}
scatterwalk_unmap(src);
scatterwalk_unmap(assoc);
scatterwalk_done(&src_sg_walk, 0, 0);
scatterwalk_done(&assoc_sg_walk, 0, 0);
} else {
scatterwalk_map_and_copy(dst, req->dst, 0,
req->cryptlen + auth_tag_len, 1);
kfree(src);
}
return 0;
}
static int __driver_rfc4106_decrypt(struct aead_request *req)
{
u8 one_entry_in_sg = 0;
u8 *src, *dst, *assoc;
unsigned long tempCipherLen = 0;
__be32 counter = cpu_to_be32(1);
int retval = 0;
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
void *aes_ctx = &(ctx->aes_key_expanded);
unsigned long auth_tag_len = crypto_aead_authsize(tfm);
u8 iv_and_authTag[32+AESNI_ALIGN];
u8 *iv = (u8 *) PTR_ALIGN((u8 *)iv_and_authTag, AESNI_ALIGN);
u8 *authTag = iv + 16;
struct scatter_walk src_sg_walk;
struct scatter_walk assoc_sg_walk;
struct scatter_walk dst_sg_walk;
unsigned int i;
if (unlikely((req->cryptlen < auth_tag_len) ||
(req->assoclen != 8 && req->assoclen != 12)))
return -EINVAL;
/* Assuming we are supporting rfc4106 64-bit extended */
/* sequence numbers We need to have the AAD length */
/* equal to 8 or 12 bytes */
tempCipherLen = (unsigned long)(req->cryptlen - auth_tag_len);
/* IV below built */
for (i = 0; i < 4; i++)
*(iv+i) = ctx->nonce[i];
for (i = 0; i < 8; i++)
*(iv+4+i) = req->iv[i];
*((__be32 *)(iv+12)) = counter;
if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) {
one_entry_in_sg = 1;
scatterwalk_start(&src_sg_walk, req->src);
scatterwalk_start(&assoc_sg_walk, req->assoc);
src = scatterwalk_map(&src_sg_walk);
assoc = scatterwalk_map(&assoc_sg_walk);
dst = src;
if (unlikely(req->src != req->dst)) {
scatterwalk_start(&dst_sg_walk, req->dst);
dst = scatterwalk_map(&dst_sg_walk);
}
} else {
/* Allocate memory for src, dst, assoc */
src = kmalloc(req->cryptlen + req->assoclen, GFP_ATOMIC);
if (!src)
return -ENOMEM;
assoc = (src + req->cryptlen + auth_tag_len);
scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0);
scatterwalk_map_and_copy(assoc, req->assoc, 0,
req->assoclen, 0);
dst = src;
}
aesni_gcm_dec(aes_ctx, dst, src, tempCipherLen, iv,
ctx->hash_subkey, assoc, (unsigned long)req->assoclen,
authTag, auth_tag_len);
/* Compare generated tag with passed in tag. */
retval = memcmp(src + tempCipherLen, authTag, auth_tag_len) ?
-EBADMSG : 0;
if (one_entry_in_sg) {
if (unlikely(req->src != req->dst)) {
scatterwalk_unmap(dst);
scatterwalk_done(&dst_sg_walk, 0, 0);
}
scatterwalk_unmap(src);
scatterwalk_unmap(assoc);
scatterwalk_done(&src_sg_walk, 0, 0);
scatterwalk_done(&assoc_sg_walk, 0, 0);
} else {
scatterwalk_map_and_copy(dst, req->dst, 0, req->cryptlen, 1);
kfree(src);
}
return retval;
}
static int qat_alg_aead_init_dec_session(struct crypto_aead *aead_tfm,
int alg,
struct crypto_authenc_keys *keys,
int mode)
{
struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(aead_tfm);
unsigned int digestsize = crypto_aead_authsize(aead_tfm);
struct qat_dec *dec_ctx = &ctx->dec_cd->qat_dec_cd;
struct icp_qat_hw_auth_algo_blk *hash = &dec_ctx->hash;
struct icp_qat_hw_cipher_algo_blk *cipher =
(struct icp_qat_hw_cipher_algo_blk *)((char *)dec_ctx +
sizeof(struct icp_qat_hw_auth_setup) +
roundup(crypto_shash_digestsize(ctx->hash_tfm), 8) * 2);
struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->dec_fw_req;
struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars;
struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr;
void *ptr = &req_tmpl->cd_ctrl;
struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = ptr;
struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr;
struct icp_qat_fw_la_auth_req_params *auth_param =
(struct icp_qat_fw_la_auth_req_params *)
((char *)&req_tmpl->serv_specif_rqpars +
sizeof(struct icp_qat_fw_la_cipher_req_params));
/* CD setup */
cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_DEC(alg, mode);
memcpy(cipher->aes.key, keys->enckey, keys->enckeylen);
hash->sha.inner_setup.auth_config.config =
ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1,
ctx->qat_hash_alg,
digestsize);
hash->sha.inner_setup.auth_counter.counter =
cpu_to_be32(crypto_shash_blocksize(ctx->hash_tfm));
if (qat_alg_do_precomputes(hash, ctx, keys->authkey, keys->authkeylen))
return -EFAULT;
/* Request setup */
qat_alg_init_common_hdr(header);
header->service_cmd_id = ICP_QAT_FW_LA_CMD_HASH_CIPHER;
ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,
ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags,
ICP_QAT_FW_LA_NO_RET_AUTH_RES);
ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags,
ICP_QAT_FW_LA_CMP_AUTH_RES);
cd_pars->u.s.content_desc_addr = ctx->dec_cd_paddr;
cd_pars->u.s.content_desc_params_sz = sizeof(struct qat_alg_cd) >> 3;
/* Cipher CD config setup */
cipher_cd_ctrl->cipher_key_sz = keys->enckeylen >> 3;
cipher_cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3;
cipher_cd_ctrl->cipher_cfg_offset =
(sizeof(struct icp_qat_hw_auth_setup) +
roundup(crypto_shash_digestsize(ctx->hash_tfm), 8) * 2) >> 3;
ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR);
/* Auth CD config setup */
hash_cd_ctrl->hash_cfg_offset = 0;
hash_cd_ctrl->hash_flags = ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED;
hash_cd_ctrl->inner_res_sz = digestsize;
hash_cd_ctrl->final_sz = digestsize;
switch (ctx->qat_hash_alg) {
case ICP_QAT_HW_AUTH_ALGO_SHA1:
hash_cd_ctrl->inner_state1_sz =
round_up(ICP_QAT_HW_SHA1_STATE1_SZ, 8);
hash_cd_ctrl->inner_state2_sz =
round_up(ICP_QAT_HW_SHA1_STATE2_SZ, 8);
break;
case ICP_QAT_HW_AUTH_ALGO_SHA256:
hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA256_STATE1_SZ;
hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA256_STATE2_SZ;
break;
case ICP_QAT_HW_AUTH_ALGO_SHA512:
hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA512_STATE1_SZ;
hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA512_STATE2_SZ;
break;
default:
break;
}
hash_cd_ctrl->inner_state2_offset = hash_cd_ctrl->hash_cfg_offset +
((sizeof(struct icp_qat_hw_auth_setup) +
round_up(hash_cd_ctrl->inner_state1_sz, 8)) >> 3);
auth_param->auth_res_sz = digestsize;
ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_AUTH);
ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
return 0;
}
static int echainiv_encrypt(struct aead_request *req)
{
struct crypto_aead *geniv = crypto_aead_reqtfm(req);
struct echainiv_ctx *ctx = crypto_aead_ctx(geniv);
struct aead_request *subreq = aead_request_ctx(req);
crypto_completion_t compl;
void *data;
u8 *info;
unsigned int ivsize = crypto_aead_ivsize(geniv);
int err;
if (req->cryptlen < ivsize)
return -EINVAL;
aead_request_set_tfm(subreq, ctx->geniv.child);
compl = echainiv_encrypt_complete;
data = req;
info = req->iv;
if (req->src != req->dst) {
struct blkcipher_desc desc = {
.tfm = ctx->null,
};
err = crypto_blkcipher_encrypt(
&desc, req->dst, req->src,
req->assoclen + req->cryptlen);
if (err)
return err;
}
if (unlikely(!IS_ALIGNED((unsigned long)info,
crypto_aead_alignmask(geniv) + 1))) {
info = kmalloc(ivsize, req->base.flags &
CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL:
GFP_ATOMIC);
if (!info)
return -ENOMEM;
memcpy(info, req->iv, ivsize);
}
aead_request_set_callback(subreq, req->base.flags, compl, data);
aead_request_set_crypt(subreq, req->dst, req->dst,
req->cryptlen - ivsize, info);
aead_request_set_ad(subreq, req->assoclen + ivsize);
crypto_xor(info, ctx->salt, ivsize);
scatterwalk_map_and_copy(info, req->dst, req->assoclen, ivsize, 1);
echainiv_read_iv(info, ivsize);
err = crypto_aead_encrypt(subreq);
echainiv_encrypt_complete2(req, err);
return err;
}
static int echainiv_decrypt(struct aead_request *req)
{
struct crypto_aead *geniv = crypto_aead_reqtfm(req);
struct echainiv_ctx *ctx = crypto_aead_ctx(geniv);
struct aead_request *subreq = aead_request_ctx(req);
crypto_completion_t compl;
void *data;
unsigned int ivsize = crypto_aead_ivsize(geniv);
if (req->cryptlen < ivsize + crypto_aead_authsize(geniv))
return -EINVAL;
aead_request_set_tfm(subreq, ctx->geniv.child);
compl = req->base.complete;
data = req->base.data;
aead_request_set_callback(subreq, req->base.flags, compl, data);
aead_request_set_crypt(subreq, req->src, req->dst,
req->cryptlen - ivsize, req->iv);
aead_request_set_ad(subreq, req->assoclen + ivsize);
scatterwalk_map_and_copy(req->iv, req->src, req->assoclen, ivsize, 0);
if (req->src != req->dst)
scatterwalk_map_and_copy(req->iv, req->dst,
req->assoclen, ivsize, 1);
return crypto_aead_decrypt(subreq);
}
static int echainiv_init(struct crypto_tfm *tfm)
{
struct crypto_aead *geniv = __crypto_aead_cast(tfm);
struct echainiv_ctx *ctx = crypto_aead_ctx(geniv);
int err;
spin_lock_init(&ctx->geniv.lock);
crypto_aead_set_reqsize(geniv, sizeof(struct aead_request));
err = crypto_get_default_rng();
if (err)
goto out;
err = crypto_rng_get_bytes(crypto_default_rng, ctx->salt,
crypto_aead_ivsize(geniv));
crypto_put_default_rng();
if (err)
goto out;
ctx->null = crypto_get_default_null_skcipher();
err = PTR_ERR(ctx->null);
if (IS_ERR(ctx->null))
goto out;
err = aead_geniv_init(tfm);
if (err)
goto drop_null;
ctx->geniv.child = geniv->child;
geniv->child = geniv;
out:
return err;
drop_null:
crypto_put_default_null_skcipher();
goto out;
}
static void echainiv_exit(struct crypto_tfm *tfm)
{
struct echainiv_ctx *ctx = crypto_tfm_ctx(tfm);
crypto_free_aead(ctx->geniv.child);
crypto_put_default_null_skcipher();
}
static int echainiv_aead_create(struct crypto_template *tmpl,
struct rtattr **tb)
{
struct aead_instance *inst;
struct crypto_aead_spawn *spawn;
struct aead_alg *alg;
int err;
inst = aead_geniv_alloc(tmpl, tb, 0, 0);
if (IS_ERR(inst))
return PTR_ERR(inst);
spawn = aead_instance_ctx(inst);
alg = crypto_spawn_aead_alg(spawn);
if (alg->base.cra_aead.encrypt)
goto done;
err = -EINVAL;
if (inst->alg.ivsize & (sizeof(u32) - 1) ||
inst->alg.ivsize > MAX_IV_SIZE)
goto free_inst;
inst->alg.encrypt = echainiv_encrypt;
inst->alg.decrypt = echainiv_decrypt;
inst->alg.base.cra_init = echainiv_init;
inst->alg.base.cra_exit = echainiv_exit;
inst->alg.base.cra_alignmask |= __alignof__(u32) - 1;
inst->alg.base.cra_ctxsize = sizeof(struct echainiv_ctx);
inst->alg.base.cra_ctxsize += inst->alg.ivsize;
done:
err = aead_register_instance(tmpl, inst);
if (err)
goto free_inst;
out:
return err;
free_inst:
aead_geniv_free(inst);
goto out;
}
static void echainiv_free(struct crypto_instance *inst)
{
aead_geniv_free(aead_instance(inst));
}
