static int crypto_gcm_decrypt(struct aead_request *req) { struct crypto_aead *aead = crypto_aead_reqtfm(req); struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); struct ablkcipher_request *abreq = &pctx->u.abreq; struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; unsigned int authsize = crypto_aead_authsize(aead); unsigned int cryptlen = req->cryptlen; int err; if (cryptlen < authsize) return -EINVAL; cryptlen -= authsize; gctx->src = req->src; gctx->cryptlen = cryptlen; gctx->complete = gcm_dec_hash_done; err = gcm_hash(req, pctx); if (err) return err; ablkcipher_request_set_callback(abreq, aead_request_flags(req), gcm_decrypt_done, req); crypto_gcm_init_crypt(abreq, req, cryptlen); err = crypto_ablkcipher_decrypt(abreq); if (err) return err; return crypto_gcm_verify(req, pctx); }
static void crypto_gcm_init_crypt(struct ablkcipher_request *ablk_req, struct aead_request *req, unsigned int cryptlen) { struct crypto_aead *aead = crypto_aead_reqtfm(req); struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead); struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); struct scatterlist *dst; __be32 counter = cpu_to_be32(1); memset(pctx->auth_tag, 0, sizeof(pctx->auth_tag)); memcpy(req->iv + 12, &counter, 4); sg_init_table(pctx->src, 2); sg_set_buf(pctx->src, pctx->auth_tag, sizeof(pctx->auth_tag)); scatterwalk_sg_chain(pctx->src, 2, req->src); dst = pctx->src; if (req->src != req->dst) { sg_init_table(pctx->dst, 2); sg_set_buf(pctx->dst, pctx->auth_tag, sizeof(pctx->auth_tag)); scatterwalk_sg_chain(pctx->dst, 2, req->dst); dst = pctx->dst; } ablkcipher_request_set_tfm(ablk_req, ctx->ctr); ablkcipher_request_set_crypt(ablk_req, pctx->src, dst, cryptlen + sizeof(pctx->auth_tag), req->iv); }
static int crypto_gcm_encrypt(struct aead_request *req) { struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); struct ablkcipher_request *abreq = &pctx->u.abreq; struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; int err; crypto_gcm_init_crypt(abreq, req, req->cryptlen); ablkcipher_request_set_callback(abreq, aead_request_flags(req), gcm_encrypt_done, req); gctx->src = req->dst; gctx->cryptlen = req->cryptlen; gctx->complete = gcm_enc_hash_done; err = crypto_ablkcipher_encrypt(abreq); if (err) return err; err = gcm_hash(req, pctx); if (err) return err; crypto_xor(pctx->auth_tag, pctx->iauth_tag, 16); gcm_enc_copy_hash(req, pctx); return 0; }
static int crypto_gcm_decrypt(struct aead_request *req) { struct crypto_aead *aead = crypto_aead_reqtfm(req); struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); struct ablkcipher_request *abreq = &pctx->abreq; struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash; unsigned int cryptlen = req->cryptlen; unsigned int authsize = crypto_aead_authsize(aead); int err; if (cryptlen < authsize) return -EINVAL; cryptlen -= authsize; crypto_gcm_init_crypt(abreq, req, cryptlen); ablkcipher_request_set_callback(abreq, aead_request_flags(req), crypto_gcm_decrypt_done, req); crypto_gcm_ghash_update_sg(ghash, req->src, cryptlen); err = crypto_ablkcipher_decrypt(abreq); if (err) return err; return crypto_gcm_verify(req); }
static void gcm_enc_hash_done(struct aead_request *req, int err) { struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); if (!err) gcm_enc_copy_hash(req, pctx); aead_request_complete(req, err); }
static void __gcm_hash_final_done(struct aead_request *req, int err) { struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; if (!err) crypto_xor(pctx->auth_tag, pctx->iauth_tag, 16); gctx->complete(req, err); }
static void gcm_decrypt_done(struct crypto_async_request *areq, int err) { struct aead_request *req = areq->data; struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); if (!err) err = crypto_gcm_verify(req, pctx); aead_request_complete(req, err); }
static void __gcm_hash_crypt_remain_done(struct aead_request *req, int err) { struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); if (!err) { err = gcm_hash_len(req, pctx); if (err == -EINPROGRESS || err == -EBUSY) return; } __gcm_hash_len_done(req, err); }
static void gcm_encrypt_done(struct crypto_async_request *areq, int err) { struct aead_request *req = areq->data; struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); if (!err) { err = gcm_hash(req, pctx); if (err == -EINPROGRESS || err == -EBUSY) return; } gcm_enc_hash_done(areq, err); }
static int crypto_gcm_hash(struct aead_request *req) { struct crypto_aead *aead = crypto_aead_reqtfm(req); struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); u8 *auth_tag = pctx->auth_tag; struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash; crypto_gcm_ghash_update_sg(ghash, req->dst, req->cryptlen); crypto_gcm_ghash_final_xor(ghash, req->assoclen, req->cryptlen, auth_tag); scatterwalk_map_and_copy(auth_tag, req->dst, req->cryptlen, crypto_aead_authsize(aead), 1); return 0; }
static int crypto_gcm_verify(struct aead_request *req) { struct crypto_aead *aead = crypto_aead_reqtfm(req); struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash; u8 *auth_tag = pctx->auth_tag; u8 *iauth_tag = pctx->iauth_tag; unsigned int authsize = crypto_aead_authsize(aead); unsigned int cryptlen = req->cryptlen - authsize; crypto_gcm_ghash_final_xor(ghash, req->assoclen, cryptlen, auth_tag); authsize = crypto_aead_authsize(aead); scatterwalk_map_and_copy(iauth_tag, req->src, cryptlen, authsize, 0); return memcmp(iauth_tag, auth_tag, authsize) ? -EBADMSG : 0; }
static int crypto_gcm_encrypt(struct aead_request *req) { struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); struct ablkcipher_request *abreq = &pctx->abreq; int err; crypto_gcm_init_crypt(abreq, req, req->cryptlen); ablkcipher_request_set_callback(abreq, aead_request_flags(req), crypto_gcm_encrypt_done, req); err = crypto_ablkcipher_encrypt(abreq); if (err) return err; return crypto_gcm_hash(req); }
static void __gcm_hash_assoc_done(struct aead_request *req, int err) { struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); unsigned int remain; if (!err) { remain = gcm_remain(req->assoclen); BUG_ON(!remain); err = gcm_hash_remain(req, pctx, remain, gcm_hash_assoc_remain_done); if (err == -EINPROGRESS || err == -EBUSY) return; } __gcm_hash_assoc_remain_done(req, err); }
static void gcm_encrypt_done(struct crypto_async_request *areq, int err) { struct aead_request *req = areq->data; struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); if (!err) { err = gcm_hash(req, pctx); if (err == -EINPROGRESS || err == -EBUSY) return; else if (!err) { crypto_xor(pctx->auth_tag, pctx->iauth_tag, 16); gcm_enc_copy_hash(req, pctx); } } aead_request_complete(req, err); }
static void gcm_dec_hash_done(struct crypto_async_request *areq, int err) { struct aead_request *req = areq->data; struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); struct ablkcipher_request *abreq = &pctx->u.abreq; struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; if (!err) { ablkcipher_request_set_callback(abreq, aead_request_flags(req), gcm_decrypt_done, req); crypto_gcm_init_crypt(abreq, req, gctx->cryptlen); err = crypto_ablkcipher_decrypt(abreq); if (err == -EINPROGRESS || err == -EBUSY) return; } gcm_decrypt_done(areq, err); }
static void gcm_hash_crypt_done(struct crypto_async_request *areq, int err) { struct aead_request *req = areq->data; struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; unsigned int remain; if (!err) { remain = gcm_remain(gctx->cryptlen); BUG_ON(!remain); err = gcm_hash_remain(req, pctx, remain, gcm_hash_crypt_remain_done); if (err == -EINPROGRESS || err == -EBUSY) return; } gcm_hash_crypt_remain_done(areq, err); }
static void __gcm_hash_init_done(struct aead_request *req, int err) { struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); crypto_completion_t compl; unsigned int remain = 0; if (!err && req->assoclen) { remain = gcm_remain(req->assoclen); compl = remain ? gcm_hash_assoc_done : gcm_hash_assoc_remain_done; err = gcm_hash_update(req, pctx, compl, req->assoc, req->assoclen); if (err == -EINPROGRESS || err == -EBUSY) return; } if (remain) __gcm_hash_assoc_done(req, err); else __gcm_hash_assoc_remain_done(req, err); }
static void __gcm_hash_assoc_remain_done(struct aead_request *req, int err) { struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; crypto_completion_t complete; unsigned int remain = 0; if (!err && gctx->cryptlen) { remain = gcm_remain(gctx->cryptlen); complete = remain ? gcm_hash_crypt_done : gcm_hash_crypt_remain_done; err = gcm_hash_update(req, pctx, complete, gctx->src, gctx->cryptlen); if (err == -EINPROGRESS || err == -EBUSY) return; } if (remain) __gcm_hash_crypt_done(req, err); else __gcm_hash_crypt_remain_done(req, err); }
static void crypto_gcm_init_crypt(struct ablkcipher_request *ablk_req, struct aead_request *req, unsigned int cryptlen) { struct crypto_aead *aead = crypto_aead_reqtfm(req); struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead); struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); u32 flags = req->base.tfm->crt_flags; struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash; struct scatterlist *dst; __be32 counter = cpu_to_be32(1); memset(pctx->auth_tag, 0, sizeof(pctx->auth_tag)); memcpy(req->iv + 12, &counter, 4); sg_init_table(pctx->src, 2); sg_set_buf(pctx->src, pctx->auth_tag, sizeof(pctx->auth_tag)); scatterwalk_sg_chain(pctx->src, 2, req->src); dst = pctx->src; if (req->src != req->dst) { sg_init_table(pctx->dst, 2); sg_set_buf(pctx->dst, pctx->auth_tag, sizeof(pctx->auth_tag)); scatterwalk_sg_chain(pctx->dst, 2, req->dst); dst = pctx->dst; } ablkcipher_request_set_tfm(ablk_req, ctx->ctr); ablkcipher_request_set_crypt(ablk_req, pctx->src, dst, cryptlen + sizeof(pctx->auth_tag), req->iv); crypto_gcm_ghash_init(ghash, flags, ctx->gf128); crypto_gcm_ghash_update_sg(ghash, req->assoc, req->assoclen); crypto_gcm_ghash_flush(ghash); }