static int crypto_gcm_decrypt(struct aead_request *req)
{
struct crypto_aead *aead = crypto_aead_reqtfm(req);
struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
struct ablkcipher_request *abreq = &pctx->u.abreq;
struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
unsigned int authsize = crypto_aead_authsize(aead);
unsigned int cryptlen = req->cryptlen;
int err;
if (cryptlen < authsize)
return -EINVAL;
cryptlen -= authsize;
gctx->src = req->src;
gctx->cryptlen = cryptlen;
gctx->complete = gcm_dec_hash_done;
err = gcm_hash(req, pctx);
if (err)
return err;
ablkcipher_request_set_callback(abreq, aead_request_flags(req),
gcm_decrypt_done, req);
crypto_gcm_init_crypt(abreq, req, cryptlen);
err = crypto_ablkcipher_decrypt(abreq);
if (err)
return err;
return crypto_gcm_verify(req, pctx);
}
static void crypto_gcm_init_crypt(struct ablkcipher_request *ablk_req,
struct aead_request *req,
unsigned int cryptlen)
{
struct crypto_aead *aead = crypto_aead_reqtfm(req);
struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead);
struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
struct scatterlist *dst;
__be32 counter = cpu_to_be32(1);
memset(pctx->auth_tag, 0, sizeof(pctx->auth_tag));
memcpy(req->iv + 12, &counter, 4);
sg_init_table(pctx->src, 2);
sg_set_buf(pctx->src, pctx->auth_tag, sizeof(pctx->auth_tag));
scatterwalk_sg_chain(pctx->src, 2, req->src);
dst = pctx->src;
if (req->src != req->dst) {
sg_init_table(pctx->dst, 2);
sg_set_buf(pctx->dst, pctx->auth_tag, sizeof(pctx->auth_tag));
scatterwalk_sg_chain(pctx->dst, 2, req->dst);
dst = pctx->dst;
}
ablkcipher_request_set_tfm(ablk_req, ctx->ctr);
ablkcipher_request_set_crypt(ablk_req, pctx->src, dst,
cryptlen + sizeof(pctx->auth_tag),
req->iv);
}
static int crypto_gcm_encrypt(struct aead_request *req)
{
struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
struct ablkcipher_request *abreq = &pctx->u.abreq;
struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
int err;
crypto_gcm_init_crypt(abreq, req, req->cryptlen);
ablkcipher_request_set_callback(abreq, aead_request_flags(req),
gcm_encrypt_done, req);
gctx->src = req->dst;
gctx->cryptlen = req->cryptlen;
gctx->complete = gcm_enc_hash_done;
err = crypto_ablkcipher_encrypt(abreq);
if (err)
return err;
err = gcm_hash(req, pctx);
if (err)
return err;
crypto_xor(pctx->auth_tag, pctx->iauth_tag, 16);
gcm_enc_copy_hash(req, pctx);
return 0;
}
static int crypto_gcm_decrypt(struct aead_request *req)
{
struct crypto_aead *aead = crypto_aead_reqtfm(req);
struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
struct ablkcipher_request *abreq = &pctx->abreq;
struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash;
unsigned int cryptlen = req->cryptlen;
unsigned int authsize = crypto_aead_authsize(aead);
int err;
if (cryptlen < authsize)
return -EINVAL;
cryptlen -= authsize;
crypto_gcm_init_crypt(abreq, req, cryptlen);
ablkcipher_request_set_callback(abreq, aead_request_flags(req),
crypto_gcm_decrypt_done, req);
crypto_gcm_ghash_update_sg(ghash, req->src, cryptlen);
err = crypto_ablkcipher_decrypt(abreq);
if (err)
return err;
return crypto_gcm_verify(req);
}
static void gcm_enc_hash_done(struct aead_request *req, int err)
{
struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
if (!err)
gcm_enc_copy_hash(req, pctx);
aead_request_complete(req, err);
}
static void __gcm_hash_final_done(struct aead_request *req, int err)
{
struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
if (!err)
crypto_xor(pctx->auth_tag, pctx->iauth_tag, 16);
gctx->complete(req, err);
}
static void gcm_decrypt_done(struct crypto_async_request *areq, int err)
{
struct aead_request *req = areq->data;
struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
if (!err)
err = crypto_gcm_verify(req, pctx);
aead_request_complete(req, err);
}
static void __gcm_hash_crypt_remain_done(struct aead_request *req, int err)
{
struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
if (!err) {
err = gcm_hash_len(req, pctx);
if (err == -EINPROGRESS || err == -EBUSY)
return;
}
__gcm_hash_len_done(req, err);
}
static void gcm_encrypt_done(struct crypto_async_request *areq,
int err)
{
struct aead_request *req = areq->data;
struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
if (!err) {
err = gcm_hash(req, pctx);
if (err == -EINPROGRESS || err == -EBUSY)
return;
}
gcm_enc_hash_done(areq, err);
}
static int crypto_gcm_hash(struct aead_request *req)
{
struct crypto_aead *aead = crypto_aead_reqtfm(req);
struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
u8 *auth_tag = pctx->auth_tag;
struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash;
crypto_gcm_ghash_update_sg(ghash, req->dst, req->cryptlen);
crypto_gcm_ghash_final_xor(ghash, req->assoclen, req->cryptlen,
auth_tag);
scatterwalk_map_and_copy(auth_tag, req->dst, req->cryptlen,
crypto_aead_authsize(aead), 1);
return 0;
}
static int crypto_gcm_verify(struct aead_request *req)
{
struct crypto_aead *aead = crypto_aead_reqtfm(req);
struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash;
u8 *auth_tag = pctx->auth_tag;
u8 *iauth_tag = pctx->iauth_tag;
unsigned int authsize = crypto_aead_authsize(aead);
unsigned int cryptlen = req->cryptlen - authsize;
crypto_gcm_ghash_final_xor(ghash, req->assoclen, cryptlen, auth_tag);
authsize = crypto_aead_authsize(aead);
scatterwalk_map_and_copy(iauth_tag, req->src, cryptlen, authsize, 0);
return memcmp(iauth_tag, auth_tag, authsize) ? -EBADMSG : 0;
}
static int crypto_gcm_encrypt(struct aead_request *req)
{
struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
struct ablkcipher_request *abreq = &pctx->abreq;
int err;
crypto_gcm_init_crypt(abreq, req, req->cryptlen);
ablkcipher_request_set_callback(abreq, aead_request_flags(req),
crypto_gcm_encrypt_done, req);
err = crypto_ablkcipher_encrypt(abreq);
if (err)
return err;
return crypto_gcm_hash(req);
}
static void __gcm_hash_assoc_done(struct aead_request *req, int err)
{
struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
unsigned int remain;
if (!err) {
remain = gcm_remain(req->assoclen);
BUG_ON(!remain);
err = gcm_hash_remain(req, pctx, remain,
gcm_hash_assoc_remain_done);
if (err == -EINPROGRESS || err == -EBUSY)
return;
}
__gcm_hash_assoc_remain_done(req, err);
}
static void gcm_encrypt_done(struct crypto_async_request *areq, int err)
{
struct aead_request *req = areq->data;
struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
if (!err) {
err = gcm_hash(req, pctx);
if (err == -EINPROGRESS || err == -EBUSY)
return;
else if (!err) {
crypto_xor(pctx->auth_tag, pctx->iauth_tag, 16);
gcm_enc_copy_hash(req, pctx);
}
}
aead_request_complete(req, err);
}
static void gcm_dec_hash_done(struct crypto_async_request *areq, int err)
{
struct aead_request *req = areq->data;
struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
struct ablkcipher_request *abreq = &pctx->u.abreq;
struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
if (!err) {
ablkcipher_request_set_callback(abreq, aead_request_flags(req),
gcm_decrypt_done, req);
crypto_gcm_init_crypt(abreq, req, gctx->cryptlen);
err = crypto_ablkcipher_decrypt(abreq);
if (err == -EINPROGRESS || err == -EBUSY)
return;
}
gcm_decrypt_done(areq, err);
}
static void gcm_hash_crypt_done(struct crypto_async_request *areq,
int err)
{
struct aead_request *req = areq->data;
struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
unsigned int remain;
if (!err) {
remain = gcm_remain(gctx->cryptlen);
BUG_ON(!remain);
err = gcm_hash_remain(req, pctx, remain,
gcm_hash_crypt_remain_done);
if (err == -EINPROGRESS || err == -EBUSY)
return;
}
gcm_hash_crypt_remain_done(areq, err);
}
static void __gcm_hash_init_done(struct aead_request *req, int err)
{
struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
crypto_completion_t compl;
unsigned int remain = 0;
if (!err && req->assoclen) {
remain = gcm_remain(req->assoclen);
compl = remain ? gcm_hash_assoc_done :
gcm_hash_assoc_remain_done;
err = gcm_hash_update(req, pctx, compl,
req->assoc, req->assoclen);
if (err == -EINPROGRESS || err == -EBUSY)
return;
}
if (remain)
__gcm_hash_assoc_done(req, err);
else
__gcm_hash_assoc_remain_done(req, err);
}
static void __gcm_hash_assoc_remain_done(struct aead_request *req, int err)
{
struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
crypto_completion_t complete;
unsigned int remain = 0;
if (!err && gctx->cryptlen) {
remain = gcm_remain(gctx->cryptlen);
complete = remain ? gcm_hash_crypt_done :
gcm_hash_crypt_remain_done;
err = gcm_hash_update(req, pctx, complete,
gctx->src, gctx->cryptlen);
if (err == -EINPROGRESS || err == -EBUSY)
return;
}
if (remain)
__gcm_hash_crypt_done(req, err);
else
__gcm_hash_crypt_remain_done(req, err);
}
static void crypto_gcm_init_crypt(struct ablkcipher_request *ablk_req,
struct aead_request *req,
unsigned int cryptlen)
{
struct crypto_aead *aead = crypto_aead_reqtfm(req);
struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead);
struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
u32 flags = req->base.tfm->crt_flags;
struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash;
struct scatterlist *dst;
__be32 counter = cpu_to_be32(1);
memset(pctx->auth_tag, 0, sizeof(pctx->auth_tag));
memcpy(req->iv + 12, &counter, 4);
sg_init_table(pctx->src, 2);
sg_set_buf(pctx->src, pctx->auth_tag, sizeof(pctx->auth_tag));
scatterwalk_sg_chain(pctx->src, 2, req->src);
dst = pctx->src;
if (req->src != req->dst) {
sg_init_table(pctx->dst, 2);
sg_set_buf(pctx->dst, pctx->auth_tag, sizeof(pctx->auth_tag));
scatterwalk_sg_chain(pctx->dst, 2, req->dst);
dst = pctx->dst;
}
ablkcipher_request_set_tfm(ablk_req, ctx->ctr);
ablkcipher_request_set_crypt(ablk_req, pctx->src, dst,
cryptlen + sizeof(pctx->auth_tag),
req->iv);
crypto_gcm_ghash_init(ghash, flags, ctx->gf128);
crypto_gcm_ghash_update_sg(ghash, req->assoc, req->assoclen);
crypto_gcm_ghash_flush(ghash);
}
