static int
rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len)
{
struct crypto_ablkcipher *ctr_tfm;
struct ablkcipher_request *req;
int ret = -EINVAL;
struct aesni_hash_subkey_req_data *req_data;
ctr_tfm = crypto_alloc_ablkcipher("ctr(aes)", 0, 0);
if (IS_ERR(ctr_tfm))
return PTR_ERR(ctr_tfm);
crypto_ablkcipher_clear_flags(ctr_tfm, ~0);
ret = crypto_ablkcipher_setkey(ctr_tfm, key, key_len);
if (ret)
goto out_free_ablkcipher;
ret = -ENOMEM;
req = ablkcipher_request_alloc(ctr_tfm, GFP_KERNEL);
if (!req)
goto out_free_ablkcipher;
req_data = kmalloc(sizeof(*req_data), GFP_KERNEL);
if (!req_data)
goto out_free_request;
memset(req_data->iv, 0, sizeof(req_data->iv));
/* Clear the data in the hash sub key container to zero.*/
/* We want to cipher all zeros to create the hash sub key. */
memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE);
init_completion(&req_data->result.completion);
sg_init_one(&req_data->sg, hash_subkey, RFC4106_HASH_SUBKEY_SIZE);
ablkcipher_request_set_tfm(req, ctr_tfm);
ablkcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP |
CRYPTO_TFM_REQ_MAY_BACKLOG,
rfc4106_set_hash_subkey_done,
&req_data->result);
ablkcipher_request_set_crypt(req, &req_data->sg,
&req_data->sg, RFC4106_HASH_SUBKEY_SIZE, req_data->iv);
ret = crypto_ablkcipher_encrypt(req);
if (ret == -EINPROGRESS || ret == -EBUSY) {
ret = wait_for_completion_interruptible
(&req_data->result.completion);
if (!ret)
ret = req_data->result.err;
}
kfree(req_data);
out_free_request:
ablkcipher_request_free(req);
out_free_ablkcipher:
crypto_free_ablkcipher(ctr_tfm);
return ret;
}
static int ext4_page_crypto(struct ext4_crypto_ctx *ctx,
struct inode *inode,
ext4_direction_t rw,
pgoff_t index,
struct page *src_page,
struct page *dest_page)
{
u8 xts_tweak[EXT4_XTS_TWEAK_SIZE];
struct ablkcipher_request *req = NULL;
DECLARE_EXT4_COMPLETION_RESULT(ecr);
struct scatterlist dst, src;
struct ext4_crypt_info *ci = EXT4_I(inode)->i_crypt_info;
struct crypto_ablkcipher *tfm = ci->ci_ctfm;
int res = 0;
req = ablkcipher_request_alloc(tfm, GFP_NOFS);
if (!req) {
printk_ratelimited(KERN_ERR
"%s: crypto_request_alloc() failed\n",
__func__);
return -ENOMEM;
}
ablkcipher_request_set_callback(
req, CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP,
ext4_crypt_complete, &ecr);
BUILD_BUG_ON(EXT4_XTS_TWEAK_SIZE < sizeof(index));
memcpy(xts_tweak, &index, sizeof(index));
memset(&xts_tweak[sizeof(index)], 0,
EXT4_XTS_TWEAK_SIZE - sizeof(index));
sg_init_table(&dst, 1);
sg_set_page(&dst, dest_page, PAGE_CACHE_SIZE, 0);
sg_init_table(&src, 1);
sg_set_page(&src, src_page, PAGE_CACHE_SIZE, 0);
ablkcipher_request_set_crypt(req, &src, &dst, PAGE_CACHE_SIZE,
xts_tweak);
if (rw == EXT4_DECRYPT)
res = crypto_ablkcipher_decrypt(req);
else
res = crypto_ablkcipher_encrypt(req);
if (res == -EINPROGRESS || res == -EBUSY) {
BUG_ON(req->base.data != &ecr);
wait_for_completion(&ecr.completion);
res = ecr.res;
}
ablkcipher_request_free(req);
if (res) {
printk_ratelimited(
KERN_ERR
"%s: crypto_ablkcipher_encrypt() returned %d\n",
__func__, res);
return res;
}
return 0;
}
/*
* ext4_fname_decrypt()
* This function decrypts the input filename, and returns
* the length of the plaintext.
* Errors are returned as negative numbers.
* We trust the caller to allocate sufficient memory to oname string.
*/
static int ext4_fname_decrypt(struct inode *inode,
const struct ext4_str *iname,
struct ext4_str *oname)
{
struct ext4_str tmp_in[2], tmp_out[1];
struct ablkcipher_request *req = NULL;
DECLARE_EXT4_COMPLETION_RESULT(ecr);
struct scatterlist src_sg, dst_sg;
struct ext4_crypt_info *ci = EXT4_I(inode)->i_crypt_info;
struct crypto_ablkcipher *tfm = ci->ci_ctfm;
int res = 0;
char iv[EXT4_CRYPTO_BLOCK_SIZE];
unsigned lim = max_name_len(inode);
if (iname->len <= 0 || iname->len > lim)
return -EIO;
tmp_in[0].name = iname->name;
tmp_in[0].len = iname->len;
tmp_out[0].name = oname->name;
/* Allocate request */
req = ablkcipher_request_alloc(tfm, GFP_NOFS);
if (!req) {
printk_ratelimited(
KERN_ERR "%s: crypto_request_alloc() failed\n", __func__);
return -ENOMEM;
}
ablkcipher_request_set_callback(req,
CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP,
ext4_dir_crypt_complete, &ecr);
/* Initialize IV */
memset(iv, 0, EXT4_CRYPTO_BLOCK_SIZE);
/* Create encryption request */
sg_init_one(&src_sg, iname->name, iname->len);
sg_init_one(&dst_sg, oname->name, oname->len);
ablkcipher_request_set_crypt(req, &src_sg, &dst_sg, iname->len, iv);
res = crypto_ablkcipher_decrypt(req);
if (res == -EINPROGRESS || res == -EBUSY) {
BUG_ON(req->base.data != &ecr);
wait_for_completion(&ecr.completion);
res = ecr.res;
}
ablkcipher_request_free(req);
if (res < 0) {
printk_ratelimited(
KERN_ERR "%s: Error in ext4_fname_encrypt (error code %d)\n",
__func__, res);
return res;
}
oname->len = strnlen(oname->name, iname->len);
return oname->len;
}
/**
* fname_decrypt() - decrypt a filename
*
* The caller must have allocated sufficient memory for the @oname string.
*
* Return: 0 on success, -errno on failure
*/
static int fname_decrypt(struct inode *inode,
const struct fscrypt_str *iname,
struct fscrypt_str *oname)
{
struct ablkcipher_request *req = NULL;
DECLARE_FS_COMPLETION_RESULT(ecr);
struct scatterlist src_sg, dst_sg;
struct fscrypt_info *ci = inode->i_crypt_info;
struct crypto_ablkcipher *tfm = ci->ci_ctfm;
int res = 0;
char iv[FS_CRYPTO_BLOCK_SIZE];
unsigned lim;
lim = inode->i_sb->s_cop->max_namelen(inode);
if (iname->len <= 0 || iname->len > lim)
return -EIO;
/* Allocate request */
req = ablkcipher_request_alloc(tfm, GFP_NOFS);
if (!req) {
printk_ratelimited(KERN_ERR
"%s: crypto_request_alloc() failed\n", __func__);
return -ENOMEM;
}
ablkcipher_request_set_callback(req,
CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP,
fname_crypt_complete, &ecr);
/* Initialize IV */
memset(iv, 0, FS_CRYPTO_BLOCK_SIZE);
/* Create decryption request */
sg_init_one(&src_sg, iname->name, iname->len);
sg_init_one(&dst_sg, oname->name, oname->len);
ablkcipher_request_set_crypt(req, &src_sg, &dst_sg, iname->len, iv);
res = crypto_ablkcipher_decrypt(req);
if (res == -EINPROGRESS || res == -EBUSY) {
wait_for_completion(&ecr.completion);
res = ecr.res;
}
ablkcipher_request_free(req);
if (res < 0) {
printk_ratelimited(KERN_ERR
"%s: Error (error code %d)\n", __func__, res);
return res;
}
oname->len = strnlen(oname->name, iname->len);
return 0;
}
void ctr_aes_deinit(struct ctr_drbg_ctx_s *ctx)
{
if (ctx->aes_ctx.req) {
ablkcipher_request_free(ctx->aes_ctx.req);
ctx->aes_ctx.req = NULL;
}
if (ctx->aes_ctx.tfm) {
crypto_free_ablkcipher(ctx->aes_ctx.tfm);
ctx->aes_ctx.tfm = NULL;
}
if (ctx->aes_ctx.input.virt_addr) {
kzfree(ctx->aes_ctx.input.virt_addr);
ctx->aes_ctx.input.virt_addr = NULL;
}
if (ctx->aes_ctx.output.virt_addr) {
kzfree(ctx->aes_ctx.output.virt_addr);
ctx->aes_ctx.output.virt_addr = NULL;
}
if (ctx->df_aes_ctx.req) {
ablkcipher_request_free(ctx->df_aes_ctx.req);
ctx->df_aes_ctx.req = NULL;
}
if (ctx->df_aes_ctx.tfm) {
crypto_free_ablkcipher(ctx->df_aes_ctx.tfm);
ctx->df_aes_ctx.tfm = NULL;
}
if (ctx->df_aes_ctx.input.virt_addr) {
kzfree(ctx->df_aes_ctx.input.virt_addr);
ctx->df_aes_ctx.input.virt_addr = NULL;
}
if (ctx->df_aes_ctx.output.virt_addr) {
kzfree(ctx->df_aes_ctx.output.virt_addr);
ctx->df_aes_ctx.output.virt_addr = NULL;
}
}
/**
* f2fs_derive_key_aes() - Derive a key using AES-128-ECB
* @deriving_key: Encryption key used for derivatio.
* @source_key: Source key to which to apply derivation.
* @derived_key: Derived key.
*
* Return: Zero on success; non-zero otherwise.
*/
static int f2fs_derive_key_aes(char deriving_key[F2FS_AES_128_ECB_KEY_SIZE],
char source_key[F2FS_AES_256_XTS_KEY_SIZE],
char derived_key[F2FS_AES_256_XTS_KEY_SIZE])
{
int res = 0;
struct ablkcipher_request *req = NULL;
DECLARE_F2FS_COMPLETION_RESULT(ecr);
struct scatterlist src_sg, dst_sg;
struct crypto_ablkcipher *tfm = crypto_alloc_ablkcipher("ecb(aes)", 0,
0);
if (IS_ERR(tfm)) {
res = PTR_ERR(tfm);
tfm = NULL;
goto out;
}
crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_REQ_WEAK_KEY);
req = ablkcipher_request_alloc(tfm, GFP_NOFS);
if (!req) {
res = -ENOMEM;
goto out;
}
ablkcipher_request_set_callback(req,
CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP,
derive_crypt_complete, &ecr);
res = crypto_ablkcipher_setkey(tfm, deriving_key,
F2FS_AES_128_ECB_KEY_SIZE);
if (res < 0)
goto out;
sg_init_one(&src_sg, source_key, F2FS_AES_256_XTS_KEY_SIZE);
sg_init_one(&dst_sg, derived_key, F2FS_AES_256_XTS_KEY_SIZE);
ablkcipher_request_set_crypt(req, &src_sg, &dst_sg,
F2FS_AES_256_XTS_KEY_SIZE, NULL);
res = crypto_ablkcipher_encrypt(req);
if (res == -EINPROGRESS || res == -EBUSY) {
BUG_ON(req->base.data != &ecr);
wait_for_completion(&ecr.completion);
res = ecr.res;
}
out:
if (req)
ablkcipher_request_free(req);
if (tfm)
crypto_free_ablkcipher(tfm);
return res;
}
void cryptodev_cipher_deinit(struct cipher_data *cdata)
{
if (cdata->init) {
if (cdata->aead == 0) {
if (cdata->async.request)
ablkcipher_request_free(cdata->async.request);
if (cdata->async.s)
crypto_free_ablkcipher(cdata->async.s);
} else {
if (cdata->async.arequest)
aead_request_free(cdata->async.arequest);
if (cdata->async.as)
crypto_free_aead(cdata->async.as);
}
kfree(cdata->async.result);
cdata->init = 0;
}
}
static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
{
struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
be128 buf[8];
struct xts_crypt_req req = {
.tbuf = buf,
.tbuflen = sizeof(buf),
.tweak_ctx = aes_ctx(ctx->raw_tweak_ctx),
.tweak_fn = aesni_xts_tweak,
.crypt_ctx = aes_ctx(ctx->raw_crypt_ctx),
.crypt_fn = lrw_xts_encrypt_callback,
};
int ret;
desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
kernel_fpu_begin();
ret = xts_crypt(desc, dst, src, nbytes, &req);
kernel_fpu_end();
return ret;
}
static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
{
struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
be128 buf[8];
struct xts_crypt_req req = {
.tbuf = buf,
.tbuflen = sizeof(buf),
.tweak_ctx = aes_ctx(ctx->raw_tweak_ctx),
.tweak_fn = aesni_xts_tweak,
.crypt_ctx = aes_ctx(ctx->raw_crypt_ctx),
.crypt_fn = lrw_xts_decrypt_callback,
};
int ret;
desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
kernel_fpu_begin();
ret = xts_crypt(desc, dst, src, nbytes, &req);
kernel_fpu_end();
return ret;
}
#endif
#ifdef CONFIG_X86_64
static int rfc4106_init(struct crypto_tfm *tfm)
{
struct cryptd_aead *cryptd_tfm;
struct aesni_rfc4106_gcm_ctx *ctx = (struct aesni_rfc4106_gcm_ctx *)
PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN);
struct crypto_aead *cryptd_child;
struct aesni_rfc4106_gcm_ctx *child_ctx;
cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni",
CRYPTO_ALG_INTERNAL,
CRYPTO_ALG_INTERNAL);
if (IS_ERR(cryptd_tfm))
return PTR_ERR(cryptd_tfm);
cryptd_child = cryptd_aead_child(cryptd_tfm);
child_ctx = aesni_rfc4106_gcm_ctx_get(cryptd_child);
memcpy(child_ctx, ctx, sizeof(*ctx));
ctx->cryptd_tfm = cryptd_tfm;
tfm->crt_aead.reqsize = sizeof(struct aead_request)
+ crypto_aead_reqsize(&cryptd_tfm->base);
return 0;
}
static void rfc4106_exit(struct crypto_tfm *tfm)
{
struct aesni_rfc4106_gcm_ctx *ctx =
(struct aesni_rfc4106_gcm_ctx *)
PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN);
if (!IS_ERR(ctx->cryptd_tfm))
cryptd_free_aead(ctx->cryptd_tfm);
return;
}
static void
rfc4106_set_hash_subkey_done(struct crypto_async_request *req, int err)
{
struct aesni_gcm_set_hash_subkey_result *result = req->data;
if (err == -EINPROGRESS)
return;
result->err = err;
complete(&result->completion);
}
static int
rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len)
{
struct crypto_ablkcipher *ctr_tfm;
struct ablkcipher_request *req;
int ret = -EINVAL;
struct aesni_hash_subkey_req_data *req_data;
ctr_tfm = crypto_alloc_ablkcipher("ctr(aes)", 0, 0);
if (IS_ERR(ctr_tfm))
return PTR_ERR(ctr_tfm);
crypto_ablkcipher_clear_flags(ctr_tfm, ~0);
ret = crypto_ablkcipher_setkey(ctr_tfm, key, key_len);
if (ret)
goto out_free_ablkcipher;
ret = -ENOMEM;
req = ablkcipher_request_alloc(ctr_tfm, GFP_KERNEL);
if (!req)
goto out_free_ablkcipher;
req_data = kmalloc(sizeof(*req_data), GFP_KERNEL);
if (!req_data)
goto out_free_request;
memset(req_data->iv, 0, sizeof(req_data->iv));
/* Clear the data in the hash sub key container to zero.*/
/* We want to cipher all zeros to create the hash sub key. */
memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE);
init_completion(&req_data->result.completion);
sg_init_one(&req_data->sg, hash_subkey, RFC4106_HASH_SUBKEY_SIZE);
ablkcipher_request_set_tfm(req, ctr_tfm);
ablkcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP |
CRYPTO_TFM_REQ_MAY_BACKLOG,
rfc4106_set_hash_subkey_done,
&req_data->result);
ablkcipher_request_set_crypt(req, &req_data->sg,
&req_data->sg, RFC4106_HASH_SUBKEY_SIZE, req_data->iv);
ret = crypto_ablkcipher_encrypt(req);
if (ret == -EINPROGRESS || ret == -EBUSY) {
ret = wait_for_completion_interruptible
(&req_data->result.completion);
if (!ret)
ret = req_data->result.err;
}
kfree(req_data);
out_free_request:
ablkcipher_request_free(req);
out_free_ablkcipher:
crypto_free_ablkcipher(ctr_tfm);
return ret;
}
static int common_rfc4106_set_key(struct crypto_aead *aead, const u8 *key,
unsigned int key_len)
{
int ret = 0;
struct crypto_tfm *tfm = crypto_aead_tfm(aead);
struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(aead);
u8 *new_key_align, *new_key_mem = NULL;
if (key_len < 4) {
crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
return -EINVAL;
}
/*Account for 4 byte nonce at the end.*/
key_len -= 4;
if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 &&
key_len != AES_KEYSIZE_256) {
crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
return -EINVAL;
}
memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce));
/*This must be on a 16 byte boundary!*/
if ((unsigned long)(&(ctx->aes_key_expanded.key_enc[0])) % AESNI_ALIGN)
return -EINVAL;
if ((unsigned long)key % AESNI_ALIGN) {
/*key is not aligned: use an auxuliar aligned pointer*/
new_key_mem = kmalloc(key_len+AESNI_ALIGN, GFP_KERNEL);
if (!new_key_mem)
return -ENOMEM;
new_key_align = PTR_ALIGN(new_key_mem, AESNI_ALIGN);
memcpy(new_key_align, key, key_len);
key = new_key_align;
}
if (!irq_fpu_usable())
ret = crypto_aes_expand_key(&(ctx->aes_key_expanded),
key, key_len);
else {
kernel_fpu_begin();
ret = aesni_set_key(&(ctx->aes_key_expanded), key, key_len);
kernel_fpu_end();
}
/*This must be on a 16 byte boundary!*/
if ((unsigned long)(&(ctx->hash_subkey[0])) % AESNI_ALIGN) {
ret = -EINVAL;
goto exit;
}
ret = rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
exit:
kfree(new_key_mem);
return ret;
}
static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key,
unsigned int key_len)
{
struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent);
struct crypto_aead *child = cryptd_aead_child(ctx->cryptd_tfm);
struct aesni_rfc4106_gcm_ctx *c_ctx = aesni_rfc4106_gcm_ctx_get(child);
struct cryptd_aead *cryptd_tfm = ctx->cryptd_tfm;
int ret;
ret = crypto_aead_setkey(child, key, key_len);
if (!ret) {
memcpy(ctx, c_ctx, sizeof(*ctx));
ctx->cryptd_tfm = cryptd_tfm;
}
return ret;
}
static int common_rfc4106_set_authsize(struct crypto_aead *aead,
unsigned int authsize)
{
switch (authsize) {
case 8:
case 12:
case 16:
break;
default:
return -EINVAL;
}
crypto_aead_crt(aead)->authsize = authsize;
return 0;
}
/* This is the Integrity Check Value (aka the authentication tag length and can
* be 8, 12 or 16 bytes long. */
static int rfc4106_set_authsize(struct crypto_aead *parent,
unsigned int authsize)
{
struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent);
struct crypto_aead *child = cryptd_aead_child(ctx->cryptd_tfm);
int ret;
ret = crypto_aead_setauthsize(child, authsize);
if (!ret)
crypto_aead_crt(parent)->authsize = authsize;
return ret;
}
static int __driver_rfc4106_encrypt(struct aead_request *req)
{
u8 one_entry_in_sg = 0;
u8 *src, *dst, *assoc;
__be32 counter = cpu_to_be32(1);
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
u32 key_len = ctx->aes_key_expanded.key_length;
void *aes_ctx = &(ctx->aes_key_expanded);
unsigned long auth_tag_len = crypto_aead_authsize(tfm);
u8 iv_tab[16+AESNI_ALIGN];
u8* iv = (u8 *) PTR_ALIGN((u8 *)iv_tab, AESNI_ALIGN);
struct scatter_walk src_sg_walk;
struct scatter_walk assoc_sg_walk;
struct scatter_walk dst_sg_walk;
unsigned int i;
/* Assuming we are supporting rfc4106 64-bit extended */
/* sequence numbers We need to have the AAD length equal */
/* to 8 or 12 bytes */
if (unlikely(req->assoclen != 8 && req->assoclen != 12))
return -EINVAL;
if (unlikely(auth_tag_len != 8 && auth_tag_len != 12 && auth_tag_len != 16))
return -EINVAL;
if (unlikely(key_len != AES_KEYSIZE_128 &&
key_len != AES_KEYSIZE_192 &&
key_len != AES_KEYSIZE_256))
return -EINVAL;
/* IV below built */
for (i = 0; i < 4; i++)
*(iv+i) = ctx->nonce[i];
for (i = 0; i < 8; i++)
*(iv+4+i) = req->iv[i];
*((__be32 *)(iv+12)) = counter;
if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) {
one_entry_in_sg = 1;
scatterwalk_start(&src_sg_walk, req->src);
scatterwalk_start(&assoc_sg_walk, req->assoc);
src = scatterwalk_map(&src_sg_walk);
assoc = scatterwalk_map(&assoc_sg_walk);
dst = src;
if (unlikely(req->src != req->dst)) {
scatterwalk_start(&dst_sg_walk, req->dst);
dst = scatterwalk_map(&dst_sg_walk);
}
} else {
/* Allocate memory for src, dst, assoc */
src = kmalloc(req->cryptlen + auth_tag_len + req->assoclen,
GFP_ATOMIC);
if (unlikely(!src))
return -ENOMEM;
assoc = (src + req->cryptlen + auth_tag_len);
scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0);
scatterwalk_map_and_copy(assoc, req->assoc, 0,
req->assoclen, 0);
dst = src;
}
aesni_gcm_enc_tfm(aes_ctx, dst, src, (unsigned long)req->cryptlen, iv,
ctx->hash_subkey, assoc, (unsigned long)req->assoclen, dst
+ ((unsigned long)req->cryptlen), auth_tag_len);
/* The authTag (aka the Integrity Check Value) needs to be written
* back to the packet. */
if (one_entry_in_sg) {
if (unlikely(req->src != req->dst)) {
scatterwalk_unmap(dst);
scatterwalk_done(&dst_sg_walk, 0, 0);
}
scatterwalk_unmap(src);
scatterwalk_unmap(assoc);
scatterwalk_done(&src_sg_walk, 0, 0);
scatterwalk_done(&assoc_sg_walk, 0, 0);
} else {
scatterwalk_map_and_copy(dst, req->dst, 0,
req->cryptlen + auth_tag_len, 1);
kfree(src);
}
return 0;
}
int cryptodev_cipher_init(struct cipher_data *out, const char *alg_name,
uint8_t *keyp, size_t keylen, int stream, int aead)
{
int ret;
if (aead == 0) {
struct ablkcipher_alg *alg;
out->async.s = crypto_alloc_ablkcipher(alg_name, 0, 0);
if (unlikely(IS_ERR(out->async.s))) {
ddebug(1, "Failed to load cipher %s", alg_name);
return -EINVAL;
}
alg = crypto_ablkcipher_alg(out->async.s);
if (alg != NULL) {
/* Was correct key length supplied? */
if (alg->max_keysize > 0 &&
unlikely((keylen < alg->min_keysize) ||
(keylen > alg->max_keysize))) {
ddebug(1, "Wrong keylen '%zu' for algorithm '%s'. Use %u to %u.",
keylen, alg_name, alg->min_keysize, alg->max_keysize);
ret = -EINVAL;
goto error;
}
}
out->blocksize = crypto_ablkcipher_blocksize(out->async.s);
out->ivsize = crypto_ablkcipher_ivsize(out->async.s);
out->alignmask = crypto_ablkcipher_alignmask(out->async.s);
ret = crypto_ablkcipher_setkey(out->async.s, keyp, keylen);
} else {
out->async.as = crypto_alloc_aead(alg_name, 0, 0);
if (unlikely(IS_ERR(out->async.as))) {
ddebug(1, "Failed to load cipher %s", alg_name);
return -EINVAL;
}
out->blocksize = crypto_aead_blocksize(out->async.as);
out->ivsize = crypto_aead_ivsize(out->async.as);
out->alignmask = crypto_aead_alignmask(out->async.as);
ret = crypto_aead_setkey(out->async.as, keyp, keylen);
}
if (unlikely(ret)) {
ddebug(1, "Setting key failed for %s-%zu.", alg_name, keylen*8);
ret = -EINVAL;
goto error;
}
out->stream = stream;
out->aead = aead;
out->async.result = kzalloc(sizeof(*out->async.result), GFP_KERNEL);
if (unlikely(!out->async.result)) {
ret = -ENOMEM;
goto error;
}
init_completion(&out->async.result->completion);
if (aead == 0) {
out->async.request = ablkcipher_request_alloc(out->async.s, GFP_KERNEL);
if (unlikely(!out->async.request)) {
derr(1, "error allocating async crypto request");
ret = -ENOMEM;
goto error;
}
ablkcipher_request_set_callback(out->async.request,
CRYPTO_TFM_REQ_MAY_BACKLOG,
cryptodev_complete, out->async.result);
} else {
out->async.arequest = aead_request_alloc(out->async.as, GFP_KERNEL);
if (unlikely(!out->async.arequest)) {
derr(1, "error allocating async crypto request");
ret = -ENOMEM;
goto error;
}
aead_request_set_callback(out->async.arequest,
CRYPTO_TFM_REQ_MAY_BACKLOG,
cryptodev_complete, out->async.result);
}
out->init = 1;
return 0;
error:
if (aead == 0) {
if (out->async.request)
ablkcipher_request_free(out->async.request);
if (out->async.s)
crypto_free_ablkcipher(out->async.s);
} else {
if (out->async.arequest)
aead_request_free(out->async.arequest);
if (out->async.as)
crypto_free_aead(out->async.as);
}
kfree(out->async.result);
return ret;
}
VOS_STATUS vos_decrypt_AES(v_U32_t cryptHandle, /* Handle */
v_U8_t *pText, /* pointer to data stream */
v_U8_t *pDecrypted,
v_U8_t *pKey) /* pointer to authentication key */
{
// VOS_STATUS uResult = VOS_STATUS_E_FAILURE;
struct ecb_aes_result result;
struct ablkcipher_request *req;
struct crypto_ablkcipher *tfm;
int ret = 0;
char iv[IV_SIZE_AES_128];
struct scatterlist sg_in;
struct scatterlist sg_out;
init_completion(&result.completion);
#if !defined(CONFIG_CNSS) && (defined(HIF_USB) || defined(HIF_SDIO))
tfm = crypto_alloc_ablkcipher( "cbc(aes)", 0, 0);
#else
tfm = wcnss_wlan_crypto_alloc_ablkcipher( "cbc(aes)", 0, 0);
#endif
if (IS_ERR(tfm)) {
VOS_TRACE(VOS_MODULE_ID_VOSS,VOS_TRACE_LEVEL_ERROR, "crypto_alloc_ablkcipher failed");
ret = PTR_ERR(tfm);
goto err_tfm;
}
req = ablkcipher_request_alloc(tfm, GFP_KERNEL);
if (!req) {
VOS_TRACE( VOS_MODULE_ID_VOSS, VOS_TRACE_LEVEL_ERROR, "Failed to allocate request for cbc(aes)");
ret = -ENOMEM;
goto err_req;
}
ablkcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
ecb_aes_complete, &result);
crypto_ablkcipher_clear_flags(tfm, ~0);
ret = crypto_ablkcipher_setkey(tfm, pKey, AES_KEYSIZE_128);
if (ret) {
VOS_TRACE(VOS_MODULE_ID_VOSS,VOS_TRACE_LEVEL_ERROR, "crypto_cipher_setkey failed");
goto err_setkey;
}
memset(iv, 0, IV_SIZE_AES_128);
sg_init_one(&sg_in, pText, AES_BLOCK_SIZE);
sg_init_one(&sg_out, pDecrypted, AES_BLOCK_SIZE);
ablkcipher_request_set_crypt(req, &sg_in, &sg_out, AES_BLOCK_SIZE, iv);
crypto_ablkcipher_decrypt(req);
// -------------------------------------
err_setkey:
#if !defined(CONFIG_CNSS) && (defined(HIF_USB) || defined(HIF_SDIO))
ablkcipher_request_free(req);
#else
wcnss_wlan_ablkcipher_request_free(req);
#endif
err_req:
#if !defined(CONFIG_CNSS) && (defined(HIF_USB) || defined(HIF_SDIO))
crypto_free_ablkcipher(tfm);
#else
wcnss_wlan_crypto_free_ablkcipher(tfm);
#endif
err_tfm:
//return ret;
if (ret != 0) {
VOS_TRACE(VOS_MODULE_ID_VOSS,VOS_TRACE_LEVEL_ERROR,"%s() call failed", __func__);
return VOS_STATUS_E_FAULT;
}
return VOS_STATUS_SUCCESS;
}
/**
* fname_encrypt() - encrypt a filename
*
* The caller must have allocated sufficient memory for the @oname string.
*
* Return: 0 on success, -errno on failure
*/
static int fname_encrypt(struct inode *inode,
const struct qstr *iname, struct fscrypt_str *oname)
{
struct ablkcipher_request *req = NULL;
DECLARE_FS_COMPLETION_RESULT(ecr);
struct fscrypt_info *ci = inode->i_crypt_info;
struct crypto_ablkcipher *tfm = ci->ci_ctfm;
int res = 0;
char iv[FS_CRYPTO_BLOCK_SIZE];
struct scatterlist sg;
int padding = 4 << (ci->ci_flags & FS_POLICY_FLAGS_PAD_MASK);
unsigned int lim;
unsigned int cryptlen;
lim = inode->i_sb->s_cop->max_namelen(inode);
if (iname->len <= 0 || iname->len > lim)
return -EIO;
/*
* Copy the filename to the output buffer for encrypting in-place and
* pad it with the needed number of NUL bytes.
*/
cryptlen = max_t(unsigned int, iname->len, FS_CRYPTO_BLOCK_SIZE);
cryptlen = round_up(cryptlen, padding);
cryptlen = min(cryptlen, lim);
memcpy(oname->name, iname->name, iname->len);
memset(oname->name + iname->len, 0, cryptlen - iname->len);
/* Initialize the IV */
memset(iv, 0, FS_CRYPTO_BLOCK_SIZE);
/* Set up the encryption request */
req = ablkcipher_request_alloc(tfm, GFP_NOFS);
if (!req) {
printk_ratelimited(KERN_ERR
"%s: ablkcipher_request_alloc() failed\n", __func__);
return -ENOMEM;
}
ablkcipher_request_set_callback(req,
CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP,
fname_crypt_complete, &ecr);
sg_init_one(&sg, oname->name, cryptlen);
ablkcipher_request_set_crypt(req, &sg, &sg, cryptlen, iv);
/* Do the encryption */
res = crypto_ablkcipher_encrypt(req);
if (res == -EINPROGRESS || res == -EBUSY) {
/* Request is being completed asynchronously; wait for it */
wait_for_completion(&ecr.completion);
res = ecr.res;
}
ablkcipher_request_free(req);
if (res < 0) {
printk_ratelimited(KERN_ERR
"%s: Error (error code %d)\n", __func__, res);
return res;
}
oname->len = cryptlen;
return 0;
}
void mmc_encrypt_req(struct mmc_host *host, struct mmc_request *mrq)
{
struct crypto_ablkcipher *tfm;
struct ablkcipher_request *req;
struct mmc_tcrypt_result result;
struct scatterlist *in_sg = mrq->data->sg;
struct scatterlist *out_sg = NULL;
u8 *dst_data = NULL;
unsigned long data_len = 0;
uint32_t bytes = 0;
int rc = 0;
u8 IV[MMC_AES_XTS_IV_LEN];
sector_t sector = mrq->data->sector;
tfm = crypto_alloc_ablkcipher("xts(aes)", 0, 0);
if (IS_ERR(tfm)) {
pr_err("%s:%s ablkcipher tfm allocation failed : error = %lu\n",
mmc_hostname(host), __func__, PTR_ERR(tfm));
return;
}
req = ablkcipher_request_alloc(tfm, GFP_KERNEL);
if (!req) {
pr_err("%s:%s ablkcipher request allocation failed\n",
mmc_hostname(host), __func__);
goto ablkcipher_req_alloc_failure;
}
ablkcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
mmc_crypto_cipher_complete, &result);
init_completion(&result.completion);
qcrypto_cipher_set_flag(req,
QCRYPTO_CTX_USE_PIPE_KEY | QCRYPTO_CTX_XTS_DU_SIZE_512B);
crypto_ablkcipher_clear_flags(tfm, ~0);
crypto_ablkcipher_setkey(tfm, NULL, MMC_KEY_SIZE_XTS);
data_len = mrq->data->blksz * mrq->data->blocks;
if (data_len > MMC_512_KB) {
pr_err("%s:%s Encryption operation aborted: req size > 512K\n",
mmc_hostname(host), __func__);
goto crypto_operation_failure;
}
if (mmc_crypto_buf_idx != MAX_ENCRYPTION_BUFFERS) {
dst_data = mmc_crypto_bufs[mmc_crypto_buf_idx];
out_sg = mmc_crypto_out_sg[mmc_crypto_buf_idx];
mmc_crypto_buf_idx = 1-mmc_crypto_buf_idx;
} else {
pr_err("%s:%s encryption buffers not available\n",
mmc_hostname(host), __func__);
goto crypto_operation_failure;
}
bytes = sg_copy_to_buffer(in_sg, mrq->data->sg_len, dst_data, data_len);
if (bytes != data_len) {
pr_err("%s:%s error in copying data from sglist to buffer\n",
mmc_hostname(host), __func__);
goto crypto_operation_failure;
}
if (!mmc_copy_sglist(in_sg, mrq->data->sg_len, out_sg, dst_data)) {
pr_err("%s:%s could not create dst sglist from in sglist\n",
mmc_hostname(host), __func__);
goto crypto_operation_failure;
}
memset(IV, 0, MMC_AES_XTS_IV_LEN);
memcpy(IV, §or, sizeof(sector_t));
ablkcipher_request_set_crypt(req, in_sg, out_sg, data_len,
(void *) IV);
rc = crypto_ablkcipher_encrypt(req);
switch (rc) {
case 0:
break;
case -EBUSY:
/*
* Lets make this synchronous request by waiting on
* in progress as well
*/
case -EINPROGRESS:
wait_for_completion_interruptible(&result.completion);
if (result.err)
pr_err("%s:%s error = %d encrypting the request\n",
mmc_hostname(host), __func__, result.err);
break;
default:
goto crypto_operation_failure;
}
mrq->data->sg = out_sg;
mrq->data->sg_len = mmc_count_sg(out_sg, data_len);
crypto_operation_failure:
ablkcipher_request_free(req);
ablkcipher_req_alloc_failure:
crypto_free_ablkcipher(tfm);
return;
}
/*
* Cipher algorithm self tests
*/
int _fips_qcrypto_cipher_selftest(struct fips_selftest_data *selftest_d)
{
int rc = 0, err, tv_index, num_tv;
struct crypto_ablkcipher *tfm;
struct ablkcipher_request *ablkcipher_req;
struct _fips_completion fips_completion;
char *k_align_src = NULL;
struct scatterlist fips_sg;
struct _fips_test_vector_cipher tv_cipher;
num_tv = (sizeof(fips_test_vector_cipher)) /
(sizeof(struct _fips_test_vector_cipher));
/* One-by-one testing */
for (tv_index = 0; tv_index < num_tv; tv_index++) {
memcpy(&tv_cipher, &fips_test_vector_cipher[tv_index],
(sizeof(struct _fips_test_vector_cipher)));
/* Single buffer allocation for in place operation */
k_align_src = kzalloc(tv_cipher.pln_txt_len, GFP_KERNEL);
if (k_align_src == NULL) {
pr_err("qcrypto:, Failed to allocate memory for k_align_src %ld\n",
PTR_ERR(k_align_src));
return -ENOMEM;
}
memcpy(&k_align_src[0], tv_cipher.pln_txt,
tv_cipher.pln_txt_len);
/* use_sw flags are set in dtsi file which makes
default Linux API calls to go to s/w crypto instead
of h/w crypto. This code makes sure that all selftests
calls always go to h/w, independent of DTSI flags. */
if (!strcmp(tv_cipher.mod_alg, "xts(aes)")) {
if (selftest_d->prefix_aes_xts_algo)
if (_fips_get_alg_cra_name(
tv_cipher.mod_alg,
selftest_d->algo_prefix,
strlen(tv_cipher.mod_alg))) {
rc = -1;
pr_err("Algo Name is too long for tv %d\n",
tv_index);
goto clr_buf;
}
} else {
if (selftest_d->prefix_aes_cbc_ecb_ctr_algo)
if (_fips_get_alg_cra_name(
tv_cipher.mod_alg,
selftest_d->algo_prefix,
strlen(tv_cipher.mod_alg))) {
rc = -1;
pr_err("Algo Name is too long for tv %d\n",
tv_index);
goto clr_buf;
}
}
tfm = crypto_alloc_ablkcipher(tv_cipher.mod_alg, 0, 0);
if (IS_ERR(tfm)) {
pr_err("qcrypto: %s algorithm not found\n",
tv_cipher.mod_alg);
rc = -ENOMEM;
goto clr_buf;
}
ablkcipher_req = ablkcipher_request_alloc(tfm, GFP_KERNEL);
if (!ablkcipher_req) {
pr_err("qcrypto: ablkcipher_request_alloc failed\n");
rc = -ENOMEM;
goto clr_tfm;
}
rc = qcrypto_cipher_set_device(ablkcipher_req,
selftest_d->ce_device);
if (rc != 0) {
pr_err("%s qcrypto_cipher_set_device failed with err %d\n",
__func__, rc);
goto clr_ablkcipher_req;
}
ablkcipher_request_set_callback(ablkcipher_req,
CRYPTO_TFM_REQ_MAY_BACKLOG,
_fips_cb, &fips_completion);
crypto_ablkcipher_clear_flags(tfm, ~0);
rc = crypto_ablkcipher_setkey(tfm, tv_cipher.key,
tv_cipher.klen);
if (rc) {
pr_err("qcrypto: crypto_ablkcipher_setkey failed\n");
goto clr_ablkcipher_req;
}
sg_set_buf(&fips_sg, k_align_src, tv_cipher.enc_txt_len);
sg_mark_end(&fips_sg);
ablkcipher_request_set_crypt(ablkcipher_req,
&fips_sg, &fips_sg, tv_cipher.pln_txt_len,
tv_cipher.iv);
/**** Encryption Test ****/
init_completion(&fips_completion.completion);
rc = crypto_ablkcipher_encrypt(ablkcipher_req);
if (rc == -EINPROGRESS || rc == -EBUSY) {
rc = wait_for_completion_interruptible(
&fips_completion.completion);
err = fips_completion.err;
if (!rc && !err) {
INIT_COMPLETION(fips_completion.completion);
} else {
pr_err("qcrypto:cipher:ENC, wait_for_completion failed\n");
goto clr_ablkcipher_req;
}
}
if (memcmp(k_align_src, tv_cipher.enc_txt,
tv_cipher.enc_txt_len)) {
rc = -1;
goto clr_ablkcipher_req;
}
/**** Decryption test ****/
init_completion(&fips_completion.completion);
rc = crypto_ablkcipher_decrypt(ablkcipher_req);
if (rc == -EINPROGRESS || rc == -EBUSY) {
rc = wait_for_completion_interruptible(
&fips_completion.completion);
err = fips_completion.err;
if (!rc && !err) {
INIT_COMPLETION(fips_completion.completion);
} else {
pr_err("qcrypto:cipher:DEC, wait_for_completion failed\n");
goto clr_ablkcipher_req;
}
}
if (memcmp(k_align_src, tv_cipher.pln_txt,
tv_cipher.pln_txt_len))
rc = -1;
clr_ablkcipher_req:
ablkcipher_request_free(ablkcipher_req);
clr_tfm:
crypto_free_ablkcipher(tfm);
clr_buf:
kzfree(k_align_src);
if (rc)
return rc;
}
return rc;
}
static int process_crypt_req(struct tegra_crypto_ctx *ctx, struct tegra_crypt_req *crypt_req)
{
struct crypto_ablkcipher *tfm;
struct ablkcipher_request *req = NULL;
struct scatterlist in_sg;
struct scatterlist out_sg;
unsigned long *xbuf[NBUFS];
int ret = 0, size = 0;
unsigned long total = 0;
const u8 *key = NULL;
struct tegra_crypto_completion tcrypt_complete;
if (crypt_req->op & TEGRA_CRYPTO_ECB) {
req = ablkcipher_request_alloc(ctx->ecb_tfm, GFP_KERNEL);
tfm = ctx->ecb_tfm;
} else if (crypt_req->op & TEGRA_CRYPTO_CBC) {
req = ablkcipher_request_alloc(ctx->cbc_tfm, GFP_KERNEL);
tfm = ctx->cbc_tfm;
} else if ((crypt_req->op & TEGRA_CRYPTO_OFB) &&
(tegra_get_chipid() != TEGRA_CHIPID_TEGRA2)) {
req = ablkcipher_request_alloc(ctx->ofb_tfm, GFP_KERNEL);
tfm = ctx->ofb_tfm;
} else if ((crypt_req->op & TEGRA_CRYPTO_CTR) &&
(tegra_get_chipid() != TEGRA_CHIPID_TEGRA2)) {
req = ablkcipher_request_alloc(ctx->ctr_tfm, GFP_KERNEL);
tfm = ctx->ctr_tfm;
}
if (!req) {
pr_err("%s: Failed to allocate request\n", __func__);
return -ENOMEM;
}
if ((crypt_req->keylen < 0) || (crypt_req->keylen > AES_MAX_KEY_SIZE)) {
ret = -EINVAL;
pr_err("crypt_req keylen invalid");
goto process_req_out;
}
crypto_ablkcipher_clear_flags(tfm, ~0);
if (!ctx->use_ssk)
key = crypt_req->key;
if (!crypt_req->skip_key) {
ret = crypto_ablkcipher_setkey(tfm, key, crypt_req->keylen);
if (ret < 0) {
pr_err("setkey failed");
goto process_req_out;
}
}
ret = alloc_bufs(xbuf);
if (ret < 0) {
pr_err("alloc_bufs failed");
goto process_req_out;
}
init_completion(&tcrypt_complete.restart);
ablkcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
tegra_crypt_complete, &tcrypt_complete);
total = crypt_req->plaintext_sz;
while (total > 0) {
size = min(total, PAGE_SIZE);
ret = copy_from_user((void *)xbuf[0],
(void __user *)crypt_req->plaintext, size);
if (ret) {
ret = -EFAULT;
pr_debug("%s: copy_from_user failed (%d)\n", __func__, ret);
goto process_req_buf_out;
}
sg_init_one(&in_sg, xbuf[0], size);
sg_init_one(&out_sg, xbuf[1], size);
if (!crypt_req->skip_iv)
ablkcipher_request_set_crypt(req, &in_sg,
&out_sg, size, crypt_req->iv);
else
ablkcipher_request_set_crypt(req, &in_sg,
&out_sg, size, NULL);
INIT_COMPLETION(tcrypt_complete.restart);
tcrypt_complete.req_err = 0;
ret = crypt_req->encrypt ?
crypto_ablkcipher_encrypt(req) :
crypto_ablkcipher_decrypt(req);
if ((ret == -EINPROGRESS) || (ret == -EBUSY)) {
/* crypto driver is asynchronous */
ret = wait_for_completion_interruptible(&tcrypt_complete.restart);
if (ret < 0)
goto process_req_buf_out;
if (tcrypt_complete.req_err < 0) {
ret = tcrypt_complete.req_err;
goto process_req_buf_out;
}
} else if (ret < 0) {
pr_debug("%scrypt failed (%d)\n",
crypt_req->encrypt ? "en" : "de", ret);
goto process_req_buf_out;
}
ret = copy_to_user((void __user *)crypt_req->result,
(const void *)xbuf[1], size);
if (ret) {
ret = -EFAULT;
pr_debug("%s: copy_to_user failed (%d)\n", __func__,
ret);
goto process_req_buf_out;
}
total -= size;
crypt_req->result += size;
crypt_req->plaintext += size;
}
process_req_buf_out:
free_bufs(xbuf);
process_req_out:
ablkcipher_request_free(req);
return ret;
}
static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
{
struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
be128 buf[8];
struct xts_crypt_req req = {
.tbuf = buf,
.tbuflen = sizeof(buf),
.tweak_ctx = aes_ctx(ctx->raw_tweak_ctx),
.tweak_fn = aesni_xts_tweak,
.crypt_ctx = aes_ctx(ctx->raw_crypt_ctx),
.crypt_fn = lrw_xts_encrypt_callback,
};
int ret;
desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
kernel_fpu_begin();
ret = xts_crypt(desc, dst, src, nbytes, &req);
kernel_fpu_end();
return ret;
}
static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
{
struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
be128 buf[8];
struct xts_crypt_req req = {
.tbuf = buf,
.tbuflen = sizeof(buf),
.tweak_ctx = aes_ctx(ctx->raw_tweak_ctx),
.tweak_fn = aesni_xts_tweak,
.crypt_ctx = aes_ctx(ctx->raw_crypt_ctx),
.crypt_fn = lrw_xts_decrypt_callback,
};
int ret;
desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
kernel_fpu_begin();
ret = xts_crypt(desc, dst, src, nbytes, &req);
kernel_fpu_end();
return ret;
}
#endif
#ifdef CONFIG_X86_64
static int rfc4106_init(struct crypto_tfm *tfm)
{
struct cryptd_aead *cryptd_tfm;
struct aesni_rfc4106_gcm_ctx *ctx = (struct aesni_rfc4106_gcm_ctx *)
PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN);
struct crypto_aead *cryptd_child;
struct aesni_rfc4106_gcm_ctx *child_ctx;
cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni", 0, 0);
if (IS_ERR(cryptd_tfm))
return PTR_ERR(cryptd_tfm);
cryptd_child = cryptd_aead_child(cryptd_tfm);
child_ctx = aesni_rfc4106_gcm_ctx_get(cryptd_child);
memcpy(child_ctx, ctx, sizeof(*ctx));
ctx->cryptd_tfm = cryptd_tfm;
tfm->crt_aead.reqsize = sizeof(struct aead_request)
+ crypto_aead_reqsize(&cryptd_tfm->base);
return 0;
}
static void rfc4106_exit(struct crypto_tfm *tfm)
{
struct aesni_rfc4106_gcm_ctx *ctx =
(struct aesni_rfc4106_gcm_ctx *)
PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN);
if (!IS_ERR(ctx->cryptd_tfm))
cryptd_free_aead(ctx->cryptd_tfm);
return;
}
static void
rfc4106_set_hash_subkey_done(struct crypto_async_request *req, int err)
{
struct aesni_gcm_set_hash_subkey_result *result = req->data;
if (err == -EINPROGRESS)
return;
result->err = err;
complete(&result->completion);
}
static int
rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len)
{
struct crypto_ablkcipher *ctr_tfm;
struct ablkcipher_request *req;
int ret = -EINVAL;
struct aesni_hash_subkey_req_data *req_data;
ctr_tfm = crypto_alloc_ablkcipher("ctr(aes)", 0, 0);
if (IS_ERR(ctr_tfm))
return PTR_ERR(ctr_tfm);
crypto_ablkcipher_clear_flags(ctr_tfm, ~0);
ret = crypto_ablkcipher_setkey(ctr_tfm, key, key_len);
if (ret)
goto out_free_ablkcipher;
ret = -ENOMEM;
req = ablkcipher_request_alloc(ctr_tfm, GFP_KERNEL);
if (!req)
goto out_free_ablkcipher;
req_data = kmalloc(sizeof(*req_data), GFP_KERNEL);
if (!req_data)
goto out_free_request;
memset(req_data->iv, 0, sizeof(req_data->iv));
/* Clear the data in the hash sub key container to zero.*/
/* We want to cipher all zeros to create the hash sub key. */
memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE);
init_completion(&req_data->result.completion);
sg_init_one(&req_data->sg, hash_subkey, RFC4106_HASH_SUBKEY_SIZE);
ablkcipher_request_set_tfm(req, ctr_tfm);
ablkcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP |
CRYPTO_TFM_REQ_MAY_BACKLOG,
rfc4106_set_hash_subkey_done,
&req_data->result);
ablkcipher_request_set_crypt(req, &req_data->sg,
&req_data->sg, RFC4106_HASH_SUBKEY_SIZE, req_data->iv);
ret = crypto_ablkcipher_encrypt(req);
if (ret == -EINPROGRESS || ret == -EBUSY) {
ret = wait_for_completion_interruptible
(&req_data->result.completion);
if (!ret)
ret = req_data->result.err;
}
kfree(req_data);
out_free_request:
ablkcipher_request_free(req);
out_free_ablkcipher:
crypto_free_ablkcipher(ctr_tfm);
return ret;
}
static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key,
unsigned int key_len)
{
int ret = 0;
struct crypto_tfm *tfm = crypto_aead_tfm(parent);
struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent);
struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
struct aesni_rfc4106_gcm_ctx *child_ctx =
aesni_rfc4106_gcm_ctx_get(cryptd_child);
u8 *new_key_align, *new_key_mem = NULL;
if (key_len < 4) {
crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
return -EINVAL;
}
/*Account for 4 byte nonce at the end.*/
key_len -= 4;
if (key_len != AES_KEYSIZE_128) {
crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
return -EINVAL;
}
memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce));
/*This must be on a 16 byte boundary!*/
if ((unsigned long)(&(ctx->aes_key_expanded.key_enc[0])) % AESNI_ALIGN)
return -EINVAL;
if ((unsigned long)key % AESNI_ALIGN) {
/*key is not aligned: use an auxuliar aligned pointer*/
new_key_mem = kmalloc(key_len+AESNI_ALIGN, GFP_KERNEL);
if (!new_key_mem)
return -ENOMEM;
new_key_align = PTR_ALIGN(new_key_mem, AESNI_ALIGN);
memcpy(new_key_align, key, key_len);
key = new_key_align;
}
if (!irq_fpu_usable())
ret = crypto_aes_expand_key(&(ctx->aes_key_expanded),
key, key_len);
else {
kernel_fpu_begin();
ret = aesni_set_key(&(ctx->aes_key_expanded), key, key_len);
kernel_fpu_end();
}
/*This must be on a 16 byte boundary!*/
if ((unsigned long)(&(ctx->hash_subkey[0])) % AESNI_ALIGN) {
ret = -EINVAL;
goto exit;
}
ret = rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
memcpy(child_ctx, ctx, sizeof(*ctx));
exit:
kfree(new_key_mem);
return ret;
}
/* This is the Integrity Check Value (aka the authentication tag length and can
* be 8, 12 or 16 bytes long. */
static int rfc4106_set_authsize(struct crypto_aead *parent,
unsigned int authsize)
{
struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent);
struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
switch (authsize) {
case 8:
case 12:
case 16:
break;
default:
return -EINVAL;
}
crypto_aead_crt(parent)->authsize = authsize;
crypto_aead_crt(cryptd_child)->authsize = authsize;
return 0;
}
static int rfc4106_encrypt(struct aead_request *req)
{
int ret;
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
if (!irq_fpu_usable()) {
struct aead_request *cryptd_req =
(struct aead_request *) aead_request_ctx(req);
memcpy(cryptd_req, req, sizeof(*req));
aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
return crypto_aead_encrypt(cryptd_req);
} else {
struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
kernel_fpu_begin();
ret = cryptd_child->base.crt_aead.encrypt(req);
kernel_fpu_end();
return ret;
}
}
void mmc_decrypt_req(struct mmc_host *host, struct mmc_request *mrq)
{
struct crypto_ablkcipher *tfm;
struct ablkcipher_request *req;
struct mmc_tcrypt_result result;
struct scatterlist *in_sg = mrq->data->sg;
int rc = 0;
u8 IV[MMC_AES_XTS_IV_LEN];
sector_t sector = mrq->data->sector;
tfm = crypto_alloc_ablkcipher("xts(aes)", 0, 0);
if (IS_ERR(tfm)) {
pr_err("%s:%s ablkcipher tfm allocation failed : error = %lu\n",
mmc_hostname(host), __func__, PTR_ERR(tfm));
return;
}
req = ablkcipher_request_alloc(tfm, GFP_KERNEL);
if (!req) {
pr_err("%s:%s ablkcipher request allocation failed\n",
mmc_hostname(host), __func__);
goto ablkcipher_req_alloc_failure;
}
ablkcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
mmc_crypto_cipher_complete, &result);
init_completion(&result.completion);
qcrypto_cipher_set_flag(req,
QCRYPTO_CTX_USE_PIPE_KEY | QCRYPTO_CTX_XTS_DU_SIZE_512B);
crypto_ablkcipher_clear_flags(tfm, ~0);
crypto_ablkcipher_setkey(tfm, NULL, MMC_KEY_SIZE_XTS);
memset(IV, 0, MMC_AES_XTS_IV_LEN);
memcpy(IV, §or, sizeof(sector_t));
ablkcipher_request_set_crypt(req, in_sg, in_sg,
mrq->data->blksz * mrq->data->blocks, (void *) IV);
rc = crypto_ablkcipher_decrypt(req);
switch (rc) {
case 0:
break;
case -EBUSY:
/*
* Lets make this synchronous request by waiting on
* in progress as well
*/
case -EINPROGRESS:
wait_for_completion_interruptible(&result.completion);
if (result.err)
pr_err("%s:%s error = %d decrypting the request\n",
mmc_hostname(host), __func__, result.err);
break;
default:
goto crypto_operation_failure;
}
crypto_operation_failure:
ablkcipher_request_free(req);
ablkcipher_req_alloc_failure:
crypto_free_ablkcipher(tfm);
return;
}
/**
* ext4_fname_encrypt() -
*
* This function encrypts the input filename, and returns the length of the
* ciphertext. Errors are returned as negative numbers. We trust the caller to
* allocate sufficient memory to oname string.
*/
static int ext4_fname_encrypt(struct inode *inode,
const struct qstr *iname,
struct ext4_str *oname)
{
u32 ciphertext_len;
struct ablkcipher_request *req = NULL;
DECLARE_EXT4_COMPLETION_RESULT(ecr);
struct ext4_crypt_info *ci = EXT4_I(inode)->i_crypt_info;
struct crypto_ablkcipher *tfm = ci->ci_ctfm;
int res = 0;
char iv[EXT4_CRYPTO_BLOCK_SIZE];
struct scatterlist src_sg, dst_sg;
int padding = 4 << (ci->ci_flags & EXT4_POLICY_FLAGS_PAD_MASK);
char *workbuf, buf[32], *alloc_buf = NULL;
unsigned lim = max_name_len(inode);
if (iname->len <= 0 || iname->len > lim)
return -EIO;
ciphertext_len = (iname->len < EXT4_CRYPTO_BLOCK_SIZE) ?
EXT4_CRYPTO_BLOCK_SIZE : iname->len;
ciphertext_len = ext4_fname_crypto_round_up(ciphertext_len, padding);
ciphertext_len = (ciphertext_len > lim)
? lim : ciphertext_len;
if (ciphertext_len <= sizeof(buf)) {
workbuf = buf;
} else {
alloc_buf = kmalloc(ciphertext_len, GFP_NOFS);
if (!alloc_buf)
return -ENOMEM;
workbuf = alloc_buf;
}
/* Allocate request */
req = ablkcipher_request_alloc(tfm, GFP_NOFS);
if (!req) {
printk_ratelimited(
KERN_ERR "%s: crypto_request_alloc() failed\n", __func__);
kfree(alloc_buf);
return -ENOMEM;
}
ablkcipher_request_set_callback(req,
CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP,
ext4_dir_crypt_complete, &ecr);
/* Copy the input */
memcpy(workbuf, iname->name, iname->len);
if (iname->len < ciphertext_len)
memset(workbuf + iname->len, 0, ciphertext_len - iname->len);
/* Initialize IV */
memset(iv, 0, EXT4_CRYPTO_BLOCK_SIZE);
/* Create encryption request */
sg_init_one(&src_sg, workbuf, ciphertext_len);
sg_init_one(&dst_sg, oname->name, ciphertext_len);
ablkcipher_request_set_crypt(req, &src_sg, &dst_sg, ciphertext_len, iv);
res = crypto_ablkcipher_encrypt(req);
if (res == -EINPROGRESS || res == -EBUSY) {
BUG_ON(req->base.data != &ecr);
wait_for_completion(&ecr.completion);
res = ecr.res;
}
kfree(alloc_buf);
ablkcipher_request_free(req);
if (res < 0) {
printk_ratelimited(
KERN_ERR "%s: Error (error code %d)\n", __func__, res);
}
oname->len = ciphertext_len;
return res;
}
static int run_test_case(const struct eme2_test_case *c, unsigned int number)
{
int err = 0;
int failed = 0;
struct crypto_ablkcipher *cipher = NULL;
struct ablkcipher_request *req = NULL;
u8 *buffer = NULL;
struct scatterlist sg[1];
buffer = kmalloc(c->plaintext_len, GFP_KERNEL);
if (!buffer) {
printk("eme2_test: ERROR allocating buffer!\n");
goto out;
}
sg_init_one(sg, buffer, c->plaintext_len);
cipher = crypto_alloc_ablkcipher("eme2(aes)", 0, 0);
if (IS_ERR(cipher)) {
printk("eme2_test: ERROR allocating cipher!\n");
err = PTR_ERR(cipher);
cipher = NULL;
goto out;
}
err = crypto_ablkcipher_setkey(cipher, c->key, c->key_len);
if (err) {
printk("eme2_test: ERROR setting key!\n");
goto out;
}
req = ablkcipher_request_alloc(cipher, GFP_KERNEL);
if (IS_ERR(req)) {
printk("eme2_test: ERROR allocating request!\n");
err = PTR_ERR(req);
req = NULL;
goto out;
}
ablkcipher_request_set_tfm(req, cipher);
ablkcipher_request_set_crypt(req, sg, sg, c->plaintext_len, (u8 *)c->assoc_data);
memcpy(buffer, c->plaintext, c->plaintext_len);
err = eme2_encrypt_sync(req, c->assoc_data_len);
if (err) {
printk("eme2_test: ERROR encrypting!\n");
goto out;
}
if (memcmp(buffer, c->ciphertext, c->plaintext_len) != 0) {
failed += 1;
printk("eme2_test: encryption-%u: Testcase failed!\n", number);
}
memcpy(buffer, c->ciphertext, c->plaintext_len);
err = eme2_decrypt_sync(req, c->assoc_data_len);
if (err) {
printk("eme2_test: ERROR decrypting!\n");
goto out;
}
if (memcmp(buffer, c->plaintext, c->plaintext_len) != 0) {
failed += 1;
printk("eme2_test: decryption-%u: Testcase failed!\n", number);
}
out:
if (buffer)
kfree(buffer);
if (cipher)
crypto_free_ablkcipher(cipher);
if (req)
ablkcipher_request_free(req);
return err < 0 ? err : failed;
}
static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
{
struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
be128 buf[8];
struct xts_crypt_req req = {
.tbuf = buf,
.tbuflen = sizeof(buf),
.tweak_ctx = aes_ctx(ctx->raw_tweak_ctx),
.tweak_fn = aesni_xts_tweak,
.crypt_ctx = aes_ctx(ctx->raw_crypt_ctx),
.crypt_fn = lrw_xts_encrypt_callback,
};
int ret;
desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
kernel_fpu_begin();
ret = xts_crypt(desc, dst, src, nbytes, &req);
kernel_fpu_end();
return ret;
}
static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
{
struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
be128 buf[8];
struct xts_crypt_req req = {
.tbuf = buf,
.tbuflen = sizeof(buf),
.tweak_ctx = aes_ctx(ctx->raw_tweak_ctx),
.tweak_fn = aesni_xts_tweak,
.crypt_ctx = aes_ctx(ctx->raw_crypt_ctx),
.crypt_fn = lrw_xts_decrypt_callback,
};
int ret;
desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
kernel_fpu_begin();
ret = xts_crypt(desc, dst, src, nbytes, &req);
kernel_fpu_end();
return ret;
}
#endif
#ifdef CONFIG_X86_64
static int rfc4106_init(struct crypto_aead *aead)
{
struct cryptd_aead *cryptd_tfm;
struct cryptd_aead **ctx = crypto_aead_ctx(aead);
cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni",
CRYPTO_ALG_INTERNAL,
CRYPTO_ALG_INTERNAL);
if (IS_ERR(cryptd_tfm))
return PTR_ERR(cryptd_tfm);
*ctx = cryptd_tfm;
crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base));
return 0;
}
static void rfc4106_exit(struct crypto_aead *aead)
{
struct cryptd_aead **ctx = crypto_aead_ctx(aead);
cryptd_free_aead(*ctx);
}
static void
rfc4106_set_hash_subkey_done(struct crypto_async_request *req, int err)
{
struct aesni_gcm_set_hash_subkey_result *result = req->data;
if (err == -EINPROGRESS)
return;
result->err = err;
complete(&result->completion);
}
static int
rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len)
{
struct crypto_ablkcipher *ctr_tfm;
struct ablkcipher_request *req;
int ret = -EINVAL;
struct aesni_hash_subkey_req_data *req_data;
ctr_tfm = crypto_alloc_ablkcipher("ctr(aes)", 0, 0);
if (IS_ERR(ctr_tfm))
return PTR_ERR(ctr_tfm);
ret = crypto_ablkcipher_setkey(ctr_tfm, key, key_len);
if (ret)
goto out_free_ablkcipher;
ret = -ENOMEM;
req = ablkcipher_request_alloc(ctr_tfm, GFP_KERNEL);
if (!req)
goto out_free_ablkcipher;
req_data = kmalloc(sizeof(*req_data), GFP_KERNEL);
if (!req_data)
goto out_free_request;
memset(req_data->iv, 0, sizeof(req_data->iv));
/* Clear the data in the hash sub key container to zero.*/
/* We want to cipher all zeros to create the hash sub key. */
memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE);
init_completion(&req_data->result.completion);
sg_init_one(&req_data->sg, hash_subkey, RFC4106_HASH_SUBKEY_SIZE);
ablkcipher_request_set_tfm(req, ctr_tfm);
ablkcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP |
CRYPTO_TFM_REQ_MAY_BACKLOG,
rfc4106_set_hash_subkey_done,
&req_data->result);
ablkcipher_request_set_crypt(req, &req_data->sg,
&req_data->sg, RFC4106_HASH_SUBKEY_SIZE, req_data->iv);
ret = crypto_ablkcipher_encrypt(req);
if (ret == -EINPROGRESS || ret == -EBUSY) {
ret = wait_for_completion_interruptible
(&req_data->result.completion);
if (!ret)
ret = req_data->result.err;
}
kfree(req_data);
out_free_request:
ablkcipher_request_free(req);
out_free_ablkcipher:
crypto_free_ablkcipher(ctr_tfm);
return ret;
}
static int common_rfc4106_set_key(struct crypto_aead *aead, const u8 *key,
unsigned int key_len)
{
struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(aead);
if (key_len < 4) {
crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
return -EINVAL;
}
/*Account for 4 byte nonce at the end.*/
key_len -= 4;
memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce));
return aes_set_key_common(crypto_aead_tfm(aead),
&ctx->aes_key_expanded, key, key_len) ?:
rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
}
static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key,
unsigned int key_len)
{
struct cryptd_aead **ctx = crypto_aead_ctx(parent);
struct cryptd_aead *cryptd_tfm = *ctx;
return crypto_aead_setkey(&cryptd_tfm->base, key, key_len);
}
static int common_rfc4106_set_authsize(struct crypto_aead *aead,
unsigned int authsize)
{
switch (authsize) {
case 8:
case 12:
case 16:
break;
default:
return -EINVAL;
}
return 0;
}
/* This is the Integrity Check Value (aka the authentication tag length and can
* be 8, 12 or 16 bytes long. */
static int rfc4106_set_authsize(struct crypto_aead *parent,
unsigned int authsize)
{
struct cryptd_aead **ctx = crypto_aead_ctx(parent);
struct cryptd_aead *cryptd_tfm = *ctx;
return crypto_aead_setauthsize(&cryptd_tfm->base, authsize);
}
static int helper_rfc4106_encrypt(struct aead_request *req)
{
u8 one_entry_in_sg = 0;
u8 *src, *dst, *assoc;
__be32 counter = cpu_to_be32(1);
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
void *aes_ctx = &(ctx->aes_key_expanded);
unsigned long auth_tag_len = crypto_aead_authsize(tfm);
u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
struct scatter_walk src_sg_walk;
struct scatter_walk dst_sg_walk;
unsigned int i;
/* Assuming we are supporting rfc4106 64-bit extended */
/* sequence numbers We need to have the AAD length equal */
/* to 16 or 20 bytes */
if (unlikely(req->assoclen != 16 && req->assoclen != 20))
return -EINVAL;
/* IV below built */
for (i = 0; i < 4; i++)
*(iv+i) = ctx->nonce[i];
for (i = 0; i < 8; i++)
*(iv+4+i) = req->iv[i];
*((__be32 *)(iv+12)) = counter;
if (sg_is_last(req->src) &&
req->src->offset + req->src->length <= PAGE_SIZE &&
sg_is_last(req->dst) &&
req->dst->offset + req->dst->length <= PAGE_SIZE) {
one_entry_in_sg = 1;
scatterwalk_start(&src_sg_walk, req->src);
assoc = scatterwalk_map(&src_sg_walk);
src = assoc + req->assoclen;
dst = src;
if (unlikely(req->src != req->dst)) {
scatterwalk_start(&dst_sg_walk, req->dst);
dst = scatterwalk_map(&dst_sg_walk) + req->assoclen;
}
} else {
/* Allocate memory for src, dst, assoc */
assoc = kmalloc(req->cryptlen + auth_tag_len + req->assoclen,
GFP_ATOMIC);
if (unlikely(!assoc))
return -ENOMEM;
scatterwalk_map_and_copy(assoc, req->src, 0,
req->assoclen + req->cryptlen, 0);
src = assoc + req->assoclen;
dst = src;
}
kernel_fpu_begin();
aesni_gcm_enc_tfm(aes_ctx, dst, src, req->cryptlen, iv,
ctx->hash_subkey, assoc, req->assoclen - 8,
dst + req->cryptlen, auth_tag_len);
kernel_fpu_end();
/* The authTag (aka the Integrity Check Value) needs to be written
* back to the packet. */
if (one_entry_in_sg) {
if (unlikely(req->src != req->dst)) {
scatterwalk_unmap(dst - req->assoclen);
scatterwalk_advance(&dst_sg_walk, req->dst->length);
scatterwalk_done(&dst_sg_walk, 1, 0);
}
scatterwalk_unmap(assoc);
scatterwalk_advance(&src_sg_walk, req->src->length);
scatterwalk_done(&src_sg_walk, req->src == req->dst, 0);
} else {
scatterwalk_map_and_copy(dst, req->dst, req->assoclen,
req->cryptlen + auth_tag_len, 1);
kfree(assoc);
}
return 0;
}
static int helper_rfc4106_decrypt(struct aead_request *req)
{
u8 one_entry_in_sg = 0;
u8 *src, *dst, *assoc;
unsigned long tempCipherLen = 0;
__be32 counter = cpu_to_be32(1);
int retval = 0;
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
void *aes_ctx = &(ctx->aes_key_expanded);
unsigned long auth_tag_len = crypto_aead_authsize(tfm);
u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
u8 authTag[16];
struct scatter_walk src_sg_walk;
struct scatter_walk dst_sg_walk;
unsigned int i;
if (unlikely(req->assoclen != 16 && req->assoclen != 20))
return -EINVAL;
/* Assuming we are supporting rfc4106 64-bit extended */
/* sequence numbers We need to have the AAD length */
/* equal to 16 or 20 bytes */
tempCipherLen = (unsigned long)(req->cryptlen - auth_tag_len);
/* IV below built */
for (i = 0; i < 4; i++)
*(iv+i) = ctx->nonce[i];
for (i = 0; i < 8; i++)
*(iv+4+i) = req->iv[i];
*((__be32 *)(iv+12)) = counter;
if (sg_is_last(req->src) &&
req->src->offset + req->src->length <= PAGE_SIZE &&
sg_is_last(req->dst) &&
req->dst->offset + req->dst->length <= PAGE_SIZE) {
one_entry_in_sg = 1;
scatterwalk_start(&src_sg_walk, req->src);
assoc = scatterwalk_map(&src_sg_walk);
src = assoc + req->assoclen;
dst = src;
if (unlikely(req->src != req->dst)) {
scatterwalk_start(&dst_sg_walk, req->dst);
dst = scatterwalk_map(&dst_sg_walk) + req->assoclen;
}
} else {
/* Allocate memory for src, dst, assoc */
assoc = kmalloc(req->cryptlen + req->assoclen, GFP_ATOMIC);
if (!assoc)
return -ENOMEM;
scatterwalk_map_and_copy(assoc, req->src, 0,
req->assoclen + req->cryptlen, 0);
src = assoc + req->assoclen;
dst = src;
}
kernel_fpu_begin();
aesni_gcm_dec_tfm(aes_ctx, dst, src, tempCipherLen, iv,
ctx->hash_subkey, assoc, req->assoclen - 8,
authTag, auth_tag_len);
kernel_fpu_end();
/* Compare generated tag with passed in tag. */
retval = crypto_memneq(src + tempCipherLen, authTag, auth_tag_len) ?
-EBADMSG : 0;
if (one_entry_in_sg) {
if (unlikely(req->src != req->dst)) {
scatterwalk_unmap(dst - req->assoclen);
scatterwalk_advance(&dst_sg_walk, req->dst->length);
scatterwalk_done(&dst_sg_walk, 1, 0);
}
scatterwalk_unmap(assoc);
scatterwalk_advance(&src_sg_walk, req->src->length);
scatterwalk_done(&src_sg_walk, req->src == req->dst, 0);
} else {
scatterwalk_map_and_copy(dst, req->dst, req->assoclen,
tempCipherLen, 1);
kfree(assoc);
}
return retval;
}
static int test_acipher(const char *algo, int enc, char *data_in,
char *data_out, size_t data_len, char *key, int keysize)
{
struct crypto_ablkcipher *tfm;
struct tcrypt_result tresult;
struct ablkcipher_request *req;
struct scatterlist sg[TVMEMSIZE];
unsigned int ret, i, j, iv_len;
char iv[128];
ret = -EAGAIN;
init_completion(&tresult.completion);
tfm = crypto_alloc_ablkcipher(algo, 0, 0);
if (IS_ERR(tfm)) {
printk(KERN_ERR "failed to load transform for %s: %ld\n",
algo, PTR_ERR(tfm));
return ret;
}
req = ablkcipher_request_alloc(tfm, GFP_KERNEL);
if (!req) {
printk(KERN_ERR "tcrypt: skcipher: Failed to allocate request for %s\n",
algo);
goto out;
}
ablkcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
tcrypt_complete, &tresult);
crypto_ablkcipher_clear_flags(tfm, ~0);
ret = crypto_ablkcipher_setkey(tfm, key, keysize);
if (ret) {
printk(KERN_ERR "setkey() failed flags=%x\n",
crypto_ablkcipher_get_flags(tfm));
goto out_free_req;
}
printk(KERN_INFO "KEY:\n");
hexdump(key, keysize);
sg_init_table(sg, TVMEMSIZE);
i = 0;
j = data_len;
while (j > PAGE_SIZE) {
sg_set_buf(sg + i, tvmem[i], PAGE_SIZE);
memcpy(tvmem[i], data_in + i * PAGE_SIZE, PAGE_SIZE);
i++;
j -= PAGE_SIZE;
}
sg_set_buf(sg + i, tvmem[i], j);
memcpy(tvmem[i], data_in + i * PAGE_SIZE, j);
iv_len = crypto_ablkcipher_ivsize(tfm);
memcpy(iv, iv16, iv_len);
printk(KERN_INFO "IV:\n");
hexdump(iv, iv_len);
ablkcipher_request_set_crypt(req, sg, sg, data_len, iv);
printk(KERN_INFO "IN:\n");
hexdump(data_in, data_len);
if (enc)
ret = do_one_acipher_op(req, crypto_ablkcipher_encrypt(req));
else
ret = do_one_acipher_op(req, crypto_ablkcipher_decrypt(req));
if (ret)
printk(KERN_ERR "failed flags=%x\n",
crypto_ablkcipher_get_flags(tfm));
else {
i = 0;
j = data_len;
while (j > PAGE_SIZE) {
memcpy(data_out + i * PAGE_SIZE, tvmem[i], PAGE_SIZE);
i++;
j -= PAGE_SIZE;
}
memcpy(data_out + i * PAGE_SIZE, tvmem[i], j);
printk(KERN_INFO "OUT:\n");
hexdump(data_out, data_len);
}
out_free_req:
ablkcipher_request_free(req);
out:
crypto_free_ablkcipher(tfm);
return ret;
}
void wcnss_wlan_ablkcipher_request_free(struct ablkcipher_request *req)
{
ablkcipher_request_free(req);
}
static int ctr_aes_init(struct ctr_drbg_ctx_s *ctx)
{
int status = 0;
ctx->aes_ctx.tfm = crypto_alloc_ablkcipher("qcom-ecb(aes)", 0, 0);
if (IS_ERR(ctx->aes_ctx.tfm) || (NULL == ctx->aes_ctx.tfm)) {
pr_info("%s: qcom-ecb(aes) failed", __func__);
ctx->aes_ctx.tfm = crypto_alloc_ablkcipher("ecb(aes)", 0, 0);
pr_info("ctx->aes_ctx.tfm = %p\n", ctx->aes_ctx.tfm);
if (IS_ERR(ctx->aes_ctx.tfm) || (NULL == ctx->aes_ctx.tfm)) {
pr_err("%s: qcom-ecb(aes) failed\n", __func__);
status = -E_FAILURE;
goto out;
}
}
ctx->aes_ctx.req = ablkcipher_request_alloc(ctx->aes_ctx.tfm,
GFP_KERNEL);
if (IS_ERR(ctx->aes_ctx.req) || (NULL == ctx->aes_ctx.req)) {
pr_info("%s: Failed to allocate request.\n", __func__);
status = -E_FAILURE;
goto clr_tfm;
}
ablkcipher_request_set_callback(ctx->aes_ctx.req,
CRYPTO_TFM_REQ_MAY_BACKLOG,
_crypto_cipher_test_complete,
&ctx->aes_ctx.result);
memset(&ctx->aes_ctx.input, 0, sizeof(struct msm_ctr_buffer_s));
memset(&ctx->aes_ctx.output, 0, sizeof(struct msm_ctr_buffer_s));
/* Allocate memory. */
ctx->aes_ctx.input.virt_addr = kmalloc(AES128_BLOCK_SIZE,
GFP_KERNEL | __GFP_DMA);
if (NULL == ctx->aes_ctx.input.virt_addr) {
pr_debug("%s: Failed to input memory.\n", __func__);
status = -E_FAILURE;
goto clr_req;
}
ctx->aes_ctx.output.virt_addr = kmalloc(AES128_BLOCK_SIZE,
GFP_KERNEL | __GFP_DMA);
if (NULL == ctx->aes_ctx.output.virt_addr) {
pr_debug("%s: Failed to output memory.\n", __func__);
status = -E_FAILURE;
goto clr_input;
}
/*--------------------------------------------------------------------
Set DF AES mode
----------------------------------------------------------------------*/
ctx->df_aes_ctx.tfm = crypto_alloc_ablkcipher("qcom-ecb(aes)", 0, 0);
if ((NULL == ctx->df_aes_ctx.tfm) || IS_ERR(ctx->df_aes_ctx.tfm)) {
pr_info("%s: qcom-ecb(aes) failed", __func__);
ctx->df_aes_ctx.tfm = crypto_alloc_ablkcipher("ecb(aes)", 0, 0);
if (IS_ERR(ctx->df_aes_ctx.tfm) ||
(NULL == ctx->df_aes_ctx.tfm)) {
pr_err("%s: ecb(aes) failed", __func__);
status = -E_FAILURE;
goto clr_output;
}
}
ctx->df_aes_ctx.req = ablkcipher_request_alloc(ctx->df_aes_ctx.tfm,
GFP_KERNEL);
if (IS_ERR(ctx->df_aes_ctx.req) || (NULL == ctx->df_aes_ctx.req)) {
pr_debug(": Failed to allocate request.\n");
status = -E_FAILURE;
goto clr_df_tfm;
}
ablkcipher_request_set_callback(ctx->df_aes_ctx.req,
CRYPTO_TFM_REQ_MAY_BACKLOG,
_crypto_cipher_test_complete,
&ctx->df_aes_ctx.result);
memset(&ctx->df_aes_ctx.input, 0, sizeof(struct msm_ctr_buffer_s));
memset(&ctx->df_aes_ctx.output, 0, sizeof(struct msm_ctr_buffer_s));
ctx->df_aes_ctx.input.virt_addr = kmalloc(AES128_BLOCK_SIZE,
GFP_KERNEL | __GFP_DMA);
if (NULL == ctx->df_aes_ctx.input.virt_addr) {
pr_debug(": Failed to input memory.\n");
status = -E_FAILURE;
goto clr_df_req;
}
ctx->df_aes_ctx.output.virt_addr = kmalloc(AES128_BLOCK_SIZE,
GFP_KERNEL | __GFP_DMA);
if (NULL == ctx->df_aes_ctx.output.virt_addr) {
pr_debug(": Failed to output memory.\n");
status = -E_FAILURE;
goto clr_df_input;
}
goto out;
clr_df_input:
if (ctx->df_aes_ctx.input.virt_addr) {
kzfree(ctx->df_aes_ctx.input.virt_addr);
ctx->df_aes_ctx.input.virt_addr = NULL;
}
clr_df_req:
if (ctx->df_aes_ctx.req) {
ablkcipher_request_free(ctx->df_aes_ctx.req);
ctx->df_aes_ctx.req = NULL;
}
clr_df_tfm:
if (ctx->df_aes_ctx.tfm) {
crypto_free_ablkcipher(ctx->df_aes_ctx.tfm);
ctx->df_aes_ctx.tfm = NULL;
}
clr_output:
if (ctx->aes_ctx.output.virt_addr) {
kzfree(ctx->aes_ctx.output.virt_addr);
ctx->aes_ctx.output.virt_addr = NULL;
}
clr_input:
if (ctx->aes_ctx.input.virt_addr) {
kzfree(ctx->aes_ctx.input.virt_addr);
ctx->aes_ctx.input.virt_addr = NULL;
}
clr_req:
if (ctx->aes_ctx.req) {
ablkcipher_request_free(ctx->aes_ctx.req);
ctx->aes_ctx.req = NULL;
}
clr_tfm:
if (ctx->aes_ctx.tfm) {
crypto_free_ablkcipher(ctx->aes_ctx.tfm);
ctx->aes_ctx.tfm = NULL;
}
out:
return status;
}