static int pcrypt_aead_setkey(struct crypto_aead *parent,
const u8 *key, unsigned int keylen)
{
struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(parent);
return crypto_aead_setkey(ctx->child, key, keylen);
}
int tls_sw_fallback_init(struct sock *sk,
struct tls_offload_context_tx *offload_ctx,
struct tls_crypto_info *crypto_info)
{
const u8 *key;
int rc;
offload_ctx->aead_send =
crypto_alloc_aead("gcm(aes)", 0, CRYPTO_ALG_ASYNC);
if (IS_ERR(offload_ctx->aead_send)) {
rc = PTR_ERR(offload_ctx->aead_send);
pr_err_ratelimited("crypto_alloc_aead failed rc=%d\n", rc);
offload_ctx->aead_send = NULL;
goto err_out;
}
key = ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->key;
rc = crypto_aead_setkey(offload_ctx->aead_send, key,
TLS_CIPHER_AES_GCM_128_KEY_SIZE);
if (rc)
goto free_aead;
rc = crypto_aead_setauthsize(offload_ctx->aead_send,
TLS_CIPHER_AES_GCM_128_TAG_SIZE);
if (rc)
goto free_aead;
return 0;
free_aead:
crypto_free_aead(offload_ctx->aead_send);
err_out:
return rc;
}
static int cryptd_aegis256_aesni_setkey(struct crypto_aead *aead,
const u8 *key, unsigned int keylen)
{
struct cryptd_aead **ctx = crypto_aead_ctx(aead);
struct cryptd_aead *cryptd_tfm = *ctx;
return crypto_aead_setkey(&cryptd_tfm->base, key, keylen);
}
static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key,
unsigned int key_len)
{
struct cryptd_aead **ctx = crypto_aead_ctx(parent);
struct cryptd_aead *cryptd_tfm = *ctx;
return crypto_aead_setkey(&cryptd_tfm->base, key, key_len);
}
static int simd_aead_setkey(struct crypto_aead *tfm, const u8 *key,
unsigned int key_len)
{
struct simd_aead_ctx *ctx = crypto_aead_ctx(tfm);
struct crypto_aead *child = &ctx->cryptd_tfm->base;
int err;
crypto_aead_clear_flags(child, CRYPTO_TFM_REQ_MASK);
crypto_aead_set_flags(child, crypto_aead_get_flags(tfm) &
CRYPTO_TFM_REQ_MASK);
err = crypto_aead_setkey(child, key, key_len);
crypto_aead_set_flags(tfm, crypto_aead_get_flags(child) &
CRYPTO_TFM_RES_MASK);
return err;
}
static int crypto4xx_setup_fallback(struct crypto4xx_ctx *ctx,
struct crypto_aead *cipher,
const u8 *key,
unsigned int keylen)
{
int rc;
crypto_aead_clear_flags(ctx->sw_cipher.aead, CRYPTO_TFM_REQ_MASK);
crypto_aead_set_flags(ctx->sw_cipher.aead,
crypto_aead_get_flags(cipher) & CRYPTO_TFM_REQ_MASK);
rc = crypto_aead_setkey(ctx->sw_cipher.aead, key, keylen);
crypto_aead_clear_flags(cipher, CRYPTO_TFM_RES_MASK);
crypto_aead_set_flags(cipher,
crypto_aead_get_flags(ctx->sw_cipher.aead) &
CRYPTO_TFM_RES_MASK);
return rc;
}
struct crypto_aead *ieee80211_aes_key_setup_encrypt(const u8 key[])
{
struct crypto_aead *tfm;
int err;
tfm = crypto_alloc_aead("ccm(aes)", 0, CRYPTO_ALG_ASYNC);
if (IS_ERR(tfm))
return tfm;
err = crypto_aead_setkey(tfm, key, WLAN_KEY_LEN_CCMP);
if (!err)
err = crypto_aead_setauthsize(tfm, IEEE80211_CCMP_MIC_LEN);
if (!err)
return tfm;
crypto_free_aead(tfm);
return ERR_PTR(err);
}
struct crypto_aead *ieee80211_aes_key_setup_encrypt(const u8 key[],
size_t key_len,
size_t mic_len)
{
struct crypto_aead *tfm;
int err;
tfm = crypto_alloc_aead("ccm(aes)", 0, CRYPTO_ALG_ASYNC);
if (IS_ERR(tfm))
return tfm;
err = crypto_aead_setkey(tfm, key, key_len);
if (!err)
err = crypto_aead_setauthsize(tfm, mic_len);
if (!err)
return tfm;
crypto_free_aead(tfm);
return ERR_PTR(err);
}
static int crypto_rfc4309_setkey(struct crypto_aead *parent, const u8 *key,
unsigned int keylen)
{
struct crypto_rfc4309_ctx *ctx = crypto_aead_ctx(parent);
struct crypto_aead *child = ctx->child;
int err;
if (keylen < 3)
return -EINVAL;
keylen -= 3;
memcpy(ctx->nonce, key + keylen, 3);
crypto_aead_clear_flags(child, CRYPTO_TFM_REQ_MASK);
crypto_aead_set_flags(child, crypto_aead_get_flags(parent) &
CRYPTO_TFM_REQ_MASK);
err = crypto_aead_setkey(child, key, keylen);
crypto_aead_set_flags(parent, crypto_aead_get_flags(child) &
CRYPTO_TFM_RES_MASK);
return err;
}
struct crypto_aead *ieee80211_aes_gcm_key_setup_encrypt(const u8 key[],
size_t key_len)
{
struct crypto_aead *tfm;
int err;
tfm = crypto_alloc_aead("gcm(aes)", 0, CRYPTO_ALG_ASYNC);
if (IS_ERR(tfm))
return tfm;
err = crypto_aead_setkey(tfm, key, key_len);
if (err)
goto free_aead;
err = crypto_aead_setauthsize(tfm, IEEE80211_GCMP_MIC_LEN);
if (err)
goto free_aead;
return tfm;
free_aead:
crypto_free_aead(tfm);
return ERR_PTR(err);
}
/*
* AEAD algorithm self tests
*/
int _fips_qcrypto_aead_selftest(struct fips_selftest_data *selftest_d)
{
int rc = 0, err, tv_index, num_tv, authsize, buf_length;
struct crypto_aead *tfm;
struct aead_request *aead_req;
struct _fips_completion fips_completion;
struct scatterlist fips_sg, fips_assoc_sg;
char *k_align_src = NULL;
struct _fips_test_vector_aead tv_aead;
num_tv = (sizeof(fips_test_vector_aead)) /
(sizeof(struct _fips_test_vector_aead));
/* One-by-one testing */
for (tv_index = 0; tv_index < num_tv; tv_index++) {
memcpy(&tv_aead, &fips_test_vector_aead[tv_index],
(sizeof(struct _fips_test_vector_aead)));
if (tv_aead.pln_txt_len > tv_aead.enc_txt_len)
buf_length = tv_aead.pln_txt_len;
else
buf_length = tv_aead.enc_txt_len;
/* Single buffer allocation for in place operation */
k_align_src = kzalloc(buf_length, GFP_KERNEL);
if (k_align_src == NULL) {
pr_err("qcrypto:, Failed to allocate memory for k_align_src %ld\n",
PTR_ERR(k_align_src));
return -ENOMEM;
}
memcpy(&k_align_src[0], tv_aead.pln_txt,
tv_aead.pln_txt_len);
/* use_sw flags are set in dtsi file which makes
default Linux API calls to go to s/w crypto instead
of h/w crypto. This code makes sure that all selftests
calls always go to h/w, independent of DTSI flags. */
if (selftest_d->prefix_aead_algo) {
if (_fips_get_alg_cra_name(tv_aead.mod_alg,
selftest_d->algo_prefix,
strlen(tv_aead.mod_alg))) {
rc = -1;
pr_err("Algo Name is too long for tv %d\n",
tv_index);
goto clr_buf;
}
}
tfm = crypto_alloc_aead(tv_aead.mod_alg, 0, 0);
if (IS_ERR(tfm)) {
pr_err("qcrypto: %s algorithm not found\n",
tv_aead.mod_alg);
rc = -ENOMEM;
goto clr_buf;
}
aead_req = aead_request_alloc(tfm, GFP_KERNEL);
if (!aead_req) {
pr_err("qcrypto:aead_request_alloc failed\n");
rc = -ENOMEM;
goto clr_tfm;
}
rc = qcrypto_aead_set_device(aead_req, selftest_d->ce_device);
if (rc != 0) {
pr_err("%s qcrypto_cipher_set_device failed with err %d\n",
__func__, rc);
goto clr_aead_req;
}
init_completion(&fips_completion.completion);
aead_request_set_callback(aead_req,
CRYPTO_TFM_REQ_MAY_BACKLOG,
_fips_cb, &fips_completion);
crypto_aead_clear_flags(tfm, ~0);
rc = crypto_aead_setkey(tfm, tv_aead.key, tv_aead.klen);
if (rc) {
pr_err("qcrypto:crypto_aead_setkey failed\n");
goto clr_aead_req;
}
authsize = abs(tv_aead.enc_txt_len - tv_aead.pln_txt_len);
rc = crypto_aead_setauthsize(tfm, authsize);
if (rc) {
pr_err("qcrypto:crypto_aead_setauthsize failed\n");
goto clr_aead_req;
}
sg_init_one(&fips_sg, k_align_src,
tv_aead.pln_txt_len + authsize);
aead_request_set_crypt(aead_req, &fips_sg, &fips_sg,
tv_aead.pln_txt_len , tv_aead.iv);
sg_init_one(&fips_assoc_sg, tv_aead.assoc, tv_aead.alen);
aead_request_set_assoc(aead_req, &fips_assoc_sg, tv_aead.alen);
/**** Encryption test ****/
rc = crypto_aead_encrypt(aead_req);
if (rc == -EINPROGRESS || rc == -EBUSY) {
rc = wait_for_completion_interruptible(
&fips_completion.completion);
err = fips_completion.err;
if (!rc && !err) {
INIT_COMPLETION(fips_completion.completion);
} else {
pr_err("qcrypto:aead:ENC, wait_for_completion failed\n");
goto clr_aead_req;
}
}
if (memcmp(k_align_src, tv_aead.enc_txt, tv_aead.enc_txt_len)) {
rc = -1;
goto clr_aead_req;
}
/** Decryption test **/
init_completion(&fips_completion.completion);
aead_request_set_callback(aead_req,
CRYPTO_TFM_REQ_MAY_BACKLOG,
_fips_cb, &fips_completion);
crypto_aead_clear_flags(tfm, ~0);
rc = crypto_aead_setkey(tfm, tv_aead.key, tv_aead.klen);
if (rc) {
pr_err("qcrypto:aead:DEC, crypto_aead_setkey failed\n");
goto clr_aead_req;
}
authsize = abs(tv_aead.enc_txt_len - tv_aead.pln_txt_len);
rc = crypto_aead_setauthsize(tfm, authsize);
if (rc) {
pr_err("qcrypto:aead:DEC, crypto_aead_setauthsize failed\n");
goto clr_aead_req;
}
sg_init_one(&fips_sg, k_align_src,
tv_aead.enc_txt_len + authsize);
aead_request_set_crypt(aead_req, &fips_sg, &fips_sg,
tv_aead.enc_txt_len, tv_aead.iv);
sg_init_one(&fips_assoc_sg, tv_aead.assoc, tv_aead.alen);
aead_request_set_assoc(aead_req, &fips_assoc_sg,
tv_aead.alen);
rc = crypto_aead_decrypt(aead_req);
if (rc == -EINPROGRESS || rc == -EBUSY) {
rc = wait_for_completion_interruptible(
&fips_completion.completion);
err = fips_completion.err;
if (!rc && !err) {
INIT_COMPLETION(fips_completion.completion);
} else {
pr_err("qcrypto:aead:DEC, wait_for_completion failed\n");
goto clr_aead_req;
}
}
if (memcmp(k_align_src, tv_aead.pln_txt, tv_aead.pln_txt_len)) {
rc = -1;
goto clr_aead_req;
}
clr_aead_req:
aead_request_free(aead_req);
clr_tfm:
crypto_free_aead(tfm);
clr_buf:
kzfree(k_align_src);
/* In case of any failure, return error */
if (rc)
return rc;
}
return rc;
}
static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
{
struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
be128 buf[8];
struct xts_crypt_req req = {
.tbuf = buf,
.tbuflen = sizeof(buf),
.tweak_ctx = aes_ctx(ctx->raw_tweak_ctx),
.tweak_fn = aesni_xts_tweak,
.crypt_ctx = aes_ctx(ctx->raw_crypt_ctx),
.crypt_fn = lrw_xts_encrypt_callback,
};
int ret;
desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
kernel_fpu_begin();
ret = xts_crypt(desc, dst, src, nbytes, &req);
kernel_fpu_end();
return ret;
}
static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
{
struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
be128 buf[8];
struct xts_crypt_req req = {
.tbuf = buf,
.tbuflen = sizeof(buf),
.tweak_ctx = aes_ctx(ctx->raw_tweak_ctx),
.tweak_fn = aesni_xts_tweak,
.crypt_ctx = aes_ctx(ctx->raw_crypt_ctx),
.crypt_fn = lrw_xts_decrypt_callback,
};
int ret;
desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
kernel_fpu_begin();
ret = xts_crypt(desc, dst, src, nbytes, &req);
kernel_fpu_end();
return ret;
}
#endif
#ifdef CONFIG_X86_64
static int rfc4106_init(struct crypto_aead *aead)
{
struct cryptd_aead *cryptd_tfm;
struct cryptd_aead **ctx = crypto_aead_ctx(aead);
cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni",
CRYPTO_ALG_INTERNAL,
CRYPTO_ALG_INTERNAL);
if (IS_ERR(cryptd_tfm))
return PTR_ERR(cryptd_tfm);
*ctx = cryptd_tfm;
crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base));
return 0;
}
static void rfc4106_exit(struct crypto_aead *aead)
{
struct cryptd_aead **ctx = crypto_aead_ctx(aead);
cryptd_free_aead(*ctx);
}
static int
rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len)
{
struct crypto_cipher *tfm;
int ret;
tfm = crypto_alloc_cipher("aes", 0, 0);
if (IS_ERR(tfm))
return PTR_ERR(tfm);
ret = crypto_cipher_setkey(tfm, key, key_len);
if (ret)
goto out_free_cipher;
/* Clear the data in the hash sub key container to zero.*/
/* We want to cipher all zeros to create the hash sub key. */
memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE);
crypto_cipher_encrypt_one(tfm, hash_subkey, hash_subkey);
out_free_cipher:
crypto_free_cipher(tfm);
return ret;
}
static int common_rfc4106_set_key(struct crypto_aead *aead, const u8 *key,
unsigned int key_len)
{
struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(aead);
if (key_len < 4) {
crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
return -EINVAL;
}
/*Account for 4 byte nonce at the end.*/
key_len -= 4;
memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce));
return aes_set_key_common(crypto_aead_tfm(aead),
&ctx->aes_key_expanded, key, key_len) ?:
rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
}
static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key,
unsigned int key_len)
{
struct cryptd_aead **ctx = crypto_aead_ctx(parent);
struct cryptd_aead *cryptd_tfm = *ctx;
return crypto_aead_setkey(&cryptd_tfm->base, key, key_len);
}
static int common_rfc4106_set_authsize(struct crypto_aead *aead,
unsigned int authsize)
{
switch (authsize) {
case 8:
case 12:
case 16:
break;
default:
return -EINVAL;
}
return 0;
}
/* This is the Integrity Check Value (aka the authentication tag length and can
* be 8, 12 or 16 bytes long. */
static int rfc4106_set_authsize(struct crypto_aead *parent,
unsigned int authsize)
{
struct cryptd_aead **ctx = crypto_aead_ctx(parent);
struct cryptd_aead *cryptd_tfm = *ctx;
return crypto_aead_setauthsize(&cryptd_tfm->base, authsize);
}
static int helper_rfc4106_encrypt(struct aead_request *req)
{
u8 one_entry_in_sg = 0;
u8 *src, *dst, *assoc;
__be32 counter = cpu_to_be32(1);
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
void *aes_ctx = &(ctx->aes_key_expanded);
unsigned long auth_tag_len = crypto_aead_authsize(tfm);
u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
struct scatter_walk src_sg_walk;
struct scatter_walk dst_sg_walk = {};
unsigned int i;
/* Assuming we are supporting rfc4106 64-bit extended */
/* sequence numbers We need to have the AAD length equal */
/* to 16 or 20 bytes */
if (unlikely(req->assoclen != 16 && req->assoclen != 20))
return -EINVAL;
/* IV below built */
for (i = 0; i < 4; i++)
*(iv+i) = ctx->nonce[i];
for (i = 0; i < 8; i++)
*(iv+4+i) = req->iv[i];
*((__be32 *)(iv+12)) = counter;
if (sg_is_last(req->src) &&
req->src->offset + req->src->length <= PAGE_SIZE &&
sg_is_last(req->dst) &&
req->dst->offset + req->dst->length <= PAGE_SIZE) {
one_entry_in_sg = 1;
scatterwalk_start(&src_sg_walk, req->src);
assoc = scatterwalk_map(&src_sg_walk);
src = assoc + req->assoclen;
dst = src;
if (unlikely(req->src != req->dst)) {
scatterwalk_start(&dst_sg_walk, req->dst);
dst = scatterwalk_map(&dst_sg_walk) + req->assoclen;
}
} else {
/* Allocate memory for src, dst, assoc */
assoc = kmalloc(req->cryptlen + auth_tag_len + req->assoclen,
GFP_ATOMIC);
if (unlikely(!assoc))
return -ENOMEM;
scatterwalk_map_and_copy(assoc, req->src, 0,
req->assoclen + req->cryptlen, 0);
src = assoc + req->assoclen;
dst = src;
}
kernel_fpu_begin();
aesni_gcm_enc_tfm(aes_ctx, dst, src, req->cryptlen, iv,
ctx->hash_subkey, assoc, req->assoclen - 8,
dst + req->cryptlen, auth_tag_len);
kernel_fpu_end();
/* The authTag (aka the Integrity Check Value) needs to be written
* back to the packet. */
if (one_entry_in_sg) {
if (unlikely(req->src != req->dst)) {
scatterwalk_unmap(dst - req->assoclen);
scatterwalk_advance(&dst_sg_walk, req->dst->length);
scatterwalk_done(&dst_sg_walk, 1, 0);
}
scatterwalk_unmap(assoc);
scatterwalk_advance(&src_sg_walk, req->src->length);
scatterwalk_done(&src_sg_walk, req->src == req->dst, 0);
} else {
scatterwalk_map_and_copy(dst, req->dst, req->assoclen,
req->cryptlen + auth_tag_len, 1);
kfree(assoc);
}
return 0;
}
static int helper_rfc4106_decrypt(struct aead_request *req)
{
u8 one_entry_in_sg = 0;
u8 *src, *dst, *assoc;
unsigned long tempCipherLen = 0;
__be32 counter = cpu_to_be32(1);
int retval = 0;
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
void *aes_ctx = &(ctx->aes_key_expanded);
unsigned long auth_tag_len = crypto_aead_authsize(tfm);
u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
u8 authTag[16];
struct scatter_walk src_sg_walk;
struct scatter_walk dst_sg_walk = {};
unsigned int i;
if (unlikely(req->assoclen != 16 && req->assoclen != 20))
return -EINVAL;
/* Assuming we are supporting rfc4106 64-bit extended */
/* sequence numbers We need to have the AAD length */
/* equal to 16 or 20 bytes */
tempCipherLen = (unsigned long)(req->cryptlen - auth_tag_len);
/* IV below built */
for (i = 0; i < 4; i++)
*(iv+i) = ctx->nonce[i];
for (i = 0; i < 8; i++)
*(iv+4+i) = req->iv[i];
*((__be32 *)(iv+12)) = counter;
if (sg_is_last(req->src) &&
req->src->offset + req->src->length <= PAGE_SIZE &&
sg_is_last(req->dst) &&
req->dst->offset + req->dst->length <= PAGE_SIZE) {
one_entry_in_sg = 1;
scatterwalk_start(&src_sg_walk, req->src);
assoc = scatterwalk_map(&src_sg_walk);
src = assoc + req->assoclen;
dst = src;
if (unlikely(req->src != req->dst)) {
scatterwalk_start(&dst_sg_walk, req->dst);
dst = scatterwalk_map(&dst_sg_walk) + req->assoclen;
}
} else {
/* Allocate memory for src, dst, assoc */
assoc = kmalloc(req->cryptlen + req->assoclen, GFP_ATOMIC);
if (!assoc)
return -ENOMEM;
scatterwalk_map_and_copy(assoc, req->src, 0,
req->assoclen + req->cryptlen, 0);
src = assoc + req->assoclen;
dst = src;
}
kernel_fpu_begin();
aesni_gcm_dec_tfm(aes_ctx, dst, src, tempCipherLen, iv,
ctx->hash_subkey, assoc, req->assoclen - 8,
authTag, auth_tag_len);
kernel_fpu_end();
/* Compare generated tag with passed in tag. */
retval = crypto_memneq(src + tempCipherLen, authTag, auth_tag_len) ?
-EBADMSG : 0;
if (one_entry_in_sg) {
if (unlikely(req->src != req->dst)) {
scatterwalk_unmap(dst - req->assoclen);
scatterwalk_advance(&dst_sg_walk, req->dst->length);
scatterwalk_done(&dst_sg_walk, 1, 0);
}
scatterwalk_unmap(assoc);
scatterwalk_advance(&src_sg_walk, req->src->length);
scatterwalk_done(&src_sg_walk, req->src == req->dst, 0);
} else {
scatterwalk_map_and_copy(dst, req->dst, req->assoclen,
tempCipherLen, 1);
kfree(assoc);
}
return retval;
}
int tls_set_sw_offload(struct sock *sk, struct tls_context *ctx)
{
char keyval[TLS_CIPHER_AES_GCM_128_KEY_SIZE];
struct tls_crypto_info *crypto_info;
struct tls12_crypto_info_aes_gcm_128 *gcm_128_info;
struct tls_sw_context *sw_ctx;
u16 nonce_size, tag_size, iv_size, rec_seq_size;
char *iv, *rec_seq;
int rc = 0;
if (!ctx) {
rc = -EINVAL;
goto out;
}
if (ctx->priv_ctx) {
rc = -EEXIST;
goto out;
}
sw_ctx = kzalloc(sizeof(*sw_ctx), GFP_KERNEL);
if (!sw_ctx) {
rc = -ENOMEM;
goto out;
}
ctx->priv_ctx = (struct tls_offload_context *)sw_ctx;
crypto_info = &ctx->crypto_send;
switch (crypto_info->cipher_type) {
case TLS_CIPHER_AES_GCM_128: {
nonce_size = TLS_CIPHER_AES_GCM_128_IV_SIZE;
tag_size = TLS_CIPHER_AES_GCM_128_TAG_SIZE;
iv_size = TLS_CIPHER_AES_GCM_128_IV_SIZE;
iv = ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->iv;
rec_seq_size = TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE;
rec_seq =
((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->rec_seq;
gcm_128_info =
(struct tls12_crypto_info_aes_gcm_128 *)crypto_info;
break;
}
default:
rc = -EINVAL;
goto out;
}
ctx->prepend_size = TLS_HEADER_SIZE + nonce_size;
ctx->tag_size = tag_size;
ctx->overhead_size = ctx->prepend_size + ctx->tag_size;
ctx->iv_size = iv_size;
ctx->iv = kmalloc(iv_size + TLS_CIPHER_AES_GCM_128_SALT_SIZE,
GFP_KERNEL);
if (!ctx->iv) {
rc = -ENOMEM;
goto out;
}
memcpy(ctx->iv, gcm_128_info->salt, TLS_CIPHER_AES_GCM_128_SALT_SIZE);
memcpy(ctx->iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE, iv, iv_size);
ctx->rec_seq_size = rec_seq_size;
ctx->rec_seq = kmalloc(rec_seq_size, GFP_KERNEL);
if (!ctx->rec_seq) {
rc = -ENOMEM;
goto free_iv;
}
memcpy(ctx->rec_seq, rec_seq, rec_seq_size);
sg_init_table(sw_ctx->sg_encrypted_data,
ARRAY_SIZE(sw_ctx->sg_encrypted_data));
sg_init_table(sw_ctx->sg_plaintext_data,
ARRAY_SIZE(sw_ctx->sg_plaintext_data));
sg_init_table(sw_ctx->sg_aead_in, 2);
sg_set_buf(&sw_ctx->sg_aead_in[0], sw_ctx->aad_space,
sizeof(sw_ctx->aad_space));
sg_unmark_end(&sw_ctx->sg_aead_in[1]);
sg_chain(sw_ctx->sg_aead_in, 2, sw_ctx->sg_plaintext_data);
sg_init_table(sw_ctx->sg_aead_out, 2);
sg_set_buf(&sw_ctx->sg_aead_out[0], sw_ctx->aad_space,
sizeof(sw_ctx->aad_space));
sg_unmark_end(&sw_ctx->sg_aead_out[1]);
sg_chain(sw_ctx->sg_aead_out, 2, sw_ctx->sg_encrypted_data);
if (!sw_ctx->aead_send) {
sw_ctx->aead_send = crypto_alloc_aead("gcm(aes)", 0, 0);
if (IS_ERR(sw_ctx->aead_send)) {
rc = PTR_ERR(sw_ctx->aead_send);
sw_ctx->aead_send = NULL;
goto free_rec_seq;
}
}
ctx->push_pending_record = tls_sw_push_pending_record;
memcpy(keyval, gcm_128_info->key, TLS_CIPHER_AES_GCM_128_KEY_SIZE);
rc = crypto_aead_setkey(sw_ctx->aead_send, keyval,
TLS_CIPHER_AES_GCM_128_KEY_SIZE);
if (rc)
goto free_aead;
rc = crypto_aead_setauthsize(sw_ctx->aead_send, ctx->tag_size);
if (!rc)
goto out;
free_aead:
crypto_free_aead(sw_ctx->aead_send);
sw_ctx->aead_send = NULL;
free_rec_seq:
kfree(ctx->rec_seq);
ctx->rec_seq = NULL;
free_iv:
kfree(ctx->iv);
ctx->iv = NULL;
out:
return rc;
}
int cryptodev_cipher_init(struct cipher_data *out, const char *alg_name,
uint8_t *keyp, size_t keylen, int stream, int aead)
{
int ret;
if (aead == 0) {
struct ablkcipher_alg *alg;
out->async.s = crypto_alloc_ablkcipher(alg_name, 0, 0);
if (unlikely(IS_ERR(out->async.s))) {
ddebug(1, "Failed to load cipher %s", alg_name);
return -EINVAL;
}
alg = crypto_ablkcipher_alg(out->async.s);
if (alg != NULL) {
/* Was correct key length supplied? */
if (alg->max_keysize > 0 &&
unlikely((keylen < alg->min_keysize) ||
(keylen > alg->max_keysize))) {
ddebug(1, "Wrong keylen '%zu' for algorithm '%s'. Use %u to %u.",
keylen, alg_name, alg->min_keysize, alg->max_keysize);
ret = -EINVAL;
goto error;
}
}
out->blocksize = crypto_ablkcipher_blocksize(out->async.s);
out->ivsize = crypto_ablkcipher_ivsize(out->async.s);
out->alignmask = crypto_ablkcipher_alignmask(out->async.s);
ret = crypto_ablkcipher_setkey(out->async.s, keyp, keylen);
} else {
out->async.as = crypto_alloc_aead(alg_name, 0, 0);
if (unlikely(IS_ERR(out->async.as))) {
ddebug(1, "Failed to load cipher %s", alg_name);
return -EINVAL;
}
out->blocksize = crypto_aead_blocksize(out->async.as);
out->ivsize = crypto_aead_ivsize(out->async.as);
out->alignmask = crypto_aead_alignmask(out->async.as);
ret = crypto_aead_setkey(out->async.as, keyp, keylen);
}
if (unlikely(ret)) {
ddebug(1, "Setting key failed for %s-%zu.", alg_name, keylen*8);
ret = -EINVAL;
goto error;
}
out->stream = stream;
out->aead = aead;
out->async.result = kzalloc(sizeof(*out->async.result), GFP_KERNEL);
if (unlikely(!out->async.result)) {
ret = -ENOMEM;
goto error;
}
init_completion(&out->async.result->completion);
if (aead == 0) {
out->async.request = ablkcipher_request_alloc(out->async.s, GFP_KERNEL);
if (unlikely(!out->async.request)) {
derr(1, "error allocating async crypto request");
ret = -ENOMEM;
goto error;
}
ablkcipher_request_set_callback(out->async.request,
CRYPTO_TFM_REQ_MAY_BACKLOG,
cryptodev_complete, out->async.result);
} else {
out->async.arequest = aead_request_alloc(out->async.as, GFP_KERNEL);
if (unlikely(!out->async.arequest)) {
derr(1, "error allocating async crypto request");
ret = -ENOMEM;
goto error;
}
aead_request_set_callback(out->async.arequest,
CRYPTO_TFM_REQ_MAY_BACKLOG,
cryptodev_complete, out->async.result);
}
out->init = 1;
return 0;
error:
if (aead == 0) {
if (out->async.request)
ablkcipher_request_free(out->async.request);
if (out->async.s)
crypto_free_ablkcipher(out->async.s);
} else {
if (out->async.arequest)
aead_request_free(out->async.arequest);
if (out->async.as)
crypto_free_aead(out->async.as);
}
kfree(out->async.result);
return ret;
}
static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
{
struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
be128 buf[8];
struct xts_crypt_req req = {
.tbuf = buf,
.tbuflen = sizeof(buf),
.tweak_ctx = aes_ctx(ctx->raw_tweak_ctx),
.tweak_fn = aesni_xts_tweak,
.crypt_ctx = aes_ctx(ctx->raw_crypt_ctx),
.crypt_fn = lrw_xts_encrypt_callback,
};
int ret;
desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
kernel_fpu_begin();
ret = xts_crypt(desc, dst, src, nbytes, &req);
kernel_fpu_end();
return ret;
}
static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
{
struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
be128 buf[8];
struct xts_crypt_req req = {
.tbuf = buf,
.tbuflen = sizeof(buf),
.tweak_ctx = aes_ctx(ctx->raw_tweak_ctx),
.tweak_fn = aesni_xts_tweak,
.crypt_ctx = aes_ctx(ctx->raw_crypt_ctx),
.crypt_fn = lrw_xts_decrypt_callback,
};
int ret;
desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
kernel_fpu_begin();
ret = xts_crypt(desc, dst, src, nbytes, &req);
kernel_fpu_end();
return ret;
}
#endif
#ifdef CONFIG_X86_64
static int rfc4106_init(struct crypto_tfm *tfm)
{
struct cryptd_aead *cryptd_tfm;
struct aesni_rfc4106_gcm_ctx *ctx = (struct aesni_rfc4106_gcm_ctx *)
PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN);
struct crypto_aead *cryptd_child;
struct aesni_rfc4106_gcm_ctx *child_ctx;
cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni",
CRYPTO_ALG_INTERNAL,
CRYPTO_ALG_INTERNAL);
if (IS_ERR(cryptd_tfm))
return PTR_ERR(cryptd_tfm);
cryptd_child = cryptd_aead_child(cryptd_tfm);
child_ctx = aesni_rfc4106_gcm_ctx_get(cryptd_child);
memcpy(child_ctx, ctx, sizeof(*ctx));
ctx->cryptd_tfm = cryptd_tfm;
tfm->crt_aead.reqsize = sizeof(struct aead_request)
+ crypto_aead_reqsize(&cryptd_tfm->base);
return 0;
}
static void rfc4106_exit(struct crypto_tfm *tfm)
{
struct aesni_rfc4106_gcm_ctx *ctx =
(struct aesni_rfc4106_gcm_ctx *)
PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN);
if (!IS_ERR(ctx->cryptd_tfm))
cryptd_free_aead(ctx->cryptd_tfm);
return;
}
static void
rfc4106_set_hash_subkey_done(struct crypto_async_request *req, int err)
{
struct aesni_gcm_set_hash_subkey_result *result = req->data;
if (err == -EINPROGRESS)
return;
result->err = err;
complete(&result->completion);
}
static int
rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len)
{
struct crypto_ablkcipher *ctr_tfm;
struct ablkcipher_request *req;
int ret = -EINVAL;
struct aesni_hash_subkey_req_data *req_data;
ctr_tfm = crypto_alloc_ablkcipher("ctr(aes)", 0, 0);
if (IS_ERR(ctr_tfm))
return PTR_ERR(ctr_tfm);
crypto_ablkcipher_clear_flags(ctr_tfm, ~0);
ret = crypto_ablkcipher_setkey(ctr_tfm, key, key_len);
if (ret)
goto out_free_ablkcipher;
ret = -ENOMEM;
req = ablkcipher_request_alloc(ctr_tfm, GFP_KERNEL);
if (!req)
goto out_free_ablkcipher;
req_data = kmalloc(sizeof(*req_data), GFP_KERNEL);
if (!req_data)
goto out_free_request;
memset(req_data->iv, 0, sizeof(req_data->iv));
/* Clear the data in the hash sub key container to zero.*/
/* We want to cipher all zeros to create the hash sub key. */
memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE);
init_completion(&req_data->result.completion);
sg_init_one(&req_data->sg, hash_subkey, RFC4106_HASH_SUBKEY_SIZE);
ablkcipher_request_set_tfm(req, ctr_tfm);
ablkcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP |
CRYPTO_TFM_REQ_MAY_BACKLOG,
rfc4106_set_hash_subkey_done,
&req_data->result);
ablkcipher_request_set_crypt(req, &req_data->sg,
&req_data->sg, RFC4106_HASH_SUBKEY_SIZE, req_data->iv);
ret = crypto_ablkcipher_encrypt(req);
if (ret == -EINPROGRESS || ret == -EBUSY) {
ret = wait_for_completion_interruptible
(&req_data->result.completion);
if (!ret)
ret = req_data->result.err;
}
kfree(req_data);
out_free_request:
ablkcipher_request_free(req);
out_free_ablkcipher:
crypto_free_ablkcipher(ctr_tfm);
return ret;
}
static int common_rfc4106_set_key(struct crypto_aead *aead, const u8 *key,
unsigned int key_len)
{
int ret = 0;
struct crypto_tfm *tfm = crypto_aead_tfm(aead);
struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(aead);
u8 *new_key_align, *new_key_mem = NULL;
if (key_len < 4) {
crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
return -EINVAL;
}
/*Account for 4 byte nonce at the end.*/
key_len -= 4;
if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 &&
key_len != AES_KEYSIZE_256) {
crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
return -EINVAL;
}
memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce));
/*This must be on a 16 byte boundary!*/
if ((unsigned long)(&(ctx->aes_key_expanded.key_enc[0])) % AESNI_ALIGN)
return -EINVAL;
if ((unsigned long)key % AESNI_ALIGN) {
/*key is not aligned: use an auxuliar aligned pointer*/
new_key_mem = kmalloc(key_len+AESNI_ALIGN, GFP_KERNEL);
if (!new_key_mem)
return -ENOMEM;
new_key_align = PTR_ALIGN(new_key_mem, AESNI_ALIGN);
memcpy(new_key_align, key, key_len);
key = new_key_align;
}
if (!irq_fpu_usable())
ret = crypto_aes_expand_key(&(ctx->aes_key_expanded),
key, key_len);
else {
kernel_fpu_begin();
ret = aesni_set_key(&(ctx->aes_key_expanded), key, key_len);
kernel_fpu_end();
}
/*This must be on a 16 byte boundary!*/
if ((unsigned long)(&(ctx->hash_subkey[0])) % AESNI_ALIGN) {
ret = -EINVAL;
goto exit;
}
ret = rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
exit:
kfree(new_key_mem);
return ret;
}
static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key,
unsigned int key_len)
{
struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent);
struct crypto_aead *child = cryptd_aead_child(ctx->cryptd_tfm);
struct aesni_rfc4106_gcm_ctx *c_ctx = aesni_rfc4106_gcm_ctx_get(child);
struct cryptd_aead *cryptd_tfm = ctx->cryptd_tfm;
int ret;
ret = crypto_aead_setkey(child, key, key_len);
if (!ret) {
memcpy(ctx, c_ctx, sizeof(*ctx));
ctx->cryptd_tfm = cryptd_tfm;
}
return ret;
}
static int common_rfc4106_set_authsize(struct crypto_aead *aead,
unsigned int authsize)
{
switch (authsize) {
case 8:
case 12:
case 16:
break;
default:
return -EINVAL;
}
crypto_aead_crt(aead)->authsize = authsize;
return 0;
}
/* This is the Integrity Check Value (aka the authentication tag length and can
* be 8, 12 or 16 bytes long. */
static int rfc4106_set_authsize(struct crypto_aead *parent,
unsigned int authsize)
{
struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent);
struct crypto_aead *child = cryptd_aead_child(ctx->cryptd_tfm);
int ret;
ret = crypto_aead_setauthsize(child, authsize);
if (!ret)
crypto_aead_crt(parent)->authsize = authsize;
return ret;
}
static int __driver_rfc4106_encrypt(struct aead_request *req)
{
u8 one_entry_in_sg = 0;
u8 *src, *dst, *assoc;
__be32 counter = cpu_to_be32(1);
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
u32 key_len = ctx->aes_key_expanded.key_length;
void *aes_ctx = &(ctx->aes_key_expanded);
unsigned long auth_tag_len = crypto_aead_authsize(tfm);
u8 iv_tab[16+AESNI_ALIGN];
u8* iv = (u8 *) PTR_ALIGN((u8 *)iv_tab, AESNI_ALIGN);
struct scatter_walk src_sg_walk;
struct scatter_walk assoc_sg_walk;
struct scatter_walk dst_sg_walk;
unsigned int i;
/* Assuming we are supporting rfc4106 64-bit extended */
/* sequence numbers We need to have the AAD length equal */
/* to 8 or 12 bytes */
if (unlikely(req->assoclen != 8 && req->assoclen != 12))
return -EINVAL;
if (unlikely(auth_tag_len != 8 && auth_tag_len != 12 && auth_tag_len != 16))
return -EINVAL;
if (unlikely(key_len != AES_KEYSIZE_128 &&
key_len != AES_KEYSIZE_192 &&
key_len != AES_KEYSIZE_256))
return -EINVAL;
/* IV below built */
for (i = 0; i < 4; i++)
*(iv+i) = ctx->nonce[i];
for (i = 0; i < 8; i++)
*(iv+4+i) = req->iv[i];
*((__be32 *)(iv+12)) = counter;
if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) {
one_entry_in_sg = 1;
scatterwalk_start(&src_sg_walk, req->src);
scatterwalk_start(&assoc_sg_walk, req->assoc);
src = scatterwalk_map(&src_sg_walk);
assoc = scatterwalk_map(&assoc_sg_walk);
dst = src;
if (unlikely(req->src != req->dst)) {
scatterwalk_start(&dst_sg_walk, req->dst);
dst = scatterwalk_map(&dst_sg_walk);
}
} else {
/* Allocate memory for src, dst, assoc */
src = kmalloc(req->cryptlen + auth_tag_len + req->assoclen,
GFP_ATOMIC);
if (unlikely(!src))
return -ENOMEM;
assoc = (src + req->cryptlen + auth_tag_len);
scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0);
scatterwalk_map_and_copy(assoc, req->assoc, 0,
req->assoclen, 0);
dst = src;
}
aesni_gcm_enc_tfm(aes_ctx, dst, src, (unsigned long)req->cryptlen, iv,
ctx->hash_subkey, assoc, (unsigned long)req->assoclen, dst
+ ((unsigned long)req->cryptlen), auth_tag_len);
/* The authTag (aka the Integrity Check Value) needs to be written
* back to the packet. */
if (one_entry_in_sg) {
if (unlikely(req->src != req->dst)) {
scatterwalk_unmap(dst);
scatterwalk_done(&dst_sg_walk, 0, 0);
}
scatterwalk_unmap(src);
scatterwalk_unmap(assoc);
scatterwalk_done(&src_sg_walk, 0, 0);
scatterwalk_done(&assoc_sg_walk, 0, 0);
} else {
scatterwalk_map_and_copy(dst, req->dst, 0,
req->cryptlen + auth_tag_len, 1);
kfree(src);
}
return 0;
}