/* Set up per cpu cipher state */
static struct crypto_cipher *setup_essiv_cpu(struct crypt_config *cc,
struct dm_target *ti,
u8 *salt, unsigned saltsize)
{
struct crypto_cipher *essiv_tfm;
int err;
/* Setup the essiv_tfm with the given salt */
essiv_tfm = crypto_alloc_cipher(cc->cipher, 0, CRYPTO_ALG_ASYNC);
if (IS_ERR(essiv_tfm)) {
ti->error = "Error allocating crypto tfm for ESSIV";
return essiv_tfm;
}
if (crypto_cipher_blocksize(essiv_tfm) !=
crypto_ablkcipher_ivsize(any_tfm(cc))) {
ti->error = "Block size of ESSIV cipher does "
"not match IV size of block cipher";
crypto_free_cipher(essiv_tfm);
return ERR_PTR(-EINVAL);
}
err = crypto_cipher_setkey(essiv_tfm, salt, saltsize);
if (err) {
ti->error = "Failed to set key for ESSIV cipher";
crypto_free_cipher(essiv_tfm);
return ERR_PTR(err);
}
return essiv_tfm;
}
static int eseqiv_init(struct crypto_tfm *tfm)
{
struct crypto_ablkcipher *geniv = __crypto_ablkcipher_cast(tfm);
struct eseqiv_ctx *ctx = crypto_ablkcipher_ctx(geniv);
unsigned long alignmask;
unsigned int reqsize;
spin_lock_init(&ctx->lock);
alignmask = crypto_tfm_ctx_alignment() - 1;
reqsize = sizeof(struct eseqiv_request_ctx);
if (alignmask & reqsize) {
alignmask &= reqsize;
alignmask--;
}
alignmask = ~alignmask;
alignmask &= crypto_ablkcipher_alignmask(geniv);
reqsize += alignmask;
reqsize += crypto_ablkcipher_ivsize(geniv);
reqsize = ALIGN(reqsize, crypto_tfm_ctx_alignment());
ctx->reqoff = reqsize - sizeof(struct eseqiv_request_ctx);
tfm->crt_ablkcipher.reqsize = reqsize +
sizeof(struct ablkcipher_request);
return skcipher_geniv_init(tfm);
}
static int async_chainiv_givencrypt_first(struct skcipher_givcrypt_request *req)
{
struct crypto_ablkcipher *geniv = skcipher_givcrypt_reqtfm(req);
struct async_chainiv_ctx *ctx = crypto_ablkcipher_ctx(geniv);
int err = 0;
if (test_and_set_bit(CHAINIV_STATE_INUSE, &ctx->state))
goto out;
if (crypto_ablkcipher_crt(geniv)->givencrypt !=
async_chainiv_givencrypt_first)
goto unlock;
crypto_ablkcipher_crt(geniv)->givencrypt = async_chainiv_givencrypt;
err = crypto_rng_get_bytes(crypto_default_rng, ctx->iv,
crypto_ablkcipher_ivsize(geniv));
unlock:
clear_bit(CHAINIV_STATE_INUSE, &ctx->state);
if (err)
return err;
out:
return async_chainiv_givencrypt(req);
}
static int chainiv_givencrypt(struct skcipher_givcrypt_request *req)
{
struct crypto_ablkcipher *geniv = skcipher_givcrypt_reqtfm(req);
struct chainiv_ctx *ctx = crypto_ablkcipher_ctx(geniv);
struct ablkcipher_request *subreq = skcipher_givcrypt_reqctx(req);
unsigned int ivsize;
int err;
ablkcipher_request_set_tfm(subreq, skcipher_geniv_cipher(geniv));
ablkcipher_request_set_callback(subreq, req->creq.base.flags &
~CRYPTO_TFM_REQ_MAY_SLEEP,
req->creq.base.complete,
req->creq.base.data);
ablkcipher_request_set_crypt(subreq, req->creq.src, req->creq.dst,
req->creq.nbytes, req->creq.info);
spin_lock_bh(&ctx->lock);
ivsize = crypto_ablkcipher_ivsize(geniv);
memcpy(req->giv, ctx->iv, ivsize);
memcpy(subreq->info, ctx->iv, ivsize);
err = crypto_ablkcipher_encrypt(subreq);
if (err)
goto unlock;
memcpy(ctx->iv, subreq->info, ivsize);
unlock:
spin_unlock_bh(&ctx->lock);
return err;
}
static int crypt_iv_essiv_ctr(struct crypt_config *cc, struct dm_target *ti,
const char *opts)
{
struct crypto_cipher *essiv_tfm = NULL;
struct crypto_hash *hash_tfm = NULL;
u8 *salt = NULL;
int err;
if (!opts) {
ti->error = "Digest algorithm missing for ESSIV mode";
return -EINVAL;
}
/* Allocate hash algorithm */
hash_tfm = crypto_alloc_hash(opts, 0, CRYPTO_ALG_ASYNC);
if (IS_ERR(hash_tfm)) {
ti->error = "Error initializing ESSIV hash";
err = PTR_ERR(hash_tfm);
goto bad;
}
salt = kzalloc(crypto_hash_digestsize(hash_tfm), GFP_KERNEL);
if (!salt) {
ti->error = "Error kmallocing salt storage in ESSIV";
err = -ENOMEM;
goto bad;
}
/* Allocate essiv_tfm */
essiv_tfm = crypto_alloc_cipher(cc->cipher, 0, CRYPTO_ALG_ASYNC);
if (IS_ERR(essiv_tfm)) {
ti->error = "Error allocating crypto tfm for ESSIV";
err = PTR_ERR(essiv_tfm);
goto bad;
}
if (crypto_cipher_blocksize(essiv_tfm) !=
crypto_ablkcipher_ivsize(cc->tfm)) {
ti->error = "Block size of ESSIV cipher does "
"not match IV size of block cipher";
err = -EINVAL;
goto bad;
}
cc->iv_gen_private.essiv.salt = salt;
cc->iv_gen_private.essiv.tfm = essiv_tfm;
cc->iv_gen_private.essiv.hash_tfm = hash_tfm;
return 0;
bad:
if (essiv_tfm && !IS_ERR(essiv_tfm))
crypto_free_cipher(essiv_tfm);
if (hash_tfm && !IS_ERR(hash_tfm))
crypto_free_hash(hash_tfm);
kfree(salt);
return err;
}
static void eseqiv_complete2(struct skcipher_givcrypt_request *req)
{
struct crypto_ablkcipher *geniv = skcipher_givcrypt_reqtfm(req);
struct eseqiv_request_ctx *reqctx = skcipher_givcrypt_reqctx(req);
memcpy(req->giv, PTR_ALIGN((u8 *)reqctx->tail,
crypto_ablkcipher_alignmask(geniv) + 1),
crypto_ablkcipher_ivsize(geniv));
}
int crypto4xx_decrypt(struct ablkcipher_request *req)
{
struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
unsigned int ivlen = crypto_ablkcipher_ivsize(
crypto_ablkcipher_reqtfm(req));
__le32 iv[ivlen];
if (ivlen)
crypto4xx_memcpy_to_le32(iv, req->info, ivlen);
return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst,
req->nbytes, iv, ivlen, ctx->sa_in, ctx->sa_len, 0);
}
static int seqiv_init(struct crypto_tfm *tfm)
{
struct crypto_ablkcipher *geniv = __crypto_ablkcipher_cast(tfm);
struct seqiv_ctx *ctx = crypto_ablkcipher_ctx(geniv);
spin_lock_init(&ctx->lock);
tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request);
return crypto_rng_get_bytes(crypto_default_rng, ctx->salt,
crypto_ablkcipher_ivsize(geniv)) ?:
skcipher_geniv_init(tfm);
}
static int eseqiv_init(struct crypto_tfm *tfm)
{
struct crypto_ablkcipher *geniv = __crypto_ablkcipher_cast(tfm);
struct eseqiv_ctx *ctx = crypto_ablkcipher_ctx(geniv);
unsigned long alignmask;
unsigned int reqsize;
#ifndef CONFIG_CRYPTO_DRBG
spin_lock_init(&ctx->lock);
#endif
alignmask = crypto_tfm_ctx_alignment() - 1;
reqsize = sizeof(struct eseqiv_request_ctx);
if (alignmask & reqsize) {
alignmask &= reqsize;
alignmask--;
}
alignmask = ~alignmask;
alignmask &= crypto_ablkcipher_alignmask(geniv);
reqsize += alignmask;
reqsize += crypto_ablkcipher_ivsize(geniv);
reqsize = ALIGN(reqsize, crypto_tfm_ctx_alignment());
ctx->reqoff = reqsize - sizeof(struct eseqiv_request_ctx);
tfm->crt_ablkcipher.reqsize = reqsize +
sizeof(struct ablkcipher_request);
#ifdef CONFIG_CRYPTO_DRBG
crypto_rng_get_bytes(crypto_default_rng, ctx->salt,
crypto_ablkcipher_ivsize(geniv));
#endif
return skcipher_geniv_init(tfm);
}
static int eseqiv_givencrypt_first(struct skcipher_givcrypt_request *req)
{
struct crypto_ablkcipher *geniv = skcipher_givcrypt_reqtfm(req);
struct eseqiv_ctx *ctx = crypto_ablkcipher_ctx(geniv);
spin_lock_bh(&ctx->lock);
if (crypto_ablkcipher_crt(geniv)->givencrypt != eseqiv_givencrypt_first)
goto unlock;
crypto_ablkcipher_crt(geniv)->givencrypt = eseqiv_givencrypt;
get_random_bytes(ctx->salt, crypto_ablkcipher_ivsize(geniv));
unlock:
spin_unlock_bh(&ctx->lock);
return eseqiv_givencrypt(req);
}
static void seqiv_complete2(struct skcipher_givcrypt_request *req, int err)
{
struct ablkcipher_request *subreq = skcipher_givcrypt_reqctx(req);
struct crypto_ablkcipher *geniv;
if (err == -EINPROGRESS)
return;
if (err)
goto out;
geniv = skcipher_givcrypt_reqtfm(req);
memcpy(req->creq.info, subreq->info, crypto_ablkcipher_ivsize(geniv));
out:
kfree(subreq->info);
}
static int seqiv_givencrypt(struct skcipher_givcrypt_request *req)
{
struct crypto_ablkcipher *geniv = skcipher_givcrypt_reqtfm(req);
struct seqiv_ctx *ctx = crypto_ablkcipher_ctx(geniv);
struct ablkcipher_request *subreq = skcipher_givcrypt_reqctx(req);
crypto_completion_t compl;
void *data;
u8 *info;
unsigned int ivsize;
int err;
ablkcipher_request_set_tfm(subreq, skcipher_geniv_cipher(geniv));
compl = req->creq.base.complete;
data = req->creq.base.data;
info = req->creq.info;
ivsize = crypto_ablkcipher_ivsize(geniv);
if (unlikely(!IS_ALIGNED((unsigned long)info,
crypto_ablkcipher_alignmask(geniv) + 1))) {
info = kmalloc(ivsize, req->creq.base.flags &
CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL:
GFP_ATOMIC);
if (!info)
return -ENOMEM;
compl = seqiv_complete;
data = req;
}
ablkcipher_request_set_callback(subreq, req->creq.base.flags, compl,
data);
ablkcipher_request_set_crypt(subreq, req->creq.src, req->creq.dst,
req->creq.nbytes, info);
seqiv_geniv(ctx, info, req->seq, ivsize);
memcpy(req->giv, info, ivsize);
err = crypto_ablkcipher_encrypt(subreq);
if (unlikely(info != req->creq.info))
seqiv_complete2(req, err);
return err;
}
static int async_chainiv_givencrypt_tail(struct skcipher_givcrypt_request *req)
{
struct crypto_ablkcipher *geniv = skcipher_givcrypt_reqtfm(req);
struct async_chainiv_ctx *ctx = crypto_ablkcipher_ctx(geniv);
struct ablkcipher_request *subreq = skcipher_givcrypt_reqctx(req);
unsigned int ivsize = crypto_ablkcipher_ivsize(geniv);
memcpy(req->giv, ctx->iv, ivsize);
memcpy(subreq->info, ctx->iv, ivsize);
ctx->err = crypto_ablkcipher_encrypt(subreq);
if (ctx->err)
goto out;
memcpy(ctx->iv, subreq->info, ivsize);
out:
return async_chainiv_schedule_work(ctx);
}
static int seqiv_givencrypt_first(struct skcipher_givcrypt_request *req)
{
struct crypto_ablkcipher *geniv = skcipher_givcrypt_reqtfm(req);
struct seqiv_ctx *ctx = crypto_ablkcipher_ctx(geniv);
int err = 0;
spin_lock_bh(&ctx->lock);
if (crypto_ablkcipher_crt(geniv)->givencrypt != seqiv_givencrypt_first)
goto unlock;
crypto_ablkcipher_crt(geniv)->givencrypt = seqiv_givencrypt;
err = crypto_rng_get_bytes(crypto_default_rng, ctx->salt,
crypto_ablkcipher_ivsize(geniv));
unlock:
spin_unlock_bh(&ctx->lock);
if (err)
return err;
return seqiv_givencrypt(req);
}
void unmap_ablkcipher_request(struct device *dev,
struct ablkcipher_request *req)
{
struct ablkcipher_req_ctx *areq_ctx;
unsigned int iv_size;
areq_ctx = ablkcipher_request_ctx(req);
iv_size = crypto_ablkcipher_ivsize(crypto_ablkcipher_reqtfm(req));
if (likely(areq_ctx->gen_ctx.iv_dma_addr != 0)) {
DX_LOG_DEBUG("Unmapped iv: iv_dma_addr=0x%08lX iv_size=%d\n",
(unsigned long)areq_ctx->gen_ctx.iv_dma_addr, iv_size);
dma_unmap_single(dev, areq_ctx->gen_ctx.iv_dma_addr,
iv_size, DMA_TO_DEVICE);
}
/*In case a pool was set, a table was allocated and should be released */
if (areq_ctx->dma_buf_type == DX_DMA_BUF_MLLI) {
dma_pool_free(areq_ctx->mlli_params.curr_pool,
areq_ctx->mlli_params.mlli_virt_addr,
areq_ctx->mlli_params.mlli_dma_addr);
}
if (areq_ctx->sec_dir != DX_SRC_DMA_IS_SECURE) {
dma_unmap_sg(dev, req->src,
areq_ctx->in_nents, DMA_BIDIRECTIONAL);
}
DX_LOG_DEBUG("Unmapped sg src: req->src=0x%08lX\n",
(unsigned long)sg_virt(req->src));
if (likely(req->src != req->dst)) {
if (areq_ctx->sec_dir != DX_DST_DMA_IS_SECURE) {
dma_unmap_sg(dev, req->dst,
areq_ctx->out_nents, DMA_BIDIRECTIONAL);
DX_LOG_DEBUG("Unmapped sg dst: req->dst=0x%08lX\n",
(unsigned long)sg_virt(req->dst));
}
}
}
static int skcipher_crypt_blkcipher(struct skcipher_request *req,
int (*crypt)(struct blkcipher_desc *,
struct scatterlist *,
struct scatterlist *,
unsigned int))
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
struct crypto_blkcipher **ctx = crypto_skcipher_ctx(tfm);
struct blkcipher_desc desc = {
.tfm = *ctx,
.info = req->iv,
.flags = req->base.flags,
};
return crypt(&desc, req->dst, req->src, req->cryptlen);
}
static int skcipher_encrypt_blkcipher(struct skcipher_request *req)
{
struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
struct crypto_tfm *tfm = crypto_skcipher_tfm(skcipher);
struct blkcipher_alg *alg = &tfm->__crt_alg->cra_blkcipher;
return skcipher_crypt_blkcipher(req, alg->encrypt);
}
static int skcipher_decrypt_blkcipher(struct skcipher_request *req)
{
struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
struct crypto_tfm *tfm = crypto_skcipher_tfm(skcipher);
struct blkcipher_alg *alg = &tfm->__crt_alg->cra_blkcipher;
return skcipher_crypt_blkcipher(req, alg->decrypt);
}
static void crypto_exit_skcipher_ops_blkcipher(struct crypto_tfm *tfm)
{
struct crypto_blkcipher **ctx = crypto_tfm_ctx(tfm);
crypto_free_blkcipher(*ctx);
}
static int crypto_init_skcipher_ops_blkcipher(struct crypto_tfm *tfm)
{
struct crypto_alg *calg = tfm->__crt_alg;
struct crypto_skcipher *skcipher = __crypto_skcipher_cast(tfm);
struct crypto_blkcipher **ctx = crypto_tfm_ctx(tfm);
struct crypto_blkcipher *blkcipher;
struct crypto_tfm *btfm;
if (!crypto_mod_get(calg))
return -EAGAIN;
btfm = __crypto_alloc_tfm(calg, CRYPTO_ALG_TYPE_BLKCIPHER,
CRYPTO_ALG_TYPE_MASK);
if (IS_ERR(btfm)) {
crypto_mod_put(calg);
return PTR_ERR(btfm);
}
blkcipher = __crypto_blkcipher_cast(btfm);
*ctx = blkcipher;
tfm->exit = crypto_exit_skcipher_ops_blkcipher;
skcipher->setkey = skcipher_setkey_blkcipher;
skcipher->encrypt = skcipher_encrypt_blkcipher;
skcipher->decrypt = skcipher_decrypt_blkcipher;
skcipher->ivsize = crypto_blkcipher_ivsize(blkcipher);
skcipher->keysize = calg->cra_blkcipher.max_keysize;
return 0;
}
static int skcipher_setkey_ablkcipher(struct crypto_skcipher *tfm,
const u8 *key, unsigned int keylen)
{
struct crypto_ablkcipher **ctx = crypto_skcipher_ctx(tfm);
struct crypto_ablkcipher *ablkcipher = *ctx;
int err;
crypto_ablkcipher_clear_flags(ablkcipher, ~0);
crypto_ablkcipher_set_flags(ablkcipher,
crypto_skcipher_get_flags(tfm) &
CRYPTO_TFM_REQ_MASK);
err = crypto_ablkcipher_setkey(ablkcipher, key, keylen);
crypto_skcipher_set_flags(tfm,
crypto_ablkcipher_get_flags(ablkcipher) &
CRYPTO_TFM_RES_MASK);
return err;
}
static int skcipher_crypt_ablkcipher(struct skcipher_request *req,
int (*crypt)(struct ablkcipher_request *))
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
struct crypto_ablkcipher **ctx = crypto_skcipher_ctx(tfm);
struct ablkcipher_request *subreq = skcipher_request_ctx(req);
ablkcipher_request_set_tfm(subreq, *ctx);
ablkcipher_request_set_callback(subreq, skcipher_request_flags(req),
req->base.complete, req->base.data);
ablkcipher_request_set_crypt(subreq, req->src, req->dst, req->cryptlen,
req->iv);
return crypt(subreq);
}
static int skcipher_encrypt_ablkcipher(struct skcipher_request *req)
{
struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
struct crypto_tfm *tfm = crypto_skcipher_tfm(skcipher);
struct ablkcipher_alg *alg = &tfm->__crt_alg->cra_ablkcipher;
return skcipher_crypt_ablkcipher(req, alg->encrypt);
}
static int skcipher_decrypt_ablkcipher(struct skcipher_request *req)
{
struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
struct crypto_tfm *tfm = crypto_skcipher_tfm(skcipher);
struct ablkcipher_alg *alg = &tfm->__crt_alg->cra_ablkcipher;
return skcipher_crypt_ablkcipher(req, alg->decrypt);
}
static void crypto_exit_skcipher_ops_ablkcipher(struct crypto_tfm *tfm)
{
struct crypto_ablkcipher **ctx = crypto_tfm_ctx(tfm);
crypto_free_ablkcipher(*ctx);
}
static int crypto_init_skcipher_ops_ablkcipher(struct crypto_tfm *tfm)
{
struct crypto_alg *calg = tfm->__crt_alg;
struct crypto_skcipher *skcipher = __crypto_skcipher_cast(tfm);
struct crypto_ablkcipher **ctx = crypto_tfm_ctx(tfm);
struct crypto_ablkcipher *ablkcipher;
struct crypto_tfm *abtfm;
if (!crypto_mod_get(calg))
return -EAGAIN;
abtfm = __crypto_alloc_tfm(calg, 0, 0);
if (IS_ERR(abtfm)) {
crypto_mod_put(calg);
return PTR_ERR(abtfm);
}
ablkcipher = __crypto_ablkcipher_cast(abtfm);
*ctx = ablkcipher;
tfm->exit = crypto_exit_skcipher_ops_ablkcipher;
skcipher->setkey = skcipher_setkey_ablkcipher;
skcipher->encrypt = skcipher_encrypt_ablkcipher;
skcipher->decrypt = skcipher_decrypt_ablkcipher;
skcipher->ivsize = crypto_ablkcipher_ivsize(ablkcipher);
skcipher->reqsize = crypto_ablkcipher_reqsize(ablkcipher) +
sizeof(struct ablkcipher_request);
skcipher->keysize = calg->cra_ablkcipher.max_keysize;
return 0;
}
static int crypto_skcipher_init_tfm(struct crypto_tfm *tfm)
{
if (tfm->__crt_alg->cra_type == &crypto_blkcipher_type)
return crypto_init_skcipher_ops_blkcipher(tfm);
BUG_ON(tfm->__crt_alg->cra_type != &crypto_ablkcipher_type &&
tfm->__crt_alg->cra_type != &crypto_givcipher_type);
return crypto_init_skcipher_ops_ablkcipher(tfm);
}
static const struct crypto_type crypto_skcipher_type2 = {
.extsize = crypto_skcipher_extsize,
.init_tfm = crypto_skcipher_init_tfm,
.maskclear = ~CRYPTO_ALG_TYPE_MASK,
.maskset = CRYPTO_ALG_TYPE_BLKCIPHER_MASK,
.type = CRYPTO_ALG_TYPE_BLKCIPHER,
.tfmsize = offsetof(struct crypto_skcipher, base),
};
struct crypto_skcipher *crypto_alloc_skcipher(const char *alg_name,
u32 type, u32 mask)
{
return crypto_alloc_tfm(alg_name, &crypto_skcipher_type2, type, mask);
}
EXPORT_SYMBOL_GPL(crypto_alloc_skcipher);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Symmetric key cipher type");
static int
qce_ablkcipher_async_req_handle(struct crypto_async_request *async_req)
{
struct ablkcipher_request *req = ablkcipher_request_cast(async_req);
struct qce_cipher_reqctx *rctx = ablkcipher_request_ctx(req);
struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
struct qce_alg_template *tmpl = to_cipher_tmpl(async_req->tfm);
struct qce_device *qce = tmpl->qce;
enum dma_data_direction dir_src, dir_dst;
struct scatterlist *sg;
bool diff_dst;
gfp_t gfp;
int ret;
rctx->iv = req->info;
rctx->ivsize = crypto_ablkcipher_ivsize(ablkcipher);
rctx->cryptlen = req->nbytes;
diff_dst = (req->src != req->dst) ? true : false;
dir_src = diff_dst ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL;
dir_dst = diff_dst ? DMA_FROM_DEVICE : DMA_BIDIRECTIONAL;
rctx->src_nents = sg_nents_for_len(req->src, req->nbytes);
if (diff_dst)
rctx->dst_nents = sg_nents_for_len(req->dst, req->nbytes);
else
rctx->dst_nents = rctx->src_nents;
if (rctx->src_nents < 0) {
dev_err(qce->dev, "Invalid numbers of src SG.\n");
return rctx->src_nents;
}
if (rctx->dst_nents < 0) {
dev_err(qce->dev, "Invalid numbers of dst SG.\n");
return -rctx->dst_nents;
}
rctx->dst_nents += 1;
gfp = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
GFP_KERNEL : GFP_ATOMIC;
ret = sg_alloc_table(&rctx->dst_tbl, rctx->dst_nents, gfp);
if (ret)
return ret;
sg_init_one(&rctx->result_sg, qce->dma.result_buf, QCE_RESULT_BUF_SZ);
sg = qce_sgtable_add(&rctx->dst_tbl, req->dst);
if (IS_ERR(sg)) {
ret = PTR_ERR(sg);
goto error_free;
}
sg = qce_sgtable_add(&rctx->dst_tbl, &rctx->result_sg);
if (IS_ERR(sg)) {
ret = PTR_ERR(sg);
goto error_free;
}
sg_mark_end(sg);
rctx->dst_sg = rctx->dst_tbl.sgl;
ret = dma_map_sg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst);
if (ret < 0)
goto error_free;
if (diff_dst) {
ret = dma_map_sg(qce->dev, req->src, rctx->src_nents, dir_src);
if (ret < 0)
goto error_unmap_dst;
rctx->src_sg = req->src;
} else {
rctx->src_sg = rctx->dst_sg;
}
ret = qce_dma_prep_sgs(&qce->dma, rctx->src_sg, rctx->src_nents,
rctx->dst_sg, rctx->dst_nents,
qce_ablkcipher_done, async_req);
if (ret)
goto error_unmap_src;
qce_dma_issue_pending(&qce->dma);
ret = qce_start(async_req, tmpl->crypto_alg_type, req->nbytes, 0);
if (ret)
goto error_terminate;
return 0;
error_terminate:
qce_dma_terminate_all(&qce->dma);
error_unmap_src:
if (diff_dst)
dma_unmap_sg(qce->dev, req->src, rctx->src_nents, dir_src);
error_unmap_dst:
dma_unmap_sg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst);
error_free:
sg_free_table(&rctx->dst_tbl);
return ret;
}
static int test_acipher(const char *algo, int enc, char *data_in,
char *data_out, size_t data_len, char *key, int keysize)
{
struct crypto_ablkcipher *tfm;
struct tcrypt_result tresult;
struct ablkcipher_request *req;
struct scatterlist sg[TVMEMSIZE];
unsigned int ret, i, j, iv_len;
char iv[128];
ret = -EAGAIN;
init_completion(&tresult.completion);
tfm = crypto_alloc_ablkcipher(algo, 0, 0);
if (IS_ERR(tfm)) {
printk(KERN_ERR "failed to load transform for %s: %ld\n",
algo, PTR_ERR(tfm));
return ret;
}
req = ablkcipher_request_alloc(tfm, GFP_KERNEL);
if (!req) {
printk(KERN_ERR "tcrypt: skcipher: Failed to allocate request for %s\n",
algo);
goto out;
}
ablkcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
tcrypt_complete, &tresult);
crypto_ablkcipher_clear_flags(tfm, ~0);
ret = crypto_ablkcipher_setkey(tfm, key, keysize);
if (ret) {
printk(KERN_ERR "setkey() failed flags=%x\n",
crypto_ablkcipher_get_flags(tfm));
goto out_free_req;
}
printk(KERN_INFO "KEY:\n");
hexdump(key, keysize);
sg_init_table(sg, TVMEMSIZE);
i = 0;
j = data_len;
while (j > PAGE_SIZE) {
sg_set_buf(sg + i, tvmem[i], PAGE_SIZE);
memcpy(tvmem[i], data_in + i * PAGE_SIZE, PAGE_SIZE);
i++;
j -= PAGE_SIZE;
}
sg_set_buf(sg + i, tvmem[i], j);
memcpy(tvmem[i], data_in + i * PAGE_SIZE, j);
iv_len = crypto_ablkcipher_ivsize(tfm);
memcpy(iv, iv16, iv_len);
printk(KERN_INFO "IV:\n");
hexdump(iv, iv_len);
ablkcipher_request_set_crypt(req, sg, sg, data_len, iv);
printk(KERN_INFO "IN:\n");
hexdump(data_in, data_len);
if (enc)
ret = do_one_acipher_op(req, crypto_ablkcipher_encrypt(req));
else
ret = do_one_acipher_op(req, crypto_ablkcipher_decrypt(req));
if (ret)
printk(KERN_ERR "failed flags=%x\n",
crypto_ablkcipher_get_flags(tfm));
else {
i = 0;
j = data_len;
while (j > PAGE_SIZE) {
memcpy(data_out + i * PAGE_SIZE, tvmem[i], PAGE_SIZE);
i++;
j -= PAGE_SIZE;
}
memcpy(data_out + i * PAGE_SIZE, tvmem[i], j);
printk(KERN_INFO "OUT:\n");
hexdump(data_out, data_len);
}
out_free_req:
ablkcipher_request_free(req);
out:
crypto_free_ablkcipher(tfm);
return ret;
}
int cryptodev_cipher_init(struct cipher_data *out, const char *alg_name,
uint8_t *keyp, size_t keylen, int stream, int aead)
{
int ret;
if (aead == 0) {
struct ablkcipher_alg *alg;
out->async.s = crypto_alloc_ablkcipher(alg_name, 0, 0);
if (unlikely(IS_ERR(out->async.s))) {
ddebug(1, "Failed to load cipher %s", alg_name);
return -EINVAL;
}
alg = crypto_ablkcipher_alg(out->async.s);
if (alg != NULL) {
/* Was correct key length supplied? */
if (alg->max_keysize > 0 &&
unlikely((keylen < alg->min_keysize) ||
(keylen > alg->max_keysize))) {
ddebug(1, "Wrong keylen '%zu' for algorithm '%s'. Use %u to %u.",
keylen, alg_name, alg->min_keysize, alg->max_keysize);
ret = -EINVAL;
goto error;
}
}
out->blocksize = crypto_ablkcipher_blocksize(out->async.s);
out->ivsize = crypto_ablkcipher_ivsize(out->async.s);
out->alignmask = crypto_ablkcipher_alignmask(out->async.s);
ret = crypto_ablkcipher_setkey(out->async.s, keyp, keylen);
} else {
out->async.as = crypto_alloc_aead(alg_name, 0, 0);
if (unlikely(IS_ERR(out->async.as))) {
ddebug(1, "Failed to load cipher %s", alg_name);
return -EINVAL;
}
out->blocksize = crypto_aead_blocksize(out->async.as);
out->ivsize = crypto_aead_ivsize(out->async.as);
out->alignmask = crypto_aead_alignmask(out->async.as);
ret = crypto_aead_setkey(out->async.as, keyp, keylen);
}
if (unlikely(ret)) {
ddebug(1, "Setting key failed for %s-%zu.", alg_name, keylen*8);
ret = -EINVAL;
goto error;
}
out->stream = stream;
out->aead = aead;
out->async.result = kzalloc(sizeof(*out->async.result), GFP_KERNEL);
if (unlikely(!out->async.result)) {
ret = -ENOMEM;
goto error;
}
init_completion(&out->async.result->completion);
if (aead == 0) {
out->async.request = ablkcipher_request_alloc(out->async.s, GFP_KERNEL);
if (unlikely(!out->async.request)) {
derr(1, "error allocating async crypto request");
ret = -ENOMEM;
goto error;
}
ablkcipher_request_set_callback(out->async.request,
CRYPTO_TFM_REQ_MAY_BACKLOG,
cryptodev_complete, out->async.result);
} else {
out->async.arequest = aead_request_alloc(out->async.as, GFP_KERNEL);
if (unlikely(!out->async.arequest)) {
derr(1, "error allocating async crypto request");
ret = -ENOMEM;
goto error;
}
aead_request_set_callback(out->async.arequest,
CRYPTO_TFM_REQ_MAY_BACKLOG,
cryptodev_complete, out->async.result);
}
out->init = 1;
return 0;
error:
if (aead == 0) {
if (out->async.request)
ablkcipher_request_free(out->async.request);
if (out->async.s)
crypto_free_ablkcipher(out->async.s);
} else {
if (out->async.arequest)
aead_request_free(out->async.arequest);
if (out->async.as)
crypto_free_aead(out->async.as);
}
kfree(out->async.result);
return ret;
}
static int
__virtio_crypto_ablkcipher_do_req(struct virtio_crypto_request *vc_req,
struct ablkcipher_request *req,
struct data_queue *data_vq,
__u8 op)
{
struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
unsigned int ivsize = crypto_ablkcipher_ivsize(tfm);
struct virtio_crypto_ablkcipher_ctx *ctx = vc_req->ablkcipher_ctx;
struct virtio_crypto *vcrypto = ctx->vcrypto;
struct virtio_crypto_op_data_req *req_data;
int src_nents, dst_nents;
int err;
unsigned long flags;
struct scatterlist outhdr, iv_sg, status_sg, **sgs;
int i;
u64 dst_len;
unsigned int num_out = 0, num_in = 0;
int sg_total;
uint8_t *iv;
src_nents = sg_nents_for_len(req->src, req->nbytes);
dst_nents = sg_nents(req->dst);
pr_debug("virtio_crypto: Number of sgs (src_nents: %d, dst_nents: %d)\n",
src_nents, dst_nents);
/* Why 3? outhdr + iv + inhdr */
sg_total = src_nents + dst_nents + 3;
sgs = kzalloc_node(sg_total * sizeof(*sgs), GFP_ATOMIC,
dev_to_node(&vcrypto->vdev->dev));
if (!sgs)
return -ENOMEM;
req_data = kzalloc_node(sizeof(*req_data), GFP_ATOMIC,
dev_to_node(&vcrypto->vdev->dev));
if (!req_data) {
kfree(sgs);
return -ENOMEM;
}
vc_req->req_data = req_data;
vc_req->type = VIRTIO_CRYPTO_SYM_OP_CIPHER;
/* Head of operation */
if (op) {
req_data->header.session_id =
cpu_to_le64(ctx->enc_sess_info.session_id);
req_data->header.opcode =
cpu_to_le32(VIRTIO_CRYPTO_CIPHER_ENCRYPT);
} else {
req_data->header.session_id =
cpu_to_le64(ctx->dec_sess_info.session_id);
req_data->header.opcode =
cpu_to_le32(VIRTIO_CRYPTO_CIPHER_DECRYPT);
}
req_data->u.sym_req.op_type = cpu_to_le32(VIRTIO_CRYPTO_SYM_OP_CIPHER);
req_data->u.sym_req.u.cipher.para.iv_len = cpu_to_le32(ivsize);
req_data->u.sym_req.u.cipher.para.src_data_len =
cpu_to_le32(req->nbytes);
dst_len = virtio_crypto_alg_sg_nents_length(req->dst);
if (unlikely(dst_len > U32_MAX)) {
pr_err("virtio_crypto: The dst_len is beyond U32_MAX\n");
err = -EINVAL;
goto free;
}
pr_debug("virtio_crypto: src_len: %u, dst_len: %llu\n",
req->nbytes, dst_len);
if (unlikely(req->nbytes + dst_len + ivsize +
sizeof(vc_req->status) > vcrypto->max_size)) {
pr_err("virtio_crypto: The length is too big\n");
err = -EINVAL;
goto free;
}
req_data->u.sym_req.u.cipher.para.dst_data_len =
cpu_to_le32((uint32_t)dst_len);
/* Outhdr */
sg_init_one(&outhdr, req_data, sizeof(*req_data));
sgs[num_out++] = &outhdr;
/* IV */
/*
* Avoid to do DMA from the stack, switch to using
* dynamically-allocated for the IV
*/
iv = kzalloc_node(ivsize, GFP_ATOMIC,
dev_to_node(&vcrypto->vdev->dev));
if (!iv) {
err = -ENOMEM;
goto free;
}
memcpy(iv, req->info, ivsize);
sg_init_one(&iv_sg, iv, ivsize);
sgs[num_out++] = &iv_sg;
vc_req->iv = iv;
/* Source data */
for (i = 0; i < src_nents; i++)
sgs[num_out++] = &req->src[i];
/* Destination data */
for (i = 0; i < dst_nents; i++)
sgs[num_out + num_in++] = &req->dst[i];
/* Status */
sg_init_one(&status_sg, &vc_req->status, sizeof(vc_req->status));
sgs[num_out + num_in++] = &status_sg;
vc_req->sgs = sgs;
spin_lock_irqsave(&data_vq->lock, flags);
err = virtqueue_add_sgs(data_vq->vq, sgs, num_out,
num_in, vc_req, GFP_ATOMIC);
virtqueue_kick(data_vq->vq);
spin_unlock_irqrestore(&data_vq->lock, flags);
if (unlikely(err < 0))
goto free_iv;
return 0;
free_iv:
kzfree(iv);
free:
kzfree(req_data);
kfree(sgs);
return err;
}
int ss_aes_one_req(sunxi_ss_t *sss, struct ablkcipher_request *req)
{
int ret = 0;
struct crypto_ablkcipher *tfm = NULL;
ss_aes_ctx_t *ctx = NULL;
ss_aes_req_ctx_t *req_ctx = NULL;
SS_ENTER();
if (!req->src || !req->dst) {
SS_ERR("Invalid sg: src = %p, dst = %p\n", req->src, req->dst);
return -EINVAL;
}
ss_dev_lock();
tfm = crypto_ablkcipher_reqtfm(req);
req_ctx = ablkcipher_request_ctx(req);
ctx = crypto_ablkcipher_ctx(tfm);
/* A31 SS need update key each cycle in decryption. */
if ((ctx->comm.flags & SS_FLAG_NEW_KEY) || (req_ctx->dir == SS_DIR_DECRYPT)) {
SS_DBG("KEY address = %p, size = %d\n", ctx->key, ctx->key_size);
ss_key_set(ctx->key, ctx->key_size);
ctx->comm.flags &= ~SS_FLAG_NEW_KEY;
}
#ifdef SS_CTS_MODE_ENABLE
if (((req_ctx->mode == SS_AES_MODE_CBC)
|| (req_ctx->mode == SS_AES_MODE_CTS)) && (req->info != NULL)) {
#else
if ((req_ctx->mode == SS_AES_MODE_CBC) && (req->info != NULL)) {
#endif
SS_DBG("IV address = %p, size = %d\n", req->info, crypto_ablkcipher_ivsize(tfm));
ss_iv_set(req->info, crypto_ablkcipher_ivsize(tfm));
}
#ifdef SS_CTR_MODE_ENABLE
if (req_ctx->mode == SS_AES_MODE_CTR) {
SS_DBG("Cnt address = %p, size = %d\n", req->info, crypto_ablkcipher_ivsize(tfm));
if (ctx->cnt == 0)
memcpy(ctx->iv, req->info, crypto_ablkcipher_ivsize(tfm));
SS_DBG("CNT: %08x %08x %08x %08x \n", *(int *)&ctx->iv[0],
*(int *)&ctx->iv[4], *(int *)&ctx->iv[8], *(int *)&ctx->iv[12]);
ss_cnt_set(ctx->iv, crypto_ablkcipher_ivsize(tfm));
}
#endif
req_ctx->dma_src.sg = req->src;
req_ctx->dma_dst.sg = req->dst;
ret = ss_aes_start(ctx, req_ctx, req->nbytes);
if (ret < 0)
SS_ERR("ss_aes_start fail(%d)\n", ret);
ss_dev_unlock();
#ifdef SS_CTR_MODE_ENABLE
if (req_ctx->mode == SS_AES_MODE_CTR) {
ss_cnt_get(ctx->comm.flow, ctx->iv, crypto_ablkcipher_ivsize(tfm));
SS_DBG("CNT: %08x %08x %08x %08x \n", *(int *)&ctx->iv[0],
*(int *)&ctx->iv[4], *(int *)&ctx->iv[8], *(int *)&ctx->iv[12]);
}
#endif
ctx->cnt += req->nbytes;
if (req->base.complete)
req->base.complete(&req->base, ret);
return ret;
}
irqreturn_t sunxi_ss_irq_handler(int irq, void *dev_id)
{
sunxi_ss_t *sss = (sunxi_ss_t *)dev_id;
unsigned long flags = 0;
int pending = 0;
spin_lock_irqsave(&sss->lock, flags);
pending = ss_pending_get();
SS_DBG("SS pending %#x\n", pending);
spin_unlock_irqrestore(&sss->lock, flags);
return IRQ_HANDLED;
}
static int ss_aes_one_req(sunxi_ss_t *sss, struct ablkcipher_request *req)
{
int ret = 0;
struct crypto_ablkcipher *tfm = NULL;
ss_aes_ctx_t *ctx = NULL;
ss_aes_req_ctx_t *req_ctx = NULL;
int key_map_flag = 0;
int iv_map_flag = 0;
SS_ENTER();
if (!req->src || !req->dst) {
SS_ERR("Invalid sg: src = %p, dst = %p\n", req->src, req->dst);
return -EINVAL;
}
ss_dev_lock();
tfm = crypto_ablkcipher_reqtfm(req);
req_ctx = ablkcipher_request_ctx(req);
ctx = crypto_ablkcipher_ctx(tfm);
/* A31 SS need update key each cycle in decryption. */
if ((ctx->comm.flags & SS_FLAG_NEW_KEY) || (req_ctx->dir == SS_DIR_DECRYPT)) {
SS_DBG("KEY address = %p, size = %d\n", ctx->key, ctx->key_size);
ss_key_set(ctx->key, ctx->key_size);
dma_map_single(&sss->pdev->dev, ctx->key, ctx->key_size, DMA_MEM_TO_DEV);
key_map_flag = 1;
ctx->comm.flags &= ~SS_FLAG_NEW_KEY;
}
#ifdef SS_CTS_MODE_ENABLE
if (((req_ctx->mode == SS_AES_MODE_CBC)
|| (req_ctx->mode == SS_AES_MODE_CTS)) && (req->info != NULL)) {
#else
if ((req_ctx->mode == SS_AES_MODE_CBC) && (req->info != NULL)) {
#endif
SS_DBG("IV address = %p, size = %d\n", req->info, crypto_ablkcipher_ivsize(tfm));
memcpy(ctx->iv, req->info, crypto_ablkcipher_ivsize(tfm));
ss_iv_set(ctx->iv, crypto_ablkcipher_ivsize(tfm));
dma_map_single(&sss->pdev->dev, ctx->iv, crypto_ablkcipher_ivsize(tfm), DMA_MEM_TO_DEV);
iv_map_flag = 1;
}
#ifdef SS_CTR_MODE_ENABLE
if (req_ctx->mode == SS_AES_MODE_CTR) {
SS_DBG("Cnt address = %p, size = %d\n", req->info, crypto_ablkcipher_ivsize(tfm));
if (ctx->cnt == 0)
memcpy(ctx->iv, req->info, crypto_ablkcipher_ivsize(tfm));
SS_DBG("CNT: %08x %08x %08x %08x \n", *(int *)&ctx->iv[0],
*(int *)&ctx->iv[4], *(int *)&ctx->iv[8], *(int *)&ctx->iv[12]);
ss_cnt_set(ctx->iv, crypto_ablkcipher_ivsize(tfm));
dma_map_single(&sss->pdev->dev, ctx->iv, crypto_ablkcipher_ivsize(tfm), DMA_MEM_TO_DEV);
iv_map_flag = 1;
}
#endif
if (req_ctx->type == SS_METHOD_RSA)
ss_rsa_width_set(crypto_ablkcipher_ivsize(tfm));
req_ctx->dma_src.sg = req->src;
req_ctx->dma_dst.sg = req->dst;
ret = ss_aes_start(ctx, req_ctx, req->nbytes);
if (ret < 0)
SS_ERR("ss_aes_start fail(%d)\n", ret);
ss_dev_unlock();
if (req->base.complete)
req->base.complete(&req->base, ret);
if (key_map_flag == 1)
dma_unmap_single(&ss_dev->pdev->dev, virt_to_phys(ctx->key), ctx->key_size, DMA_MEM_TO_DEV);
if (iv_map_flag == 1)
dma_unmap_single(&sss->pdev->dev, virt_to_phys(ctx->iv), crypto_ablkcipher_ivsize(tfm), DMA_MEM_TO_DEV);
#ifdef SS_CTR_MODE_ENABLE
if (req_ctx->mode == SS_AES_MODE_CTR) {
ss_cnt_get(ctx->comm.flow, ctx->iv, crypto_ablkcipher_ivsize(tfm));
SS_DBG("CNT: %08x %08x %08x %08x \n", *(int *)&ctx->iv[0],
*(int *)&ctx->iv[4], *(int *)&ctx->iv[8], *(int *)&ctx->iv[12]);
}
#endif
ctx->cnt += req->nbytes;
return ret;
}
static int ss_hash_one_req(sunxi_ss_t *sss, struct ahash_request *req)
{
int ret = 0;
ss_aes_req_ctx_t *req_ctx = NULL;
ss_hash_ctx_t *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
SS_ENTER();
if (!req->src) {
SS_ERR("Invalid sg: src = %p\n", req->src);
return -EINVAL;
}
ss_dev_lock();
req_ctx = ahash_request_ctx(req);
req_ctx->dma_src.sg = req->src;
ss_hash_padding_data_prepare(ctx, req->result, req->nbytes);
ret = ss_hash_start(ctx, req_ctx, req->nbytes);
if (ret < 0)
SS_ERR("ss_hash_start fail(%d)\n", ret);
ss_dev_unlock();
if (req->base.complete)
req->base.complete(&req->base, ret);
return ret;
}
static int eseqiv_givencrypt(struct skcipher_givcrypt_request *req)
{
struct crypto_ablkcipher *geniv = skcipher_givcrypt_reqtfm(req);
struct eseqiv_ctx *ctx = crypto_ablkcipher_ctx(geniv);
struct eseqiv_request_ctx *reqctx = skcipher_givcrypt_reqctx(req);
struct ablkcipher_request *subreq;
crypto_completion_t complete;
void *data;
struct scatterlist *osrc, *odst;
struct scatterlist *dst;
struct page *srcp;
struct page *dstp;
u8 *giv;
u8 *vsrc;
u8 *vdst;
__be64 seq;
unsigned int ivsize;
unsigned int len;
int err;
subreq = (void *)(reqctx->tail + ctx->reqoff);
ablkcipher_request_set_tfm(subreq, skcipher_geniv_cipher(geniv));
giv = req->giv;
complete = req->creq.base.complete;
data = req->creq.base.data;
osrc = req->creq.src;
odst = req->creq.dst;
srcp = sg_page(osrc);
dstp = sg_page(odst);
vsrc = PageHighMem(srcp) ? NULL : page_address(srcp) + osrc->offset;
vdst = PageHighMem(dstp) ? NULL : page_address(dstp) + odst->offset;
ivsize = crypto_ablkcipher_ivsize(geniv);
if (vsrc != giv + ivsize && vdst != giv + ivsize) {
giv = PTR_ALIGN((u8 *)reqctx->tail,
crypto_ablkcipher_alignmask(geniv) + 1);
complete = eseqiv_complete;
data = req;
}
ablkcipher_request_set_callback(subreq, req->creq.base.flags, complete,
data);
sg_init_table(reqctx->src, 2);
sg_set_buf(reqctx->src, giv, ivsize);
scatterwalk_crypto_chain(reqctx->src, osrc, vsrc == giv + ivsize, 2);
dst = reqctx->src;
if (osrc != odst) {
sg_init_table(reqctx->dst, 2);
sg_set_buf(reqctx->dst, giv, ivsize);
scatterwalk_crypto_chain(reqctx->dst, odst, vdst == giv + ivsize, 2);
dst = reqctx->dst;
}
ablkcipher_request_set_crypt(subreq, reqctx->src, dst,
req->creq.nbytes + ivsize,
req->creq.info);
memcpy(req->creq.info, ctx->salt, ivsize);
len = ivsize;
if (ivsize > sizeof(u64)) {
memset(req->giv, 0, ivsize - sizeof(u64));
len = sizeof(u64);
}
seq = cpu_to_be64(req->seq);
memcpy(req->giv + ivsize - len, &seq, len);
err = crypto_ablkcipher_encrypt(subreq);
if (err)
goto out;
if (giv != req->giv)
eseqiv_complete2(req);
out:
return err;
}
int map_ablkcipher_request(struct device *dev, struct ablkcipher_request *req)
{
struct ablkcipher_req_ctx *areq_ctx = ablkcipher_request_ctx(req);
unsigned int iv_size = crypto_ablkcipher_ivsize(crypto_ablkcipher_reqtfm(req));
struct mlli_params *mlli_params = &areq_ctx->mlli_params;
struct sg_data_array sg_data;
struct buff_mgr_handle *buff_mgr = crypto_drvdata->buff_mgr_handle;
int dummy = 0;
int rc = 0;
areq_ctx->sec_dir = 0;
areq_ctx->dma_buf_type = DX_DMA_BUF_DLLI;
mlli_params->curr_pool = NULL;
sg_data.num_of_sg = 0;
/* Map IV buffer */
if (likely(iv_size != 0) ) {
dump_byte_array("iv", (uint8_t *)req->info, iv_size);
areq_ctx->gen_ctx.iv_dma_addr =
dma_map_single(dev, (void *)req->info,
iv_size, DMA_TO_DEVICE);
if (unlikely(dma_mapping_error(dev,
areq_ctx->gen_ctx.iv_dma_addr))) {
DX_LOG_ERR("Mapping iv %u B at va=0x%08lX "
"for DMA failed\n",iv_size,
(unsigned long)req->info);
return -ENOMEM;
}
DX_LOG_DEBUG("Mapped iv %u B at va=0x%08lX to dma=0x%08lX\n",
iv_size, (unsigned long)req->info,
(unsigned long)areq_ctx->gen_ctx.iv_dma_addr);
} else {
areq_ctx->gen_ctx.iv_dma_addr = 0;
}
/* Map the src sg */
if ( sg_is_last(req->src) &&
(sg_page(req->src) == NULL) &&
sg_dma_address(req->src)) {
/* The source is secure no mapping is needed */
areq_ctx->sec_dir = DX_SRC_DMA_IS_SECURE;
areq_ctx->in_nents = 1;
} else {
if ( unlikely( dx_map_sg( dev,req->src, req->nbytes,
DMA_BIDIRECTIONAL,
&areq_ctx->in_nents,
LLI_MAX_NUM_OF_DATA_ENTRIES,
&dummy))){
rc = -ENOMEM;
goto fail_unmap_iv;
}
if ( areq_ctx->in_nents > 1 ) {
areq_ctx->dma_buf_type = DX_DMA_BUF_MLLI;
}
}
if ( unlikely(req->src == req->dst)) {
if ( areq_ctx->sec_dir == DX_SRC_DMA_IS_SECURE ) {
DX_LOG_ERR("Secure key inplace operation "
"is not supported \n");
/* both sides are secure */
rc = -ENOMEM;
goto fail_unmap_din;
}
/* Handle inplace operation */
if ( unlikely(areq_ctx->dma_buf_type == DX_DMA_BUF_MLLI) ) {
areq_ctx->out_nents = 0;
buffer_mgr_set_sg_entry(&sg_data,
areq_ctx->in_nents,
req->src,
req->nbytes,
true);
}
} else {
if ( sg_is_last(req->dst) &&
(sg_page(req->dst) == NULL) &&
sg_dma_address(req->dst)) {
if ( areq_ctx->sec_dir == DX_SRC_DMA_IS_SECURE ) {
DX_LOG_ERR("Secure key in both sides is"
"not supported \n");
/* both sides are secure */
rc = -ENOMEM;
goto fail_unmap_din;
}
/* The dest is secure no mapping is needed */
areq_ctx->sec_dir = DX_DST_DMA_IS_SECURE;
areq_ctx->out_nents = 1;
} else {
/* Map the dst sg */
if ( unlikely( dx_map_sg(dev,req->dst, req->nbytes,
DMA_BIDIRECTIONAL,
&areq_ctx->out_nents,
LLI_MAX_NUM_OF_DATA_ENTRIES,
&dummy))){
rc = -ENOMEM;
goto fail_unmap_din;
}
if ( areq_ctx->out_nents > 1 ) {
areq_ctx->dma_buf_type = DX_DMA_BUF_MLLI;
}
}
if ( unlikely( (areq_ctx->dma_buf_type == DX_DMA_BUF_MLLI) ) ) {
if (areq_ctx->sec_dir != DX_SRC_DMA_IS_SECURE) {
buffer_mgr_set_sg_entry(&sg_data,
areq_ctx->in_nents,
req->src,
req->nbytes,
true);
}
if (areq_ctx->sec_dir != DX_DST_DMA_IS_SECURE) {
buffer_mgr_set_sg_entry(&sg_data,
areq_ctx->out_nents,
req->dst,
req->nbytes,
true);
}
} /*few entries */
} /* !inplace */
if (unlikely(areq_ctx->dma_buf_type == DX_DMA_BUF_MLLI)) {
#if (DX_DEV_SIGNATURE == DX_CC441P_SIG)
if (areq_ctx->sec_dir) {
/* one of the sides is secure, can't use MLLI*/
rc = -EINVAL;
goto fail_unmap_dout;
}
#endif
mlli_params->curr_pool = buff_mgr->mlli_buffs_pool;
if (unlikely(buffer_mgr_build_mlli(dev, &sg_data, mlli_params))) {
rc = -ENOMEM;
goto fail_unmap_dout;
}
} /*MLLI case*/
DX_LOG_DEBUG(" buf type = %s \n",
dx_get_buff_type(areq_ctx->dma_buf_type));
return 0;
fail_unmap_dout:
if (areq_ctx->sec_dir != DX_DST_DMA_IS_SECURE) {
dma_unmap_sg(dev, req->dst,
areq_ctx->out_nents, DMA_BIDIRECTIONAL);
}
fail_unmap_din:
if (areq_ctx->sec_dir != DX_SRC_DMA_IS_SECURE) {
dma_unmap_sg(dev, req->src,
areq_ctx->in_nents, DMA_BIDIRECTIONAL);
}
fail_unmap_iv:
if (areq_ctx->gen_ctx.iv_dma_addr != 0) {
dma_unmap_single(dev, areq_ctx->gen_ctx.iv_dma_addr,
iv_size, DMA_TO_DEVICE);
}
return rc;
}