static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int ring,
struct crypto_async_request *async,
struct scatterlist *src,
struct scatterlist *dst,
unsigned int cryptlen,
struct safexcel_cipher_req *sreq,
bool *should_complete, int *ret)
{
struct safexcel_result_desc *rdesc;
int ndesc = 0;
*ret = 0;
spin_lock_bh(&priv->ring[ring].egress_lock);
do {
rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr);
if (IS_ERR(rdesc)) {
dev_err(priv->dev,
"cipher: result: could not retrieve the result descriptor\n");
*ret = PTR_ERR(rdesc);
break;
}
if (likely(!*ret))
*ret = safexcel_rdesc_check_errors(priv, rdesc);
ndesc++;
} while (!rdesc->last_seg);
safexcel_complete(priv, ring);
spin_unlock_bh(&priv->ring[ring].egress_lock);
if (src == dst) {
dma_unmap_sg(priv->dev, src,
sg_nents_for_len(src, cryptlen),
DMA_BIDIRECTIONAL);
} else {
dma_unmap_sg(priv->dev, src,
sg_nents_for_len(src, cryptlen),
DMA_TO_DEVICE);
dma_unmap_sg(priv->dev, dst,
sg_nents_for_len(dst, cryptlen),
DMA_FROM_DEVICE);
}
*should_complete = true;
return ndesc;
}
static int pkcs1pad_verify_complete(struct akcipher_request *req, int err)
{
struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
struct pkcs1pad_ctx *ctx = akcipher_tfm_ctx(tfm);
struct pkcs1pad_request *req_ctx = akcipher_request_ctx(req);
const struct rsa_asn1_template *digest_info;
unsigned int pos;
if (err == -EOVERFLOW)
/* Decrypted value had no leading 0 byte */
err = -EINVAL;
if (err)
goto done;
if (req_ctx->child_req.dst_len != ctx->key_size - 1) {
err = -EINVAL;
goto done;
}
err = -EBADMSG;
if (req_ctx->out_buf[0] != 0x01)
goto done;
for (pos = 1; pos < req_ctx->child_req.dst_len; pos++)
if (req_ctx->out_buf[pos] != 0xff)
break;
if (pos < 9 || pos == req_ctx->child_req.dst_len ||
req_ctx->out_buf[pos] != 0x00)
goto done;
pos++;
if (ctx->hash_name) {
digest_info = rsa_lookup_asn1(ctx->hash_name);
if (!digest_info)
goto done;
if (memcmp(req_ctx->out_buf + pos, digest_info->data,
digest_info->size))
goto done;
pos += digest_info->size;
}
err = 0;
if (req->dst_len < req_ctx->child_req.dst_len - pos)
err = -EOVERFLOW;
req->dst_len = req_ctx->child_req.dst_len - pos;
if (!err)
sg_copy_from_buffer(req->dst,
sg_nents_for_len(req->dst, req->dst_len),
req_ctx->out_buf + pos, req->dst_len);
done:
kzfree(req_ctx->out_buf);
return err;
}
static int qce_ahash_async_req_handle(struct crypto_async_request *async_req)
{
struct ahash_request *req = ahash_request_cast(async_req);
struct qce_sha_reqctx *rctx = ahash_request_ctx(req);
struct qce_sha_ctx *ctx = crypto_tfm_ctx(async_req->tfm);
struct qce_alg_template *tmpl = to_ahash_tmpl(async_req->tfm);
struct qce_device *qce = tmpl->qce;
unsigned long flags = rctx->flags;
int ret;
if (IS_SHA_HMAC(flags)) {
rctx->authkey = ctx->authkey;
rctx->authklen = QCE_SHA_HMAC_KEY_SIZE;
} else if (IS_CMAC(flags)) {
rctx->authkey = ctx->authkey;
rctx->authklen = AES_KEYSIZE_128;
}
rctx->src_nents = sg_nents_for_len(req->src, req->nbytes);
if (rctx->src_nents < 0) {
dev_err(qce->dev, "Invalid numbers of src SG.\n");
return rctx->src_nents;
}
ret = dma_map_sg(qce->dev, req->src, rctx->src_nents, DMA_TO_DEVICE);
if (ret < 0)
return ret;
sg_init_one(&rctx->result_sg, qce->dma.result_buf, QCE_RESULT_BUF_SZ);
ret = dma_map_sg(qce->dev, &rctx->result_sg, 1, DMA_FROM_DEVICE);
if (ret < 0)
goto error_unmap_src;
ret = qce_dma_prep_sgs(&qce->dma, req->src, rctx->src_nents,
&rctx->result_sg, 1, qce_ahash_done, async_req);
if (ret)
goto error_unmap_dst;
qce_dma_issue_pending(&qce->dma);
ret = qce_start(async_req, tmpl->crypto_alg_type, 0, 0);
if (ret)
goto error_terminate;
return 0;
error_terminate:
qce_dma_terminate_all(&qce->dma);
error_unmap_dst:
dma_unmap_sg(qce->dev, &rctx->result_sg, 1, DMA_FROM_DEVICE);
error_unmap_src:
dma_unmap_sg(qce->dev, req->src, rctx->src_nents, DMA_TO_DEVICE);
return ret;
}
static int pkcs1pad_decrypt_complete(struct akcipher_request *req, int err)
{
struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
struct pkcs1pad_ctx *ctx = akcipher_tfm_ctx(tfm);
struct pkcs1pad_request *req_ctx = akcipher_request_ctx(req);
unsigned int dst_len;
unsigned int pos;
u8 *out_buf;
if (err)
goto done;
err = -EINVAL;
dst_len = req_ctx->child_req.dst_len;
if (dst_len < ctx->key_size - 1)
goto done;
out_buf = req_ctx->out_buf;
if (dst_len == ctx->key_size) {
if (out_buf[0] != 0x00)
/* Decrypted value had no leading 0 byte */
goto done;
dst_len--;
out_buf++;
}
if (out_buf[0] != 0x02)
goto done;
for (pos = 1; pos < dst_len; pos++)
if (out_buf[pos] == 0x00)
break;
if (pos < 9 || pos == dst_len)
goto done;
pos++;
err = 0;
if (req->dst_len < dst_len - pos)
err = -EOVERFLOW;
req->dst_len = dst_len - pos;
if (!err)
sg_copy_from_buffer(req->dst,
sg_nents_for_len(req->dst, req->dst_len),
out_buf + pos, req->dst_len);
done:
kzfree(req_ctx->out_buf);
return err;
}
static int pkcs1pad_encrypt_sign_complete(struct akcipher_request *req, int err)
{
struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
struct pkcs1pad_ctx *ctx = akcipher_tfm_ctx(tfm);
struct pkcs1pad_request *req_ctx = akcipher_request_ctx(req);
size_t pad_len = ctx->key_size - req_ctx->child_req.dst_len;
size_t chunk_len, pad_left;
struct sg_mapping_iter miter;
if (!err) {
if (pad_len) {
sg_miter_start(&miter, req->dst,
sg_nents_for_len(req->dst, pad_len),
SG_MITER_ATOMIC | SG_MITER_TO_SG);
pad_left = pad_len;
while (pad_left) {
sg_miter_next(&miter);
chunk_len = min(miter.length, pad_left);
memset(miter.addr, 0, chunk_len);
pad_left -= chunk_len;
}
sg_miter_stop(&miter);
}
sg_pcopy_from_buffer(req->dst,
sg_nents_for_len(req->dst, ctx->key_size),
req_ctx->out_buf, req_ctx->child_req.dst_len,
pad_len);
}
req->dst_len = ctx->key_size;
kfree(req_ctx->in_buf);
kzfree(req_ctx->out_buf);
return err;
}
static int pkcs1pad_encrypt_sign_complete(struct akcipher_request *req, int err)
{
struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
struct pkcs1pad_ctx *ctx = akcipher_tfm_ctx(tfm);
struct pkcs1pad_request *req_ctx = akcipher_request_ctx(req);
unsigned int pad_len;
unsigned int len;
u8 *out_buf;
if (err)
goto out;
len = req_ctx->child_req.dst_len;
pad_len = ctx->key_size - len;
/* Four billion to one */
if (likely(!pad_len))
goto out;
out_buf = kzalloc(ctx->key_size, GFP_KERNEL);
err = -ENOMEM;
if (!out_buf)
goto out;
sg_copy_to_buffer(req->dst, sg_nents_for_len(req->dst, len),
out_buf + pad_len, len);
sg_copy_from_buffer(req->dst,
sg_nents_for_len(req->dst, ctx->key_size),
out_buf, ctx->key_size);
kzfree(out_buf);
out:
req->dst_len = ctx->key_size;
kfree(req_ctx->in_buf);
return err;
}
static int caam_rsa_count_leading_zeros(struct scatterlist *sgl,
unsigned int nbytes,
unsigned int flags)
{
struct sg_mapping_iter miter;
int lzeros, ents;
unsigned int len;
unsigned int tbytes = nbytes;
const u8 *buff;
ents = sg_nents_for_len(sgl, nbytes);
if (ents < 0)
return ents;
sg_miter_start(&miter, sgl, ents, SG_MITER_FROM_SG | flags);
lzeros = 0;
len = 0;
while (nbytes > 0) {
while (len && !*buff) {
lzeros++;
len--;
buff++;
}
if (len && *buff)
break;
sg_miter_next(&miter);
buff = miter.addr;
len = miter.length;
nbytes -= lzeros;
lzeros = 0;
}
miter.consumed = lzeros;
sg_miter_stop(&miter);
nbytes -= lzeros;
return tbytes - nbytes;
}
static int alloc_src_sglist(struct nitrox_kcrypt_request *nkreq,
struct scatterlist *src, char *iv, int ivsize,
int buflen)
{
int nents = sg_nents_for_len(src, buflen);
int ret;
if (nents < 0)
return nents;
/* IV entry */
nents += 1;
/* Allocate buffer to hold IV and input scatterlist array */
ret = alloc_src_req_buf(nkreq, nents, ivsize);
if (ret)
return ret;
nitrox_creq_copy_iv(nkreq->src, iv, ivsize);
nitrox_creq_set_src_sg(nkreq, nents, ivsize, src, buflen);
return 0;
}
static int mv_cesa_ahash_req_init(struct ahash_request *req, bool *cached)
{
struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);
int ret;
creq->src_nents = sg_nents_for_len(req->src, req->nbytes);
if (creq->src_nents < 0) {
dev_err(cesa_dev->dev, "Invalid number of src SG");
return creq->src_nents;
}
ret = mv_cesa_ahash_cache_req(req, cached);
if (ret)
return ret;
if (*cached)
return 0;
if (cesa_dev->caps->has_tdma)
ret = mv_cesa_ahash_dma_req_init(req);
return ret;
}
static int alloc_dst_sglist(struct nitrox_kcrypt_request *nkreq,
struct scatterlist *dst, int ivsize, int buflen)
{
int nents = sg_nents_for_len(dst, buflen);
int ret;
if (nents < 0)
return nents;
/* IV, ORH, COMPLETION entries */
nents += 3;
/* Allocate buffer to hold ORH, COMPLETION and output scatterlist
* array
*/
ret = alloc_dst_req_buf(nkreq, nents);
if (ret)
return ret;
nitrox_creq_set_orh(nkreq);
nitrox_creq_set_comp(nkreq);
nitrox_creq_set_dst_sg(nkreq, nents, ivsize, dst, buflen);
return 0;
}
static int
__virtio_crypto_ablkcipher_do_req(struct virtio_crypto_request *vc_req,
struct ablkcipher_request *req,
struct data_queue *data_vq,
__u8 op)
{
struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
unsigned int ivsize = crypto_ablkcipher_ivsize(tfm);
struct virtio_crypto_ablkcipher_ctx *ctx = vc_req->ablkcipher_ctx;
struct virtio_crypto *vcrypto = ctx->vcrypto;
struct virtio_crypto_op_data_req *req_data;
int src_nents, dst_nents;
int err;
unsigned long flags;
struct scatterlist outhdr, iv_sg, status_sg, **sgs;
int i;
u64 dst_len;
unsigned int num_out = 0, num_in = 0;
int sg_total;
uint8_t *iv;
src_nents = sg_nents_for_len(req->src, req->nbytes);
dst_nents = sg_nents(req->dst);
pr_debug("virtio_crypto: Number of sgs (src_nents: %d, dst_nents: %d)\n",
src_nents, dst_nents);
/* Why 3? outhdr + iv + inhdr */
sg_total = src_nents + dst_nents + 3;
sgs = kzalloc_node(sg_total * sizeof(*sgs), GFP_ATOMIC,
dev_to_node(&vcrypto->vdev->dev));
if (!sgs)
return -ENOMEM;
req_data = kzalloc_node(sizeof(*req_data), GFP_ATOMIC,
dev_to_node(&vcrypto->vdev->dev));
if (!req_data) {
kfree(sgs);
return -ENOMEM;
}
vc_req->req_data = req_data;
vc_req->type = VIRTIO_CRYPTO_SYM_OP_CIPHER;
/* Head of operation */
if (op) {
req_data->header.session_id =
cpu_to_le64(ctx->enc_sess_info.session_id);
req_data->header.opcode =
cpu_to_le32(VIRTIO_CRYPTO_CIPHER_ENCRYPT);
} else {
req_data->header.session_id =
cpu_to_le64(ctx->dec_sess_info.session_id);
req_data->header.opcode =
cpu_to_le32(VIRTIO_CRYPTO_CIPHER_DECRYPT);
}
req_data->u.sym_req.op_type = cpu_to_le32(VIRTIO_CRYPTO_SYM_OP_CIPHER);
req_data->u.sym_req.u.cipher.para.iv_len = cpu_to_le32(ivsize);
req_data->u.sym_req.u.cipher.para.src_data_len =
cpu_to_le32(req->nbytes);
dst_len = virtio_crypto_alg_sg_nents_length(req->dst);
if (unlikely(dst_len > U32_MAX)) {
pr_err("virtio_crypto: The dst_len is beyond U32_MAX\n");
err = -EINVAL;
goto free;
}
pr_debug("virtio_crypto: src_len: %u, dst_len: %llu\n",
req->nbytes, dst_len);
if (unlikely(req->nbytes + dst_len + ivsize +
sizeof(vc_req->status) > vcrypto->max_size)) {
pr_err("virtio_crypto: The length is too big\n");
err = -EINVAL;
goto free;
}
req_data->u.sym_req.u.cipher.para.dst_data_len =
cpu_to_le32((uint32_t)dst_len);
/* Outhdr */
sg_init_one(&outhdr, req_data, sizeof(*req_data));
sgs[num_out++] = &outhdr;
/* IV */
/*
* Avoid to do DMA from the stack, switch to using
* dynamically-allocated for the IV
*/
iv = kzalloc_node(ivsize, GFP_ATOMIC,
dev_to_node(&vcrypto->vdev->dev));
if (!iv) {
err = -ENOMEM;
goto free;
}
memcpy(iv, req->info, ivsize);
sg_init_one(&iv_sg, iv, ivsize);
sgs[num_out++] = &iv_sg;
vc_req->iv = iv;
/* Source data */
for (i = 0; i < src_nents; i++)
sgs[num_out++] = &req->src[i];
/* Destination data */
for (i = 0; i < dst_nents; i++)
sgs[num_out + num_in++] = &req->dst[i];
/* Status */
sg_init_one(&status_sg, &vc_req->status, sizeof(vc_req->status));
sgs[num_out + num_in++] = &status_sg;
vc_req->sgs = sgs;
spin_lock_irqsave(&data_vq->lock, flags);
err = virtqueue_add_sgs(data_vq->vq, sgs, num_out,
num_in, vc_req, GFP_ATOMIC);
virtqueue_kick(data_vq->vq);
spin_unlock_irqrestore(&data_vq->lock, flags);
if (unlikely(err < 0))
goto free_iv;
return 0;
free_iv:
kzfree(iv);
free:
kzfree(req_data);
kfree(sgs);
return err;
}
static int
qce_ablkcipher_async_req_handle(struct crypto_async_request *async_req)
{
struct ablkcipher_request *req = ablkcipher_request_cast(async_req);
struct qce_cipher_reqctx *rctx = ablkcipher_request_ctx(req);
struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
struct qce_alg_template *tmpl = to_cipher_tmpl(async_req->tfm);
struct qce_device *qce = tmpl->qce;
enum dma_data_direction dir_src, dir_dst;
struct scatterlist *sg;
bool diff_dst;
gfp_t gfp;
int ret;
rctx->iv = req->info;
rctx->ivsize = crypto_ablkcipher_ivsize(ablkcipher);
rctx->cryptlen = req->nbytes;
diff_dst = (req->src != req->dst) ? true : false;
dir_src = diff_dst ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL;
dir_dst = diff_dst ? DMA_FROM_DEVICE : DMA_BIDIRECTIONAL;
rctx->src_nents = sg_nents_for_len(req->src, req->nbytes);
if (diff_dst)
rctx->dst_nents = sg_nents_for_len(req->dst, req->nbytes);
else
rctx->dst_nents = rctx->src_nents;
if (rctx->src_nents < 0) {
dev_err(qce->dev, "Invalid numbers of src SG.\n");
return rctx->src_nents;
}
if (rctx->dst_nents < 0) {
dev_err(qce->dev, "Invalid numbers of dst SG.\n");
return -rctx->dst_nents;
}
rctx->dst_nents += 1;
gfp = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
GFP_KERNEL : GFP_ATOMIC;
ret = sg_alloc_table(&rctx->dst_tbl, rctx->dst_nents, gfp);
if (ret)
return ret;
sg_init_one(&rctx->result_sg, qce->dma.result_buf, QCE_RESULT_BUF_SZ);
sg = qce_sgtable_add(&rctx->dst_tbl, req->dst);
if (IS_ERR(sg)) {
ret = PTR_ERR(sg);
goto error_free;
}
sg = qce_sgtable_add(&rctx->dst_tbl, &rctx->result_sg);
if (IS_ERR(sg)) {
ret = PTR_ERR(sg);
goto error_free;
}
sg_mark_end(sg);
rctx->dst_sg = rctx->dst_tbl.sgl;
ret = dma_map_sg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst);
if (ret < 0)
goto error_free;
if (diff_dst) {
ret = dma_map_sg(qce->dev, req->src, rctx->src_nents, dir_src);
if (ret < 0)
goto error_unmap_dst;
rctx->src_sg = req->src;
} else {
rctx->src_sg = rctx->dst_sg;
}
ret = qce_dma_prep_sgs(&qce->dma, rctx->src_sg, rctx->src_nents,
rctx->dst_sg, rctx->dst_nents,
qce_ablkcipher_done, async_req);
if (ret)
goto error_unmap_src;
qce_dma_issue_pending(&qce->dma);
ret = qce_start(async_req, tmpl->crypto_alg_type, req->nbytes, 0);
if (ret)
goto error_terminate;
return 0;
error_terminate:
qce_dma_terminate_all(&qce->dma);
error_unmap_src:
if (diff_dst)
dma_unmap_sg(qce->dev, req->src, rctx->src_nents, dir_src);
error_unmap_dst:
dma_unmap_sg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst);
error_free:
sg_free_table(&rctx->dst_tbl);
return ret;
}
static int safexcel_aes_send(struct crypto_async_request *base, int ring,
struct safexcel_request *request,
struct safexcel_cipher_req *sreq,
struct scatterlist *src, struct scatterlist *dst,
unsigned int cryptlen, unsigned int assoclen,
unsigned int digestsize, u8 *iv, int *commands,
int *results)
{
struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
struct safexcel_crypto_priv *priv = ctx->priv;
struct safexcel_command_desc *cdesc;
struct safexcel_result_desc *rdesc;
struct scatterlist *sg;
unsigned int totlen = cryptlen + assoclen;
int nr_src, nr_dst, n_cdesc = 0, n_rdesc = 0, queued = totlen;
int i, ret = 0;
if (src == dst) {
nr_src = dma_map_sg(priv->dev, src,
sg_nents_for_len(src, totlen),
DMA_BIDIRECTIONAL);
nr_dst = nr_src;
if (!nr_src)
return -EINVAL;
} else {
nr_src = dma_map_sg(priv->dev, src,
sg_nents_for_len(src, totlen),
DMA_TO_DEVICE);
if (!nr_src)
return -EINVAL;
nr_dst = dma_map_sg(priv->dev, dst,
sg_nents_for_len(dst, totlen),
DMA_FROM_DEVICE);
if (!nr_dst) {
dma_unmap_sg(priv->dev, src,
sg_nents_for_len(src, totlen),
DMA_TO_DEVICE);
return -EINVAL;
}
}
memcpy(ctx->base.ctxr->data, ctx->key, ctx->key_len);
if (ctx->aead) {
memcpy(ctx->base.ctxr->data + ctx->key_len / sizeof(u32),
ctx->ipad, ctx->state_sz);
memcpy(ctx->base.ctxr->data + (ctx->key_len + ctx->state_sz) / sizeof(u32),
ctx->opad, ctx->state_sz);
}
spin_lock_bh(&priv->ring[ring].egress_lock);
/* command descriptors */
for_each_sg(src, sg, nr_src, i) {
int len = sg_dma_len(sg);
/* Do not overflow the request */
if (queued - len < 0)
len = queued;
cdesc = safexcel_add_cdesc(priv, ring, !n_cdesc, !(queued - len),
sg_dma_address(sg), len, totlen,
ctx->base.ctxr_dma);
if (IS_ERR(cdesc)) {
/* No space left in the command descriptor ring */
ret = PTR_ERR(cdesc);
goto cdesc_rollback;
}
n_cdesc++;
if (n_cdesc == 1) {
safexcel_context_control(ctx, base, sreq, cdesc);
if (ctx->aead)
safexcel_aead_token(ctx, iv, cdesc,
sreq->direction, cryptlen,
assoclen, digestsize);
else
safexcel_skcipher_token(ctx, iv, cdesc,
cryptlen);
}
queued -= len;
if (!queued)
break;
}
static int pkcs1pad_verify_complete(struct akcipher_request *req, int err)
{
struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
struct pkcs1pad_ctx *ctx = akcipher_tfm_ctx(tfm);
struct pkcs1pad_request *req_ctx = akcipher_request_ctx(req);
struct akcipher_instance *inst = akcipher_alg_instance(tfm);
struct pkcs1pad_inst_ctx *ictx = akcipher_instance_ctx(inst);
const struct rsa_asn1_template *digest_info = ictx->digest_info;
unsigned int dst_len;
unsigned int pos;
u8 *out_buf;
if (err)
goto done;
err = -EINVAL;
dst_len = req_ctx->child_req.dst_len;
if (dst_len < ctx->key_size - 1)
goto done;
out_buf = req_ctx->out_buf;
if (dst_len == ctx->key_size) {
if (out_buf[0] != 0x00)
/* Decrypted value had no leading 0 byte */
goto done;
dst_len--;
out_buf++;
}
err = -EBADMSG;
if (out_buf[0] != 0x01)
goto done;
for (pos = 1; pos < dst_len; pos++)
if (out_buf[pos] != 0xff)
break;
if (pos < 9 || pos == dst_len || out_buf[pos] != 0x00)
goto done;
pos++;
if (digest_info) {
if (crypto_memneq(out_buf + pos, digest_info->data,
digest_info->size))
goto done;
pos += digest_info->size;
}
err = 0;
if (req->dst_len < dst_len - pos)
err = -EOVERFLOW;
req->dst_len = dst_len - pos;
if (!err)
sg_copy_from_buffer(req->dst,
sg_nents_for_len(req->dst, req->dst_len),
out_buf + pos, req->dst_len);
done:
kzfree(req_ctx->out_buf);
return err;
}
static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req,
size_t desclen)
{
struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm);
struct device *dev = ctx->dev;
struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req);
struct rsa_edesc *edesc;
gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
GFP_KERNEL : GFP_ATOMIC;
int sg_flags = (flags == GFP_ATOMIC) ? SG_MITER_ATOMIC : 0;
int sgc;
int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes;
int src_nents, dst_nents;
int lzeros;
lzeros = caam_rsa_count_leading_zeros(req->src, req->src_len, sg_flags);
if (lzeros < 0)
return ERR_PTR(lzeros);
req->src_len -= lzeros;
req->src = scatterwalk_ffwd(req_ctx->src, req->src, lzeros);
src_nents = sg_nents_for_len(req->src, req->src_len);
dst_nents = sg_nents_for_len(req->dst, req->dst_len);
if (src_nents > 1)
sec4_sg_len = src_nents;
if (dst_nents > 1)
sec4_sg_len += dst_nents;
sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry);
/* allocate space for base edesc, hw desc commands and link tables */
edesc = kzalloc(sizeof(*edesc) + desclen + sec4_sg_bytes,
GFP_DMA | flags);
if (!edesc)
return ERR_PTR(-ENOMEM);
sgc = dma_map_sg(dev, req->src, src_nents, DMA_TO_DEVICE);
if (unlikely(!sgc)) {
dev_err(dev, "unable to map source\n");
goto src_fail;
}
sgc = dma_map_sg(dev, req->dst, dst_nents, DMA_FROM_DEVICE);
if (unlikely(!sgc)) {
dev_err(dev, "unable to map destination\n");
goto dst_fail;
}
edesc->sec4_sg = (void *)edesc + sizeof(*edesc) + desclen;
sec4_sg_index = 0;
if (src_nents > 1) {
sg_to_sec4_sg_last(req->src, src_nents, edesc->sec4_sg, 0);
sec4_sg_index += src_nents;
}
if (dst_nents > 1)
sg_to_sec4_sg_last(req->dst, dst_nents,
edesc->sec4_sg + sec4_sg_index, 0);
/* Save nents for later use in Job Descriptor */
edesc->src_nents = src_nents;
edesc->dst_nents = dst_nents;
if (!sec4_sg_bytes)
return edesc;
edesc->sec4_sg_dma = dma_map_single(dev, edesc->sec4_sg,
sec4_sg_bytes, DMA_TO_DEVICE);
if (dma_mapping_error(dev, edesc->sec4_sg_dma)) {
dev_err(dev, "unable to map S/G table\n");
goto sec4_sg_fail;
}
edesc->sec4_sg_bytes = sec4_sg_bytes;
return edesc;
sec4_sg_fail:
dma_unmap_sg(dev, req->dst, dst_nents, DMA_FROM_DEVICE);
dst_fail:
dma_unmap_sg(dev, req->src, src_nents, DMA_TO_DEVICE);
src_fail:
kfree(edesc);
return ERR_PTR(-ENOMEM);
}
static int ecdh_compute_value(struct kpp_request *req)
{
struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
struct ecdh_ctx *ctx = ecdh_get_ctx(tfm);
u64 *public_key;
u64 *shared_secret = NULL;
void *buf;
size_t copied, nbytes, public_key_sz;
int ret = -ENOMEM;
nbytes = ctx->ndigits << ECC_DIGITS_TO_BYTES_SHIFT;
/* Public part is a point thus it has both coordinates */
public_key_sz = 2 * nbytes;
public_key = kmalloc(public_key_sz, GFP_KERNEL);
if (!public_key)
return -ENOMEM;
if (req->src) {
shared_secret = kmalloc(nbytes, GFP_KERNEL);
if (!shared_secret)
goto free_pubkey;
/* from here on it's invalid parameters */
ret = -EINVAL;
/* must have exactly two points to be on the curve */
if (public_key_sz != req->src_len)
goto free_all;
copied = sg_copy_to_buffer(req->src,
sg_nents_for_len(req->src,
public_key_sz),
public_key, public_key_sz);
if (copied != public_key_sz)
goto free_all;
ret = crypto_ecdh_shared_secret(ctx->curve_id, ctx->ndigits,
ctx->private_key, public_key,
shared_secret);
buf = shared_secret;
} else {
ret = ecc_make_pub_key(ctx->curve_id, ctx->ndigits,
ctx->private_key, public_key);
buf = public_key;
nbytes = public_key_sz;
}
if (ret < 0)
goto free_all;
/* might want less than we've got */
nbytes = min_t(size_t, nbytes, req->dst_len);
copied = sg_copy_from_buffer(req->dst, sg_nents_for_len(req->dst,
nbytes),
buf, nbytes);
if (copied != nbytes)
ret = -EINVAL;
/* fall through */
free_all:
kzfree(shared_secret);
free_pubkey:
kfree(public_key);
return ret;
}
static int omap_aes_gcm_copy_buffers(struct omap_aes_dev *dd,
struct aead_request *req)
{
int alen, clen, cryptlen, assoclen, ret;
struct crypto_aead *aead = crypto_aead_reqtfm(req);
unsigned int authlen = crypto_aead_authsize(aead);
struct scatterlist *tmp, sg_arr[2];
int nsg;
u16 flags;
assoclen = req->assoclen;
cryptlen = req->cryptlen;
if (dd->flags & FLAGS_RFC4106_GCM)
assoclen -= 8;
if (!(dd->flags & FLAGS_ENCRYPT))
cryptlen -= authlen;
alen = ALIGN(assoclen, AES_BLOCK_SIZE);
clen = ALIGN(cryptlen, AES_BLOCK_SIZE);
nsg = !!(assoclen && cryptlen);
omap_aes_clear_copy_flags(dd);
sg_init_table(dd->in_sgl, nsg + 1);
if (assoclen) {
tmp = req->src;
ret = omap_crypto_align_sg(&tmp, assoclen,
AES_BLOCK_SIZE, dd->in_sgl,
OMAP_CRYPTO_COPY_DATA |
OMAP_CRYPTO_ZERO_BUF |
OMAP_CRYPTO_FORCE_SINGLE_ENTRY,
FLAGS_ASSOC_DATA_ST_SHIFT,
&dd->flags);
}
if (cryptlen) {
tmp = scatterwalk_ffwd(sg_arr, req->src, req->assoclen);
ret = omap_crypto_align_sg(&tmp, cryptlen,
AES_BLOCK_SIZE, &dd->in_sgl[nsg],
OMAP_CRYPTO_COPY_DATA |
OMAP_CRYPTO_ZERO_BUF |
OMAP_CRYPTO_FORCE_SINGLE_ENTRY,
FLAGS_IN_DATA_ST_SHIFT,
&dd->flags);
}
dd->in_sg = dd->in_sgl;
dd->total = cryptlen;
dd->assoc_len = assoclen;
dd->authsize = authlen;
dd->out_sg = req->dst;
dd->orig_out = req->dst;
dd->out_sg = scatterwalk_ffwd(sg_arr, req->dst, assoclen);
flags = 0;
if (req->src == req->dst || dd->out_sg == sg_arr)
flags |= OMAP_CRYPTO_FORCE_COPY;
ret = omap_crypto_align_sg(&dd->out_sg, cryptlen,
AES_BLOCK_SIZE, &dd->out_sgl,
flags,
FLAGS_OUT_DATA_ST_SHIFT, &dd->flags);
if (ret)
return ret;
dd->in_sg_len = sg_nents_for_len(dd->in_sg, alen + clen);
dd->out_sg_len = sg_nents_for_len(dd->out_sg, clen);
return 0;
}
