static int async_chainiv_postpone_request(struct skcipher_givcrypt_request *req)
{
struct crypto_ablkcipher *geniv = skcipher_givcrypt_reqtfm(req);
struct async_chainiv_ctx *ctx = crypto_ablkcipher_ctx(geniv);
int err;
spin_lock_bh(&ctx->lock);
err = skcipher_enqueue_givcrypt(&ctx->queue, req);
spin_unlock_bh(&ctx->lock);
if (test_and_set_bit(CHAINIV_STATE_INUSE, &ctx->state))
return err;
ctx->err = err;
return async_chainiv_schedule_work(ctx);
}
static int s5p_aes_crypt(struct ablkcipher_request *req, unsigned long mode)
{
struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
struct s5p_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
struct s5p_aes_reqctx *reqctx = ablkcipher_request_ctx(req);
struct s5p_aes_dev *dev = ctx->dev;
if (!IS_ALIGNED(req->nbytes, AES_BLOCK_SIZE)) {
pr_err("request size is not exact amount of AES blocks\n");
return -EINVAL;
}
reqctx->mode = mode;
return s5p_aes_handle_req(dev, req);
}
static int eseqiv_givencrypt_first(struct skcipher_givcrypt_request *req)
{
struct crypto_ablkcipher *geniv = skcipher_givcrypt_reqtfm(req);
struct eseqiv_ctx *ctx = crypto_ablkcipher_ctx(geniv);
spin_lock_bh(&ctx->lock);
if (crypto_ablkcipher_crt(geniv)->givencrypt != eseqiv_givencrypt_first)
goto unlock;
crypto_ablkcipher_crt(geniv)->givencrypt = eseqiv_givencrypt;
get_random_bytes(ctx->salt, crypto_ablkcipher_ivsize(geniv));
unlock:
spin_unlock_bh(&ctx->lock);
return eseqiv_givencrypt(req);
}
static int sahara_aes_cbc_decrypt(struct ablkcipher_request *req)
{
struct crypto_tfm *tfm =
crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req));
struct sahara_ctx *ctx = crypto_ablkcipher_ctx(
crypto_ablkcipher_reqtfm(req));
int err;
if (unlikely(ctx->keylen != AES_KEYSIZE_128)) {
ablkcipher_request_set_tfm(req, ctx->fallback);
err = crypto_ablkcipher_decrypt(req);
ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm));
return err;
}
return sahara_aes_crypt(req, FLAGS_CBC);
}
static int dcp_aes_cbc_encrypt(struct ablkcipher_request *req)
{
struct crypto_tfm *tfm =
crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req));
struct dcp_op *ctx = crypto_ablkcipher_ctx(
crypto_ablkcipher_reqtfm(req));
if (unlikely(ctx->keylen != AES_KEYSIZE_128)) {
int err = 0;
ablkcipher_request_set_tfm(req, ctx->fallback);
err = crypto_ablkcipher_encrypt(req);
ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm));
return err;
}
return dcp_aes_cbc_crypt(req, DCP_AES | DCP_ENC | DCP_CBC);
}
int ablk_encrypt(struct ablkcipher_request *req)
{
struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
struct async_helper_ctx *ctx = crypto_ablkcipher_ctx(tfm);
if (!may_use_simd()) {
struct ablkcipher_request *cryptd_req =
ablkcipher_request_ctx(req);
memcpy(cryptd_req, req, sizeof(*req));
ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
return crypto_ablkcipher_encrypt(cryptd_req);
} else {
return __ablk_encrypt(req);
}
}
static int seqiv_givencrypt(struct skcipher_givcrypt_request *req)
{
struct crypto_ablkcipher *geniv = skcipher_givcrypt_reqtfm(req);
struct seqiv_ctx *ctx = crypto_ablkcipher_ctx(geniv);
struct ablkcipher_request *subreq = skcipher_givcrypt_reqctx(req);
crypto_completion_t compl;
void *data;
u8 *info;
unsigned int ivsize;
int err;
ablkcipher_request_set_tfm(subreq, skcipher_geniv_cipher(geniv));
compl = req->creq.base.complete;
data = req->creq.base.data;
info = req->creq.info;
ivsize = crypto_ablkcipher_ivsize(geniv);
if (unlikely(!IS_ALIGNED((unsigned long)info,
crypto_ablkcipher_alignmask(geniv) + 1))) {
info = kmalloc(ivsize, req->creq.base.flags &
CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL:
GFP_ATOMIC);
if (!info)
return -ENOMEM;
compl = seqiv_complete;
data = req;
}
ablkcipher_request_set_callback(subreq, req->creq.base.flags, compl,
data);
ablkcipher_request_set_crypt(subreq, req->creq.src, req->creq.dst,
req->creq.nbytes, info);
seqiv_geniv(ctx, info, req->seq, ivsize);
memcpy(req->giv, info, ivsize);
err = crypto_ablkcipher_encrypt(subreq);
if (unlikely(info != req->creq.info))
seqiv_complete2(req, err);
return err;
}
static int async_chainiv_givencrypt_tail(struct skcipher_givcrypt_request *req)
{
struct crypto_ablkcipher *geniv = skcipher_givcrypt_reqtfm(req);
struct async_chainiv_ctx *ctx = crypto_ablkcipher_ctx(geniv);
struct ablkcipher_request *subreq = skcipher_givcrypt_reqctx(req);
unsigned int ivsize = crypto_ablkcipher_ivsize(geniv);
memcpy(req->giv, ctx->iv, ivsize);
memcpy(subreq->info, ctx->iv, ivsize);
ctx->err = crypto_ablkcipher_encrypt(subreq);
if (ctx->err)
goto out;
memcpy(ctx->iv, subreq->info, ivsize);
out:
return async_chainiv_schedule_work(ctx);
}
static int aes_setkey(struct crypto_ablkcipher *tfm, const u8 *in_key,
unsigned int keylen)
{
struct aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
unsigned long *flags = (unsigned long *) &tfm->base.crt_flags;
DPRINTF(2, "set_key in %s\n", __FILE__);
if (keylen != 16 && keylen != 24 && keylen != 32) {
*flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
return -EINVAL;
}
ctx->key_length = keylen;
DPRINTF(0, "ctx @%p, keylen %d, ctx->key_length %d\n", ctx, keylen, ctx->key_length);
memcpy ((u8 *) (ctx->buf), in_key, keylen);
return 0;
}
static int virtio_crypto_ablkcipher_encrypt(struct ablkcipher_request *req)
{
struct crypto_ablkcipher *atfm = crypto_ablkcipher_reqtfm(req);
struct virtio_crypto_ablkcipher_ctx *ctx = crypto_ablkcipher_ctx(atfm);
struct virtio_crypto_request *vc_req = ablkcipher_request_ctx(req);
struct virtio_crypto *vcrypto = ctx->vcrypto;
int ret;
/* Use the first data virtqueue as default */
struct data_queue *data_vq = &vcrypto->data_vq[0];
vc_req->ablkcipher_ctx = ctx;
vc_req->ablkcipher_req = req;
ret = __virtio_crypto_ablkcipher_do_req(vc_req, req, data_vq, 1);
if (ret < 0) {
pr_err("virtio_crypto: Encryption failed!\n");
return ret;
}
return -EINPROGRESS;
}
static int rfc3686_aes_encrypt(struct ablkcipher_request *areq)
{
struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
struct aes_ctx *ctx = crypto_ablkcipher_ctx(cipher);
int ret;
u8 *info = areq->info;
u8 rfc3686_iv[16];
memcpy(rfc3686_iv, ctx->nonce, CTR_RFC3686_NONCE_SIZE);
memcpy(rfc3686_iv + CTR_RFC3686_NONCE_SIZE, info, CTR_RFC3686_IV_SIZE);
/* initialize counter portion of counter block */
*(__be32 *)(rfc3686_iv + CTR_RFC3686_NONCE_SIZE + CTR_RFC3686_IV_SIZE) =
cpu_to_be32(1);
areq->info = rfc3686_iv;
ret = lq_aes_queue_mgr(ctx, areq, areq->info, CRYPTO_DIR_ENCRYPT, 4);
areq->info = info;
return ret;
}
static int ablk_decrypt(struct ablkcipher_request *req)
{
struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
if (!irq_fpu_usable()) {
struct ablkcipher_request *cryptd_req =
ablkcipher_request_ctx(req);
memcpy(cryptd_req, req, sizeof(*req));
ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
return crypto_ablkcipher_decrypt(cryptd_req);
} else {
struct blkcipher_desc desc;
desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
desc.info = req->info;
desc.flags = 0;
return crypto_blkcipher_crt(desc.tfm)->decrypt(
&desc, req->dst, req->src, req->nbytes);
}
}
static void dcp_queue_task(unsigned long data)
{
struct dcp_dev *dev = (struct dcp_dev *) data;
struct crypto_async_request *async_req, *backlog;
struct crypto_ablkcipher *tfm;
struct dcp_op *ctx;
struct dcp_dev_req_ctx *rctx;
struct ablkcipher_request *req;
unsigned long flags;
spin_lock_irqsave(&dev->queue_lock, flags);
backlog = crypto_get_backlog(&dev->queue);
async_req = crypto_dequeue_request(&dev->queue);
spin_unlock_irqrestore(&dev->queue_lock, flags);
if (!async_req)
goto ret_nothing_done;
if (backlog)
backlog->complete(backlog, -EINPROGRESS);
req = ablkcipher_request_cast(async_req);
tfm = crypto_ablkcipher_reqtfm(req);
rctx = ablkcipher_request_ctx(req);
ctx = crypto_ablkcipher_ctx(tfm);
if (!req->src || !req->dst)
goto ret_nothing_done;
ctx->flags |= rctx->mode;
ctx->req = req;
dcp_crypt(dev, ctx);
return;
ret_nothing_done:
clear_bit(DCP_FLAG_BUSY, &dev->flags);
}
static int async_chainiv_givencrypt_first(struct skcipher_givcrypt_request *req)
{
struct crypto_ablkcipher *geniv = skcipher_givcrypt_reqtfm(req);
struct async_chainiv_ctx *ctx = crypto_ablkcipher_ctx(geniv);
if (test_and_set_bit(CHAINIV_STATE_INUSE, &ctx->state))
goto out;
if (crypto_ablkcipher_crt(geniv)->givencrypt !=
async_chainiv_givencrypt_first)
goto unlock;
crypto_ablkcipher_crt(geniv)->givencrypt = async_chainiv_givencrypt;
get_random_bytes(ctx->iv, crypto_ablkcipher_ivsize(geniv));
unlock:
clear_bit(CHAINIV_STATE_INUSE, &ctx->state);
out:
return async_chainiv_givencrypt(req);
}
static int crypto_rfc3686_crypt(struct ablkcipher_request *req)
{
struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
struct crypto_rfc3686_ctx *ctx = crypto_ablkcipher_ctx(tfm);
struct crypto_ablkcipher *child = ctx->child;
unsigned long align = crypto_ablkcipher_alignmask(tfm);
struct crypto_rfc3686_req_ctx *rctx =
(void *)PTR_ALIGN((u8 *)ablkcipher_request_ctx(req), align + 1);
struct ablkcipher_request *subreq = &rctx->subreq;
u8 *iv = rctx->iv;
/* set up counter block */
memcpy(iv, ctx->nonce, CTR_RFC3686_NONCE_SIZE);
memcpy(iv + CTR_RFC3686_NONCE_SIZE, req->info, CTR_RFC3686_IV_SIZE);
/* initialize counter portion of counter block */
*(__be32 *)(iv + CTR_RFC3686_NONCE_SIZE + CTR_RFC3686_IV_SIZE) =
cpu_to_be32(1);
ablkcipher_request_set_tfm(subreq, child);
ablkcipher_re
static int seqiv_givencrypt_first(struct skcipher_givcrypt_request *req)
{
struct crypto_ablkcipher *geniv = skcipher_givcrypt_reqtfm(req);
struct seqiv_ctx *ctx = crypto_ablkcipher_ctx(geniv);
int err = 0;
spin_lock_bh(&ctx->lock);
if (crypto_ablkcipher_crt(geniv)->givencrypt != seqiv_givencrypt_first)
goto unlock;
crypto_ablkcipher_crt(geniv)->givencrypt = seqiv_givencrypt;
err = crypto_rng_get_bytes(crypto_default_rng, ctx->salt,
crypto_ablkcipher_ivsize(geniv));
unlock:
spin_unlock_bh(&ctx->lock);
if (err)
return err;
return seqiv_givencrypt(req);
}
static int eseqiv_init(struct crypto_tfm *tfm)
{
struct crypto_ablkcipher *geniv = __crypto_ablkcipher_cast(tfm);
struct eseqiv_ctx *ctx = crypto_ablkcipher_ctx(geniv);
unsigned long alignmask;
unsigned int reqsize;
#ifndef CONFIG_CRYPTO_DRBG
spin_lock_init(&ctx->lock);
#endif
alignmask = crypto_tfm_ctx_alignment() - 1;
reqsize = sizeof(struct eseqiv_request_ctx);
if (alignmask & reqsize) {
alignmask &= reqsize;
alignmask--;
}
alignmask = ~alignmask;
alignmask &= crypto_ablkcipher_alignmask(geniv);
reqsize += alignmask;
reqsize += crypto_ablkcipher_ivsize(geniv);
reqsize = ALIGN(reqsize, crypto_tfm_ctx_alignment());
ctx->reqoff = reqsize - sizeof(struct eseqiv_request_ctx);
tfm->crt_ablkcipher.reqsize = reqsize +
sizeof(struct ablkcipher_request);
#ifdef CONFIG_CRYPTO_DRBG
crypto_rng_get_bytes(crypto_default_rng, ctx->salt,
crypto_ablkcipher_ivsize(geniv));
#endif
return skcipher_geniv_init(tfm);
}
static int rfc3686_aes_setkey(struct crypto_ablkcipher *tfm,
const u8 *in_key, unsigned int keylen)
{
struct aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
unsigned long *flags = (unsigned long *)&tfm->base.crt_flags;
DPRINTF(2, "ctr_rfc3686_aes_set_key in %s\n", __FILE__);
memcpy(ctx->nonce, in_key + (keylen - CTR_RFC3686_NONCE_SIZE),
CTR_RFC3686_NONCE_SIZE);
keylen -= CTR_RFC3686_NONCE_SIZE; // remove 4 bytes of nonce
if (keylen != 16 && keylen != 24 && keylen != 32) {
*flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
return -EINVAL;
}
ctx->key_length = keylen;
memcpy ((u8 *) (ctx->buf), in_key, keylen);
return 0;
}
static int sahara_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
unsigned int keylen)
{
struct sahara_ctx *ctx = crypto_ablkcipher_ctx(tfm);
int ret;
ctx->keylen = keylen;
/* SAHARA only supports 128bit keys */
if (keylen == AES_KEYSIZE_128) {
memcpy(ctx->key, key, keylen);
ctx->flags |= FLAGS_NEW_KEY;
return 0;
}
if (keylen != AES_KEYSIZE_128 &&
keylen != AES_KEYSIZE_192 && keylen != AES_KEYSIZE_256)
return -EINVAL;
/*
* The requested key size is not supported by HW, do a fallback.
*/
ctx->fallback->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
ctx->fallback->base.crt_flags |=
(tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK);
ret = crypto_ablkcipher_setkey(ctx->fallback, key, keylen);
if (ret) {
struct crypto_tfm *tfm_aux = crypto_ablkcipher_tfm(tfm);
tfm_aux->crt_flags &= ~CRYPTO_TFM_RES_MASK;
tfm_aux->crt_flags |=
(ctx->fallback->base.crt_flags & CRYPTO_TFM_RES_MASK);
}
return ret;
}
int ablk_decrypt(struct ablkcipher_request *req)
{
struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
struct async_helper_ctx *ctx = crypto_ablkcipher_ctx(tfm);
if (!may_use_simd()) {
struct ablkcipher_request *cryptd_req =
ablkcipher_request_ctx(req);
*cryptd_req = *req;
ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
return crypto_ablkcipher_decrypt(cryptd_req);
} else {
struct blkcipher_desc desc;
desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
desc.info = req->info;
desc.flags = 0;
return crypto_blkcipher_crt(desc.tfm)->decrypt(
&desc, req->dst, req->src, req->nbytes);
}
}
static int ss_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
unsigned int keylen)
{
int ret = 0;
ss_aes_ctx_t *ctx = crypto_ablkcipher_ctx(tfm);
SS_DBG("keylen = %d\n", keylen);
if (ctx->comm.flags & SS_FLAG_NEW_KEY) {
SS_ERR("The key has already update.\n");
return -EBUSY;
}
ret = ss_aes_key_valid(tfm, keylen);
if (ret != 0)
return ret;
ctx->key_size = keylen;
memcpy(ctx->key, key, keylen);
if (keylen < AES_KEYSIZE_256)
memset(&ctx->key[keylen], 0, AES_KEYSIZE_256 - keylen);
ctx->comm.flags |= SS_FLAG_NEW_KEY;
return 0;
}
static int process_next_packet(struct aes_container *aes_con, struct ablkcipher_request *areq,
int state)
{
u8 *iv;
int mode, dir, err = -EINVAL;
unsigned long queue_flag;
u32 inc, nbytes, remain, chunk_size;
struct scatterlist *src = NULL;
struct scatterlist *dst = NULL;
struct crypto_ablkcipher *cipher;
struct aes_ctx *ctx;
spin_lock_irqsave(&aes_queue->lock, queue_flag);
dir = aes_con->encdec;
mode = aes_con->mode;
iv = aes_con->iv;
if (state & PROCESS_SCATTER) {
src = scatterwalk_sg_next(areq->src);
dst = scatterwalk_sg_next(areq->dst);
if (!src || !dst) {
spin_unlock_irqrestore(&aes_queue->lock, queue_flag);
return 1;
}
}
else if (state & PROCESS_NEW_PACKET) {
src = areq->src;
dst = areq->dst;
}
remain = aes_con->bytes_processed;
chunk_size = src->length;
if (remain > DEU_MAX_PACKET_SIZE)
inc = DEU_MAX_PACKET_SIZE;
else if (remain > chunk_size)
inc = chunk_size;
else
inc = remain;
remain -= inc;
aes_con->nbytes = inc;
if (state & PROCESS_SCATTER) {
aes_con->src_buf += aes_con->nbytes;
aes_con->dst_buf += aes_con->nbytes;
}
lq_sg_init(aes_con, src, dst);
nbytes = aes_con->nbytes;
//printk("debug - Line: %d, func: %s, reqsize: %d, scattersize: %d\n",
// __LINE__, __func__, nbytes, chunk_size);
cipher = crypto_ablkcipher_reqtfm(areq);
ctx = crypto_ablkcipher_ctx(cipher);
if (aes_queue->hw_status == AES_IDLE)
aes_queue->hw_status = AES_STARTED;
aes_con->bytes_processed -= aes_con->nbytes;
err = ablkcipher_enqueue_request(&aes_queue->list, &aes_con->arequest);
if (err == -EBUSY) {
spin_unlock_irqrestore(&aes_queue->lock, queue_flag);
printk("Failed to enqueue request, ln: %d, err: %d\n",
__LINE__, err);
return -EINVAL;
}
spin_unlock_irqrestore(&aes_queue->lock, queue_flag);
err = lq_deu_aes_core(ctx, aes_con->dst_buf, aes_con->src_buf, iv, nbytes, dir, mode);
return err;
}
static int ss_aes_one_req(sunxi_ss_t *sss, struct ablkcipher_request *req)
{
int ret = 0;
struct crypto_ablkcipher *tfm = NULL;
ss_aes_ctx_t *ctx = NULL;
ss_aes_req_ctx_t *req_ctx = NULL;
int key_map_flag = 0;
int iv_map_flag = 0;
SS_ENTER();
if (!req->src || !req->dst) {
SS_ERR("Invalid sg: src = %p, dst = %p\n", req->src, req->dst);
return -EINVAL;
}
ss_dev_lock();
tfm = crypto_ablkcipher_reqtfm(req);
req_ctx = ablkcipher_request_ctx(req);
ctx = crypto_ablkcipher_ctx(tfm);
/* A31 SS need update key each cycle in decryption. */
if ((ctx->comm.flags & SS_FLAG_NEW_KEY) || (req_ctx->dir == SS_DIR_DECRYPT)) {
SS_DBG("KEY address = %p, size = %d\n", ctx->key, ctx->key_size);
ss_key_set(ctx->key, ctx->key_size);
dma_map_single(&sss->pdev->dev, ctx->key, ctx->key_size, DMA_MEM_TO_DEV);
key_map_flag = 1;
ctx->comm.flags &= ~SS_FLAG_NEW_KEY;
}
#ifdef SS_CTS_MODE_ENABLE
if (((req_ctx->mode == SS_AES_MODE_CBC)
|| (req_ctx->mode == SS_AES_MODE_CTS)) && (req->info != NULL)) {
#else
if ((req_ctx->mode == SS_AES_MODE_CBC) && (req->info != NULL)) {
#endif
SS_DBG("IV address = %p, size = %d\n", req->info, crypto_ablkcipher_ivsize(tfm));
memcpy(ctx->iv, req->info, crypto_ablkcipher_ivsize(tfm));
ss_iv_set(ctx->iv, crypto_ablkcipher_ivsize(tfm));
dma_map_single(&sss->pdev->dev, ctx->iv, crypto_ablkcipher_ivsize(tfm), DMA_MEM_TO_DEV);
iv_map_flag = 1;
}
#ifdef SS_CTR_MODE_ENABLE
if (req_ctx->mode == SS_AES_MODE_CTR) {
SS_DBG("Cnt address = %p, size = %d\n", req->info, crypto_ablkcipher_ivsize(tfm));
if (ctx->cnt == 0)
memcpy(ctx->iv, req->info, crypto_ablkcipher_ivsize(tfm));
SS_DBG("CNT: %08x %08x %08x %08x \n", *(int *)&ctx->iv[0],
*(int *)&ctx->iv[4], *(int *)&ctx->iv[8], *(int *)&ctx->iv[12]);
ss_cnt_set(ctx->iv, crypto_ablkcipher_ivsize(tfm));
dma_map_single(&sss->pdev->dev, ctx->iv, crypto_ablkcipher_ivsize(tfm), DMA_MEM_TO_DEV);
iv_map_flag = 1;
}
#endif
if (req_ctx->type == SS_METHOD_RSA)
ss_rsa_width_set(crypto_ablkcipher_ivsize(tfm));
req_ctx->dma_src.sg = req->src;
req_ctx->dma_dst.sg = req->dst;
ret = ss_aes_start(ctx, req_ctx, req->nbytes);
if (ret < 0)
SS_ERR("ss_aes_start fail(%d)\n", ret);
ss_dev_unlock();
if (req->base.complete)
req->base.complete(&req->base, ret);
if (key_map_flag == 1)
dma_unmap_single(&ss_dev->pdev->dev, virt_to_phys(ctx->key), ctx->key_size, DMA_MEM_TO_DEV);
if (iv_map_flag == 1)
dma_unmap_single(&sss->pdev->dev, virt_to_phys(ctx->iv), crypto_ablkcipher_ivsize(tfm), DMA_MEM_TO_DEV);
#ifdef SS_CTR_MODE_ENABLE
if (req_ctx->mode == SS_AES_MODE_CTR) {
ss_cnt_get(ctx->comm.flow, ctx->iv, crypto_ablkcipher_ivsize(tfm));
SS_DBG("CNT: %08x %08x %08x %08x \n", *(int *)&ctx->iv[0],
*(int *)&ctx->iv[4], *(int *)&ctx->iv[8], *(int *)&ctx->iv[12]);
}
#endif
ctx->cnt += req->nbytes;
return ret;
}
static int ss_hash_one_req(sunxi_ss_t *sss, struct ahash_request *req)
{
int ret = 0;
ss_aes_req_ctx_t *req_ctx = NULL;
ss_hash_ctx_t *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
SS_ENTER();
if (!req->src) {
SS_ERR("Invalid sg: src = %p\n", req->src);
return -EINVAL;
}
ss_dev_lock();
req_ctx = ahash_request_ctx(req);
req_ctx->dma_src.sg = req->src;
ss_hash_padding_data_prepare(ctx, req->result, req->nbytes);
ret = ss_hash_start(ctx, req_ctx, req->nbytes);
if (ret < 0)
SS_ERR("ss_hash_start fail(%d)\n", ret);
ss_dev_unlock();
if (req->base.complete)
req->base.complete(&req->base, ret);
return ret;
}
static int eseqiv_givencrypt(struct skcipher_givcrypt_request *req)
{
struct crypto_ablkcipher *geniv = skcipher_givcrypt_reqtfm(req);
struct eseqiv_ctx *ctx = crypto_ablkcipher_ctx(geniv);
struct eseqiv_request_ctx *reqctx = skcipher_givcrypt_reqctx(req);
struct ablkcipher_request *subreq;
crypto_completion_t complete;
void *data;
struct scatterlist *osrc, *odst;
struct scatterlist *dst;
struct page *srcp;
struct page *dstp;
u8 *giv;
u8 *vsrc;
u8 *vdst;
__be64 seq;
unsigned int ivsize;
unsigned int len;
int err;
subreq = (void *)(reqctx->tail + ctx->reqoff);
ablkcipher_request_set_tfm(subreq, skcipher_geniv_cipher(geniv));
giv = req->giv;
complete = req->creq.base.complete;
data = req->creq.base.data;
osrc = req->creq.src;
odst = req->creq.dst;
srcp = sg_page(osrc);
dstp = sg_page(odst);
vsrc = PageHighMem(srcp) ? NULL : page_address(srcp) + osrc->offset;
vdst = PageHighMem(dstp) ? NULL : page_address(dstp) + odst->offset;
ivsize = crypto_ablkcipher_ivsize(geniv);
if (vsrc != giv + ivsize && vdst != giv + ivsize) {
giv = PTR_ALIGN((u8 *)reqctx->tail,
crypto_ablkcipher_alignmask(geniv) + 1);
complete = eseqiv_complete;
data = req;
}
ablkcipher_request_set_callback(subreq, req->creq.base.flags, complete,
data);
sg_init_table(reqctx->src, 2);
sg_set_buf(reqctx->src, giv, ivsize);
scatterwalk_crypto_chain(reqctx->src, osrc, vsrc == giv + ivsize, 2);
dst = reqctx->src;
if (osrc != odst) {
sg_init_table(reqctx->dst, 2);
sg_set_buf(reqctx->dst, giv, ivsize);
scatterwalk_crypto_chain(reqctx->dst, odst, vdst == giv + ivsize, 2);
dst = reqctx->dst;
}
ablkcipher_request_set_crypt(subreq, reqctx->src, dst,
req->creq.nbytes + ivsize,
req->creq.info);
memcpy(req->creq.info, ctx->salt, ivsize);
len = ivsize;
if (ivsize > sizeof(u64)) {
memset(req->giv, 0, ivsize - sizeof(u64));
len = sizeof(u64);
}
seq = cpu_to_be64(req->seq);
memcpy(req->giv + ivsize - len, &seq, len);
err = crypto_ablkcipher_encrypt(subreq);
if (err)
goto out;
if (giv != req->giv)
eseqiv_complete2(req);
out:
return err;
}
int ss_aes_one_req(sunxi_ss_t *sss, struct ablkcipher_request *req)
{
int ret = 0;
struct crypto_ablkcipher *tfm = NULL;
ss_aes_ctx_t *ctx = NULL;
ss_aes_req_ctx_t *req_ctx = NULL;
SS_ENTER();
if (!req->src || !req->dst) {
SS_ERR("Invalid sg: src = %p, dst = %p\n", req->src, req->dst);
return -EINVAL;
}
ss_dev_lock();
tfm = crypto_ablkcipher_reqtfm(req);
req_ctx = ablkcipher_request_ctx(req);
ctx = crypto_ablkcipher_ctx(tfm);
/* A31 SS need update key each cycle in decryption. */
if ((ctx->comm.flags & SS_FLAG_NEW_KEY) || (req_ctx->dir == SS_DIR_DECRYPT)) {
SS_DBG("KEY address = %p, size = %d\n", ctx->key, ctx->key_size);
ss_key_set(ctx->key, ctx->key_size);
ctx->comm.flags &= ~SS_FLAG_NEW_KEY;
}
#ifdef SS_CTS_MODE_ENABLE
if (((req_ctx->mode == SS_AES_MODE_CBC)
|| (req_ctx->mode == SS_AES_MODE_CTS)) && (req->info != NULL)) {
#else
if ((req_ctx->mode == SS_AES_MODE_CBC) && (req->info != NULL)) {
#endif
SS_DBG("IV address = %p, size = %d\n", req->info, crypto_ablkcipher_ivsize(tfm));
ss_iv_set(req->info, crypto_ablkcipher_ivsize(tfm));
}
#ifdef SS_CTR_MODE_ENABLE
if (req_ctx->mode == SS_AES_MODE_CTR) {
SS_DBG("Cnt address = %p, size = %d\n", req->info, crypto_ablkcipher_ivsize(tfm));
if (ctx->cnt == 0)
memcpy(ctx->iv, req->info, crypto_ablkcipher_ivsize(tfm));
SS_DBG("CNT: %08x %08x %08x %08x \n", *(int *)&ctx->iv[0],
*(int *)&ctx->iv[4], *(int *)&ctx->iv[8], *(int *)&ctx->iv[12]);
ss_cnt_set(ctx->iv, crypto_ablkcipher_ivsize(tfm));
}
#endif
req_ctx->dma_src.sg = req->src;
req_ctx->dma_dst.sg = req->dst;
ret = ss_aes_start(ctx, req_ctx, req->nbytes);
if (ret < 0)
SS_ERR("ss_aes_start fail(%d)\n", ret);
ss_dev_unlock();
#ifdef SS_CTR_MODE_ENABLE
if (req_ctx->mode == SS_AES_MODE_CTR) {
ss_cnt_get(ctx->comm.flow, ctx->iv, crypto_ablkcipher_ivsize(tfm));
SS_DBG("CNT: %08x %08x %08x %08x \n", *(int *)&ctx->iv[0],
*(int *)&ctx->iv[4], *(int *)&ctx->iv[8], *(int *)&ctx->iv[12]);
}
#endif
ctx->cnt += req->nbytes;
if (req->base.complete)
req->base.complete(&req->base, ret);
return ret;
}
irqreturn_t sunxi_ss_irq_handler(int irq, void *dev_id)
{
sunxi_ss_t *sss = (sunxi_ss_t *)dev_id;
unsigned long flags = 0;
int pending = 0;
spin_lock_irqsave(&sss->lock, flags);
pending = ss_pending_get();
SS_DBG("SS pending %#x\n", pending);
spin_unlock_irqrestore(&sss->lock, flags);
return IRQ_HANDLED;
}
static int aes_dma_start(struct aes_hwa_ctx *ctx)
{
int err, fast = 0, in, out;
size_t count;
dma_addr_t addr_in, addr_out;
struct omap_dma_channel_params dma_params;
struct tf_crypto_aes_operation_state *state =
crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(ctx->req));
static size_t last_count;
unsigned long flags;
in = IS_ALIGNED((u32)ctx->in_sg->offset, sizeof(u32));
out = IS_ALIGNED((u32)ctx->out_sg->offset, sizeof(u32));
fast = in && out;
if (fast) {
count = min(ctx->total, sg_dma_len(ctx->in_sg));
count = min(count, sg_dma_len(ctx->out_sg));
if (count != ctx->total)
return -EINVAL;
/* Only call dma_map_sg if it has not yet been done */
if (!(ctx->req->base.flags & CRYPTO_TFM_REQ_DMA_VISIBLE)) {
err = dma_map_sg(NULL, ctx->in_sg, 1, DMA_TO_DEVICE);
if (!err)
return -EINVAL;
err = dma_map_sg(NULL, ctx->out_sg, 1, DMA_FROM_DEVICE);
if (!err) {
dma_unmap_sg(
NULL, ctx->in_sg, 1, DMA_TO_DEVICE);
return -EINVAL;
}
}
ctx->req->base.flags &= ~CRYPTO_TFM_REQ_DMA_VISIBLE;
addr_in = sg_dma_address(ctx->in_sg);
addr_out = sg_dma_address(ctx->out_sg);
ctx->flags |= FLAGS_FAST;
} else {
count = sg_copy(&ctx->in_sg, &ctx->in_offset, ctx->buf_in,
ctx->buflen, ctx->total, 0);
addr_in = ctx->dma_addr_in;
addr_out = ctx->dma_addr_out;
ctx->flags &= ~FLAGS_FAST;
}
ctx->total -= count;
/* Configure HWA */
tf_crypto_enable_clock(PUBLIC_CRYPTO_AES1_CLOCK_REG);
tf_aes_restore_registers(state, ctx->flags & FLAGS_ENCRYPT ? 1 : 0);
OUTREG32(&paes_reg->AES_SYSCONFIG, INREG32(&paes_reg->AES_SYSCONFIG)
| AES_SYSCONFIG_DMA_REQ_OUT_EN_BIT
| AES_SYSCONFIG_DMA_REQ_IN_EN_BIT);
ctx->dma_size = count;
if (!fast)
dma_sync_single_for_device(NULL, addr_in, count,
DMA_TO_DEVICE);
dma_params.data_type = OMAP_DMA_DATA_TYPE_S32;
dma_params.frame_count = count / AES_BLOCK_SIZE;
dma_params.elem_count = DMA_CEN_Elts_per_Frame_AES;
dma_params.src_ei = 0;
dma_params.src_fi = 0;
dma_params.dst_ei = 0;
dma_params.dst_fi = 0;
dma_params.sync_mode = OMAP_DMA_SYNC_FRAME;
dma_params.read_prio = 0;
dma_params.write_prio = 0;
/* IN */
dma_params.trigger = ctx->dma_in;
dma_params.src_or_dst_synch = OMAP_DMA_DST_SYNC;
dma_params.dst_start = AES1_REGS_HW_ADDR + 0x60;
dma_params.dst_amode = OMAP_DMA_AMODE_CONSTANT;
dma_params.src_start = addr_in;
dma_params.src_amode = OMAP_DMA_AMODE_POST_INC;
if (reconfigure_dma) {
omap_set_dma_params(ctx->dma_lch_in, &dma_params);
omap_set_dma_dest_burst_mode(ctx->dma_lch_in,
OMAP_DMA_DATA_BURST_8);
omap_set_dma_src_burst_mode(ctx->dma_lch_in,
OMAP_DMA_DATA_BURST_8);
omap_set_dma_src_data_pack(ctx->dma_lch_in, 1);
} else {
if (last_count != count)
omap_set_dma_transfer_params(ctx->dma_lch_in,
dma_params.data_type,
dma_params.elem_count, dma_params.frame_count,
dma_params.sync_mode, dma_params.trigger,
dma_params.src_or_dst_synch);
/* Configure input start address */
__raw_writel(dma_params.src_start,
omap_dma_base + (0x60 * (ctx->dma_lch_in) + 0x9c));
}
/* OUT */
dma_params.trigger = ctx->dma_out;
dma_params.src_or_dst_synch = OMAP_DMA_SRC_SYNC;
dma_params.src_start = AES1_REGS_HW_ADDR + 0x60;
dma_params.src_amode = OMAP_DMA_AMODE_CONSTANT;
dma_params.dst_start = addr_out;
dma_params.dst_amode = OMAP_DMA_AMODE_POST_INC;
if (reconfigure_dma) {
omap_set_dma_params(ctx->dma_lch_out, &dma_params);
omap_set_dma_dest_burst_mode(ctx->dma_lch_out,
OMAP_DMA_DATA_BURST_8);
omap_set_dma_src_burst_mode(ctx->dma_lch_out,
OMAP_DMA_DATA_BURST_8);
omap_set_dma_dest_data_pack(ctx->dma_lch_out, 1);
reconfigure_dma = false;
} else {
if (last_count != count) {
omap_set_dma_transfer_params(ctx->dma_lch_out,
dma_params.data_type,
dma_params.elem_count, dma_params.frame_count,
dma_params.sync_mode, dma_params.trigger,
dma_params.src_or_dst_synch);
last_count = count;
}
/* Configure output start address */
__raw_writel(dma_params.dst_start,
omap_dma_base + (0x60 * (ctx->dma_lch_out) + 0xa0));
}
/* Is this really needed? */
omap_enable_dma_irq(ctx->dma_lch_in, OMAP_DMA_BLOCK_IRQ);
omap_enable_dma_irq(ctx->dma_lch_out, OMAP_DMA_BLOCK_IRQ);
wmb();
omap_start_dma(ctx->dma_lch_in);
omap_start_dma(ctx->dma_lch_out);
spin_lock_irqsave(&ctx->lock, flags);
if (ctx->next_req) {
struct ablkcipher_request *req =
ablkcipher_request_cast(ctx->next_req);
if (!(ctx->next_req->flags & CRYPTO_TFM_REQ_DMA_VISIBLE)) {
err = dma_map_sg(NULL, req->src, 1, DMA_TO_DEVICE);
if (!err) {
/* Silently fail for now... */
spin_unlock_irqrestore(&ctx->lock, flags);
return 0;
}
err = dma_map_sg(NULL, req->dst, 1, DMA_FROM_DEVICE);
if (!err) {
dma_unmap_sg(NULL, req->src, 1, DMA_TO_DEVICE);
/* Silently fail for now... */
spin_unlock_irqrestore(&ctx->lock, flags);
return 0;
}
ctx->next_req->flags |= CRYPTO_TFM_REQ_DMA_VISIBLE;
ctx->next_req = NULL;
}
}
if (ctx->backlog) {
ctx->backlog->complete(ctx->backlog, -EINPROGRESS);
ctx->backlog = NULL;
}
spin_unlock_irqrestore(&ctx->lock, flags);
return 0;
}
