static int aes_dma_stop(struct aes_hwa_ctx *ctx)
{
struct tf_crypto_aes_operation_state *state =
crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(ctx->req));
int err = 0;
size_t count;
dprintk(KERN_INFO "aes_dma_stop(%p)\n", ctx);
tf_aes_save_registers(state);
if (!AES_CTRL_IS_MODE_ECB(state->CTRL)) {
u32 *ptr = (u32 *) ctx->req->info;
ptr[0] = state->AES_IV_0;
ptr[1] = state->AES_IV_1;
ptr[2] = state->AES_IV_2;
ptr[3] = state->AES_IV_3;
}
OUTREG32(&paes_reg->AES_SYSCONFIG, 0);
omap_stop_dma(ctx->dma_lch_in);
omap_stop_dma(ctx->dma_lch_out);
tf_crypto_disable_clock(PUBLIC_CRYPTO_AES1_CLOCK_REG);
if (!(ctx->flags & FLAGS_FAST)) {
dma_sync_single_for_device(NULL, ctx->dma_addr_out,
ctx->dma_size, DMA_FROM_DEVICE);
#ifdef CONFIG_TF_DRIVER_FAULT_INJECTION
tf_aes_fault_injection(paes_reg->AES_CTRL, ctx->buf_out);
#endif
/* Copy data */
count = sg_copy(&ctx->out_sg, &ctx->out_offset, ctx->buf_out,
ctx->buflen, ctx->dma_size, 1);
if (count != ctx->dma_size)
err = -EINVAL;
} else {
dma_unmap_sg(NULL, ctx->out_sg, 1, DMA_FROM_DEVICE);
dma_unmap_sg(NULL, ctx->in_sg, 1, DMA_TO_DEVICE);
#ifdef CONFIG_TF_DRIVER_FAULT_INJECTION
tf_aes_fault_injection(paes_reg->AES_CTRL,
sg_virt(ctx->out_sg));
#endif
}
if (err || !ctx->total)
ctx->req->base.complete(&ctx->req->base, err);
return err;
}
static int aes_dma_start(struct aes_hwa_ctx *ctx)
{
int err, fast = 0, in, out;
size_t count;
dma_addr_t addr_in, addr_out;
struct omap_dma_channel_params dma_params;
struct tf_crypto_aes_operation_state *state =
crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(ctx->req));
static size_t last_count;
unsigned long flags;
in = IS_ALIGNED((u32)ctx->in_sg->offset, sizeof(u32));
out = IS_ALIGNED((u32)ctx->out_sg->offset, sizeof(u32));
fast = in && out;
if (fast) {
count = min(ctx->total, sg_dma_len(ctx->in_sg));
count = min(count, sg_dma_len(ctx->out_sg));
if (count != ctx->total)
return -EINVAL;
/* Only call dma_map_sg if it has not yet been done */
if (!(ctx->req->base.flags & CRYPTO_TFM_REQ_DMA_VISIBLE)) {
err = dma_map_sg(NULL, ctx->in_sg, 1, DMA_TO_DEVICE);
if (!err)
return -EINVAL;
err = dma_map_sg(NULL, ctx->out_sg, 1, DMA_FROM_DEVICE);
if (!err) {
dma_unmap_sg(
NULL, ctx->in_sg, 1, DMA_TO_DEVICE);
return -EINVAL;
}
}
ctx->req->base.flags &= ~CRYPTO_TFM_REQ_DMA_VISIBLE;
addr_in = sg_dma_address(ctx->in_sg);
addr_out = sg_dma_address(ctx->out_sg);
ctx->flags |= FLAGS_FAST;
} else {
count = sg_copy(&ctx->in_sg, &ctx->in_offset, ctx->buf_in,
ctx->buflen, ctx->total, 0);
addr_in = ctx->dma_addr_in;
addr_out = ctx->dma_addr_out;
ctx->flags &= ~FLAGS_FAST;
}
ctx->total -= count;
/* Configure HWA */
tf_crypto_enable_clock(PUBLIC_CRYPTO_AES1_CLOCK_REG);
tf_aes_restore_registers(state, ctx->flags & FLAGS_ENCRYPT ? 1 : 0);
OUTREG32(&paes_reg->AES_SYSCONFIG, INREG32(&paes_reg->AES_SYSCONFIG)
| AES_SYSCONFIG_DMA_REQ_OUT_EN_BIT
| AES_SYSCONFIG_DMA_REQ_IN_EN_BIT);
ctx->dma_size = count;
if (!fast)
dma_sync_single_for_device(NULL, addr_in, count,
DMA_TO_DEVICE);
dma_params.data_type = OMAP_DMA_DATA_TYPE_S32;
dma_params.frame_count = count / AES_BLOCK_SIZE;
dma_params.elem_count = DMA_CEN_Elts_per_Frame_AES;
dma_params.src_ei = 0;
dma_params.src_fi = 0;
dma_params.dst_ei = 0;
dma_params.dst_fi = 0;
dma_params.sync_mode = OMAP_DMA_SYNC_FRAME;
dma_params.read_prio = 0;
dma_params.write_prio = 0;
/* IN */
dma_params.trigger = ctx->dma_in;
dma_params.src_or_dst_synch = OMAP_DMA_DST_SYNC;
dma_params.dst_start = AES1_REGS_HW_ADDR + 0x60;
dma_params.dst_amode = OMAP_DMA_AMODE_CONSTANT;
dma_params.src_start = addr_in;
dma_params.src_amode = OMAP_DMA_AMODE_POST_INC;
if (reconfigure_dma) {
omap_set_dma_params(ctx->dma_lch_in, &dma_params);
omap_set_dma_dest_burst_mode(ctx->dma_lch_in,
OMAP_DMA_DATA_BURST_8);
omap_set_dma_src_burst_mode(ctx->dma_lch_in,
OMAP_DMA_DATA_BURST_8);
omap_set_dma_src_data_pack(ctx->dma_lch_in, 1);
} else {
if (last_count != count)
omap_set_dma_transfer_params(ctx->dma_lch_in,
dma_params.data_type,
dma_params.elem_count, dma_params.frame_count,
dma_params.sync_mode, dma_params.trigger,
dma_params.src_or_dst_synch);
/* Configure input start address */
__raw_writel(dma_params.src_start,
omap_dma_base + (0x60 * (ctx->dma_lch_in) + 0x9c));
}
/* OUT */
dma_params.trigger = ctx->dma_out;
dma_params.src_or_dst_synch = OMAP_DMA_SRC_SYNC;
dma_params.src_start = AES1_REGS_HW_ADDR + 0x60;
dma_params.src_amode = OMAP_DMA_AMODE_CONSTANT;
dma_params.dst_start = addr_out;
dma_params.dst_amode = OMAP_DMA_AMODE_POST_INC;
if (reconfigure_dma) {
omap_set_dma_params(ctx->dma_lch_out, &dma_params);
omap_set_dma_dest_burst_mode(ctx->dma_lch_out,
OMAP_DMA_DATA_BURST_8);
omap_set_dma_src_burst_mode(ctx->dma_lch_out,
OMAP_DMA_DATA_BURST_8);
omap_set_dma_dest_data_pack(ctx->dma_lch_out, 1);
reconfigure_dma = false;
} else {
if (last_count != count) {
omap_set_dma_transfer_params(ctx->dma_lch_out,
dma_params.data_type,
dma_params.elem_count, dma_params.frame_count,
dma_params.sync_mode, dma_params.trigger,
dma_params.src_or_dst_synch);
last_count = count;
}
/* Configure output start address */
__raw_writel(dma_params.dst_start,
omap_dma_base + (0x60 * (ctx->dma_lch_out) + 0xa0));
}
/* Is this really needed? */
omap_enable_dma_irq(ctx->dma_lch_in, OMAP_DMA_BLOCK_IRQ);
omap_enable_dma_irq(ctx->dma_lch_out, OMAP_DMA_BLOCK_IRQ);
wmb();
omap_start_dma(ctx->dma_lch_in);
omap_start_dma(ctx->dma_lch_out);
spin_lock_irqsave(&ctx->lock, flags);
if (ctx->next_req) {
struct ablkcipher_request *req =
ablkcipher_request_cast(ctx->next_req);
if (!(ctx->next_req->flags & CRYPTO_TFM_REQ_DMA_VISIBLE)) {
err = dma_map_sg(NULL, req->src, 1, DMA_TO_DEVICE);
if (!err) {
/* Silently fail for now... */
spin_unlock_irqrestore(&ctx->lock, flags);
return 0;
}
err = dma_map_sg(NULL, req->dst, 1, DMA_FROM_DEVICE);
if (!err) {
dma_unmap_sg(NULL, req->src, 1, DMA_TO_DEVICE);
/* Silently fail for now... */
spin_unlock_irqrestore(&ctx->lock, flags);
return 0;
}
ctx->next_req->flags |= CRYPTO_TFM_REQ_DMA_VISIBLE;
ctx->next_req = NULL;
}
}
if (ctx->backlog) {
ctx->backlog->complete(ctx->backlog, -EINPROGRESS);
ctx->backlog = NULL;
}
spin_unlock_irqrestore(&ctx->lock, flags);
return 0;
}
/* Makes caop->auth_src available as scatterlist.
* It also provides a pointer to caop->dst, which however,
* is assumed to be within the caop->auth_src buffer. If not
* (if their difference exceeds MAX_SRTP_AUTH_DATA_DIFF) it
* returns error.
*/
static int get_userbuf_srtp(struct csession *ses, struct kernel_crypt_auth_op *kcaop,
struct scatterlist **auth_sg, struct scatterlist **dst_sg)
{
int pagecount, diff;
int auth_pagecount = 0;
struct crypt_auth_op *caop = &kcaop->caop;
int rc;
if (caop->dst == NULL && caop->auth_src == NULL) {
dprintk(1, KERN_ERR, "dst and auth_src cannot be both null\n");
return -EINVAL;
}
if (ses->alignmask) {
if (!IS_ALIGNED((unsigned long)caop->dst, ses->alignmask))
dprintk(2, KERN_WARNING, "careful - source address %lx is not %d byte aligned\n",
(unsigned long)caop->dst, ses->alignmask + 1);
if (!IS_ALIGNED((unsigned long)caop->auth_src, ses->alignmask))
dprintk(2, KERN_WARNING, "careful - source address %lx is not %d byte aligned\n",
(unsigned long)caop->auth_src, ses->alignmask + 1);
}
if (unlikely(kcaop->dst_len == 0 || caop->auth_len == 0)) {
dprintk(1, KERN_WARNING, "Destination length cannot be zero\n");
return -EINVAL;
}
/* Note that in SRTP auth data overlap with data to be encrypted (dst)
*/
auth_pagecount = PAGECOUNT(caop->auth_src, caop->auth_len);
diff = (int)(caop->src - caop->auth_src);
if (diff > MAX_SRTP_AUTH_DATA_DIFF || diff < 0) {
dprintk(1, KERN_WARNING, "auth_src must overlap with src (diff: %d).\n", diff);
return -EINVAL;
}
pagecount = auth_pagecount;
rc = adjust_sg_array(ses, pagecount*2); /* double pages to have pages for dst(=auth_src) */
if (rc) {
dprintk(1, KERN_ERR, "cannot adjust sg array\n");
return rc;
}
rc = __get_userbuf(caop->auth_src, caop->auth_len, 1, auth_pagecount,
ses->pages, ses->sg, kcaop->task, kcaop->mm);
if (unlikely(rc)) {
dprintk(1, KERN_ERR,
"failed to get user pages for data input\n");
return -EINVAL;
}
ses->used_pages = pagecount;
ses->readonly_pages = 0;
(*auth_sg) = ses->sg;
(*dst_sg) = ses->sg + auth_pagecount;
sg_init_table(*dst_sg, auth_pagecount);
sg_copy(ses->sg, (*dst_sg), caop->auth_len);
(*dst_sg) = sg_advance(*dst_sg, diff);
if (*dst_sg == NULL) {
release_user_pages(ses);
dprintk(1, KERN_ERR,
"failed to get enough pages for auth data\n");
return -EINVAL;
}
return 0;
}
