/* This is the main crypto function - zero-copy edition */
static int
__crypto_run_zc(struct csession *ses_ptr, struct kernel_crypt_op *kcop)
{
struct scatterlist *src_sg, *dst_sg;
struct crypt_op *cop = &kcop->cop;
int ret = 0;
ret = get_userbuf(ses_ptr, cop->src, cop->len, cop->dst, cop->len,
kcop->task, kcop->mm, &src_sg, &dst_sg);
if (unlikely(ret)) {
derr(1, "Error getting user pages. Falling back to non zero copy.");
return __crypto_run_std(ses_ptr, cop);
}
ret = hash_n_crypt(ses_ptr, cop, src_sg, dst_sg, cop->len);
release_user_pages(ses_ptr);
return ret;
}
static int transfer_max_buffers(struct goldfish_pipe *pipe,
unsigned long address,
unsigned long address_end,
int is_write,
unsigned long last_page,
unsigned int last_page_size,
s32 *consumed_size,
int *status)
{
unsigned long first_page = address & PAGE_MASK;
unsigned int iter_last_page_size;
int pages_count;
/* Serialize access to the pipe command buffers */
if (mutex_lock_interruptible(&pipe->lock))
return -ERESTARTSYS;
pages_count = pin_user_pages(first_page, last_page,
last_page_size, is_write,
pipe->pages, &iter_last_page_size);
if (pages_count < 0) {
mutex_unlock(&pipe->lock);
return pages_count;
}
populate_rw_params(pipe->pages, pages_count, address, address_end,
first_page, last_page, iter_last_page_size, is_write,
pipe->command_buffer);
/* Transfer the data */
*status = goldfish_pipe_cmd_locked(pipe,
is_write ? PIPE_CMD_WRITE : PIPE_CMD_READ);
*consumed_size = pipe->command_buffer->rw_params.consumed_size;
release_user_pages(pipe->pages, pages_count, is_write, *consumed_size);
mutex_unlock(&pipe->lock);
return 0;
}
/* This is the main crypto function - zero-copy edition */
static int
__crypto_auth_run_zc(struct csession *ses_ptr, struct kernel_crypt_auth_op *kcaop)
{
struct scatterlist *dst_sg, *auth_sg, *src_sg;
struct crypt_auth_op *caop = &kcaop->caop;
int ret = 0;
if (caop->flags & COP_FLAG_AEAD_SRTP_TYPE) {
if (unlikely(ses_ptr->cdata.init != 0 &&
(ses_ptr->cdata.stream == 0 || ses_ptr->cdata.aead != 0)))
{
dprintk(0, KERN_ERR, "Only stream modes are allowed in SRTP mode (but not AEAD)\n");
return -EINVAL;
}
ret = get_userbuf_srtp(ses_ptr, kcaop, &auth_sg, &dst_sg);
if (unlikely(ret)) {
dprintk(1, KERN_ERR, "get_userbuf_srtp(): Error getting user pages.\n");
return ret;
}
ret = srtp_auth_n_crypt(ses_ptr, kcaop, auth_sg, caop->auth_len,
dst_sg, caop->len);
release_user_pages(ses_ptr);
} else { /* TLS and normal cases. Here auth data are usually small
* so we just copy them to a free page, instead of trying
* to map them.
*/
unsigned char* auth_buf = NULL;
struct scatterlist tmp;
if (unlikely(caop->auth_len > PAGE_SIZE)) {
dprintk(1, KERN_ERR, "auth data len is excessive.\n");
return -EINVAL;
}
auth_buf = (char *)__get_free_page(GFP_KERNEL);
if (unlikely(!auth_buf)) {
dprintk(1, KERN_ERR, "unable to get a free page.\n");
return -ENOMEM;
}
if (caop->auth_src && caop->auth_len > 0) {
if (unlikely(copy_from_user(auth_buf, caop->auth_src, caop->auth_len))) {
dprintk(1, KERN_ERR, "unable to copy auth data from userspace.\n");
ret = -EFAULT;
goto free_auth_buf;
}
sg_init_one(&tmp, auth_buf, caop->auth_len);
auth_sg = &tmp;
} else {
auth_sg = NULL;
}
if (caop->flags & COP_FLAG_AEAD_TLS_TYPE && ses_ptr->cdata.aead == 0) {
ret = get_userbuf_tls(ses_ptr, kcaop, &dst_sg);
if (unlikely(ret)) {
dprintk(1, KERN_ERR, "get_userbuf_tls(): Error getting user pages.\n");
goto free_auth_buf;
}
ret = tls_auth_n_crypt(ses_ptr, kcaop, auth_sg, caop->auth_len,
dst_sg, caop->len);
} else {
int dst_len;
if (unlikely(ses_ptr->cdata.init == 0 ||
ses_ptr->cdata.stream == 0 ||
ses_ptr->cdata.aead == 0))
{
dprintk(0, KERN_ERR, "Only stream and AEAD ciphers are allowed for authenc\n");
ret = -EINVAL;
goto free_auth_buf;
}
if (caop->op == COP_ENCRYPT) dst_len = caop->len + cryptodev_cipher_get_tag_size(&ses_ptr->cdata);
else dst_len = caop->len;
ret = get_userbuf(ses_ptr, caop->src, caop->len, caop->dst, dst_len,
kcaop->task, kcaop->mm, &src_sg, &dst_sg);
if (unlikely(ret)) {
dprintk(1, KERN_ERR, "get_userbuf(): Error getting user pages.\n");
goto free_auth_buf;
}
ret = auth_n_crypt(ses_ptr, kcaop, auth_sg, caop->auth_len,
src_sg, dst_sg, caop->len);
}
release_user_pages(ses_ptr);
free_auth_buf:
free_page((unsigned long)auth_buf);
}
return ret;
}
/* Makes caop->auth_src available as scatterlist.
* It also provides a pointer to caop->dst, which however,
* is assumed to be within the caop->auth_src buffer. If not
* (if their difference exceeds MAX_SRTP_AUTH_DATA_DIFF) it
* returns error.
*/
static int get_userbuf_srtp(struct csession *ses, struct kernel_crypt_auth_op *kcaop,
struct scatterlist **auth_sg, struct scatterlist **dst_sg)
{
int pagecount, diff;
int auth_pagecount = 0;
struct crypt_auth_op *caop = &kcaop->caop;
int rc;
if (caop->dst == NULL && caop->auth_src == NULL) {
dprintk(1, KERN_ERR, "dst and auth_src cannot be both null\n");
return -EINVAL;
}
if (ses->alignmask) {
if (!IS_ALIGNED((unsigned long)caop->dst, ses->alignmask))
dprintk(2, KERN_WARNING, "careful - source address %lx is not %d byte aligned\n",
(unsigned long)caop->dst, ses->alignmask + 1);
if (!IS_ALIGNED((unsigned long)caop->auth_src, ses->alignmask))
dprintk(2, KERN_WARNING, "careful - source address %lx is not %d byte aligned\n",
(unsigned long)caop->auth_src, ses->alignmask + 1);
}
if (unlikely(kcaop->dst_len == 0 || caop->auth_len == 0)) {
dprintk(1, KERN_WARNING, "Destination length cannot be zero\n");
return -EINVAL;
}
/* Note that in SRTP auth data overlap with data to be encrypted (dst)
*/
auth_pagecount = PAGECOUNT(caop->auth_src, caop->auth_len);
diff = (int)(caop->src - caop->auth_src);
if (diff > MAX_SRTP_AUTH_DATA_DIFF || diff < 0) {
dprintk(1, KERN_WARNING, "auth_src must overlap with src (diff: %d).\n", diff);
return -EINVAL;
}
pagecount = auth_pagecount;
rc = adjust_sg_array(ses, pagecount*2); /* double pages to have pages for dst(=auth_src) */
if (rc) {
dprintk(1, KERN_ERR, "cannot adjust sg array\n");
return rc;
}
rc = __get_userbuf(caop->auth_src, caop->auth_len, 1, auth_pagecount,
ses->pages, ses->sg, kcaop->task, kcaop->mm);
if (unlikely(rc)) {
dprintk(1, KERN_ERR,
"failed to get user pages for data input\n");
return -EINVAL;
}
ses->used_pages = pagecount;
ses->readonly_pages = 0;
(*auth_sg) = ses->sg;
(*dst_sg) = ses->sg + auth_pagecount;
sg_init_table(*dst_sg, auth_pagecount);
sg_copy(ses->sg, (*dst_sg), caop->auth_len);
(*dst_sg) = sg_advance(*dst_sg, diff);
if (*dst_sg == NULL) {
release_user_pages(ses);
dprintk(1, KERN_ERR,
"failed to get enough pages for auth data\n");
return -EINVAL;
}
return 0;
}
