static int virtio_crypto_alg_ablkcipher_close_session( struct virtio_crypto_ablkcipher_ctx *ctx, int encrypt) { struct scatterlist outhdr, status_sg, *sgs[2]; unsigned int tmp; struct virtio_crypto_destroy_session_req *destroy_session; struct virtio_crypto *vcrypto = ctx->vcrypto; int err; unsigned int num_out = 0, num_in = 0; spin_lock(&vcrypto->ctrl_lock); vcrypto->ctrl_status.status = VIRTIO_CRYPTO_ERR; /* Pad ctrl header */ vcrypto->ctrl.header.opcode = cpu_to_le32(VIRTIO_CRYPTO_CIPHER_DESTROY_SESSION); /* Set the default virtqueue id to 0 */ vcrypto->ctrl.header.queue_id = 0; destroy_session = &vcrypto->ctrl.u.destroy_session; if (encrypt) destroy_session->session_id = cpu_to_le64(ctx->enc_sess_info.session_id); else destroy_session->session_id = cpu_to_le64(ctx->dec_sess_info.session_id); sg_init_one(&outhdr, &vcrypto->ctrl, sizeof(vcrypto->ctrl)); sgs[num_out++] = &outhdr; /* Return status and session id back */ sg_init_one(&status_sg, &vcrypto->ctrl_status.status, sizeof(vcrypto->ctrl_status.status)); sgs[num_out + num_in++] = &status_sg; err = virtqueue_add_sgs(vcrypto->ctrl_vq, sgs, num_out, num_in, vcrypto, GFP_ATOMIC); if (err < 0) { spin_unlock(&vcrypto->ctrl_lock); return err; } virtqueue_kick(vcrypto->ctrl_vq); while (!virtqueue_get_buf(vcrypto->ctrl_vq, &tmp) && !virtqueue_is_broken(vcrypto->ctrl_vq)) cpu_relax(); if (vcrypto->ctrl_status.status != VIRTIO_CRYPTO_OK) { spin_unlock(&vcrypto->ctrl_lock); pr_err("virtio_crypto: Close session failed status: %u, session_id: 0x%llx\n", vcrypto->ctrl_status.status, destroy_session->session_id); return -EINVAL; } spin_unlock(&vcrypto->ctrl_lock); return 0; }
static int virtblk_add_req_scsi(struct virtqueue *vq, struct virtblk_req *vbr, struct scatterlist *data_sg, bool have_data) { struct scatterlist hdr, status, cmd, sense, inhdr, *sgs[6]; unsigned int num_out = 0, num_in = 0; sg_init_one(&hdr, &vbr->out_hdr, sizeof(vbr->out_hdr)); sgs[num_out++] = &hdr; sg_init_one(&cmd, vbr->sreq.cmd, vbr->sreq.cmd_len); sgs[num_out++] = &cmd; if (have_data) { if (vbr->out_hdr.type & cpu_to_virtio32(vq->vdev, VIRTIO_BLK_T_OUT)) sgs[num_out++] = data_sg; else sgs[num_out + num_in++] = data_sg; } sg_init_one(&sense, vbr->sense, SCSI_SENSE_BUFFERSIZE); sgs[num_out + num_in++] = &sense; sg_init_one(&inhdr, &vbr->in_hdr, sizeof(vbr->in_hdr)); sgs[num_out + num_in++] = &inhdr; sg_init_one(&status, &vbr->status, sizeof(vbr->status)); sgs[num_out + num_in++] = &status; return virtqueue_add_sgs(vq, sgs, num_out, num_in, vbr, GFP_ATOMIC); }
static int p9_virtio_request(struct p9_client *client, struct p9_req_t *req) { int err; int in, out, out_sgs, in_sgs; unsigned long flags; struct virtio_chan *chan = client->trans; struct scatterlist *sgs[2]; p9_debug(P9_DEBUG_TRANS, "9p debug: virtio request\n"); req->status = REQ_STATUS_SENT; req_retry: spin_lock_irqsave(&chan->lock, flags); out_sgs = in_sgs = 0; /* Handle out VirtIO ring buffers */ out = pack_sg_list(chan->sg, 0, VIRTQUEUE_NUM, req->tc->sdata, req->tc->size); if (out) sgs[out_sgs++] = chan->sg; in = pack_sg_list(chan->sg, out, VIRTQUEUE_NUM, req->rc->sdata, req->rc->capacity); if (in) sgs[out_sgs + in_sgs++] = chan->sg + out; err = virtqueue_add_sgs(chan->vq, sgs, out_sgs, in_sgs, req->tc, GFP_ATOMIC); if (err < 0) { if (err == -ENOSPC) { chan->ring_bufs_avail = 0; spin_unlock_irqrestore(&chan->lock, flags); err = wait_event_interruptible(*chan->vc_wq, chan->ring_bufs_avail); if (err == -ERESTARTSYS) return err; p9_debug(P9_DEBUG_TRANS, "Retry virtio request\n"); goto req_retry; } else { spin_unlock_irqrestore(&chan->lock, flags); p9_debug(P9_DEBUG_TRANS, "virtio rpc add_sgs returned failure\n"); return -EIO; } } virtqueue_kick(chan->vq); spin_unlock_irqrestore(&chan->lock, flags); p9_debug(P9_DEBUG_TRANS, "virtio request kicked\n"); return 0; }
static int __virtblk_add_req(struct virtqueue *vq, struct virtblk_req *vbr, struct scatterlist *data_sg, bool have_data) { struct scatterlist hdr, status, cmd, sense, inhdr, *sgs[6]; unsigned int num_out = 0, num_in = 0; __virtio32 type = vbr->out_hdr.type & ~cpu_to_virtio32(vq->vdev, VIRTIO_BLK_T_OUT); sg_init_one(&hdr, &vbr->out_hdr, sizeof(vbr->out_hdr)); sgs[num_out++] = &hdr; /* * If this is a packet command we need a couple of additional headers. * Behind the normal outhdr we put a segment with the scsi command * block, and before the normal inhdr we put the sense data and the * inhdr with additional status information. */ if (type == cpu_to_virtio32(vq->vdev, VIRTIO_BLK_T_SCSI_CMD)) { sg_init_one(&cmd, vbr->req->cmd, vbr->req->cmd_len); sgs[num_out++] = &cmd; } if (have_data) { if (vbr->out_hdr.type & cpu_to_virtio32(vq->vdev, VIRTIO_BLK_T_OUT)) sgs[num_out++] = data_sg; else sgs[num_out + num_in++] = data_sg; } if (type == cpu_to_virtio32(vq->vdev, VIRTIO_BLK_T_SCSI_CMD)) { sg_init_one(&sense, vbr->req->sense, SCSI_SENSE_BUFFERSIZE); sgs[num_out + num_in++] = &sense; sg_init_one(&inhdr, &vbr->in_hdr, sizeof(vbr->in_hdr)); sgs[num_out + num_in++] = &inhdr; } sg_init_one(&status, &vbr->status, sizeof(vbr->status)); sgs[num_out + num_in++] = &status; return virtqueue_add_sgs(vq, sgs, num_out, num_in, vbr, GFP_ATOMIC); }
static int virtblk_add_req(struct virtqueue *vq, struct virtblk_req *vbr, struct scatterlist *data_sg, bool have_data) { struct scatterlist hdr, status, *sgs[3]; unsigned int num_out = 0, num_in = 0; sg_init_one(&hdr, &vbr->out_hdr, sizeof(vbr->out_hdr)); sgs[num_out++] = &hdr; if (have_data) { if (vbr->out_hdr.type & cpu_to_virtio32(vq->vdev, VIRTIO_BLK_T_OUT)) sgs[num_out++] = data_sg; else sgs[num_out + num_in++] = data_sg; } sg_init_one(&status, &vbr->status, sizeof(vbr->status)); sgs[num_out + num_in++] = &status; return virtqueue_add_sgs(vq, sgs, num_out, num_in, vbr, GFP_ATOMIC); }
static int virtio_crypto_alg_ablkcipher_init_session( struct virtio_crypto_ablkcipher_ctx *ctx, uint32_t alg, const uint8_t *key, unsigned int keylen, int encrypt) { struct scatterlist outhdr, key_sg, inhdr, *sgs[3]; unsigned int tmp; struct virtio_crypto *vcrypto = ctx->vcrypto; int op = encrypt ? VIRTIO_CRYPTO_OP_ENCRYPT : VIRTIO_CRYPTO_OP_DECRYPT; int err; unsigned int num_out = 0, num_in = 0; /* * Avoid to do DMA from the stack, switch to using * dynamically-allocated for the key */ uint8_t *cipher_key = kmalloc(keylen, GFP_ATOMIC); if (!cipher_key) return -ENOMEM; memcpy(cipher_key, key, keylen); spin_lock(&vcrypto->ctrl_lock); /* Pad ctrl header */ vcrypto->ctrl.header.opcode = cpu_to_le32(VIRTIO_CRYPTO_CIPHER_CREATE_SESSION); vcrypto->ctrl.header.algo = cpu_to_le32(alg); /* Set the default dataqueue id to 0 */ vcrypto->ctrl.header.queue_id = 0; vcrypto->input.status = cpu_to_le32(VIRTIO_CRYPTO_ERR); /* Pad cipher's parameters */ vcrypto->ctrl.u.sym_create_session.op_type = cpu_to_le32(VIRTIO_CRYPTO_SYM_OP_CIPHER); vcrypto->ctrl.u.sym_create_session.u.cipher.para.algo = vcrypto->ctrl.header.algo; vcrypto->ctrl.u.sym_create_session.u.cipher.para.keylen = cpu_to_le32(keylen); vcrypto->ctrl.u.sym_create_session.u.cipher.para.op = cpu_to_le32(op); sg_init_one(&outhdr, &vcrypto->ctrl, sizeof(vcrypto->ctrl)); sgs[num_out++] = &outhdr; /* Set key */ sg_init_one(&key_sg, cipher_key, keylen); sgs[num_out++] = &key_sg; /* Return status and session id back */ sg_init_one(&inhdr, &vcrypto->input, sizeof(vcrypto->input)); sgs[num_out + num_in++] = &inhdr; err = virtqueue_add_sgs(vcrypto->ctrl_vq, sgs, num_out, num_in, vcrypto, GFP_ATOMIC); if (err < 0) { spin_unlock(&vcrypto->ctrl_lock); kzfree(cipher_key); return err; } virtqueue_kick(vcrypto->ctrl_vq); /* * Trapping into the hypervisor, so the request should be * handled immediately. */ while (!virtqueue_get_buf(vcrypto->ctrl_vq, &tmp) && !virtqueue_is_broken(vcrypto->ctrl_vq)) cpu_relax(); if (le32_to_cpu(vcrypto->input.status) != VIRTIO_CRYPTO_OK) { spin_unlock(&vcrypto->ctrl_lock); pr_err("virtio_crypto: Create session failed status: %u\n", le32_to_cpu(vcrypto->input.status)); kzfree(cipher_key); return -EINVAL; } if (encrypt) ctx->enc_sess_info.session_id = le64_to_cpu(vcrypto->input.session_id); else ctx->dec_sess_info.session_id = le64_to_cpu(vcrypto->input.session_id); spin_unlock(&vcrypto->ctrl_lock); kzfree(cipher_key); return 0; }
static int __virtio_crypto_ablkcipher_do_req(struct virtio_crypto_request *vc_req, struct ablkcipher_request *req, struct data_queue *data_vq, __u8 op) { struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); unsigned int ivsize = crypto_ablkcipher_ivsize(tfm); struct virtio_crypto_ablkcipher_ctx *ctx = vc_req->ablkcipher_ctx; struct virtio_crypto *vcrypto = ctx->vcrypto; struct virtio_crypto_op_data_req *req_data; int src_nents, dst_nents; int err; unsigned long flags; struct scatterlist outhdr, iv_sg, status_sg, **sgs; int i; u64 dst_len; unsigned int num_out = 0, num_in = 0; int sg_total; uint8_t *iv; src_nents = sg_nents_for_len(req->src, req->nbytes); dst_nents = sg_nents(req->dst); pr_debug("virtio_crypto: Number of sgs (src_nents: %d, dst_nents: %d)\n", src_nents, dst_nents); /* Why 3? outhdr + iv + inhdr */ sg_total = src_nents + dst_nents + 3; sgs = kzalloc_node(sg_total * sizeof(*sgs), GFP_ATOMIC, dev_to_node(&vcrypto->vdev->dev)); if (!sgs) return -ENOMEM; req_data = kzalloc_node(sizeof(*req_data), GFP_ATOMIC, dev_to_node(&vcrypto->vdev->dev)); if (!req_data) { kfree(sgs); return -ENOMEM; } vc_req->req_data = req_data; vc_req->type = VIRTIO_CRYPTO_SYM_OP_CIPHER; /* Head of operation */ if (op) { req_data->header.session_id = cpu_to_le64(ctx->enc_sess_info.session_id); req_data->header.opcode = cpu_to_le32(VIRTIO_CRYPTO_CIPHER_ENCRYPT); } else { req_data->header.session_id = cpu_to_le64(ctx->dec_sess_info.session_id); req_data->header.opcode = cpu_to_le32(VIRTIO_CRYPTO_CIPHER_DECRYPT); } req_data->u.sym_req.op_type = cpu_to_le32(VIRTIO_CRYPTO_SYM_OP_CIPHER); req_data->u.sym_req.u.cipher.para.iv_len = cpu_to_le32(ivsize); req_data->u.sym_req.u.cipher.para.src_data_len = cpu_to_le32(req->nbytes); dst_len = virtio_crypto_alg_sg_nents_length(req->dst); if (unlikely(dst_len > U32_MAX)) { pr_err("virtio_crypto: The dst_len is beyond U32_MAX\n"); err = -EINVAL; goto free; } pr_debug("virtio_crypto: src_len: %u, dst_len: %llu\n", req->nbytes, dst_len); if (unlikely(req->nbytes + dst_len + ivsize + sizeof(vc_req->status) > vcrypto->max_size)) { pr_err("virtio_crypto: The length is too big\n"); err = -EINVAL; goto free; } req_data->u.sym_req.u.cipher.para.dst_data_len = cpu_to_le32((uint32_t)dst_len); /* Outhdr */ sg_init_one(&outhdr, req_data, sizeof(*req_data)); sgs[num_out++] = &outhdr; /* IV */ /* * Avoid to do DMA from the stack, switch to using * dynamically-allocated for the IV */ iv = kzalloc_node(ivsize, GFP_ATOMIC, dev_to_node(&vcrypto->vdev->dev)); if (!iv) { err = -ENOMEM; goto free; } memcpy(iv, req->info, ivsize); sg_init_one(&iv_sg, iv, ivsize); sgs[num_out++] = &iv_sg; vc_req->iv = iv; /* Source data */ for (i = 0; i < src_nents; i++) sgs[num_out++] = &req->src[i]; /* Destination data */ for (i = 0; i < dst_nents; i++) sgs[num_out + num_in++] = &req->dst[i]; /* Status */ sg_init_one(&status_sg, &vc_req->status, sizeof(vc_req->status)); sgs[num_out + num_in++] = &status_sg; vc_req->sgs = sgs; spin_lock_irqsave(&data_vq->lock, flags); err = virtqueue_add_sgs(data_vq->vq, sgs, num_out, num_in, vc_req, GFP_ATOMIC); virtqueue_kick(data_vq->vq); spin_unlock_irqrestore(&data_vq->lock, flags); if (unlikely(err < 0)) goto free_iv; return 0; free_iv: kzfree(iv); free: kzfree(req_data); kfree(sgs); return err; }
/** * p9_virtio_zc_request - issue a zero copy request * @client: client instance issuing the request * @req: request to be issued * @uidata: user bffer that should be ued for zero copy read * @uodata: user buffer that shoud be user for zero copy write * @inlen: read buffer size * @olen: write buffer size * @hdrlen: reader header size, This is the size of response protocol data * */ static int p9_virtio_zc_request(struct p9_client *client, struct p9_req_t *req, char *uidata, char *uodata, int inlen, int outlen, int in_hdr_len, int kern_buf) { int in, out, err, out_sgs, in_sgs; unsigned long flags; int in_nr_pages = 0, out_nr_pages = 0; struct page **in_pages = NULL, **out_pages = NULL; struct virtio_chan *chan = client->trans; struct scatterlist *sgs[4]; p9_debug(P9_DEBUG_TRANS, "virtio request\n"); if (uodata) { out_nr_pages = p9_nr_pages(uodata, outlen); out_pages = kmalloc(sizeof(struct page *) * out_nr_pages, GFP_NOFS); if (!out_pages) { err = -ENOMEM; goto err_out; } out_nr_pages = p9_get_mapped_pages(chan, out_pages, uodata, out_nr_pages, 0, kern_buf); if (out_nr_pages < 0) { err = out_nr_pages; kfree(out_pages); out_pages = NULL; goto err_out; } } if (uidata) { in_nr_pages = p9_nr_pages(uidata, inlen); in_pages = kmalloc(sizeof(struct page *) * in_nr_pages, GFP_NOFS); if (!in_pages) { err = -ENOMEM; goto err_out; } in_nr_pages = p9_get_mapped_pages(chan, in_pages, uidata, in_nr_pages, 1, kern_buf); if (in_nr_pages < 0) { err = in_nr_pages; kfree(in_pages); in_pages = NULL; goto err_out; } } req->status = REQ_STATUS_SENT; req_retry_pinned: spin_lock_irqsave(&chan->lock, flags); out_sgs = in_sgs = 0; /* out data */ out = pack_sg_list(chan->sg, 0, VIRTQUEUE_NUM, req->tc->sdata, req->tc->size); if (out) sgs[out_sgs++] = chan->sg; if (out_pages) { sgs[out_sgs++] = chan->sg + out; out += pack_sg_list_p(chan->sg, out, VIRTQUEUE_NUM, out_pages, out_nr_pages, uodata, outlen); } /* * Take care of in data * For example TREAD have 11. * 11 is the read/write header = PDU Header(7) + IO Size (4). * Arrange in such a way that server places header in the * alloced memory and payload onto the user buffer. */ in = pack_sg_list(chan->sg, out, VIRTQUEUE_NUM, req->rc->sdata, in_hdr_len); if (in) sgs[out_sgs + in_sgs++] = chan->sg + out; if (in_pages) { sgs[out_sgs + in_sgs++] = chan->sg + out + in; in += pack_sg_list_p(chan->sg, out + in, VIRTQUEUE_NUM, in_pages, in_nr_pages, uidata, inlen); } BUG_ON(out_sgs + in_sgs > ARRAY_SIZE(sgs)); err = virtqueue_add_sgs(chan->vq, sgs, out_sgs, in_sgs, req->tc, GFP_ATOMIC); if (err < 0) { if (err == -ENOSPC) { chan->ring_bufs_avail = 0; spin_unlock_irqrestore(&chan->lock, flags); err = wait_event_interruptible(*chan->vc_wq, chan->ring_bufs_avail); if (err == -ERESTARTSYS) goto err_out; p9_debug(P9_DEBUG_TRANS, "Retry virtio request\n"); goto req_retry_pinned; } else { spin_unlock_irqrestore(&chan->lock, flags); p9_debug(P9_DEBUG_TRANS, "virtio rpc add_sgs returned failure\n"); err = -EIO; goto err_out; } } virtqueue_kick(chan->vq); spin_unlock_irqrestore(&chan->lock, flags); p9_debug(P9_DEBUG_TRANS, "virtio request kicked\n"); err = wait_event_interruptible(*req->wq, req->status >= REQ_STATUS_RCVD); /* * Non kernel buffers are pinned, unpin them */ err_out: if (!kern_buf) { if (in_pages) { p9_release_pages(in_pages, in_nr_pages); atomic_sub(in_nr_pages, &vp_pinned); } if (out_pages) { p9_release_pages(out_pages, out_nr_pages); atomic_sub(out_nr_pages, &vp_pinned); } /* wakeup anybody waiting for slots to pin pages */ wake_up(&vp_wq); } kfree(in_pages); kfree(out_pages); return err; }