struct nvme_request *
nvme_allocate_request(const struct nvme_payload *payload, uint32_t payload_size,
spdk_nvme_cmd_cb cb_fn,
void *cb_arg)
{
struct nvme_request *req = NULL;
nvme_alloc_request(&req);
if (req == NULL) {
return req;
}
/*
* Only memset up to (but not including) the children
* TAILQ_ENTRY. children, and following members, are
* only used as part of I/O splitting so we avoid
* memsetting them until it is actually needed.
* They will be initialized in nvme_request_add_child()
* if the request is split.
*/
memset(req, 0, offsetof(struct nvme_request, children));
req->cb_fn = cb_fn;
req->cb_arg = cb_arg;
req->timeout = true;
req->payload = *payload;
req->payload_size = payload_size;
return req;
}
struct nvme_request *
nvme_allocate_request(const struct nvme_payload *payload, uint32_t payload_size,
spdk_nvme_cmd_cb cb_fn,
void *cb_arg)
{
struct nvme_request *req = NULL;
nvme_alloc_request(&req);
if (req != NULL) {
memset(req, 0, offsetof(struct nvme_request, children));
req->payload = *payload;
req->payload_size = payload_size;
req->cb_fn = cb_fn;
req->cb_arg = cb_arg;
req->timeout = true;
}
struct nvme_request *
nvme_allocate_request(void *payload, uint32_t payload_size,
nvme_cb_fn_t cb_fn, void *cb_arg)
{
struct nvme_request *req = NULL;
nvme_alloc_request(&req);
if (req != NULL) {
memset(req, 0, offsetof(struct nvme_request, children));
if (payload == NULL || payload_size == 0) {
req->u.payload = NULL;
req->payload_size = 0;
} else {
req->u.payload = payload;
req->payload_size = payload_size;
}
req->cb_fn = cb_fn;
req->cb_arg = cb_arg;
req->timeout = true;
}
static void
ut_insert_cq_entry(struct nvme_qpair *qpair, uint32_t slot)
{
struct nvme_request *req;
struct nvme_tracker *tr;
struct spdk_nvme_cpl *cpl;
nvme_alloc_request(&req);
SPDK_CU_ASSERT_FATAL(req != NULL);
memset(req, 0, sizeof(*req));
req->cmd.cid = slot;
tr = LIST_FIRST(&qpair->free_tr);
LIST_REMOVE(tr, list); /* remove tr from free_tr */
LIST_INSERT_HEAD(&qpair->outstanding_tr, tr, list);
tr->cid = slot;
tr->req = req;
qpair->act_tr[tr->cid] = tr;
cpl = &qpair->cpl[slot];
cpl->status.p = qpair->phase;
cpl->cid = slot;
}
struct nvme_request *
nvme_allocate_request(void *payload, uint32_t payload_size,
nvme_cb_fn_t cb_fn, void *cb_arg)
{
struct nvme_request *req = NULL;
nvme_alloc_request(&req);
if (req == NULL) {
return req;
}
/*
* Only memset up to (but not including) the children
* TAILQ_ENTRY. children, and following members, are
* only used as part of I/O splitting so we avoid
* memsetting them until it is actually needed.
* They will be initialized in nvme_request_add_child()
* if the request is split.
*/
memset(req, 0, offsetof(struct nvme_request, children));
req->cb_fn = cb_fn;
req->cb_arg = cb_arg;
req->timeout = true;
nvme_assert((payload == NULL && payload_size == 0) ||
(payload != NULL && payload_size != 0),
("Invalid argument combination of payload and payload_size\n"));
if (payload == NULL || payload_size == 0) {
req->u.payload = NULL;
req->payload_size = 0;
} else {
req->u.payload = payload;
req->payload_size = payload_size;
}
return req;
}
static struct request *nvme_nvm_alloc_request(struct request_queue *q,
struct nvm_rq *rqd,
struct nvme_nvm_command *cmd)
{
struct nvme_ns *ns = q->queuedata;
struct request *rq;
nvme_nvm_rqtocmd(rqd, ns, cmd);
rq = nvme_alloc_request(q, (struct nvme_command *)cmd, 0, NVME_QID_ANY);
if (IS_ERR(rq))
return rq;
rq->cmd_flags &= ~REQ_FAILFAST_DRIVER;
if (rqd->bio) {
blk_init_request_from_bio(rq, rqd->bio);
} else {
rq->ioprio = IOPRIO_PRIO_VALUE(IOPRIO_CLASS_BE, IOPRIO_NORM);
rq->__data_len = 0;
}
return rq;
}
static int nvme_nvm_submit_user_cmd(struct request_queue *q,
struct nvme_ns *ns,
struct nvme_nvm_command *vcmd,
void __user *ubuf, unsigned int bufflen,
void __user *meta_buf, unsigned int meta_len,
void __user *ppa_buf, unsigned int ppa_len,
u32 *result, u64 *status, unsigned int timeout)
{
bool write = nvme_is_write((struct nvme_command *)vcmd);
struct nvm_dev *dev = ns->ndev;
struct gendisk *disk = ns->disk;
struct request *rq;
struct bio *bio = NULL;
__le64 *ppa_list = NULL;
dma_addr_t ppa_dma;
__le64 *metadata = NULL;
dma_addr_t metadata_dma;
DECLARE_COMPLETION_ONSTACK(wait);
int ret = 0;
rq = nvme_alloc_request(q, (struct nvme_command *)vcmd, 0,
NVME_QID_ANY);
if (IS_ERR(rq)) {
ret = -ENOMEM;
goto err_cmd;
}
rq->timeout = timeout ? timeout : ADMIN_TIMEOUT;
if (ppa_buf && ppa_len) {
ppa_list = dma_pool_alloc(dev->dma_pool, GFP_KERNEL, &ppa_dma);
if (!ppa_list) {
ret = -ENOMEM;
goto err_rq;
}
if (copy_from_user(ppa_list, (void __user *)ppa_buf,
sizeof(u64) * (ppa_len + 1))) {
ret = -EFAULT;
goto err_ppa;
}
vcmd->ph_rw.spba = cpu_to_le64(ppa_dma);
} else {
vcmd->ph_rw.spba = cpu_to_le64((uintptr_t)ppa_buf);
}
if (ubuf && bufflen) {
ret = blk_rq_map_user(q, rq, NULL, ubuf, bufflen, GFP_KERNEL);
if (ret)
goto err_ppa;
bio = rq->bio;
if (meta_buf && meta_len) {
metadata = dma_pool_alloc(dev->dma_pool, GFP_KERNEL,
&metadata_dma);
if (!metadata) {
ret = -ENOMEM;
goto err_map;
}
if (write) {
if (copy_from_user(metadata,
(void __user *)meta_buf,
meta_len)) {
ret = -EFAULT;
goto err_meta;
}
}
vcmd->ph_rw.metadata = cpu_to_le64(metadata_dma);
}
bio->bi_disk = disk;
}
blk_execute_rq(q, NULL, rq, 0);
if (nvme_req(rq)->flags & NVME_REQ_CANCELLED)
ret = -EINTR;
else if (nvme_req(rq)->status & 0x7ff)
ret = -EIO;
if (result)
*result = nvme_req(rq)->status & 0x7ff;
if (status)
*status = le64_to_cpu(nvme_req(rq)->result.u64);
if (metadata && !ret && !write) {
if (copy_to_user(meta_buf, (void *)metadata, meta_len))
ret = -EFAULT;
}
err_meta:
if (meta_buf && meta_len)
dma_pool_free(dev->dma_pool, metadata, metadata_dma);
err_map:
if (bio)
blk_rq_unmap_user(bio);
err_ppa:
if (ppa_buf && ppa_len)
dma_pool_free(dev->dma_pool, ppa_list, ppa_dma);
err_rq:
blk_mq_free_request(rq);
err_cmd:
return ret;
}
