static void
nvmf_direct_ctrlr_complete_cmd(void *ctx, const struct spdk_nvme_cpl *cmp)
{
struct spdk_nvmf_request *req = ctx;
spdk_trace_record(TRACE_NVMF_LIB_COMPLETE, 0, 0, (uint64_t)req, 0);
req->rsp->nvme_cpl = *cmp;
spdk_nvmf_request_complete(req);
}
static bool
nvmf_process_io_cmd(struct spdk_nvmf_request *req)
{
struct nvmf_session *session = req->conn->sess;
struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;
struct spdk_nvme_cpl *response;
struct spdk_nvmf_subsystem *subsystem = session->subsys;
struct spdk_nvmf_namespace *nvmf_ns;
struct spdk_nvme_ctrlr *ctrlr = NULL;
struct spdk_nvme_ns *ns = NULL;
struct spdk_nvme_qpair *qpair;
uint32_t nsid = 0;
struct nvme_read_cdw12 *cdw12;
uint64_t lba_address;
uint32_t lba_count;
uint32_t io_flags;
int rc = 0;
/* pre-set response details for this command */
response = &req->rsp->nvme_cpl;
response->status.sc = SPDK_NVME_SC_SUCCESS;
response->cid = cmd->cid;
/* verify subsystem */
if (subsystem == NULL) {
SPDK_ERRLOG("Subsystem Not Initialized!\n");
response->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
return true;
}
/* verify that the contoller is ready to process commands */
if (session->vcprop.csts.bits.rdy == 0) {
SPDK_ERRLOG("Subsystem Controller Not Ready!\n");
response->status.sc = SPDK_NVME_SC_NAMESPACE_NOT_READY;
return true;
}
/* verify namespace id */
if (cmd->nsid == 0 || cmd->nsid > MAX_PER_SUBSYSTEM_NAMESPACES) {
SPDK_ERRLOG("Invalid NS_ID %u\n", cmd->nsid);
response->status.sc = SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT;
return true;
}
nvmf_ns = &subsystem->ns_list_map[cmd->nsid - 1];
ctrlr = nvmf_ns->ctrlr;
nsid = nvmf_ns->nvme_ns_id;
ns = nvmf_ns->ns;
qpair = nvmf_ns->qpair;
switch (cmd->opc) {
case SPDK_NVME_OPC_READ:
case SPDK_NVME_OPC_WRITE:
cdw12 = (struct nvme_read_cdw12 *)&cmd->cdw12;
/* NVMe library read/write interface expects non-0based lba_count value */
lba_count = cdw12->nlb + 1;
lba_address = cmd->cdw11;
lba_address = (lba_address << 32) + cmd->cdw10;
io_flags = cmd->cdw12 & 0xFFFF0000U;
if (cmd->opc == SPDK_NVME_OPC_READ) {
SPDK_TRACELOG(SPDK_TRACE_NVMF, "Read LBA 0x%" PRIx64 ", 0x%x blocks\n",
lba_address, lba_count);
spdk_trace_record(TRACE_NVMF_LIB_READ_START, 0, 0, (uint64_t)req, 0);
rc = spdk_nvme_ns_cmd_read(ns, qpair,
req->data, lba_address, lba_count,
nvmf_complete_cmd,
req, io_flags);
} else {
SPDK_TRACELOG(SPDK_TRACE_NVMF, "Write LBA 0x%" PRIx64 ", 0x%x blocks\n",
lba_address, lba_count);
spdk_trace_record(TRACE_NVMF_LIB_WRITE_START, 0, 0, (uint64_t)req, 0);
rc = spdk_nvme_ns_cmd_write(ns, qpair,
req->data, lba_address, lba_count,
nvmf_complete_cmd,
req, io_flags);
}
break;
default:
SPDK_TRACELOG(SPDK_TRACE_NVMF, "io_cmd passthrough: opc 0x%02x\n", cmd->opc);
cmd->nsid = nsid;
rc = spdk_nvme_ctrlr_cmd_io_raw(ctrlr, qpair,
cmd,
req->data, req->length,
nvmf_complete_cmd,
req);
break;
}
if (rc) {
SPDK_ERRLOG("Failed to submit Opcode 0x%02x\n", cmd->opc);
response->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
return true;
}
return false;
}
static int nvmf_cq_event_handler(struct spdk_nvmf_conn *conn)
{
struct ibv_wc wc;
struct nvme_qp_tx_desc *tx_desc;
struct nvmf_request *req;
int rc;
int cq_count = 0;
int i;
for (i = 0; i < conn->sq_depth; i++) {
tx_desc = NULL;
/* if an overflow condition was hit
we want to stop all processing, but
do not disconnect.
*/
if (conn->state == CONN_STATE_OVERFLOW)
break;
rc = ibv_poll_cq(conn->cq, 1, &wc);
if (rc == 0) // No completions at this time
break;
if (rc < 0) {
SPDK_ERRLOG("Poll CQ error!(%d): %s\n",
errno, strerror(errno));
goto handler_error;
}
/* OK, process the single successful cq event */
cq_count += rc;
if (wc.status) {
SPDK_TRACELOG(SPDK_TRACE_RDMA, "CQ completion error status %d, exiting handler\n",
wc.status);
break;
}
switch (wc.opcode) {
case IBV_WC_SEND:
SPDK_TRACELOG(SPDK_TRACE_RDMA, "\nCQ send completion\n");
tx_desc = (struct nvme_qp_tx_desc *)wc.wr_id;
nvmf_deactive_tx_desc(tx_desc);
break;
case IBV_WC_RDMA_WRITE:
/*
* Will get this event only if we set IBV_SEND_SIGNALED
* flag in rdma_write, to trace rdma write latency
*/
SPDK_TRACELOG(SPDK_TRACE_RDMA, "\nCQ rdma write completion\n");
tx_desc = (struct nvme_qp_tx_desc *)wc.wr_id;
spdk_trace_record(TRACE_RDMA_WRITE_COMPLETE, 0, 0, (uint64_t)tx_desc->rx_desc, 0);
break;
case IBV_WC_RDMA_READ:
SPDK_TRACELOG(SPDK_TRACE_RDMA, "\nCQ rdma read completion\n");
tx_desc = (struct nvme_qp_tx_desc *)wc.wr_id;
spdk_trace_record(TRACE_RDMA_READ_COMPLETE, 0, 0, (uint64_t)tx_desc->rx_desc, 0);
req = &tx_desc->req_state;
if (req->pending == NVMF_PENDING_WRITE) {
req->pending = NVMF_PENDING_NONE;
rc = nvmf_io_cmd_continue(conn, tx_desc);
if (rc) {
SPDK_ERRLOG("error from io cmd continue\n");
goto handler_error;
}
/*
* Check for any pending rdma_reads to start
*/
conn->pending_rdma_read_count--;
if (!STAILQ_EMPTY(&conn->qp_pending_desc)) {
tx_desc = STAILQ_FIRST(&conn->qp_pending_desc);
STAILQ_REMOVE_HEAD(&conn->qp_pending_desc, link);
STAILQ_INSERT_TAIL(&conn->qp_tx_active_desc, tx_desc, link);
SPDK_TRACELOG(SPDK_TRACE_RDMA, "Issue rdma read from pending queue: tx_desc %p\n",
tx_desc);
rc = nvmf_post_rdma_read(conn, tx_desc);
if (rc) {
SPDK_ERRLOG("Unable to post pending rdma read descriptor\n");
goto handler_error;
}
}
} else if (req->pending == NVMF_PENDING_CONNECT) {
req->pending = NVMF_PENDING_NONE;
nvmf_connect_continue(conn, tx_desc);
}
break;
case IBV_WC_RECV:
SPDK_TRACELOG(SPDK_TRACE_RDMA, "\nCQ recv completion\n");
spdk_trace_record(TRACE_NVMF_IO_START, 0, 0, wc.wr_id, 0);
rc = nvmf_recv(conn, &wc);
if (rc) {
SPDK_ERRLOG("nvmf_recv processing failure\n");
goto handler_error;
}
break;
default:
SPDK_ERRLOG("Poll cq opcode type unknown!!!!! completion\n");
goto handler_error;
}
}
return cq_count;
handler_error:
if (tx_desc != NULL)
nvmf_deactive_tx_desc(tx_desc);
SPDK_ERRLOG("handler error, exiting!\n");
return -1;
}