int
spdk_nvmf_request_exec(struct spdk_nvmf_request *req)
{
struct spdk_nvmf_session *session = req->conn->sess;
struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;
struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl;
spdk_nvmf_request_exec_status status;
nvmf_trace_command(req->cmd, req->conn->type);
if (cmd->opc == SPDK_NVME_OPC_FABRIC) {
status = nvmf_process_fabrics_command(req);
} else if (session == NULL || !session->vcprop.cc.bits.en) {
/* Only Fabric commands are allowed when the controller is disabled */
SPDK_ERRLOG("Non-Fabric command sent to disabled controller\n");
rsp->status.sc = SPDK_NVME_SC_COMMAND_SEQUENCE_ERROR;
status = SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
} else if (req->conn->type == CONN_TYPE_AQ) {
struct spdk_nvmf_subsystem *subsystem;
subsystem = session->subsys;
assert(subsystem != NULL);
if (subsystem->subtype == SPDK_NVMF_SUBTYPE_DISCOVERY) {
status = nvmf_process_discovery_cmd(req);
} else {
status = session->subsys->ops->process_admin_cmd(req);
}
} else {
status = session->subsys->ops->process_io_cmd(req);
}
switch (status) {
case SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE:
return spdk_nvmf_request_complete(req);
case SPDK_NVMF_REQUEST_EXEC_STATUS_RELEASE:
if (req->conn->transport->req_release(req)) {
SPDK_ERRLOG("Transport request release error!\n");
return -1;
}
return 0;
case SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS:
return 0;
default:
SPDK_ERRLOG("Unknown request exec status: 0x%x\n", status);
return -1;
}
return 0;
}
int
spdk_nvmf_request_prep_data(struct spdk_nvmf_request *req,
void *in_cap_data, uint32_t in_cap_len,
void *bb, uint32_t bb_len)
{
struct spdk_nvmf_conn *conn = req->conn;
struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;
struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl;
enum spdk_nvme_data_transfer xfer;
int ret;
nvmf_trace_command(req->cmd, conn->type);
req->length = 0;
req->xfer = SPDK_NVME_DATA_NONE;
req->data = NULL;
if (cmd->opc == SPDK_NVME_OPC_FABRIC) {
xfer = spdk_nvme_opc_get_data_transfer(req->cmd->nvmf_cmd.fctype);
} else {
xfer = spdk_nvme_opc_get_data_transfer(cmd->opc);
}
if (xfer != SPDK_NVME_DATA_NONE) {
struct spdk_nvme_sgl_descriptor *sgl = (struct spdk_nvme_sgl_descriptor *)&cmd->dptr.sgl1;
if (sgl->generic.type == SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK &&
(sgl->keyed.subtype == SPDK_NVME_SGL_SUBTYPE_ADDRESS ||
sgl->keyed.subtype == SPDK_NVME_SGL_SUBTYPE_INVALIDATE_KEY)) {
if (sgl->keyed.length > bb_len) {
SPDK_ERRLOG("SGL length 0x%x exceeds BB length 0x%x\n",
sgl->keyed.length, bb_len);
rsp->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID;
return -1;
}
req->data = bb;
req->length = sgl->keyed.length;
} else if (sgl->generic.type == SPDK_NVME_SGL_TYPE_DATA_BLOCK &&
sgl->unkeyed.subtype == SPDK_NVME_SGL_SUBTYPE_OFFSET) {
uint64_t offset = sgl->address;
uint32_t max_len = in_cap_len;
SPDK_TRACELOG(SPDK_TRACE_NVMF, "In-capsule data: offset 0x%" PRIx64 ", length 0x%x\n",
offset, sgl->unkeyed.length);
if (conn->type == CONN_TYPE_AQ) {
SPDK_ERRLOG("In-capsule data not allowed for admin queue\n");
return -1;
}
if (offset > max_len) {
SPDK_ERRLOG("In-capsule offset 0x%" PRIx64 " exceeds capsule length 0x%x\n",
offset, max_len);
rsp->status.sc = SPDK_NVME_SC_INVALID_SGL_OFFSET;
return -1;
}
max_len -= (uint32_t)offset;
if (sgl->unkeyed.length > max_len) {
SPDK_ERRLOG("In-capsule data length 0x%x exceeds capsule length 0x%x\n",
sgl->unkeyed.length, max_len);
rsp->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID;
return -1;
}
req->data = in_cap_data + offset;
req->length = sgl->unkeyed.length;
} else {
SPDK_ERRLOG("Invalid NVMf I/O Command SGL: Type 0x%x, Subtype 0x%x\n",
sgl->generic.type, sgl->generic.subtype);
rsp->status.sc = SPDK_NVME_SC_SGL_DESCRIPTOR_TYPE_INVALID;
return -1;
}
if (req->length == 0) {
xfer = SPDK_NVME_DATA_NONE;
req->data = NULL;
}
req->xfer = xfer;
/*
* For any I/O that requires data to be
* pulled into target BB before processing by
* the backend NVMe device
*/
if (xfer == SPDK_NVME_DATA_HOST_TO_CONTROLLER) {
if (sgl->generic.type == SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK) {
SPDK_TRACELOG(SPDK_TRACE_NVMF, "Initiating Host to Controller data transfer\n");
ret = nvmf_post_rdma_read(conn, req);
if (ret) {
SPDK_ERRLOG("Unable to post rdma read tx descriptor\n");
rsp->status.sc = SPDK_NVME_SC_DATA_TRANSFER_ERROR;
return -1;
}
/* Wait for transfer to complete before executing command. */
return 1;
}
}
}
if (xfer == SPDK_NVME_DATA_NONE) {
SPDK_TRACELOG(SPDK_TRACE_NVMF, "No data to transfer\n");
RTE_VERIFY(req->data == NULL);
RTE_VERIFY(req->length == 0);
} else {
RTE_VERIFY(req->data != NULL);
RTE_VERIFY(req->length != 0);
SPDK_TRACELOG(SPDK_TRACE_NVMF, "%s data ready\n",
xfer == SPDK_NVME_DATA_HOST_TO_CONTROLLER ? "Host to Controller" :
"Controller to Host");
}
return 0;
}
static int nvmf_recv(struct spdk_nvmf_conn *conn, struct ibv_wc *wc)
{
struct nvme_qp_rx_desc *rx_desc;
struct nvme_qp_tx_desc *tx_desc = NULL;
struct spdk_nvmf_capsule_cmd *cap_hdr;
struct nvmf_request *req;
int ret = 0;
rx_desc = (struct nvme_qp_rx_desc *)wc->wr_id;
cap_hdr = (struct spdk_nvmf_capsule_cmd *)&rx_desc->msg_buf;
/* Update Connection SQ Tracking, increment
the SQ tail consuming a free RX recv slot.
Check for exceeding queue full - should
never happen.
*/
conn->sq_tail < (conn->sq_depth - 1) ? (conn->sq_tail++) : (conn->sq_tail = 0);
SPDK_TRACELOG(SPDK_TRACE_DEBUG, "sq_head %x, sq_tail %x, sq_depth %x\n",
conn->sq_head, conn->sq_tail, conn->sq_depth);
/* trap if initiator exceeds qdepth */
if (conn->sq_head == conn->sq_tail) {
SPDK_ERRLOG(" *** SQ Overflow !! ***\n");
/* controller fatal status condition:
set the cfs flag in controller status
and stop processing this and any I/O
on this queue.
*/
if (conn->sess) {
conn->sess->vcprop.csts.bits.cfs = 1;
conn->state = CONN_STATE_OVERFLOW;
}
if (conn->type == CONN_TYPE_IOQ) {
/* if overflow on the I/O queue
stop processing, allow for
remote host to query failure
via admin queue
*/
goto drop_recv;
} else {
/* if overflow on the admin queue
there is no recovery, error out
to trigger disconnect
*/
goto recv_error;
}
}
if (wc->byte_len < sizeof(*cap_hdr)) {
SPDK_ERRLOG("recv length less than capsule header\n");
goto recv_error;
}
rx_desc->recv_bc = wc->byte_len;
SPDK_TRACELOG(SPDK_TRACE_NVMF, "recv byte count %x\n", rx_desc->recv_bc);
/* get a response buffer */
if (STAILQ_EMPTY(&conn->qp_tx_desc)) {
SPDK_ERRLOG("tx desc pool empty!\n");
goto recv_error;
}
tx_desc = STAILQ_FIRST(&conn->qp_tx_desc);
nvmf_active_tx_desc(tx_desc);
tx_desc->rx_desc = rx_desc;
req = &tx_desc->req_state;
req->session = conn->sess;
req->fabric_tx_ctx = tx_desc;
req->fabric_rx_ctx = rx_desc;
req->cb_fn = nvmf_process_async_completion;
req->length = 0;
req->cid = cap_hdr->cid;
req->cmd = &rx_desc->msg_buf;
nvmf_trace_command(cap_hdr, conn->type);
if (cap_hdr->opcode == SPDK_NVMF_FABRIC_OPCODE) {
ret = nvmf_process_fabrics_command(conn, tx_desc);
} else if (conn->type == CONN_TYPE_AQ) {
ret = nvmf_process_admin_command(conn, tx_desc);
} else {
ret = nvmf_process_io_command(conn, tx_desc);
}
if (ret < 0) {
goto recv_error;
}
/* re-post rx_desc and re-queue tx_desc here,
there is not a delayed posting because of
command processing.
*/
if (ret == 1) {
tx_desc->rx_desc = NULL;
nvmf_deactive_tx_desc(tx_desc);
if (nvmf_post_rdma_recv(conn, rx_desc)) {
SPDK_ERRLOG("Unable to re-post aq rx descriptor\n");
goto recv_error;
}
}
drop_recv:
return 0;
recv_error:
/* recover the tx_desc */
if (tx_desc != NULL) {
tx_desc->rx_desc = NULL;
nvmf_deactive_tx_desc(tx_desc);
}
return -1;
}
