static int
reservation_ns_register(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair,
uint16_t ns_id)
{
int ret;
struct spdk_nvme_reservation_register_data rr_data;
struct spdk_nvme_ns *ns;
ns = spdk_nvme_ctrlr_get_ns(ctrlr, ns_id);
rr_data.crkey = CR_KEY;
rr_data.nrkey = CR_KEY;
outstanding_commands = 0;
reserve_command_result = -1;
ret = spdk_nvme_ns_cmd_reservation_register(ns, qpair, &rr_data, true,
SPDK_NVME_RESERVE_REGISTER_KEY,
SPDK_NVME_RESERVE_PTPL_NO_CHANGES,
reservation_ns_completion, NULL);
if (ret) {
fprintf(stderr, "Reservation Register Failed\n");
return -1;
}
outstanding_commands++;
while (outstanding_commands) {
spdk_nvme_qpair_process_completions(qpair, 100);
}
if (reserve_command_result)
fprintf(stderr, "Reservation Register Failed\n");
return 0;
}
static int
reservation_ns_release(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair, uint16_t ns_id)
{
int ret;
struct spdk_nvme_reservation_key_data cdata;
struct spdk_nvme_ns *ns;
ns = spdk_nvme_ctrlr_get_ns(ctrlr, ns_id);
cdata.crkey = CR_KEY;
outstanding_commands = 0;
reserve_command_result = -1;
ret = spdk_nvme_ns_cmd_reservation_release(ns, qpair, &cdata,
false,
SPDK_NVME_RESERVE_RELEASE,
SPDK_NVME_RESERVE_WRITE_EXCLUSIVE,
reservation_ns_completion, NULL);
if (ret) {
fprintf(stderr, "Reservation Release Failed\n");
return -1;
}
outstanding_commands++;
while (outstanding_commands) {
spdk_nvme_qpair_process_completions(qpair, 100);
}
if (reserve_command_result)
fprintf(stderr, "Reservation Release Failed\n");
return 0;
}
static void
check_io(struct ns_worker_ctx *ns_ctx)
{
#if HAVE_LIBAIO
if (ns_ctx->entry->type == ENTRY_TYPE_AIO_FILE) {
aio_check_io(ns_ctx);
} else
#endif
{
spdk_nvme_qpair_process_completions(ns_ctx->u.nvme.qpair, g_max_completions);
}
}
static int
reservation_ns_report(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair, uint16_t ns_id)
{
int ret, i;
uint8_t *payload;
struct spdk_nvme_reservation_status_data *status;
struct spdk_nvme_reservation_ctrlr_data *cdata;
struct spdk_nvme_ns *ns;
ns = spdk_nvme_ctrlr_get_ns(ctrlr, ns_id);
payload = rte_zmalloc(NULL, 0x1000, 0x1000);
outstanding_commands = 0;
reserve_command_result = -1;
ret = spdk_nvme_ns_cmd_reservation_report(ns, qpair, payload, 0x1000,
reservation_ns_completion, NULL);
if (ret) {
fprintf(stderr, "Reservation Report Failed\n");
rte_free(payload);
return -1;
}
outstanding_commands++;
while (outstanding_commands) {
spdk_nvme_qpair_process_completions(qpair, 100);
}
if (reserve_command_result) {
fprintf(stderr, "Reservation Report Failed\n");
rte_free(payload);
return 0;
}
status = (struct spdk_nvme_reservation_status_data *)payload;
fprintf(stdout, "Reservation Generation Counter %u\n", status->generation);
fprintf(stdout, "Reservation type %u\n", status->type);
fprintf(stdout, "Reservation Number of Registered Controllers %u\n", status->nr_regctl);
fprintf(stdout, "Reservation Persist Through Power Loss State %u\n", status->ptpl_state);
for (i = 0; i < status->nr_regctl; i++) {
cdata = (struct spdk_nvme_reservation_ctrlr_data *)(payload + sizeof(struct
spdk_nvme_reservation_status_data) * (i + 1));
fprintf(stdout, "Controller ID %u\n", cdata->ctrlr_id);
fprintf(stdout, "Controller Reservation Status %u\n", cdata->rcsts.status);
fprintf(stdout, "Controller Host ID 0x%"PRIx64"\n", cdata->host_id);
fprintf(stdout, "Controller Reservation Key 0x%"PRIx64"\n", cdata->key);
}
rte_free(payload);
return 0;
}
static void
check_io(void)
{
uint64_t end, tsc_complete;
spdk_mb();
#if HAVE_LIBAIO
if (g_ns->type == ENTRY_TYPE_AIO_FILE) {
aio_check_io();
} else
#endif
{
spdk_nvme_qpair_process_completions(g_ns->u.nvme.qpair, 0);
}
spdk_mb();
end = spdk_get_ticks();
if (g_ns->current_queue_depth == 1) {
/*
* Account for race condition in AIO case where interrupt occurs
* after checking for queue depth. If the timestamp capture
* is too big compared to the last capture, assume that an
* interrupt fired, and do not bump the start tsc forward. This
* will ensure this extra time is accounted for next time through
* when we see current_queue_depth drop to 0.
*/
if (g_ns->type == ENTRY_TYPE_NVME_NS || (end - g_complete_tsc_start) < 500) {
g_complete_tsc_start = end;
}
} else {
tsc_complete = end - g_complete_tsc_start;
g_tsc_complete += tsc_complete;
if (tsc_complete < g_tsc_complete_min) {
g_tsc_complete_min = tsc_complete;
}
if (tsc_complete > g_tsc_complete_max) {
g_tsc_complete_max = tsc_complete;
}
g_io_completed++;
if (!g_ns->is_draining) {
submit_single_io();
}
g_complete_tsc_start = spdk_get_ticks();
}
}
static int
reservation_ns_acquire(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair, uint16_t ns_id)
{
int ret;
struct spdk_nvme_reservation_acquire_data *cdata;
struct spdk_nvme_ns *ns;
ns = spdk_nvme_ctrlr_get_ns(ctrlr, ns_id);
cdata = rte_zmalloc(NULL, sizeof(struct spdk_nvme_reservation_acquire_data), 0);
cdata->crkey = CR_KEY;
outstanding_commands = 0;
reserve_command_result = -1;
ret = spdk_nvme_ns_cmd_reservation_acquire(ns, qpair, cdata,
0,
SPDK_NVME_RESERVE_ACQUIRE,
SPDK_NVME_RESERVE_WRITE_EXCLUSIVE,
reservation_ns_completion, NULL);
if (ret) {
fprintf(stderr, "Reservation Acquire Failed\n");
rte_free(cdata);
return -1;
}
outstanding_commands++;
while (outstanding_commands) {
spdk_nvme_qpair_process_completions(qpair, 100);
}
if (reserve_command_result)
fprintf(stderr, "Reservation Acquire Failed\n");
rte_free(cdata);
return 0;
}
static void
hello_world(void)
{
struct ns_entry *ns_entry;
struct hello_world_sequence sequence;
int rc;
ns_entry = g_namespaces;
while (ns_entry != NULL) {
/*
* Allocate an I/O qpair that we can use to submit read/write requests
* to namespaces on the controller. NVMe controllers typically support
* many qpairs per controller. Any I/O qpair allocated for a controller
* can submit I/O to any namespace on that controller.
*
* The SPDK NVMe driver provides no synchronization for qpair accesses -
* the application must ensure only a single thread submits I/O to a
* qpair, and that same thread must also check for completions on that
* qpair. This enables extremely efficient I/O processing by making all
* I/O operations completely lockless.
*/
ns_entry->qpair = spdk_nvme_ctrlr_alloc_io_qpair(ns_entry->ctrlr, 0);
if (ns_entry->qpair == NULL) {
printf("ERROR: init_ns_worker_ctx() failed\n");
return;
}
/*
* Use DPDK rte_zmalloc to allocate a 4KB zeroed buffer. This memory
* will be allocated from 2MB hugepages and will be pinned. These are
* both requirements for data buffers used for SPDK NVMe I/O operations.
*/
sequence.buf = rte_zmalloc(NULL, 0x1000, 0x1000);
sequence.is_completed = 0;
sequence.ns_entry = ns_entry;
/*
* Print "Hello world!" to sequence.buf. We will write this data to LBA
* 0 on the namespace, and then later read it back into a separate buffer
* to demonstrate the full I/O path.
*/
sprintf(sequence.buf, "Hello world!\n");
/*
* Write the data buffer to LBA 0 of this namespace. "write_complete" and
* "&sequence" are specified as the completion callback function and
* argument respectively. write_complete() will be called with the
* value of &sequence as a parameter when the write I/O is completed.
* This allows users to potentially specify different completion
* callback routines for each I/O, as well as pass a unique handle
* as an argument so the application knows which I/O has completed.
*
* Note that the SPDK NVMe driver will only check for completions
* when the application calls spdk_nvme_qpair_process_completions().
* It is the responsibility of the application to trigger the polling
* process.
*/
rc = spdk_nvme_ns_cmd_write(ns_entry->ns, ns_entry->qpair, sequence.buf,
0, /* LBA start */
1, /* number of LBAs */
write_complete, &sequence, 0);
if (rc != 0) {
fprintf(stderr, "starting write I/O failed\n");
exit(1);
}
/*
* Poll for completions. 0 here means process all available completions.
* In certain usage models, the caller may specify a positive integer
* instead of 0 to signify the maximum number of completions it should
* process. This function will never block - if there are no
* completions pending on the specified qpair, it will return immediately.
*
* When the write I/O completes, write_complete() will submit a new I/O
* to read LBA 0 into a separate buffer, specifying read_complete() as its
* completion routine. When the read I/O completes, read_complete() will
* print the buffer contents and set sequence.is_completed = 1. That will
* break this loop and then exit the program.
*/
while (!sequence.is_completed) {
spdk_nvme_qpair_process_completions(ns_entry->qpair, 0);
}
/*
* Free the I/O qpair. This typically is done when an application exits.
* But SPDK does support freeing and then reallocating qpairs during
* operation. It is the responsibility of the caller to ensure all
* pending I/O are completed before trying to free the qpair.
*/
spdk_nvme_ctrlr_free_io_qpair(ns_entry->qpair);
ns_entry = ns_entry->next;
}
}
static int
write_read_e2e_dp_tests(struct dev *dev, nvme_build_io_req_fn_t build_io_fn, const char *test_name)
{
int rc = 0;
uint32_t lba_count;
uint32_t io_flags = 0;
struct io_request *req;
struct spdk_nvme_ns *ns;
struct spdk_nvme_qpair *qpair;
const struct spdk_nvme_ns_data *nsdata;
ns = spdk_nvme_ctrlr_get_ns(dev->ctrlr, 1);
if (!ns) {
fprintf(stderr, "Null namespace\n");
return 0;
}
if (!(spdk_nvme_ns_get_flags(ns) & SPDK_NVME_NS_DPS_PI_SUPPORTED))
return 0;
nsdata = spdk_nvme_ns_get_data(ns);
if (!nsdata || !spdk_nvme_ns_get_sector_size(ns)) {
fprintf(stderr, "Empty nsdata or wrong sector size\n");
return 0;
}
req = rte_zmalloc(NULL, sizeof(*req), 0);
if (!req) {
fprintf(stderr, "Allocate request failed\n");
return 0;
}
/* IO parameters setting */
lba_count = build_io_fn(ns, req, &io_flags);
if (!lba_count) {
fprintf(stderr, "%s: %s bypass the test case\n", dev->name, test_name);
free_req(req);
return 0;
}
qpair = spdk_nvme_ctrlr_alloc_io_qpair(dev->ctrlr, 0);
if (!qpair) {
free_req(req);
return -1;
}
ns_data_buffer_reset(ns, req, DATA_PATTERN);
if (req->use_extended_lba)
rc = spdk_nvme_ns_cmd_write(ns, qpair, req->contig, req->lba, lba_count,
io_complete, req, io_flags);
else
rc = spdk_nvme_ns_cmd_write_with_md(ns, qpair, req->contig, req->metadata, req->lba, lba_count,
io_complete, req, io_flags, req->apptag_mask, req->apptag);
if (rc != 0) {
fprintf(stderr, "%s: %s write submit failed\n", dev->name, test_name);
spdk_nvme_ctrlr_free_io_qpair(qpair);
free_req(req);
return -1;
}
io_complete_flag = 0;
while (!io_complete_flag)
spdk_nvme_qpair_process_completions(qpair, 1);
if (io_complete_flag != 1) {
fprintf(stderr, "%s: %s write exec failed\n", dev->name, test_name);
spdk_nvme_ctrlr_free_io_qpair(qpair);
free_req(req);
return -1;
}
/* reset completion flag */
io_complete_flag = 0;
ns_data_buffer_reset(ns, req, 0);
if (req->use_extended_lba)
rc = spdk_nvme_ns_cmd_read(ns, qpair, req->contig, req->lba, lba_count,
io_complete, req, io_flags);
else
rc = spdk_nvme_ns_cmd_read_with_md(ns, qpair, req->contig, req->metadata, req->lba, lba_count,
io_complete, req, io_flags, req->apptag_mask, req->apptag);
if (rc != 0) {
fprintf(stderr, "%s: %s read failed\n", dev->name, test_name);
spdk_nvme_ctrlr_free_io_qpair(qpair);
free_req(req);
return -1;
}
while (!io_complete_flag)
spdk_nvme_qpair_process_completions(qpair, 1);
if (io_complete_flag != 1) {
fprintf(stderr, "%s: %s read failed\n", dev->name, test_name);
spdk_nvme_ctrlr_free_io_qpair(qpair);
free_req(req);
return -1;
}
rc = ns_data_buffer_compare(ns, req, DATA_PATTERN);
if (rc < 0) {
fprintf(stderr, "%s: %s write/read success, but memcmp Failed\n", dev->name, test_name);
spdk_nvme_ctrlr_free_io_qpair(qpair);
free_req(req);
return -1;
}
fprintf(stdout, "%s: %s test passed\n", dev->name, test_name);
spdk_nvme_ctrlr_free_io_qpair(qpair);
free_req(req);
return rc;
}
static
int nvme_ns_identify_update(struct spdk_nvme_ns *ns)
{
struct nvme_completion_poll_status status;
struct spdk_nvme_ns_data *nsdata;
int rc;
nsdata = _nvme_ns_get_data(ns);
status.done = false;
rc = nvme_ctrlr_cmd_identify_namespace(ns->ctrlr, ns->id, nsdata,
nvme_completion_poll_cb, &status);
if (rc != 0) {
return rc;
}
while (status.done == false) {
pthread_mutex_lock(&ns->ctrlr->ctrlr_lock);
spdk_nvme_qpair_process_completions(&ns->ctrlr->adminq, 0);
pthread_mutex_unlock(&ns->ctrlr->ctrlr_lock);
}
if (spdk_nvme_cpl_is_error(&status.cpl)) {
/* This can occur if the namespace is not active. Simply zero the
* namespace data and continue. */
memset(nsdata, 0, sizeof(*nsdata));
ns->stripe_size = 0;
ns->sector_size = 0;
ns->md_size = 0;
ns->pi_type = 0;
ns->sectors_per_max_io = 0;
ns->sectors_per_stripe = 0;
ns->flags = 0;
return 0;
}
ns->sector_size = 1 << nsdata->lbaf[nsdata->flbas.format].lbads;
ns->sectors_per_max_io = spdk_nvme_ns_get_max_io_xfer_size(ns) / ns->sector_size;
ns->sectors_per_stripe = ns->stripe_size / ns->sector_size;
ns->flags = 0x0000;
if (ns->ctrlr->cdata.oncs.dsm) {
ns->flags |= SPDK_NVME_NS_DEALLOCATE_SUPPORTED;
}
if (ns->ctrlr->cdata.vwc.present) {
ns->flags |= SPDK_NVME_NS_FLUSH_SUPPORTED;
}
if (ns->ctrlr->cdata.oncs.write_zeroes) {
ns->flags |= SPDK_NVME_NS_WRITE_ZEROES_SUPPORTED;
}
if (nsdata->nsrescap.raw) {
ns->flags |= SPDK_NVME_NS_RESERVATION_SUPPORTED;
}
ns->md_size = nsdata->lbaf[nsdata->flbas.format].ms;
ns->pi_type = SPDK_NVME_FMT_NVM_PROTECTION_DISABLE;
if (nsdata->lbaf[nsdata->flbas.format].ms && nsdata->dps.pit) {
ns->flags |= SPDK_NVME_NS_DPS_PI_SUPPORTED;
ns->pi_type = nsdata->dps.pit;
if (nsdata->flbas.extended)
ns->flags |= SPDK_NVME_NS_EXTENDED_LBA_SUPPORTED;
}
return rc;
}
static int
u2_lat_bench(void)
{
int i, rc;
//int rc;
void *buf;
uint32_t io_size_blocks;
uint64_t offset_in_ios, size_in_ios;
uint64_t tsc_rate;
uint64_t tsc_start, tsc_elapsed;
//uint64_t tsc_end;
buf = rte_malloc(NULL, io_size, U2_BUFFER_ALIGN);
if (buf == NULL) {
fprintf(stderr, "failed to rte_malloc buffer!\n");
return 1;
}
memset(buf, 0xff, io_size);
//io_num = 0;
//io_depth = 0;
io_size_blocks = io_size / u2_ns_sector;
offset_in_ios = -1;
size_in_ios = u2_ns_size / io_size;
tsc_rate = rte_get_tsc_hz();
tsc_elapsed = 0;
tsc_start = rte_get_timer_cycles();
//tsc_end = rte_get_timer_cycles() + time_in_sec * tsc_rate;
for (i = 0; i < io_num; i++) {
//while (1) {
if (is_random) {
offset_in_ios = rand_r(&seed) % size_in_ios;
} else {
if (++offset_in_ios >= size_in_ios) {
offset_in_ios = 0;
}
}
if (is_rw) {
rc = spdk_nvme_ns_cmd_read (u2_ns, u2_qpair, buf, offset_in_ios * io_size_blocks, io_size_blocks, u2_io_complete, NULL, 0);
} else {
rc = spdk_nvme_ns_cmd_write(u2_ns, u2_qpair, buf, offset_in_ios * io_size_blocks, io_size_blocks, u2_io_complete, NULL, 0);
}
if (rc) {
fprintf(stderr, "failed to submit request %d!\n", i);
//fprintf(stderr, "failed to submit request %d!\n", io_num);
return rc;
}
io_depth++; // for latency benchmarking, queue depth stays at 1.
while (io_depth > 0) {
spdk_nvme_qpair_process_completions(u2_qpair, 0);
}
//if (rte_get_timer_cycles() > tsc_end) {
// break;
//}
}
tsc_elapsed = rte_get_timer_cycles() - tsc_start;
printf("\t\t%9.1f us", (float) (tsc_elapsed * 1000000) / (io_num * tsc_rate));
printf("\t\t%10.1f s", (float) tsc_elapsed / tsc_rate);
printf("\n");
//printf("\t\t%9.1f us", (float) (time_in_sec * 1000000) / io_num);
//printf("\t\t%12"PRIu64"\n", io_num);
rte_free(buf);
return 0;
}