static void
nvme_ns_io_test_cb(void *arg, const struct nvme_completion *cpl)
{
struct nvme_io_test_thread *tth = arg;
struct timeval t;
tth->io_completed++;
if (nvme_completion_is_error(cpl)) {
printf("%s: error occurred\n", __func__);
wakeup_one(tth);
return;
}
getmicrouptime(&t);
timevalsub(&t, &tth->start);
if (t.tv_sec >= tth->time) {
wakeup_one(tth);
return;
}
switch (tth->opc) {
case NVME_OPC_WRITE:
nvme_ns_cmd_write(tth->ns, tth->buf, tth->idx * 2048,
tth->size/nvme_ns_get_sector_size(tth->ns),
nvme_ns_io_test_cb, tth);
break;
case NVME_OPC_READ:
nvme_ns_cmd_read(tth->ns, tth->buf, tth->idx * 2048,
tth->size/nvme_ns_get_sector_size(tth->ns),
nvme_ns_io_test_cb, tth);
break;
default:
break;
}
}
int
nvme_ns_cmd_read(struct nvme_namespace *ns, void *payload, uint64_t lba,
uint32_t lba_count, nvme_cb_fn_t cb_fn, void *cb_arg)
{
struct nvme_request *req;
struct nvme_command *cmd;
req = nvme_allocate_request_vaddr(payload,
lba_count*nvme_ns_get_sector_size(ns), cb_fn, cb_arg);
if (req == NULL)
return (ENOMEM);
cmd = &req->cmd;
cmd->opc = NVME_OPC_READ;
cmd->nsid = ns->id;
/* TODO: create a read command data structure */
*(uint64_t *)&cmd->cdw10 = lba;
cmd->cdw12 = lba_count-1;
nvme_ctrlr_submit_io_request(ns->ctrlr, req);
return (0);
}
static void *
nvd_new_disk(struct nvme_namespace *ns, void *ctrlr_arg)
{
uint8_t descr[NVME_MODEL_NUMBER_LENGTH+1];
struct nvd_disk *ndisk;
struct disk *disk;
struct nvd_controller *ctrlr = ctrlr_arg;
ndisk = malloc(sizeof(struct nvd_disk), M_NVD, M_ZERO | M_WAITOK);
disk = disk_alloc();
disk->d_strategy = nvd_strategy;
disk->d_ioctl = nvd_ioctl;
disk->d_name = NVD_STR;
disk->d_drv1 = ndisk;
disk->d_maxsize = nvme_ns_get_max_io_xfer_size(ns);
disk->d_sectorsize = nvme_ns_get_sector_size(ns);
disk->d_mediasize = (off_t)nvme_ns_get_size(ns);
if (TAILQ_EMPTY(&disk_head))
disk->d_unit = 0;
else
disk->d_unit =
TAILQ_LAST(&disk_head, disk_list)->disk->d_unit + 1;
disk->d_flags = 0;
if (nvme_ns_get_flags(ns) & NVME_NS_DEALLOCATE_SUPPORTED)
disk->d_flags |= DISKFLAG_CANDELETE;
if (nvme_ns_get_flags(ns) & NVME_NS_FLUSH_SUPPORTED)
disk->d_flags |= DISKFLAG_CANFLUSHCACHE;
/* ifdef used here to ease porting to stable branches at a later point. */
#ifdef DISKFLAG_UNMAPPED_BIO
disk->d_flags |= DISKFLAG_UNMAPPED_BIO;
#endif
/*
* d_ident and d_descr are both far bigger than the length of either
* the serial or model number strings.
*/
nvme_strvis(disk->d_ident, nvme_ns_get_serial_number(ns),
sizeof(disk->d_ident), NVME_SERIAL_NUMBER_LENGTH);
nvme_strvis(descr, nvme_ns_get_model_number(ns), sizeof(descr),
NVME_MODEL_NUMBER_LENGTH);
#if __FreeBSD_version >= 900034
strlcpy(disk->d_descr, descr, sizeof(descr));
#endif
ndisk->ns = ns;
ndisk->disk = disk;
ndisk->cur_depth = 0;
mtx_init(&ndisk->bioqlock, "NVD bioq lock", NULL, MTX_DEF);
bioq_init(&ndisk->bioq);
TASK_INIT(&ndisk->bioqtask, 0, nvd_bioq_process, ndisk);
ndisk->tq = taskqueue_create("nvd_taskq", M_WAITOK,
taskqueue_thread_enqueue, &ndisk->tq);
taskqueue_start_threads(&ndisk->tq, 1, PI_DISK, "nvd taskq");
TAILQ_INSERT_TAIL(&disk_head, ndisk, global_tailq);
TAILQ_INSERT_TAIL(&ctrlr->disk_head, ndisk, ctrlr_tailq);
disk_create(disk, DISK_VERSION);
printf(NVD_STR"%u: <%s> NVMe namespace\n", disk->d_unit, descr);
printf(NVD_STR"%u: %juMB (%ju %u byte sectors)\n", disk->d_unit,
(uintmax_t)disk->d_mediasize / (1024*1024),
(uintmax_t)disk->d_mediasize / disk->d_sectorsize,
disk->d_sectorsize);
return (NULL);
}
static void
nvme_ns_bio_test(void *arg)
{
struct nvme_io_test_internal *io_test = arg;
struct cdevsw *csw;
struct mtx *mtx;
struct bio *bio;
struct cdev *dev;
void *buf;
struct timeval t;
uint64_t offset;
uint32_t idx, io_completed = 0;
#if __FreeBSD_version >= 900017
int ref;
#endif
buf = malloc(io_test->size, M_NVME, M_WAITOK);
idx = atomic_fetchadd_int(&io_test->td_idx, 1);
dev = io_test->ns->cdev;
offset = idx * 2048 * nvme_ns_get_sector_size(io_test->ns);
while (1) {
bio = g_alloc_bio();
memset(bio, 0, sizeof(*bio));
bio->bio_cmd = (io_test->opc == NVME_OPC_READ) ?
BIO_READ : BIO_WRITE;
bio->bio_done = nvme_ns_bio_test_cb;
bio->bio_dev = dev;
bio->bio_offset = offset;
bio->bio_data = buf;
bio->bio_bcount = io_test->size;
if (io_test->flags & NVME_TEST_FLAG_REFTHREAD) {
#if __FreeBSD_version >= 900017
csw = dev_refthread(dev, &ref);
#else
csw = dev_refthread(dev);
#endif
} else
csw = dev->si_devsw;
mtx = mtx_pool_find(mtxpool_sleep, bio);
mtx_lock(mtx);
(*csw->d_strategy)(bio);
msleep(bio, mtx, PRIBIO, "biotestwait", 0);
mtx_unlock(mtx);
if (io_test->flags & NVME_TEST_FLAG_REFTHREAD) {
#if __FreeBSD_version >= 900017
dev_relthread(dev, ref);
#else
dev_relthread(dev);
#endif
}
if ((bio->bio_flags & BIO_ERROR) || (bio->bio_resid > 0))
break;
g_destroy_bio(bio);
io_completed++;
getmicrouptime(&t);
timevalsub(&t, &io_test->start);
if (t.tv_sec >= io_test->time)
break;
offset += io_test->size;
if ((offset + io_test->size) > nvme_ns_get_size(io_test->ns))
offset = 0;
}
io_test->io_completed[idx] = io_completed;
wakeup_one(io_test);
free(buf, M_NVME);
atomic_subtract_int(&io_test->td_active, 1);
mb();
#if __FreeBSD_version >= 800000
kthread_exit();
#else
kthread_exit(0);
#endif
}
static void *
nvd_new_disk(struct nvme_namespace *ns, void *ctrlr_arg)
{
struct nvd_disk *ndisk;
struct disk *disk;
struct nvd_controller *ctrlr = ctrlr_arg;
ndisk = malloc(sizeof(struct nvd_disk), M_NVD, M_ZERO | M_WAITOK);
disk = disk_alloc();
disk->d_strategy = nvd_strategy;
disk->d_ioctl = nvd_ioctl;
disk->d_name = "nvd";
disk->d_drv1 = ndisk;
disk->d_maxsize = nvme_ns_get_max_io_xfer_size(ns);
disk->d_sectorsize = nvme_ns_get_sector_size(ns);
disk->d_mediasize = (off_t)nvme_ns_get_size(ns);
if (TAILQ_EMPTY(&disk_head))
disk->d_unit = 0;
else
disk->d_unit =
TAILQ_LAST(&disk_head, disk_list)->disk->d_unit + 1;
disk->d_flags = 0;
if (nvme_ns_get_flags(ns) & NVME_NS_DEALLOCATE_SUPPORTED)
disk->d_flags |= DISKFLAG_CANDELETE;
if (nvme_ns_get_flags(ns) & NVME_NS_FLUSH_SUPPORTED)
disk->d_flags |= DISKFLAG_CANFLUSHCACHE;
/* ifdef used here to ease porting to stable branches at a later point. */
#ifdef DISKFLAG_UNMAPPED_BIO
disk->d_flags |= DISKFLAG_UNMAPPED_BIO;
#endif
strlcpy(disk->d_ident, nvme_ns_get_serial_number(ns),
sizeof(disk->d_ident));
#if __FreeBSD_version >= 900034
strlcpy(disk->d_descr, nvme_ns_get_model_number(ns),
sizeof(disk->d_descr));
#endif
disk_create(disk, DISK_VERSION);
ndisk->ns = ns;
ndisk->disk = disk;
ndisk->cur_depth = 0;
mtx_init(&ndisk->bioqlock, "NVD bioq lock", NULL, MTX_DEF);
bioq_init(&ndisk->bioq);
TASK_INIT(&ndisk->bioqtask, 0, nvd_bioq_process, ndisk);
ndisk->tq = taskqueue_create("nvd_taskq", M_WAITOK,
taskqueue_thread_enqueue, &ndisk->tq);
taskqueue_start_threads(&ndisk->tq, 1, PI_DISK, "nvd taskq");
TAILQ_INSERT_TAIL(&disk_head, ndisk, global_tailq);
TAILQ_INSERT_TAIL(&ctrlr->disk_head, ndisk, ctrlr_tailq);
return (NULL);
}
