/* Used to get the required Queue entry for discontig SQ and CQ
* Returns- dma address
*/
static uint64_t find_discontig_queue_entry(uint32_t pg_size, uint16_t queue_ptr,
uint32_t cmd_size, uint64_t st_dma_addr) {
uint32_t index = 0;
uint32_t pg_no, prp_pg_no, entr_per_pg, prps_per_pg, prp_entry, pg_entry;
uint64_t dma_addr, entry_addr;
LOG_DBG("%s(): called", __func__);
/* All PRP entries start with offset 00h */
entr_per_pg = (uint32_t) (pg_size / cmd_size);
/* pg_no and prp_pg_no start with 0 */
pg_no = (uint32_t) (queue_ptr / entr_per_pg);
pg_entry = (uint32_t) (queue_ptr % entr_per_pg);
prps_per_pg = (uint32_t) (pg_size / PRP_ENTRY_SIZE);
prp_pg_no = (uint32_t) (pg_no / (prps_per_pg - 1));
prp_entry = (uint32_t) (pg_no % (prps_per_pg - 1));
/* Get to the correct page */
for (index = 1; index <= prp_pg_no; index++) {
nvme_dma_mem_read((st_dma_addr + ((prps_per_pg - 1) * PRP_ENTRY_SIZE)),
(uint8_t *)&dma_addr, PRP_ENTRY_SIZE);
st_dma_addr = dma_addr;
}
/* Correct offset within the prp list page */
dma_addr = st_dma_addr + (prp_entry * PRP_ENTRY_SIZE);
/* Reading the PRP List at required offset */
nvme_dma_mem_read(dma_addr, (uint8_t *)&entry_addr, PRP_ENTRY_SIZE);
/* Correct offset within the page */
dma_addr = entry_addr + (pg_entry * cmd_size);
return dma_addr;
}
/*********************************************************************
Function : read_dsm_ranges
Description : Read range definition data buffer specified
in cmd through prp1 and prp2.
Return Type : uint8_t
Arguments : uint64_t : PRP1
uint64_t : PRP2
uint8_t * : Pointer to target buffer location
uint64_t * : total data to be copied to target
buffer
*********************************************************************/
static uint8_t read_dsm_ranges(uint64_t range_prp1, uint64_t range_prp2,
uint8_t *buffer_addr, uint64_t *data_size_p)
{
uint64_t data_len;
/* Data Len to be written per page basis */
data_len = PAGE_SIZE - (range_prp1 % PAGE_SIZE);
if (data_len > *data_size_p) {
data_len = *data_size_p;
}
nvme_dma_mem_read(range_prp1, buffer_addr, data_len);
*data_size_p = *data_size_p - data_len;
if (*data_size_p) {
buffer_addr = buffer_addr + data_len;
nvme_dma_mem_read(range_prp2, buffer_addr, *data_size_p);
}
return NVME_SC_SUCCESS;
}
static uint8_t do_rw_prp_list(NVMEState *n, NVMECmd *command,
uint64_t *data_size_p, uint64_t *file_offset_p, uint8_t *mapping_addr)
{
uint64_t prp_list[512], prp_entries;
uint16_t i = 0;
uint8_t res = FAIL;
NVME_rw *cmd = (NVME_rw *)command;
LOG_DBG("Data Size remaining for read/write:%ld", *data_size_p);
/* Logic to find the number of PRP Entries */
prp_entries = (uint64_t) ((*data_size_p + PAGE_SIZE - 1) / PAGE_SIZE);
nvme_dma_mem_read(cmd->prp2, (uint8_t *)prp_list,
min(sizeof(prp_list), prp_entries * sizeof(uint64_t)));
/* Read/Write on PRPList */
while (*data_size_p != 0) {
if (i == 511 && *data_size_p > PAGE_SIZE) {
/* Calculate the actual number of remaining entries */
prp_entries = (uint64_t) ((*data_size_p + PAGE_SIZE - 1) /
PAGE_SIZE);
nvme_dma_mem_read(prp_list[511], (uint8_t *)prp_list,
min(sizeof(prp_list), prp_entries * sizeof(uint64_t)));
i = 0;
}
res = do_rw_prp(n, prp_list[i], data_size_p,
file_offset_p, mapping_addr, cmd->opcode);
LOG_DBG("Data Size remaining for read/write:%ld", *data_size_p);
if (res == FAIL) {
break;
}
i++;
}
return res;
}
static uint8_t do_rw_prp(NVMEState *n, uint64_t mem_addr, uint64_t *data_size_p,
uint64_t *file_offset_p, uint8_t *mapping_addr, uint8_t rw)
{
uint64_t data_len;
if (*data_size_p == 0) {
return FAIL;
}
/* Data Len to be written per page basis */
data_len = PAGE_SIZE - (mem_addr % PAGE_SIZE);
if (data_len > *data_size_p) {
data_len = *data_size_p;
}
LOG_DBG("File offset for read/write:%ld", *file_offset_p);
LOG_DBG("Length for read/write:%ld", data_len);
LOG_DBG("Address for read/write:%ld", mem_addr);
switch (rw) {
case NVME_CMD_READ:
LOG_DBG("Read cmd called");
nvme_dma_mem_write(mem_addr, (mapping_addr + *file_offset_p), data_len);
break;
case NVME_CMD_WRITE:
LOG_DBG("Write cmd called");
nvme_dma_mem_read(mem_addr, (mapping_addr + *file_offset_p), data_len);
break;
default:
LOG_ERR("Error- wrong opcode: %d", rw);
return FAIL;
}
*file_offset_p = *file_offset_p + data_len;
*data_size_p = *data_size_p - data_len;
return NVME_SC_SUCCESS;
}
uint8_t nvme_io_command(NVMEState *n, NVMECmd *sqe, NVMECQE *cqe)
{
NVME_rw *e = (NVME_rw *)sqe;
NVMEStatusField *sf = (NVMEStatusField *)&cqe->status;
uint8_t res = FAIL;
uint64_t data_size, file_offset;
uint8_t *mapping_addr;
uint32_t nvme_blk_sz;
DiskInfo *disk;
uint8_t lba_idx;
sf->sc = NVME_SC_SUCCESS;
LOG_DBG("%s(): called", __func__);
/* As of NVMe spec rev 1.0b "All NVM cmds use the CMD.DW1 (NSID) field".
* Thus all NVM cmd set cmds must check for illegal namespaces up front */
if (e->nsid == 0 || (e->nsid > n->idtfy_ctrl->nn)) {
LOG_NORM("%s(): Invalid nsid:%u", __func__, e->nsid);
sf->sc = NVME_SC_INVALID_NAMESPACE;
return FAIL;
}
if (sqe->opcode == NVME_CMD_FLUSH) {
return NVME_SC_SUCCESS;
} else if (sqe->opcode == NVME_CMD_DSM) {
return NVME_SC_SUCCESS;
} else if ((sqe->opcode != NVME_CMD_READ) &&
(sqe->opcode != NVME_CMD_WRITE)) {
LOG_NORM("%s():Wrong IO opcode:\t\t0x%02x", __func__, sqe->opcode);
sf->sc = NVME_SC_INVALID_OPCODE;
return FAIL;
}
disk = &n->disk[e->nsid - 1];
if ((e->slba + e->nlb) >= disk->idtfy_ns.nsze) {
LOG_NORM("%s(): LBA out of range", __func__);
sf->sc = NVME_SC_LBA_RANGE;
return FAIL;
} else if ((e->slba + e->nlb) >= disk->idtfy_ns.ncap) {
LOG_NORM("%s():Capacity Exceeded", __func__);
sf->sc = NVME_SC_CAP_EXCEEDED;
return FAIL;
}
lba_idx = disk->idtfy_ns.flbas & 0xf;
if ((e->mptr == 0) && /* if NOT supplying separate meta buffer */
(disk->idtfy_ns.lbafx[lba_idx].ms != 0) && /* if using metadata */
((disk->idtfy_ns.flbas & 0x10) == 0)) { /* if using separate buffer */
LOG_ERR("%s(): invalid meta-data for extended lba", __func__);
sf->sc = NVME_SC_INVALID_FIELD;
return FAIL;
}
/* Read in the command */
nvme_blk_sz = NVME_BLOCK_SIZE(disk->idtfy_ns.lbafx[lba_idx].lbads);
LOG_DBG("NVME Block size: %u", nvme_blk_sz);
data_size = (e->nlb + 1) * nvme_blk_sz;
if (disk->idtfy_ns.flbas & 0x10) {
data_size += (disk->idtfy_ns.lbafx[lba_idx].ms * (e->nlb + 1));
}
if (n->idtfy_ctrl->mdts && data_size > PAGE_SIZE *
(1 << (n->idtfy_ctrl->mdts))) {
LOG_ERR("%s(): data size:%ld exceeds max:%ld", __func__,
data_size, ((uint64_t)PAGE_SIZE) * (1 << (n->idtfy_ctrl->mdts)));
sf->sc = NVME_SC_INVALID_FIELD;
return FAIL;
}
file_offset = e->slba * nvme_blk_sz;
mapping_addr = disk->mapping_addr;
/* Namespace not ready */
if (mapping_addr == NULL) {
LOG_NORM("%s():Namespace not ready", __func__);
sf->sc = NVME_SC_NS_NOT_READY;
return FAIL;
}
/* Writing/Reading PRP1 */
res = do_rw_prp(n, e->prp1, &data_size, &file_offset, mapping_addr,
e->opcode);
if (res == FAIL) {
return FAIL;
}
if (data_size > 0) {
if (data_size <= PAGE_SIZE) {
res = do_rw_prp(n, e->prp2, &data_size, &file_offset, mapping_addr,
e->opcode);
} else {
res = do_rw_prp_list(n, sqe, &data_size, &file_offset,
mapping_addr);
}
if (res == FAIL) {
return FAIL;
}
}
/* Spec states that non-zero meta data buffers shall be ignored, i.e. no
* error reported, when the DW4&5 (MPTR) field is not in use */
if ((e->mptr != 0) && /* if supplying separate meta buffer */
(disk->idtfy_ns.lbafx[lba_idx].ms != 0) && /* if using metadata */
((disk->idtfy_ns.flbas & 0x10) == 0)) { /* if using separate buffer */
/* Then go ahead and use the separate meta data buffer */
unsigned int ms, meta_offset, meta_size;
uint8_t *meta_mapping_addr;
ms = disk->idtfy_ns.lbafx[lba_idx].ms;
meta_offset = e->slba * ms;
meta_size = (e->nlb + 1) * ms;
meta_mapping_addr = disk->meta_mapping_addr + meta_offset;
if (e->opcode == NVME_CMD_READ) {
nvme_dma_mem_write(e->mptr, meta_mapping_addr, meta_size);
} else if (e->opcode == NVME_CMD_WRITE) {
nvme_dma_mem_read(e->mptr, meta_mapping_addr, meta_size);
}
}
nvme_update_stats(n, disk, e->opcode, e->slba, e->nlb);
return res;
}
void process_sq(NVMEState *n, uint16_t sq_id)
{
target_phys_addr_t addr;
uint16_t cq_id;
NVMECmd sqe;
NVMECQE cqe;
NVMEStatusField *sf = (NVMEStatusField *) &cqe.status;
if (n->sq[sq_id].dma_addr == 0 || n->cq[n->sq[sq_id].cq_id].dma_addr
== 0) {
LOG_ERR("Required Submission/Completion Queue does not exist");
n->sq[sq_id].head = n->sq[sq_id].tail = 0;
goto exit;
}
cq_id = n->sq[sq_id].cq_id;
if (is_cq_full(n, cq_id)) {
return;
}
memset(&cqe, 0, sizeof(cqe));
LOG_DBG("%s(): called", __func__);
/* Process SQE */
if (sq_id == ASQ_ID || n->sq[sq_id].phys_contig) {
addr = n->sq[sq_id].dma_addr + n->sq[sq_id].head * sizeof(sqe);
} else {
/* PRP implementation */
addr = find_discontig_queue_entry(n->page_size, n->sq[sq_id].head,
sizeof(sqe), n->sq[sq_id].dma_addr);
}
nvme_dma_mem_read(addr, (uint8_t *)&sqe, sizeof(sqe));
if (n->abort) {
if (abort_command(n, sq_id, &sqe)) {
incr_sq_head(&n->sq[sq_id]);
return;
}
}
incr_sq_head(&n->sq[sq_id]);
if (sq_id == ASQ_ID) {
nvme_admin_command(n, &sqe, &cqe);
} else {
/* TODO add support for IO commands with different sizes of Q elements */
nvme_io_command(n, &sqe, &cqe);
}
/* Filling up the CQ entry */
cqe.sq_id = sq_id;
cqe.sq_head = n->sq[sq_id].head;
cqe.command_id = sqe.cid;
sf->p = n->cq[cq_id].phase_tag;
sf->m = 0;
sf->dnr = 0; /* TODO add support for dnr */
/* write cqe to completion queue */
if (cq_id == ACQ_ID || n->cq[cq_id].phys_contig) {
addr = n->cq[cq_id].dma_addr + n->cq[cq_id].tail * sizeof(cqe);
} else {
/* PRP implementation */
addr = find_discontig_queue_entry(n->page_size, n->cq[cq_id].tail,
sizeof(cqe), n->cq[cq_id].dma_addr);
}
nvme_dma_mem_write(addr, (uint8_t *)&cqe, sizeof(cqe));
incr_cq_tail(&n->cq[cq_id]);
if (cq_id == ACQ_ID) {
/*
3.1.9 says: "This queue is always associated
with interrupt vector 0"
*/
msix_notify(&(n->dev), 0);
return;
}
if (n->cq[cq_id].irq_enabled) {
msix_notify(&(n->dev), n->cq[cq_id].vector);
} else {
LOG_NORM("kw q: IRQ not enabled for CQ: %d", cq_id);
}
exit:
return;
}
