static int rrpc_write_rq(struct rrpc *rrpc, struct bio *bio, struct nvm_rq *rqd, unsigned long flags) { struct rrpc_rq *rrqd = nvm_rq_to_pdu(rqd); struct rrpc_addr *p; int is_gc = flags & NVM_IOTYPE_GC; sector_t laddr = rrpc_get_laddr(bio); if (!is_gc && rrpc_lock_rq(rrpc, bio, rqd)) return NVM_IO_REQUEUE; p = rrpc_map_page(rrpc, laddr, is_gc); if (!p) { BUG_ON(is_gc); rrpc_unlock_rq(rrpc, rqd); rrpc_gc_kick(rrpc); return NVM_IO_REQUEUE; } rqd->ppa_addr = rrpc_ppa_to_gaddr(rrpc->dev, p->addr); rqd->opcode = NVM_OP_HBWRITE; rrqd->addr = p; return NVM_IO_OK; }
static int rrpc_write_ppalist_rq(struct rrpc *rrpc, struct bio *bio, struct nvm_rq *rqd, unsigned long flags, int npages) { struct rrpc_inflight_rq *r = rrpc_get_inflight_rq(rqd); struct rrpc_addr *p; sector_t laddr = rrpc_get_laddr(bio); int is_gc = flags & NVM_IOTYPE_GC; int i; if (!is_gc && rrpc_lock_rq(rrpc, bio, rqd)) { nvm_dev_dma_free(rrpc->dev, rqd->ppa_list, rqd->dma_ppa_list); return NVM_IO_REQUEUE; } for (i = 0; i < npages; i++) { /* We assume that mapping occurs at 4KB granularity */ p = rrpc_map_page(rrpc, laddr + i, is_gc); if (!p) { BUG_ON(is_gc); rrpc_unlock_laddr(rrpc, r); nvm_dev_dma_free(rrpc->dev, rqd->ppa_list, rqd->dma_ppa_list); rrpc_gc_kick(rrpc); return NVM_IO_REQUEUE; } rqd->ppa_list[i] = rrpc_ppa_to_gaddr(rrpc->dev, p->addr); } rqd->opcode = NVM_OP_HBWRITE; return NVM_IO_OK; }
static int rrpc_read_ppalist_rq(struct rrpc *rrpc, struct bio *bio, struct nvm_rq *rqd, unsigned long flags, int npages) { struct rrpc_inflight_rq *r = rrpc_get_inflight_rq(rqd); struct rrpc_addr *gp; sector_t laddr = rrpc_get_laddr(bio); int is_gc = flags & NVM_IOTYPE_GC; int i; if (!is_gc && rrpc_lock_rq(rrpc, bio, rqd)) { nvm_dev_dma_free(rrpc->dev, rqd->ppa_list, rqd->dma_ppa_list); return NVM_IO_REQUEUE; } for (i = 0; i < npages; i++) { /* We assume that mapping occurs at 4KB granularity */ BUG_ON(!(laddr + i >= 0 && laddr + i < rrpc->nr_sects)); gp = &rrpc->trans_map[laddr + i]; if (gp->rblk) { rqd->ppa_list[i] = rrpc_ppa_to_gaddr(rrpc->dev, gp->addr); } else { BUG_ON(is_gc); rrpc_unlock_laddr(rrpc, r); nvm_dev_dma_free(rrpc->dev, rqd->ppa_list, rqd->dma_ppa_list); return NVM_IO_DONE; } } rqd->opcode = NVM_OP_HBREAD; return NVM_IO_OK; }
static int rrpc_read_rq(struct rrpc *rrpc, struct bio *bio, struct nvm_rq *rqd, unsigned long flags) { struct rrpc_rq *rrqd = nvm_rq_to_pdu(rqd); int is_gc = flags & NVM_IOTYPE_GC; sector_t laddr = rrpc_get_laddr(bio); struct rrpc_addr *gp; if (!is_gc && rrpc_lock_rq(rrpc, bio, rqd)) return NVM_IO_REQUEUE; BUG_ON(!(laddr >= 0 && laddr < rrpc->nr_sects)); gp = &rrpc->trans_map[laddr]; if (gp->rblk) { rqd->ppa_addr = rrpc_ppa_to_gaddr(rrpc->dev, gp->addr); } else { BUG_ON(is_gc); rrpc_unlock_rq(rrpc, rqd); return NVM_IO_DONE; } rqd->opcode = NVM_OP_HBREAD; rrqd->addr = gp; return NVM_IO_OK; }
static int rrpc_end_io(struct nvm_rq *rqd, int error) { struct rrpc *rrpc = container_of(rqd->ins, struct rrpc, instance); struct rrpc_rq *rrqd = nvm_rq_to_pdu(rqd); uint8_t npages = rqd->nr_pages; sector_t laddr = rrpc_get_laddr(rqd->bio) - npages; if (bio_data_dir(rqd->bio) == WRITE) rrpc_end_io_write(rrpc, rrqd, laddr, npages); if (rrqd->flags & NVM_IOTYPE_GC) return 0; rrpc_unlock_rq(rrpc, rqd); bio_put(rqd->bio); if (npages > 1) nvm_dev_dma_free(rrpc->dev, rqd->ppa_list, rqd->dma_ppa_list); if (rqd->metadata) nvm_dev_dma_free(rrpc->dev, rqd->metadata, rqd->dma_metadata); mempool_free(rqd, rrpc->rq_pool); return 0; }