static int rrpc_write_rq(struct rrpc *rrpc, struct bio *bio,
struct nvm_rq *rqd, unsigned long flags)
{
struct rrpc_rq *rrqd = nvm_rq_to_pdu(rqd);
struct rrpc_addr *p;
int is_gc = flags & NVM_IOTYPE_GC;
sector_t laddr = rrpc_get_laddr(bio);
if (!is_gc && rrpc_lock_rq(rrpc, bio, rqd))
return NVM_IO_REQUEUE;
p = rrpc_map_page(rrpc, laddr, is_gc);
if (!p) {
BUG_ON(is_gc);
rrpc_unlock_rq(rrpc, rqd);
rrpc_gc_kick(rrpc);
return NVM_IO_REQUEUE;
}
rqd->ppa_addr = rrpc_ppa_to_gaddr(rrpc->dev, p->addr);
rqd->opcode = NVM_OP_HBWRITE;
rrqd->addr = p;
return NVM_IO_OK;
}
static int rrpc_write_ppalist_rq(struct rrpc *rrpc, struct bio *bio,
struct nvm_rq *rqd, unsigned long flags, int npages)
{
struct rrpc_inflight_rq *r = rrpc_get_inflight_rq(rqd);
struct rrpc_addr *p;
sector_t laddr = rrpc_get_laddr(bio);
int is_gc = flags & NVM_IOTYPE_GC;
int i;
if (!is_gc && rrpc_lock_rq(rrpc, bio, rqd)) {
nvm_dev_dma_free(rrpc->dev, rqd->ppa_list, rqd->dma_ppa_list);
return NVM_IO_REQUEUE;
}
for (i = 0; i < npages; i++) {
/* We assume that mapping occurs at 4KB granularity */
p = rrpc_map_page(rrpc, laddr + i, is_gc);
if (!p) {
BUG_ON(is_gc);
rrpc_unlock_laddr(rrpc, r);
nvm_dev_dma_free(rrpc->dev, rqd->ppa_list,
rqd->dma_ppa_list);
rrpc_gc_kick(rrpc);
return NVM_IO_REQUEUE;
}
rqd->ppa_list[i] = rrpc_ppa_to_gaddr(rrpc->dev,
p->addr);
}
rqd->opcode = NVM_OP_HBWRITE;
return NVM_IO_OK;
}
static int rrpc_read_ppalist_rq(struct rrpc *rrpc, struct bio *bio,
struct nvm_rq *rqd, unsigned long flags, int npages)
{
struct rrpc_inflight_rq *r = rrpc_get_inflight_rq(rqd);
struct rrpc_addr *gp;
sector_t laddr = rrpc_get_laddr(bio);
int is_gc = flags & NVM_IOTYPE_GC;
int i;
if (!is_gc && rrpc_lock_rq(rrpc, bio, rqd)) {
nvm_dev_dma_free(rrpc->dev, rqd->ppa_list, rqd->dma_ppa_list);
return NVM_IO_REQUEUE;
}
for (i = 0; i < npages; i++) {
/* We assume that mapping occurs at 4KB granularity */
BUG_ON(!(laddr + i >= 0 && laddr + i < rrpc->nr_sects));
gp = &rrpc->trans_map[laddr + i];
if (gp->rblk) {
rqd->ppa_list[i] = rrpc_ppa_to_gaddr(rrpc->dev,
gp->addr);
} else {
BUG_ON(is_gc);
rrpc_unlock_laddr(rrpc, r);
nvm_dev_dma_free(rrpc->dev, rqd->ppa_list,
rqd->dma_ppa_list);
return NVM_IO_DONE;
}
}
rqd->opcode = NVM_OP_HBREAD;
return NVM_IO_OK;
}
static int rrpc_read_rq(struct rrpc *rrpc, struct bio *bio, struct nvm_rq *rqd,
unsigned long flags)
{
struct rrpc_rq *rrqd = nvm_rq_to_pdu(rqd);
int is_gc = flags & NVM_IOTYPE_GC;
sector_t laddr = rrpc_get_laddr(bio);
struct rrpc_addr *gp;
if (!is_gc && rrpc_lock_rq(rrpc, bio, rqd))
return NVM_IO_REQUEUE;
BUG_ON(!(laddr >= 0 && laddr < rrpc->nr_sects));
gp = &rrpc->trans_map[laddr];
if (gp->rblk) {
rqd->ppa_addr = rrpc_ppa_to_gaddr(rrpc->dev, gp->addr);
} else {
BUG_ON(is_gc);
rrpc_unlock_rq(rrpc, rqd);
return NVM_IO_DONE;
}
rqd->opcode = NVM_OP_HBREAD;
rrqd->addr = gp;
return NVM_IO_OK;
}
static int rrpc_end_io(struct nvm_rq *rqd, int error)
{
struct rrpc *rrpc = container_of(rqd->ins, struct rrpc, instance);
struct rrpc_rq *rrqd = nvm_rq_to_pdu(rqd);
uint8_t npages = rqd->nr_pages;
sector_t laddr = rrpc_get_laddr(rqd->bio) - npages;
if (bio_data_dir(rqd->bio) == WRITE)
rrpc_end_io_write(rrpc, rrqd, laddr, npages);
if (rrqd->flags & NVM_IOTYPE_GC)
return 0;
rrpc_unlock_rq(rrpc, rqd);
bio_put(rqd->bio);
if (npages > 1)
nvm_dev_dma_free(rrpc->dev, rqd->ppa_list, rqd->dma_ppa_list);
if (rqd->metadata)
nvm_dev_dma_free(rrpc->dev, rqd->metadata, rqd->dma_metadata);
mempool_free(rqd, rrpc->rq_pool);
return 0;
}