/* requires lun->lock taken */
static void rrpc_set_lun_cur(struct rrpc_lun *rlun, struct rrpc_block *new_rblk,
struct rrpc_block **cur_rblk)
{
struct rrpc *rrpc = rlun->rrpc;
if (*cur_rblk) {
spin_lock(&(*cur_rblk)->lock);
WARN_ON(!block_is_full(rrpc, *cur_rblk));
spin_unlock(&(*cur_rblk)->lock);
}
*cur_rblk = new_rblk;
}
/* requires lun->lock taken */
static void rrpc_set_lun_cur(struct rrpc_lun *rlun, struct rrpc_block *rblk)
{
struct rrpc *rrpc = rlun->rrpc;
BUG_ON(!rblk);
if (rlun->cur) {
spin_lock(&rlun->cur->lock);
WARN_ON(!block_is_full(rrpc, rlun->cur));
spin_unlock(&rlun->cur->lock);
}
rlun->cur = rblk;
}
static u64 rrpc_alloc_addr(struct rrpc *rrpc, struct rrpc_block *rblk)
{
u64 addr = ADDR_EMPTY;
spin_lock(&rblk->lock);
if (block_is_full(rrpc, rblk))
goto out;
addr = block_to_addr(rrpc, rblk) + rblk->next_page;
rblk->next_page++;
out:
spin_unlock(&rblk->lock);
return addr;
}
/*
* rrpc_move_valid_pages -- migrate live data off the block
* @rrpc: the 'rrpc' structure
* @block: the block from which to migrate live pages
*
* Description:
* GC algorithms may call this function to migrate remaining live
* pages off the block prior to erasing it. This function blocks
* further execution until the operation is complete.
*/
static int rrpc_move_valid_pages(struct rrpc *rrpc, struct rrpc_block *rblk)
{
struct request_queue *q = rrpc->dev->q;
struct rrpc_rev_addr *rev;
struct nvm_rq *rqd;
struct bio *bio;
struct page *page;
int slot;
int nr_sec_per_blk = rrpc->dev->sec_per_blk;
u64 phys_addr;
DECLARE_COMPLETION_ONSTACK(wait);
if (bitmap_full(rblk->invalid_pages, nr_sec_per_blk))
return 0;
bio = bio_alloc(GFP_NOIO, 1);
if (!bio) {
pr_err("nvm: could not alloc bio to gc\n");
return -ENOMEM;
}
page = mempool_alloc(rrpc->page_pool, GFP_NOIO);
if (!page) {
bio_put(bio);
return -ENOMEM;
}
while ((slot = find_first_zero_bit(rblk->invalid_pages,
nr_sec_per_blk)) < nr_sec_per_blk) {
/* Lock laddr */
phys_addr = rblk->parent->id * nr_sec_per_blk + slot;
try:
spin_lock(&rrpc->rev_lock);
/* Get logical address from physical to logical table */
rev = &rrpc->rev_trans_map[phys_addr - rrpc->poffset];
/* already updated by previous regular write */
if (rev->addr == ADDR_EMPTY) {
spin_unlock(&rrpc->rev_lock);
continue;
}
rqd = rrpc_inflight_laddr_acquire(rrpc, rev->addr, 1);
if (IS_ERR_OR_NULL(rqd)) {
spin_unlock(&rrpc->rev_lock);
schedule();
goto try;
}
spin_unlock(&rrpc->rev_lock);
/* Perform read to do GC */
bio->bi_iter.bi_sector = rrpc_get_sector(rev->addr);
bio->bi_rw = READ;
bio->bi_private = &wait;
bio->bi_end_io = rrpc_end_sync_bio;
/* TODO: may fail when EXP_PG_SIZE > PAGE_SIZE */
bio_add_pc_page(q, bio, page, RRPC_EXPOSED_PAGE_SIZE, 0);
if (rrpc_submit_io(rrpc, bio, rqd, NVM_IOTYPE_GC)) {
pr_err("rrpc: gc read failed.\n");
rrpc_inflight_laddr_release(rrpc, rqd);
goto finished;
}
wait_for_completion_io(&wait);
if (bio->bi_error) {
rrpc_inflight_laddr_release(rrpc, rqd);
goto finished;
}
bio_reset(bio);
reinit_completion(&wait);
bio->bi_iter.bi_sector = rrpc_get_sector(rev->addr);
bio->bi_rw = WRITE;
bio->bi_private = &wait;
bio->bi_end_io = rrpc_end_sync_bio;
bio_add_pc_page(q, bio, page, RRPC_EXPOSED_PAGE_SIZE, 0);
/* turn the command around and write the data back to a new
* address
*/
if (rrpc_submit_io(rrpc, bio, rqd, NVM_IOTYPE_GC)) {
pr_err("rrpc: gc write failed.\n");
rrpc_inflight_laddr_release(rrpc, rqd);
goto finished;
}
wait_for_completion_io(&wait);
rrpc_inflight_laddr_release(rrpc, rqd);
if (bio->bi_error)
goto finished;
bio_reset(bio);
}
finished:
mempool_free(page, rrpc->page_pool);
bio_put(bio);
if (!bitmap_full(rblk->invalid_pages, nr_sec_per_blk)) {
pr_err("nvm: failed to garbage collect block\n");
return -EIO;
}
return 0;
}
static void rrpc_block_gc(struct work_struct *work)
{
struct rrpc_block_gc *gcb = container_of(work, struct rrpc_block_gc,
ws_gc);
struct rrpc *rrpc = gcb->rrpc;
struct rrpc_block *rblk = gcb->rblk;
struct nvm_dev *dev = rrpc->dev;
struct nvm_lun *lun = rblk->parent->lun;
struct rrpc_lun *rlun = &rrpc->luns[lun->id - rrpc->lun_offset];
mempool_free(gcb, rrpc->gcb_pool);
pr_debug("nvm: block '%lu' being reclaimed\n", rblk->parent->id);
if (rrpc_move_valid_pages(rrpc, rblk))
goto put_back;
if (nvm_erase_blk(dev, rblk->parent))
goto put_back;
rrpc_put_blk(rrpc, rblk);
return;
put_back:
spin_lock(&rlun->lock);
list_add_tail(&rblk->prio, &rlun->prio_list);
spin_unlock(&rlun->lock);
}
/* the block with highest number of invalid pages, will be in the beginning
* of the list
*/
static struct rrpc_block *rblock_max_invalid(struct rrpc_block *ra,
struct rrpc_block *rb)
{
if (ra->nr_invalid_pages == rb->nr_invalid_pages)
return ra;
return (ra->nr_invalid_pages < rb->nr_invalid_pages) ? rb : ra;
}
/* linearly find the block with highest number of invalid pages
* requires lun->lock
*/
static struct rrpc_block *block_prio_find_max(struct rrpc_lun *rlun)
{
struct list_head *prio_list = &rlun->prio_list;
struct rrpc_block *rblock, *max;
BUG_ON(list_empty(prio_list));
max = list_first_entry(prio_list, struct rrpc_block, prio);
list_for_each_entry(rblock, prio_list, prio)
max = rblock_max_invalid(max, rblock);
return max;
}
static void rrpc_lun_gc(struct work_struct *work)
{
struct rrpc_lun *rlun = container_of(work, struct rrpc_lun, ws_gc);
struct rrpc *rrpc = rlun->rrpc;
struct nvm_lun *lun = rlun->parent;
struct rrpc_block_gc *gcb;
unsigned int nr_blocks_need;
nr_blocks_need = rrpc->dev->blks_per_lun / GC_LIMIT_INVERSE;
if (nr_blocks_need < rrpc->nr_luns)
nr_blocks_need = rrpc->nr_luns;
spin_lock(&rlun->lock);
while (nr_blocks_need > lun->nr_free_blocks &&
!list_empty(&rlun->prio_list)) {
struct rrpc_block *rblock = block_prio_find_max(rlun);
struct nvm_block *block = rblock->parent;
if (!rblock->nr_invalid_pages)
break;
gcb = mempool_alloc(rrpc->gcb_pool, GFP_ATOMIC);
if (!gcb)
break;
list_del_init(&rblock->prio);
BUG_ON(!block_is_full(rrpc, rblock));
pr_debug("rrpc: selected block '%lu' for GC\n", block->id);
gcb->rrpc = rrpc;
gcb->rblk = rblock;
INIT_WORK(&gcb->ws_gc, rrpc_block_gc);
queue_work(rrpc->kgc_wq, &gcb->ws_gc);
nr_blocks_need--;
}
spin_unlock(&rlun->lock);
/* TODO: Hint that request queue can be started again */
}
static void rrpc_gc_queue(struct work_struct *work)
{
struct rrpc_block_gc *gcb = container_of(work, struct rrpc_block_gc,
ws_gc);
struct rrpc *rrpc = gcb->rrpc;
struct rrpc_block *rblk = gcb->rblk;
struct nvm_lun *lun = rblk->parent->lun;
struct nvm_block *blk = rblk->parent;
struct rrpc_lun *rlun = &rrpc->luns[lun->id - rrpc->lun_offset];
spin_lock(&rlun->lock);
list_add_tail(&rblk->prio, &rlun->prio_list);
spin_unlock(&rlun->lock);
spin_lock(&lun->lock);
lun->nr_open_blocks--;
lun->nr_closed_blocks++;
blk->state &= ~NVM_BLK_ST_OPEN;
blk->state |= NVM_BLK_ST_CLOSED;
list_move_tail(&rblk->list, &rlun->closed_list);
spin_unlock(&lun->lock);
mempool_free(gcb, rrpc->gcb_pool);
pr_debug("nvm: block '%lu' is full, allow GC (sched)\n",
rblk->parent->id);
}
static const struct block_device_operations rrpc_fops = {
.owner = THIS_MODULE,
};
static struct rrpc_lun *rrpc_get_lun_rr(struct rrpc *rrpc, int is_gc)
{
unsigned int i;
struct rrpc_lun *rlun, *max_free;
if (!is_gc)
return get_next_lun(rrpc);
/* during GC, we don't care about RR, instead we want to make
* sure that we maintain evenness between the block luns.
*/
max_free = &rrpc->luns[0];
/* prevent GC-ing lun from devouring pages of a lun with
* little free blocks. We don't take the lock as we only need an
* estimate.
*/
rrpc_for_each_lun(rrpc, rlun, i) {
if (rlun->parent->nr_free_blocks >
max_free->parent->nr_free_blocks)
max_free = rlun;
}
return max_free;
}
void *
allocateForCompact (Capability *cap,
StgCompactNFData *str,
StgWord sizeW)
{
StgPtr to;
StgWord next_size;
StgCompactNFDataBlock *block;
bdescr *bd;
ASSERT(str->nursery != NULL);
ASSERT(str->hp > Bdescr((P_)str->nursery)->start);
ASSERT(str->hp <= Bdescr((P_)str->nursery)->start +
Bdescr((P_)str->nursery)->blocks * BLOCK_SIZE_W);
retry:
if (str->hp + sizeW < str->hpLim) {
to = str->hp;
str->hp += sizeW;
return to;
}
bd = Bdescr((P_)str->nursery);
bd->free = str->hp;
// We know it doesn't fit in the nursery
// if it is a large object, allocate a new block
if (sizeW > LARGE_OBJECT_THRESHOLD/sizeof(W_)) {
next_size = BLOCK_ROUND_UP(sizeW*sizeof(W_) +
sizeof(StgCompactNFData));
block = compactAppendBlock(cap, str, next_size);
bd = Bdescr((P_)block);
to = bd->free;
bd->free += sizeW;
return to;
}
// move the nursery past full blocks
if (block_is_full (str->nursery)) {
do {
str->nursery = str->nursery->next;
} while (str->nursery && block_is_full(str->nursery));
if (str->nursery == NULL) {
str->nursery = compactAppendBlock(cap, str,
str->autoBlockW * sizeof(W_));
}
bd = Bdescr((P_)str->nursery);
str->hp = bd->free;
str->hpLim = bd->start + bd->blocks * BLOCK_SIZE_W;
goto retry;
}
// try subsequent blocks
for (block = str->nursery->next; block != NULL; block = block->next) {
bd = Bdescr((P_)block);
if (has_room_for(bd,sizeW)) {
to = bd->free;
bd->free += sizeW;
return to;
}
}
// If all else fails, allocate a new block of the right size.
next_size = stg_max(str->autoBlockW * sizeof(StgWord),
BLOCK_ROUND_UP(sizeW * sizeof(StgWord)
+ sizeof(StgCompactNFDataBlock)));
block = compactAppendBlock(cap, str, next_size);
bd = Bdescr((P_)block);
to = bd->free;
bd->free += sizeW;
return to;
}
