/*Consumer*/
int kthread_consumer(void *data)
{
struct task_struct *task = NULL;
int *cs = data, ret = 0;
task = current;
while (1) {
if (!kthread_should_stop()) {
set_current_state(TASK_INTERRUPTIBLE);
if (*cs > 0) {
printk("%s cs = %d\n", task->comm, (*cs)--);
if (producer)
wake_up_process(producer);
} else {
printk("CONSUMER WILL SLEEP\n");
/* <-------------------- Here the wake up signal is lost from the Producer*/
// set_current_state(TASK_INTERRUPTIBLE);
// Must unlock before schedule
io_schedule();
}
set_current_state(TASK_RUNNING);
} else {
/*The wake up is called from the kthread_stop*/
printk("Stopping Consumer\n");
break;
}
}
return ret;
}
int ax8swap_sync_page(void *word)
{
struct address_space *mapping;
struct page *page;
page = container_of((unsigned long *)word, struct page, flags);
/*
* page_mapping() is being called without PG_locked held.
* Some knowledge of the state and use of the page is used to
* reduce the requirements down to a memory barrier.
* The danger here is of a stale page_mapping() return value
* indicating a struct address_space different from the one it's
* associated with when it is associated with one.
* After smp_mb(), it's either the correct page_mapping() for
* the page, or an old page_mapping() and the page's own
* page_mapping() has gone NULL.
* The ->sync_page() address_space operation must tolerate
* page_mapping() going NULL. By an amazing coincidence,
* this comes about because none of the users of the page
* in the ->sync_page() methods make essential use of the
* page_mapping(), merely passing the page down to the backing
* device's unplug functions when it's non-NULL, which in turn
* ignore it for all cases but swap, where only page_private(page) is
* of interest. When page_mapping() does go NULL, the entire
* call stack gracefully ignores the page and returns.
* -- wli
*/
smp_mb();
mapping = page_mapping(page);
if (mapping && mapping->a_ops && mapping->a_ops->sync_page)
mapping->a_ops->sync_page(page);
io_schedule();
return 0;
}
/*Producer*/
int kthread_producer(void *data)
{
struct task_struct *task = NULL;
int *cs = data, ret = 0;
task = current;
while (1) {
if (!kthread_should_stop()) {
set_current_state(TASK_INTERRUPTIBLE);
if (*cs <= 9) {
printk("%s cs = %d\n", task->comm, ++(*cs));
if (consumer)
wake_up_process(consumer);
} else {
printk("PRODUCER WILL SLEEP\n");
/* <-------------------- Here the wake up signal may lost from the consumer*/
//set_current_state(TASK_INTERRUPTIBLE);
// Must unlock before schedule
io_schedule();
}
set_current_state(TASK_RUNNING);
} else {
printk("Stopping Producer\n");
break;
}
}
return ret;
}
int pblk_write_to_cache(struct pblk *pblk, struct bio *bio, unsigned long flags)
{
struct request_queue *q = pblk->dev->q;
struct pblk_w_ctx w_ctx;
sector_t lba = pblk_get_lba(bio);
unsigned long start_time = jiffies;
unsigned int bpos, pos;
int nr_entries = pblk_get_secs(bio);
int i, ret;
generic_start_io_acct(q, WRITE, bio_sectors(bio), &pblk->disk->part0);
/* Update the write buffer head (mem) with the entries that we can
* write. The write in itself cannot fail, so there is no need to
* rollback from here on.
*/
retry:
ret = pblk_rb_may_write_user(&pblk->rwb, bio, nr_entries, &bpos);
switch (ret) {
case NVM_IO_REQUEUE:
io_schedule();
goto retry;
case NVM_IO_ERR:
pblk_pipeline_stop(pblk);
goto out;
}
if (unlikely(!bio_has_data(bio)))
goto out;
pblk_ppa_set_empty(&w_ctx.ppa);
w_ctx.flags = flags;
if (bio->bi_opf & REQ_PREFLUSH)
w_ctx.flags |= PBLK_FLUSH_ENTRY;
for (i = 0; i < nr_entries; i++) {
void *data = bio_data(bio);
w_ctx.lba = lba + i;
pos = pblk_rb_wrap_pos(&pblk->rwb, bpos + i);
pblk_rb_write_entry_user(&pblk->rwb, data, w_ctx, pos);
bio_advance(bio, PBLK_EXPOSED_PAGE_SIZE);
}
atomic64_add(nr_entries, &pblk->user_wa);
#ifdef CONFIG_NVM_DEBUG
atomic_long_add(nr_entries, &pblk->inflight_writes);
atomic_long_add(nr_entries, &pblk->req_writes);
#endif
pblk_rl_inserted(&pblk->rl, nr_entries);
out:
generic_end_io_acct(q, WRITE, &pblk->disk->part0, start_time);
pblk_write_should_kick(pblk);
return ret;
}
/*
* Retry logic exist outside these macros to protect from spurrious wakeups.
*/
static inline void TXN_SLEEP_DROP_LOCK(wait_queue_head_t * event)
{
DECLARE_WAITQUEUE(wait, current);
add_wait_queue(event, &wait);
set_current_state(TASK_UNINTERRUPTIBLE);
TXN_UNLOCK();
io_schedule();
__set_current_state(TASK_RUNNING);
remove_wait_queue(event, &wait);
}
/**
* mempool_alloc - allocate an element from a specific memory pool
* @pool: pointer to the memory pool which was allocated via
* mempool_create().
* @gfp_mask: the usual allocation bitmask.
*
* this function only sleeps if the alloc_fn function sleeps or
* returns NULL. Note that due to preallocation, this function
* *never* fails when called from process contexts. (it might
* fail if called from an IRQ context.)
*/
void * mempool_alloc(mempool_t *pool, int gfp_mask)
{
void *element;
unsigned long flags;
DEFINE_WAIT(wait);
int gfp_nowait = gfp_mask & ~(__GFP_WAIT | __GFP_IO);
might_sleep_if(gfp_mask & __GFP_WAIT);
repeat_alloc:
element = pool->alloc(gfp_nowait|__GFP_NOWARN, pool->pool_data);
if (likely(element != NULL))
return element;
/*
* If the pool is less than 50% full and we can perform effective
* page reclaim then try harder to allocate an element.
*/
mb();
if ((gfp_mask & __GFP_FS) && (gfp_mask != gfp_nowait) &&
(pool->curr_nr <= pool->min_nr/2)) {
element = pool->alloc(gfp_mask, pool->pool_data);
if (likely(element != NULL))
return element;
}
/*
* Kick the VM at this point.
*/
wakeup_bdflush(0);
spin_lock_irqsave(&pool->lock, flags);
if (likely(pool->curr_nr)) {
element = remove_element(pool);
spin_unlock_irqrestore(&pool->lock, flags);
return element;
}
spin_unlock_irqrestore(&pool->lock, flags);
/* We must not sleep in the GFP_ATOMIC case */
if (!(gfp_mask & __GFP_WAIT))
return NULL;
prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE);
mb();
if (!pool->curr_nr)
io_schedule();
finish_wait(&pool->wait, &wait);
goto repeat_alloc;
}
void
__xfs_iflock(
struct xfs_inode *ip)
{
wait_queue_head_t *wq = bit_waitqueue(&ip->i_flags, __XFS_IFLOCK_BIT);
DEFINE_WAIT_BIT(wait, &ip->i_flags, __XFS_IFLOCK_BIT);
do {
prepare_to_wait_exclusive(wq, &wait.wait, TASK_UNINTERRUPTIBLE);
if (xfs_isiflocked(ip))
io_schedule();
} while (!xfs_iflock_nowait(ip));
finish_wait(wq, &wait.wait);
}
static inline void __lock_metapage(struct metapage *mp)
{
DECLARE_WAITQUEUE(wait, current);
INCREMENT(mpStat.lockwait);
add_wait_queue_exclusive(&mp->wait, &wait);
do {
set_current_state(TASK_UNINTERRUPTIBLE);
if (metapage_locked(mp)) {
unlock_page(mp->page);
io_schedule();
lock_page(mp->page);
}
} while (trylock_metapage(mp));
__set_current_state(TASK_RUNNING);
remove_wait_queue(&mp->wait, &wait);
}
/*
* On GC the incoming lbas are not necessarily sequential. Also, some of the
* lbas might not be valid entries, which are marked as empty by the GC thread
*/
int pblk_write_gc_to_cache(struct pblk *pblk, struct pblk_gc_rq *gc_rq)
{
struct pblk_w_ctx w_ctx;
unsigned int bpos, pos;
void *data = gc_rq->data;
int i, valid_entries;
/* Update the write buffer head (mem) with the entries that we can
* write. The write in itself cannot fail, so there is no need to
* rollback from here on.
*/
retry:
if (!pblk_rb_may_write_gc(&pblk->rwb, gc_rq->secs_to_gc, &bpos)) {
io_schedule();
goto retry;
}
w_ctx.flags = PBLK_IOTYPE_GC;
pblk_ppa_set_empty(&w_ctx.ppa);
for (i = 0, valid_entries = 0; i < gc_rq->nr_secs; i++) {
if (gc_rq->lba_list[i] == ADDR_EMPTY)
continue;
w_ctx.lba = gc_rq->lba_list[i];
pos = pblk_rb_wrap_pos(&pblk->rwb, bpos + valid_entries);
pblk_rb_write_entry_gc(&pblk->rwb, data, w_ctx, gc_rq->line,
gc_rq->paddr_list[i], pos);
data += PBLK_EXPOSED_PAGE_SIZE;
valid_entries++;
}
WARN_ONCE(gc_rq->secs_to_gc != valid_entries,
"pblk: inconsistent GC write\n");
atomic64_add(valid_entries, &pblk->gc_wa);
#ifdef CONFIG_NVM_DEBUG
atomic_long_add(valid_entries, &pblk->inflight_writes);
atomic_long_add(valid_entries, &pblk->recov_gc_writes);
#endif
pblk_write_should_kick(pblk);
return NVM_IO_OK;
}
static int sync_io(struct dm_io_client *client, unsigned int num_regions,
struct dm_io_region *where, int rw, struct dpages *dp,
unsigned long *error_bits)
{
/*
* gcc <= 4.3 can't do the alignment for stack variables, so we must
* align it on our own.
* volatile prevents the optimizer from removing or reusing
* "io_" field from the stack frame (allowed in ANSI C).
*/
volatile char io_[sizeof(struct io) + __alignof__(struct io) - 1];
struct io *io = (struct io *)PTR_ALIGN(&io_, __alignof__(struct io));
if (num_regions > 1 && (rw & RW_MASK) != WRITE) {
WARN_ON(1);
return -EIO;
}
retry:
io->error_bits = 0;
io->eopnotsupp_bits = 0;
atomic_set(&io->count, 1); /* see dispatch_io() */
io->sleeper = current;
io->client = client;
dispatch_io(rw, num_regions, where, dp, io, 1);
while (1) {
set_current_state(TASK_UNINTERRUPTIBLE);
if (!atomic_read(&io->count))
break;
io_schedule();
}
set_current_state(TASK_RUNNING);
if (io->eopnotsupp_bits && (rw & (1 << BIO_RW_BARRIER))) {
rw &= ~(1 << BIO_RW_BARRIER);
goto retry;
}
if (error_bits)
*error_bits = io->error_bits;
return io->error_bits ? -EIO : 0;
}
/**
* mempool_alloc - allocate an element from a specific memory pool
* @pool: pointer to the memory pool which was allocated via
* mempool_create().
* @gfp_mask: the usual allocation bitmask.
*
* this function only sleeps if the alloc_fn function sleeps or
* returns NULL. Note that due to preallocation, this function
* *never* fails when called from process contexts. (it might
* fail if called from an IRQ context.)
*/
void * mempool_alloc(mempool_t *pool, gfp_t gfp_mask)
{
void *element;
unsigned long flags;
wait_queue_t wait;
gfp_t gfp_temp;
might_sleep_if(gfp_mask & __GFP_WAIT);
gfp_mask |= __GFP_NOMEMALLOC; /* don't allocate emergency reserves */
gfp_mask |= __GFP_NORETRY; /* don't loop in __alloc_pages */
gfp_mask |= __GFP_NOWARN; /* failures are OK */
gfp_temp = gfp_mask & ~(__GFP_WAIT|__GFP_IO);
repeat_alloc:
element = pool->alloc(gfp_temp, pool->pool_data);
if (likely(element != NULL))
return element;
spin_lock_irqsave(&pool->lock, flags);
if (likely(pool->curr_nr)) {
element = remove_element(pool);
spin_unlock_irqrestore(&pool->lock, flags);
return element;
}
spin_unlock_irqrestore(&pool->lock, flags);
/* We must not sleep in the GFP_ATOMIC case */
if (!(gfp_mask & __GFP_WAIT))
return NULL;
/* Now start performing page reclaim */
gfp_temp = gfp_mask;
init_wait(&wait);
prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE);
smp_mb();
if (!pool->curr_nr)
io_schedule();
finish_wait(&pool->wait, &wait);
goto repeat_alloc;
}
static void
cv_wait_common(kcondvar_t *cvp, kmutex_t *mp, int state, int io)
{
DEFINE_WAIT(wait);
SENTRY;
ASSERT(cvp);
ASSERT(mp);
ASSERT(cvp->cv_magic == CV_MAGIC);
ASSERT(mutex_owned(mp));
atomic_inc(&cvp->cv_refs);
if (cvp->cv_mutex == NULL)
cvp->cv_mutex = mp;
/* Ensure the same mutex is used by all callers */
ASSERT(cvp->cv_mutex == mp);
prepare_to_wait_exclusive(&cvp->cv_event, &wait, state);
atomic_inc(&cvp->cv_waiters);
/* Mutex should be dropped after prepare_to_wait() this
* ensures we're linked in to the waiters list and avoids the
* race where 'cvp->cv_waiters > 0' but the list is empty. */
mutex_exit(mp);
if (io)
io_schedule();
else
schedule();
mutex_enter(mp);
/* No more waiters a different mutex could be used */
if (atomic_dec_and_test(&cvp->cv_waiters)) {
cvp->cv_mutex = NULL;
wake_up(&cvp->cv_destroy);
}
finish_wait(&cvp->cv_event, &wait);
atomic_dec(&cvp->cv_refs);
SEXIT;
}
static struct nullb_cmd *alloc_cmd(struct nullb_queue *nq, int can_wait)
{
struct nullb_cmd *cmd;
DEFINE_WAIT(wait);
cmd = __alloc_cmd(nq);
if (cmd || !can_wait)
return cmd;
do {
prepare_to_wait(&nq->wait, &wait, TASK_UNINTERRUPTIBLE);
cmd = __alloc_cmd(nq);
if (cmd)
break;
io_schedule();
} while (1);
finish_wait(&nq->wait, &wait);
return cmd;
}
static int sync_io(struct dm_io_client *client, unsigned int num_regions,
struct dm_io_region *where, int rw, struct dpages *dp,
unsigned long *error_bits)
{
volatile char io_[sizeof(struct io) + __alignof__(struct io) - 1];
struct io *io = (struct io *)PTR_ALIGN(&io_, __alignof__(struct io));
if (num_regions > 1 && (rw & RW_MASK) != WRITE) {
WARN_ON(1);
return -EIO;
}
io->error_bits = 0;
atomic_set(&io->count, 1);
io->sleeper = current;
io->client = client;
io->vma_invalidate_address = dp->vma_invalidate_address;
io->vma_invalidate_size = dp->vma_invalidate_size;
dispatch_io(rw, num_regions, where, dp, io, 1);
while (1) {
set_current_state(TASK_UNINTERRUPTIBLE);
if (!atomic_read(&io->count))
break;
io_schedule();
}
set_current_state(TASK_RUNNING);
if (error_bits)
*error_bits = io->error_bits;
return io->error_bits ? -EIO : 0;
}
/*
* We need to be able to change a mapping table under a mounted
* filesystem. For example we might want to move some data in
* the background. Before the table can be swapped with
* dm_bind_table, dm_suspend must be called to flush any in
* flight bios and ensure that any further io gets deferred.
*/
int dm_suspend(struct mapped_device *md, unsigned suspend_flags)
{
struct dm_table *map = NULL;
unsigned long flags;
DECLARE_WAITQUEUE(wait, current);
struct bio *def;
int r = -EINVAL;
int do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG ? 1 : 0;
int noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG ? 1 : 0;
down(&md->suspend_lock);
if (dm_suspended(md))
goto out_unlock;
map = dm_get_table(md);
/*
* DMF_NOFLUSH_SUSPENDING must be set before presuspend.
* This flag is cleared before dm_suspend returns.
*/
if (noflush)
set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
/* This does not get reverted if there's an error later. */
dm_table_presuspend_targets(map);
/* bdget() can stall if the pending I/Os are not flushed */
if (!noflush) {
md->suspended_bdev = bdget_disk(md->disk, 0);
if (!md->suspended_bdev) {
DMWARN("bdget failed in dm_suspend");
r = -ENOMEM;
goto flush_and_out;
}
}
/*
* Flush I/O to the device.
* noflush supersedes do_lockfs, because lock_fs() needs to flush I/Os.
*/
if (do_lockfs && !noflush) {
r = lock_fs(md);
if (r)
goto out;
}
/*
* First we set the BLOCK_IO flag so no more ios will be mapped.
*/
down_write(&md->io_lock);
set_bit(DMF_BLOCK_IO, &md->flags);
add_wait_queue(&md->wait, &wait);
up_write(&md->io_lock);
/* unplug */
if (map)
dm_table_unplug_all(map);
/*
* Then we wait for the already mapped ios to
* complete.
*/
while (1) {
set_current_state(TASK_INTERRUPTIBLE);
if (!atomic_read(&md->pending) || signal_pending(current))
break;
io_schedule();
}
set_current_state(TASK_RUNNING);
down_write(&md->io_lock);
remove_wait_queue(&md->wait, &wait);
if (noflush) {
spin_lock_irqsave(&md->pushback_lock, flags);
clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
bio_list_merge_head(&md->deferred, &md->pushback);
bio_list_init(&md->pushback);
spin_unlock_irqrestore(&md->pushback_lock, flags);
}
/* were we interrupted ? */
r = -EINTR;
if (atomic_read(&md->pending)) {
clear_bit(DMF_BLOCK_IO, &md->flags);
def = bio_list_get(&md->deferred);
__flush_deferred_io(md, def);
up_write(&md->io_lock);
unlock_fs(md);
goto out; /* pushback list is already flushed, so skip flush */
}
up_write(&md->io_lock);
dm_table_postsuspend_targets(map);
set_bit(DMF_SUSPENDED, &md->flags);
r = 0;
flush_and_out:
if (r && noflush) {
/*
* Because there may be already I/Os in the pushback list,
* flush them before return.
*/
down_write(&md->io_lock);
spin_lock_irqsave(&md->pushback_lock, flags);
clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
bio_list_merge_head(&md->deferred, &md->pushback);
bio_list_init(&md->pushback);
spin_unlock_irqrestore(&md->pushback_lock, flags);
def = bio_list_get(&md->deferred);
__flush_deferred_io(md, def);
up_write(&md->io_lock);
}
out:
if (r && md->suspended_bdev) {
bdput(md->suspended_bdev);
md->suspended_bdev = NULL;
}
dm_table_put(map);
out_unlock:
up(&md->suspend_lock);
return r;
}
/*
* We need to be able to change a mapping table under a mounted
* filesystem. For example we might want to move some data in
* the background. Before the table can be swapped with
* dm_bind_table, dm_suspend must be called to flush any in
* flight bios and ensure that any further io gets deferred.
*/
int dm_suspend(struct mapped_device *md, int do_lockfs)
{
struct dm_table *map = NULL;
DECLARE_WAITQUEUE(wait, current);
struct bio *def;
int r = -EINVAL;
down(&md->suspend_lock);
if (dm_suspended(md))
goto out;
map = dm_get_table(md);
/* This does not get reverted if there's an error later. */
dm_table_presuspend_targets(map);
md->suspended_bdev = bdget_disk(md->disk, 0);
if (!md->suspended_bdev) {
DMWARN("bdget failed in dm_suspend");
r = -ENOMEM;
goto out;
}
/* Flush I/O to the device. */
if (do_lockfs) {
r = lock_fs(md);
if (r)
goto out;
}
/*
* First we set the BLOCK_IO flag so no more ios will be mapped.
*/
down_write(&md->io_lock);
set_bit(DMF_BLOCK_IO, &md->flags);
add_wait_queue(&md->wait, &wait);
up_write(&md->io_lock);
/* unplug */
if (map)
dm_table_unplug_all(map);
/*
* Then we wait for the already mapped ios to
* complete.
*/
while (1) {
set_current_state(TASK_INTERRUPTIBLE);
if (!atomic_read(&md->pending) || signal_pending(current))
break;
io_schedule();
}
set_current_state(TASK_RUNNING);
down_write(&md->io_lock);
remove_wait_queue(&md->wait, &wait);
/* were we interrupted ? */
r = -EINTR;
if (atomic_read(&md->pending)) {
clear_bit(DMF_BLOCK_IO, &md->flags);
def = bio_list_get(&md->deferred);
__flush_deferred_io(md, def);
up_write(&md->io_lock);
unlock_fs(md);
goto out;
}
up_write(&md->io_lock);
dm_table_postsuspend_targets(map);
set_bit(DMF_SUSPENDED, &md->flags);
r = 0;
out:
if (r && md->suspended_bdev) {
bdput(md->suspended_bdev);
md->suspended_bdev = NULL;
}
dm_table_put(map);
up(&md->suspend_lock);
return r;
}
int bit_wait_io(void *word)
{
io_schedule();
return 0;
}
static int nfs_wait_bit_uninterruptible(void *word)
{
io_schedule();
return 0;
}
static void wait_io(void)
{
while(atomic_read(&io_done))
io_schedule();
}
/*
* We need to be able to change a mapping table under a mounted
* filesystem. For example we might want to move some data in
* the background. Before the table can be swapped with
* dm_bind_table, dm_suspend must be called to flush any in
* flight bios and ensure that any further io gets deferred.
*/
int dm_suspend(struct mapped_device *md)
{
struct dm_table *map;
DECLARE_WAITQUEUE(wait, current);
/* Flush I/O to the device. */
down_read(&md->lock);
if (test_bit(DMF_BLOCK_IO, &md->flags)) {
up_read(&md->lock);
return -EINVAL;
}
__lock_fs(md);
up_read(&md->lock);
/*
* First we set the BLOCK_IO flag so no more ios will be
* mapped.
*/
down_write(&md->lock);
if (test_bit(DMF_BLOCK_IO, &md->flags)) {
/*
* If we get here we know another thread is
* trying to suspend as well, so we leave the fs
* locked for this thread.
*/
up_write(&md->lock);
return -EINVAL;
}
set_bit(DMF_BLOCK_IO, &md->flags);
add_wait_queue(&md->wait, &wait);
up_write(&md->lock);
/* unplug */
map = dm_get_table(md);
if (map) {
dm_table_unplug_all(map);
dm_table_put(map);
}
/*
* Then we wait for the already mapped ios to
* complete.
*/
while (1) {
set_current_state(TASK_INTERRUPTIBLE);
if (!atomic_read(&md->pending) || signal_pending(current))
break;
io_schedule();
}
set_current_state(TASK_RUNNING);
down_write(&md->lock);
remove_wait_queue(&md->wait, &wait);
/* were we interrupted ? */
if (atomic_read(&md->pending)) {
__unlock_fs(md);
clear_bit(DMF_BLOCK_IO, &md->flags);
up_write(&md->lock);
return -EINTR;
}
set_bit(DMF_SUSPENDED, &md->flags);
map = dm_get_table(md);
if (map)
dm_table_suspend_targets(map);
dm_table_put(map);
up_write(&md->lock);
return 0;
}
static int sleep_on_page(void *word)
{
io_schedule();
return 0;
}