/**
* wb_get_create - get wb for a given memcg, create if necessary
* @bdi: target bdi
* @memcg_css: cgroup_subsys_state of the target memcg (must have positive ref)
* @gfp: allocation mask to use
*
* Try to get the wb for @memcg_css on @bdi. If it doesn't exist, try to
* create one. The returned wb has its refcount incremented.
*
* This function uses css_get() on @memcg_css and thus expects its refcnt
* to be positive on invocation. IOW, rcu_read_lock() protection on
* @memcg_css isn't enough. try_get it before calling this function.
*
* A wb is keyed by its associated memcg. As blkcg implicitly enables
* memcg on the default hierarchy, memcg association is guaranteed to be
* more specific (equal or descendant to the associated blkcg) and thus can
* identify both the memcg and blkcg associations.
*
* Because the blkcg associated with a memcg may change as blkcg is enabled
* and disabled closer to root in the hierarchy, each wb keeps track of
* both the memcg and blkcg associated with it and verifies the blkcg on
* each lookup. On mismatch, the existing wb is discarded and a new one is
* created.
*/
struct bdi_writeback *wb_get_create(struct backing_dev_info *bdi,
struct cgroup_subsys_state *memcg_css,
gfp_t gfp)
{
struct bdi_writeback *wb;
might_sleep_if(gfpflags_allow_blocking(gfp));
if (!memcg_css->parent)
return &bdi->wb;
do {
rcu_read_lock();
wb = radix_tree_lookup(&bdi->cgwb_tree, memcg_css->id);
if (wb) {
struct cgroup_subsys_state *blkcg_css;
/* see whether the blkcg association has changed */
blkcg_css = cgroup_get_e_css(memcg_css->cgroup,
&io_cgrp_subsys);
if (unlikely(wb->blkcg_css != blkcg_css ||
!wb_tryget(wb)))
wb = NULL;
css_put(blkcg_css);
}
rcu_read_unlock();
} while (!wb && !cgwb_create(bdi, memcg_css, gfp));
return wb;
}
/*
* Hooks for other subsystems that check memory allocations. In a typical
* production configuration these hooks all should produce no code at all.
*/
static inline int slab_pre_alloc_hook(struct kmem_cache *s, gfp_t flags)
{
flags &= gfp_allowed_mask;
lockdep_trace_alloc(flags);
might_sleep_if(flags & __GFP_WAIT);
return should_failslab(s->objsize, flags, s->flags);
}
/**
* mempool_alloc - allocate an element from a specific memory pool
* @pool: pointer to the memory pool which was allocated via
* mempool_create().
* @gfp_mask: the usual allocation bitmask.
*
* this function only sleeps if the alloc_fn() function sleeps or
* returns NULL. Note that due to preallocation, this function
* *never* fails when called from process contexts. (it might
* fail if called from an IRQ context.)
*/
void * mempool_alloc(mempool_t *pool, gfp_t gfp_mask)
{
void *element;
unsigned long flags;
wait_queue_t wait;
gfp_t gfp_temp;
#ifdef CONFIG_KRG_EPM
struct task_struct *krg_cur;
#endif
might_sleep_if(gfp_mask & __GFP_WAIT);
gfp_mask |= __GFP_NOMEMALLOC; /* don't allocate emergency reserves */
gfp_mask |= __GFP_NORETRY; /* don't loop in __alloc_pages */
gfp_mask |= __GFP_NOWARN; /* failures are OK */
gfp_temp = gfp_mask & ~(__GFP_WAIT|__GFP_IO);
repeat_alloc:
element = pool->alloc(gfp_temp, pool->pool_data);
if (likely(element != NULL))
return element;
spin_lock_irqsave(&pool->lock, flags);
if (likely(pool->curr_nr)) {
element = remove_element(pool);
spin_unlock_irqrestore(&pool->lock, flags);
return element;
}
spin_unlock_irqrestore(&pool->lock, flags);
/* We must not sleep in the GFP_ATOMIC case */
if (!(gfp_mask & __GFP_WAIT))
return NULL;
#ifdef CONFIG_KRG_EPM
krg_current_save(krg_cur);
#endif
/* Now start performing page reclaim */
gfp_temp = gfp_mask;
init_wait(&wait);
prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE);
smp_mb();
if (!pool->curr_nr) {
/*
* FIXME: this should be io_schedule(). The timeout is there
* as a workaround for some DM problems in 2.6.18.
*/
io_schedule_timeout(5*HZ);
}
finish_wait(&pool->wait, &wait);
#ifdef CONFIG_KRG_EPM
krg_current_restore(krg_cur);
#endif
goto repeat_alloc;
}
/**
* mempool_alloc - allocate an element from a specific memory pool
* @pool: pointer to the memory pool which was allocated via
* mempool_create().
* @gfp_mask: the usual allocation bitmask.
*
* this function only sleeps if the alloc_fn function sleeps or
* returns NULL. Note that due to preallocation, this function
* *never* fails when called from process contexts. (it might
* fail if called from an IRQ context.)
*/
void * mempool_alloc(mempool_t *pool, int gfp_mask)
{
void *element;
unsigned long flags;
DEFINE_WAIT(wait);
int gfp_nowait = gfp_mask & ~(__GFP_WAIT | __GFP_IO);
might_sleep_if(gfp_mask & __GFP_WAIT);
repeat_alloc:
element = pool->alloc(gfp_nowait|__GFP_NOWARN, pool->pool_data);
if (likely(element != NULL))
return element;
/*
* If the pool is less than 50% full and we can perform effective
* page reclaim then try harder to allocate an element.
*/
mb();
if ((gfp_mask & __GFP_FS) && (gfp_mask != gfp_nowait) &&
(pool->curr_nr <= pool->min_nr/2)) {
element = pool->alloc(gfp_mask, pool->pool_data);
if (likely(element != NULL))
return element;
}
/*
* Kick the VM at this point.
*/
wakeup_bdflush(0);
spin_lock_irqsave(&pool->lock, flags);
if (likely(pool->curr_nr)) {
element = remove_element(pool);
spin_unlock_irqrestore(&pool->lock, flags);
return element;
}
spin_unlock_irqrestore(&pool->lock, flags);
/* We must not sleep in the GFP_ATOMIC case */
if (!(gfp_mask & __GFP_WAIT))
return NULL;
prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE);
mb();
if (!pool->curr_nr)
io_schedule();
finish_wait(&pool->wait, &wait);
goto repeat_alloc;
}
/**
* mempool_alloc - allocate an element from a specific memory pool
* @pool: pointer to the memory pool which was allocated via
* mempool_create().
* @gfp_mask: the usual allocation bitmask.
*
* this function only sleeps if the alloc_fn function sleeps or
* returns NULL. Note that due to preallocation, this function
* *never* fails when called from process contexts. (it might
* fail if called from an IRQ context.)
*/
void * mempool_alloc(mempool_t *pool, gfp_t gfp_mask)
{
void *element;
unsigned long flags;
wait_queue_t wait;
gfp_t gfp_temp;
might_sleep_if(gfp_mask & __GFP_WAIT);
gfp_mask |= __GFP_NOMEMALLOC; /* don't allocate emergency reserves */
gfp_mask |= __GFP_NORETRY; /* don't loop in __alloc_pages */
gfp_mask |= __GFP_NOWARN; /* failures are OK */
gfp_temp = gfp_mask & ~(__GFP_WAIT|__GFP_IO);
repeat_alloc:
element = pool->alloc(gfp_temp, pool->pool_data);
if (likely(element != NULL))
return element;
spin_lock_irqsave(&pool->lock, flags);
if (likely(pool->curr_nr)) {
element = remove_element(pool);
spin_unlock_irqrestore(&pool->lock, flags);
return element;
}
spin_unlock_irqrestore(&pool->lock, flags);
/* We must not sleep in the GFP_ATOMIC case */
if (!(gfp_mask & __GFP_WAIT))
return NULL;
/* Now start performing page reclaim */
gfp_temp = gfp_mask;
init_wait(&wait);
prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE);
smp_mb();
if (!pool->curr_nr)
io_schedule();
finish_wait(&pool->wait, &wait);
goto repeat_alloc;
}
/**
* bfq_get_io_context - return the @cic associated to @bfqd in @ioc.
* @bfqd: the search key.
* @gfp_mask: the mask to use for cic allocation.
*
* Setup general io context and cfq io context. There can be several cfq
* io contexts per general io context, if this process is doing io to more
* than one device managed by cfq.
*/
static struct cfq_io_context *bfq_get_io_context(struct bfq_data *bfqd,
gfp_t gfp_mask)
{
struct io_context *ioc = NULL;
struct cfq_io_context *cic;
might_sleep_if(gfp_mask & __GFP_WAIT);
ioc = get_io_context(gfp_mask, bfqd->queue->node);
if (ioc == NULL)
return NULL;
/* Lookup for an existing cic. */
cic = bfq_cic_lookup(bfqd, ioc);
if (cic != NULL)
goto out;
/* Alloc one if needed. */
cic = bfq_alloc_io_context(bfqd, gfp_mask);
if (cic == NULL)
goto err;
/* Link it into the ioc's radix tree and cic list. */
if (bfq_cic_link(bfqd, ioc, cic, gfp_mask) != 0)
goto err_free;
out:
/*
* test_and_clear_bit() implies a memory barrier, paired with
* the wmb() in fs/ioprio.c, so the value seen for ioprio is the
* new one.
*/
if (unlikely(test_and_clear_bit(IOC_BFQ_IOPRIO_CHANGED,
ioc->ioprio_changed)))
bfq_ioc_set_ioprio(ioc);
return cic;
err_free:
bfq_cic_free(cic);
err:
put_io_context(ioc);
return NULL;
}
/**
* get_task_io_context - get io_context of a task
* @task: task of interest
* @gfp_flags: allocation flags, used if allocation is necessary
* @node: allocation node, used if allocation is necessary
*
* Return io_context of @task. If it doesn't exist, it is created with
* @gfp_flags and @node. The returned io_context has its reference count
* incremented.
*
* This function always goes through task_lock() and it's better to use
* %current->io_context + get_io_context() for %current.
*/
struct io_context *get_task_io_context(struct task_struct *task,
gfp_t gfp_flags, int node)
{
struct io_context *ioc;
might_sleep_if(gfp_flags & __GFP_WAIT);
do {
task_lock(task);
ioc = task->io_context;
if (likely(ioc)) {
get_io_context(ioc);
task_unlock(task);
return ioc;
}
task_unlock(task);
} while (create_io_context(task, gfp_flags, node));
return NULL;
}
int ssb_pcicore_dev_irqvecs_enable(struct ssb_pcicore *pc,
struct ssb_device *dev)
{
struct ssb_device *pdev = pc->dev;
struct ssb_bus *bus;
int err = 0;
u32 tmp;
if (dev->bus->bustype != SSB_BUSTYPE_PCI) {
goto out;
}
if (!pdev)
goto out;
bus = pdev->bus;
might_sleep_if(pdev->id.coreid != SSB_DEV_PCI);
if (bus->host_pci &&
((pdev->id.revision >= 6) || (pdev->id.coreid == SSB_DEV_PCIE))) {
u32 coremask;
coremask = (1 << dev->core_index);
err = pci_read_config_dword(bus->host_pci, SSB_PCI_IRQMASK, &tmp);
if (err)
goto out;
tmp |= coremask << 8;
err = pci_write_config_dword(bus->host_pci, SSB_PCI_IRQMASK, tmp);
if (err)
goto out;
} else {
u32 intvec;
intvec = ssb_read32(pdev, SSB_INTVEC);
tmp = ssb_read32(dev, SSB_TPSFLAG);
tmp &= SSB_TPSFLAG_BPFLAG;
intvec |= (1 << tmp);
ssb_write32(pdev, SSB_INTVEC, intvec);
}
if (pc->setup_done)
goto out;
if (pdev->id.coreid == SSB_DEV_PCI) {
tmp = pcicore_read32(pc, SSB_PCICORE_SBTOPCI2);
tmp |= SSB_PCICORE_SBTOPCI_PREF;
tmp |= SSB_PCICORE_SBTOPCI_BURST;
pcicore_write32(pc, SSB_PCICORE_SBTOPCI2, tmp);
if (pdev->id.revision < 5) {
tmp = ssb_read32(pdev, SSB_IMCFGLO);
tmp &= ~SSB_IMCFGLO_SERTO;
tmp |= 2;
tmp &= ~SSB_IMCFGLO_REQTO;
tmp |= 3 << SSB_IMCFGLO_REQTO_SHIFT;
ssb_write32(pdev, SSB_IMCFGLO, tmp);
ssb_commit_settings(bus);
} else if (pdev->id.revision >= 11) {
tmp = pcicore_read32(pc, SSB_PCICORE_SBTOPCI2);
tmp |= SSB_PCICORE_SBTOPCI_MRM;
pcicore_write32(pc, SSB_PCICORE_SBTOPCI2, tmp);
}
} else {
WARN_ON(pdev->id.coreid != SSB_DEV_PCIE);
if ((pdev->id.revision == 0) || (pdev->id.revision == 1)) {
tmp = ssb_pcie_read(pc, 0x4);
tmp |= 0x8;
ssb_pcie_write(pc, 0x4, tmp);
}
if (pdev->id.revision == 0) {
const u8 serdes_rx_device = 0x1F;
ssb_pcie_mdio_write(pc, serdes_rx_device,
2 , 0x8128);
ssb_pcie_mdio_write(pc, serdes_rx_device,
6 , 0x0100);
ssb_pcie_mdio_write(pc, serdes_rx_device,
7 , 0x1466);
} else if (pdev->id.revision == 1) {
tmp = ssb_pcie_read(pc, 0x100);
tmp |= 0x40;
ssb_pcie_write(pc, 0x100, tmp);
}
}
pc->setup_done = 1;
out:
return err;
}
/**
* mempool_alloc - allocate an element from a specific memory pool
* @pool: pointer to the memory pool which was allocated via
* mempool_create().
* @gfp_mask: the usual allocation bitmask.
*
* this function only sleeps if the alloc_fn() function sleeps or
* returns NULL. Note that due to preallocation, this function
* *never* fails when called from process contexts. (it might
* fail if called from an IRQ context.)
* Note: using __GFP_ZERO is not supported.
*/
void * mempool_alloc(mempool_t *pool, gfp_t gfp_mask)
{
void *element;
unsigned long flags;
wait_queue_t wait;
gfp_t gfp_temp;
VM_WARN_ON_ONCE(gfp_mask & __GFP_ZERO);
might_sleep_if(gfp_mask & __GFP_WAIT);
gfp_mask |= __GFP_NOMEMALLOC; /* don't allocate emergency reserves */
gfp_mask |= __GFP_NORETRY; /* don't loop in __alloc_pages */
gfp_mask |= __GFP_NOWARN; /* failures are OK */
gfp_temp = gfp_mask & ~(__GFP_WAIT|__GFP_IO);
repeat_alloc:
element = pool->alloc(gfp_temp, pool->pool_data);
if (likely(element != NULL))
return element;
spin_lock_irqsave(&pool->lock, flags);
if (likely(pool->curr_nr)) {
element = remove_element(pool);
spin_unlock_irqrestore(&pool->lock, flags);
/* paired with rmb in mempool_free(), read comment there */
smp_wmb();
/*
* Update the allocation stack trace as this is more useful
* for debugging.
*/
kmemleak_update_trace(element);
return element;
}
/*
* We use gfp mask w/o __GFP_WAIT or IO for the first round. If
* alloc failed with that and @pool was empty, retry immediately.
*/
if (gfp_temp != gfp_mask) {
spin_unlock_irqrestore(&pool->lock, flags);
gfp_temp = gfp_mask;
goto repeat_alloc;
}
/* We must not sleep if !__GFP_WAIT */
if (!(gfp_mask & __GFP_WAIT)) {
spin_unlock_irqrestore(&pool->lock, flags);
return NULL;
}
/* Let's wait for someone else to return an element to @pool */
init_wait(&wait);
prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE);
spin_unlock_irqrestore(&pool->lock, flags);
/*
* FIXME: this should be io_schedule(). The timeout is there as a
* workaround for some DM problems in 2.6.18.
*/
io_schedule_timeout(5*HZ);
finish_wait(&pool->wait, &wait);
goto repeat_alloc;
}
int ssb_pcicore_dev_irqvecs_enable(struct ssb_pcicore *pc,
struct ssb_device *dev)
{
struct ssb_device *pdev = pc->dev;
struct ssb_bus *bus;
int err = 0;
u32 tmp;
if (dev->bus->bustype != SSB_BUSTYPE_PCI) {
/* This SSB device is not on a PCI host-bus. So the IRQs are
* not routed through the PCI core.
* So we must not enable routing through the PCI core. */
goto out;
}
if (!pdev)
goto out;
bus = pdev->bus;
might_sleep_if(pdev->id.coreid != SSB_DEV_PCI);
/* Enable interrupts for this device. */
if ((pdev->id.revision >= 6) || (pdev->id.coreid == SSB_DEV_PCIE)) {
u32 coremask;
/* Calculate the "coremask" for the device. */
coremask = (1 << dev->core_index);
SSB_WARN_ON(bus->bustype != SSB_BUSTYPE_PCI);
err = pci_read_config_dword(bus->host_pci, SSB_PCI_IRQMASK, &tmp);
if (err)
goto out;
tmp |= coremask << 8;
err = pci_write_config_dword(bus->host_pci, SSB_PCI_IRQMASK, tmp);
if (err)
goto out;
} else {
u32 intvec;
intvec = ssb_read32(pdev, SSB_INTVEC);
tmp = ssb_read32(dev, SSB_TPSFLAG);
tmp &= SSB_TPSFLAG_BPFLAG;
intvec |= (1 << tmp);
ssb_write32(pdev, SSB_INTVEC, intvec);
}
/* Setup PCIcore operation. */
if (pc->setup_done)
goto out;
if (pdev->id.coreid == SSB_DEV_PCI) {
tmp = pcicore_read32(pc, SSB_PCICORE_SBTOPCI2);
tmp |= SSB_PCICORE_SBTOPCI_PREF;
tmp |= SSB_PCICORE_SBTOPCI_BURST;
pcicore_write32(pc, SSB_PCICORE_SBTOPCI2, tmp);
if (pdev->id.revision < 5) {
tmp = ssb_read32(pdev, SSB_IMCFGLO);
tmp &= ~SSB_IMCFGLO_SERTO;
tmp |= 2;
tmp &= ~SSB_IMCFGLO_REQTO;
tmp |= 3 << SSB_IMCFGLO_REQTO_SHIFT;
ssb_write32(pdev, SSB_IMCFGLO, tmp);
ssb_commit_settings(bus);
} else if (pdev->id.revision >= 11) {
tmp = pcicore_read32(pc, SSB_PCICORE_SBTOPCI2);
tmp |= SSB_PCICORE_SBTOPCI_MRM;
pcicore_write32(pc, SSB_PCICORE_SBTOPCI2, tmp);
}
} else {
WARN_ON(pdev->id.coreid != SSB_DEV_PCIE);
//TODO: Better make defines for all these magic PCIE values.
if ((pdev->id.revision == 0) || (pdev->id.revision == 1)) {
/* TLP Workaround register. */
tmp = ssb_pcie_read(pc, 0x4);
tmp |= 0x8;
ssb_pcie_write(pc, 0x4, tmp);
}
if (pdev->id.revision == 0) {
const u8 serdes_rx_device = 0x1F;
ssb_pcie_mdio_write(pc, serdes_rx_device,
2 /* Timer */, 0x8128);
ssb_pcie_mdio_write(pc, serdes_rx_device,
6 /* CDR */, 0x0100);
ssb_pcie_mdio_write(pc, serdes_rx_device,
7 /* CDR BW */, 0x1466);
} else if (pdev->id.revision == 1) {
/* DLLP Link Control register. */
tmp = ssb_pcie_read(pc, 0x100);
tmp |= 0x40;
ssb_pcie_write(pc, 0x100, tmp);
}
}
pc->setup_done = 1;
out:
return err;
}
/* _VMKLNX_CODECHECK_: mempool_alloc */
void * mempool_alloc(mempool_t *pool, gfp_t gfp_mask)
{
void *element;
unsigned long flags;
wait_queue_t wait;
gfp_t gfp_temp;
#if defined(__VMKLNX__)
VMK_ASSERT(vmk_PreemptionIsEnabled() == VMK_FALSE);
#endif
might_sleep_if(gfp_mask & __GFP_WAIT);
gfp_mask |= __GFP_NOMEMALLOC; /* don't allocate emergency reserves */
gfp_mask |= __GFP_NORETRY; /* don't loop in __alloc_pages */
gfp_mask |= __GFP_NOWARN; /* failures are OK */
gfp_temp = gfp_mask & ~(__GFP_WAIT|__GFP_IO);
#if defined(__VMKLNX__) && defined(VMX86_DEBUG)
if (gfp_mask & __GFP_WAIT) {
vmk_WorldAssertIsSafeToBlock();
}
#endif /* defined(__VMKLNX__) */
repeat_alloc:
#if defined(__VMKLNX__)
VMKAPI_MODULE_CALL(pool->module_id, element, pool->alloc,
gfp_temp, pool->pool_data);
#else /* !defined(__VMKLNX__) */
element = pool->alloc(gfp_temp, pool->pool_data);
#endif /* defined(__VMKLNX__) */
if (likely(element != NULL))
return element;
spin_lock_irqsave(&pool->lock, flags);
if (likely(pool->curr_nr)) {
element = remove_element(pool);
spin_unlock_irqrestore(&pool->lock, flags);
return element;
}
spin_unlock_irqrestore(&pool->lock, flags);
/* We must not sleep in the GFP_ATOMIC case */
if (!(gfp_mask & __GFP_WAIT))
return NULL;
/* Now start performing page reclaim */
gfp_temp = gfp_mask;
init_wait(&wait);
prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE);
smp_mb();
if (!pool->curr_nr) {
/*
* FIXME: this should be io_schedule(). The timeout is there
* as a workaround for some DM problems in 2.6.18.
*/
#if defined(__VMKLNX__)
schedule_timeout(5*HZ);
#else /* !defined(__VMKLNX__) */
io_schedule_timeout(5*HZ);
#endif /* defined(__VMKLNX__) */
}
finish_wait(&pool->wait, &wait);
goto repeat_alloc;
}
