void ovs_flow_stats_update(struct sw_flow *flow, __be16 tcp_flags,
const struct sk_buff *skb)
{
struct flow_stats *stats;
int node = numa_node_id();
stats = rcu_dereference(flow->stats[node]);
/* Check if already have node-specific stats. */
if (likely(stats)) {
spin_lock(&stats->lock);
/* Mark if we write on the pre-allocated stats. */
if (node == 0 && unlikely(flow->stats_last_writer != node))
flow->stats_last_writer = node;
} else {
stats = rcu_dereference(flow->stats[0]); /* Pre-allocated. */
spin_lock(&stats->lock);
/* If the current NUMA-node is the only writer on the
* pre-allocated stats keep using them.
*/
if (unlikely(flow->stats_last_writer != node)) {
/* A previous locker may have already allocated the
* stats, so we need to check again. If node-specific
* stats were already allocated, we update the pre-
* allocated stats as we have already locked them.
*/
if (likely(flow->stats_last_writer != NUMA_NO_NODE)
&& likely(!rcu_access_pointer(flow->stats[node]))) {
/* Try to allocate node-specific stats. */
struct flow_stats *new_stats;
new_stats =
kmem_cache_alloc_node(flow_stats_cache,
GFP_THISNODE |
__GFP_NOMEMALLOC,
node);
if (likely(new_stats)) {
new_stats->used = jiffies;
new_stats->packet_count = 1;
new_stats->byte_count = skb->len;
new_stats->tcp_flags = tcp_flags;
spin_lock_init(&new_stats->lock);
rcu_assign_pointer(flow->stats[node],
new_stats);
goto unlock;
}
}
flow->stats_last_writer = node;
}
}
stats->used = jiffies;
stats->packet_count++;
stats->byte_count += skb->len;
stats->tcp_flags |= tcp_flags;
unlock:
spin_unlock(&stats->lock);
}
void *
cfs_mem_cache_cpt_alloc(struct kmem_cache *cachep, struct cfs_cpt_table *cptab,
int cpt, gfp_t flags)
{
return kmem_cache_alloc_node(cachep, flags,
cfs_cpt_spread_node(cptab, cpt));
}
struct io_context *alloc_io_context(gfp_t gfp_flags, int node)
{
struct io_context *ioc;
ioc = kmem_cache_alloc_node(iocontext_cachep, gfp_flags, node);
if (ioc) {
atomic_long_set(&ioc->refcount, 1);
atomic_set(&ioc->nr_tasks, 1);
spin_lock_init(&ioc->lock);
//* ioc->ioprio_changed = 0;
bitmap_zero(ioc->ioprio_changed, IOC_IOPRIO_CHANGED_BITS);
ioc->ioprio = 0;
ioc->last_waited = 0; /* doesn't matter... */
ioc->nr_batch_requests = 0; /* because this is 0 */
INIT_RADIX_TREE(&ioc->radix_root, GFP_ATOMIC | __GFP_HIGH);
INIT_HLIST_HEAD(&ioc->cic_list);
INIT_RADIX_TREE(&ioc->bfq_radix_root, GFP_ATOMIC | __GFP_HIGH);
INIT_HLIST_HEAD(&ioc->bfq_cic_list);
ioc->ioc_data = NULL;
#if defined(CONFIG_BLK_CGROUP) || defined(CONFIG_BLK_CGROUP_MODULE)
ioc->cgroup_changed = 0;
#endif
}
return ioc;
}
struct sw_flow *ovs_flow_alloc(void)
{
struct sw_flow *flow;
struct flow_stats *stats;
int node;
flow = kmem_cache_alloc(flow_cache, GFP_KERNEL);
if (!flow)
return ERR_PTR(-ENOMEM);
flow->sf_acts = NULL;
flow->mask = NULL;
flow->stats_last_writer = NUMA_NO_NODE;
/* Initialize the default stat node. */
stats = kmem_cache_alloc_node(flow_stats_cache,
GFP_KERNEL | __GFP_ZERO, 0);
if (!stats)
goto err;
spin_lock_init(&stats->lock);
RCU_INIT_POINTER(flow->stats[0], stats);
for_each_node(node)
if (node != 0)
RCU_INIT_POINTER(flow->stats[node], NULL);
return flow;
err:
kmem_cache_free(flow_cache, flow);
return ERR_PTR(-ENOMEM);
}
void create_io_context_slowpath(struct task_struct *task, gfp_t gfp_flags,
int node)
{
struct io_context *ioc;
ioc = kmem_cache_alloc_node(iocontext_cachep, gfp_flags | __GFP_ZERO,
node);
if (unlikely(!ioc))
return;
/* initialize */
atomic_long_set(&ioc->refcount, 1);
atomic_set(&ioc->nr_tasks, 1);
spin_lock_init(&ioc->lock);
INIT_RADIX_TREE(&ioc->icq_tree, GFP_ATOMIC | __GFP_HIGH);
INIT_HLIST_HEAD(&ioc->icq_list);
INIT_WORK(&ioc->release_work, ioc_release_fn);
/*
* Try to install. ioc shouldn't be installed if someone else
* already did or @task, which isn't %current, is exiting. Note
* that we need to allow ioc creation on exiting %current as exit
* path may issue IOs from e.g. exit_files(). The exit path is
* responsible for not issuing IO after exit_io_context().
*/
task_lock(task);
if (!task->io_context &&
(task == current || !(task->flags & PF_EXITING)))
task->io_context = ioc;
else
kmem_cache_free(iocontext_cachep, ioc);
task_unlock(task);
}
/**
* __alloc_skb - allocate a network buffer
* @size: size to allocate
* @gfp_mask: allocation mask
* @fclone: allocate from fclone cache instead of head cache
* and allocate a cloned (child) skb
* @node: numa node to allocate memory on
*
* Allocate a new &sk_buff. The returned buffer has no headroom and a
* tail room of size bytes. The object has a reference count of one.
* The return is the buffer. On a failure the return is %NULL.
*
* Buffers may only be allocated from interrupts using a @gfp_mask of
* %GFP_ATOMIC.
*/
struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
int fclone, int node)
{
struct kmem_cache *cache;
struct skb_shared_info *shinfo;
struct sk_buff *skb;
u8 *data;
cache = fclone ? skbuff_fclone_cache : skbuff_head_cache;
/* Get the HEAD */
skb = kmem_cache_alloc_node(cache, gfp_mask & ~__GFP_DMA, node);
if (!skb)
goto out;
/* Get the DATA. Size must match skb_add_mtu(). */
size = SKB_DATA_ALIGN(size);
data = kmalloc_node_track_caller(size + sizeof(struct skb_shared_info),
gfp_mask, node);
if (!data)
goto nodata;
memset(skb, 0, offsetof(struct sk_buff, truesize));
skb->truesize = size + sizeof(struct sk_buff);
atomic_set(&skb->users, 1);
skb->head = data;
skb->data = data;
skb->tail = data;
skb->end = data + size;
/* make sure we initialize shinfo sequentially */
shinfo = skb_shinfo(skb);
atomic_set(&shinfo->dataref, 1);
shinfo->nr_frags = 0;
shinfo->gso_size = 0;
shinfo->gso_segs = 0;
shinfo->gso_type = 0;
shinfo->ip6_frag_id = 0;
shinfo->frag_list = NULL;
if (fclone) {
struct sk_buff *child = skb + 1;
atomic_t *fclone_ref = (atomic_t *) (child + 1);
skb->fclone = SKB_FCLONE_ORIG;
atomic_set(fclone_ref, 1);
child->fclone = SKB_FCLONE_UNAVAILABLE;
}
out:
return skb;
nodata:
kmem_cache_free(cache, skb);
skb = NULL;
goto out;
}
struct thread_info *alloc_thread_info_node(struct task_struct *tsk, int node)
{
struct thread_info *ti;
#ifdef CONFIG_DEBUG_STACK_USAGE
gfp_t mask = GFP_KERNEL | __GFP_ZERO;
#else
gfp_t mask = GFP_KERNEL;
#endif
ti = kmem_cache_alloc_node(thread_info_cache, mask, node);
return ti;
}
struct thread_info *alloc_thread_info_node(struct task_struct *tsk, int node)
{
struct thread_info *ti;
ti = kmem_cache_alloc_node(thread_info_cache, GFP_KERNEL, node);
if (unlikely(ti == NULL))
return NULL;
#ifdef CONFIG_DEBUG_STACK_USAGE
memset(ti, 0, THREAD_SIZE);
#endif
return ti;
}
/**
* ioc_create_icq - create and link io_cq
* @q: request_queue of interest
* @gfp_mask: allocation mask
*
* Make sure io_cq linking %current->io_context and @q exists. If either
* io_context and/or icq don't exist, they will be created using @gfp_mask.
*
* The caller is responsible for ensuring @ioc won't go away and @q is
* alive and will stay alive until this function returns.
*/
struct io_cq *ioc_create_icq(struct request_queue *q, gfp_t gfp_mask)
{
struct elevator_type *et = q->elevator->type;
struct io_context *ioc;
struct io_cq *icq;
/* allocate stuff */
ioc = create_io_context(current, gfp_mask, q->node);
if (!ioc)
return NULL;
icq = kmem_cache_alloc_node(et->icq_cache, gfp_mask | __GFP_ZERO,
q->node);
if (!icq)
return NULL;
if (radix_tree_preload(gfp_mask) < 0) {
kmem_cache_free(et->icq_cache, icq);
return NULL;
}
icq->ioc = ioc;
icq->q = q;
INIT_LIST_HEAD(&icq->q_node);
INIT_HLIST_NODE(&icq->ioc_node);
/* lock both q and ioc and try to link @icq */
spin_lock_irq(q->queue_lock);
spin_lock(&ioc->lock);
if (likely(!radix_tree_insert(&ioc->icq_tree, q->id, icq))) {
hlist_add_head(&icq->ioc_node, &ioc->icq_list);
list_add(&icq->q_node, &q->icq_list);
if (et->ops.elevator_init_icq_fn)
et->ops.elevator_init_icq_fn(icq);
} else {
kmem_cache_free(et->icq_cache, icq);
icq = ioc_lookup_icq(ioc, q);
if (!icq)
printk(KERN_ERR "cfq: icq link failed!\n");
}
spin_unlock(&ioc->lock);
spin_unlock_irq(q->queue_lock);
radix_tree_preload_end();
return icq;
}
static struct cfq_io_context *bfq_alloc_io_context(struct bfq_data *bfqd,
gfp_t gfp_mask)
{
struct cfq_io_context *cic;
cic = kmem_cache_alloc_node(bfq_ioc_pool, gfp_mask | __GFP_ZERO,
bfqd->queue->node);
if (cic != NULL) {
cic->last_end_request = jiffies;
INIT_LIST_HEAD(&cic->queue_list);
INIT_HLIST_NODE(&cic->cic_list);
cic->dtor = bfq_free_io_context;
cic->exit = bfq_exit_io_context;
elv_ioc_count_inc(bfq_ioc_count);
}
return cic;
}
struct io_context *alloc_io_context(gfp_t gfp_flags, int node)
{
struct io_context *ret;
ret = kmem_cache_alloc_node(iocontext_cachep, gfp_flags, node);
if (ret) {
atomic_long_set(&ret->refcount, 1);
atomic_set(&ret->nr_tasks, 1);
spin_lock_init(&ret->lock);
ret->ioprio_changed = 0;
ret->ioprio = 0;
ret->last_waited = 0; /* doesn't matter... */
ret->nr_batch_requests = 0; /* because this is 0 */
INIT_RADIX_TREE(&ret->radix_root, GFP_ATOMIC | __GFP_HIGH);
INIT_HLIST_HEAD(&ret->cic_list);
ret->ioc_data = NULL;
}
return ret;
}
static int dtl_enable(struct dtl *dtl)
{
long int n_entries;
long int rc;
struct dtl_entry *buf = NULL;
if (!dtl_cache)
return -ENOMEM;
/* only allow one reader */
if (dtl->buf)
return -EBUSY;
n_entries = dtl_buf_entries;
buf = kmem_cache_alloc_node(dtl_cache, GFP_KERNEL, cpu_to_node(dtl->cpu));
if (!buf) {
printk(KERN_WARNING "%s: buffer alloc failed for cpu %d\n",
__func__, dtl->cpu);
return -ENOMEM;
}
spin_lock(&dtl->lock);
rc = -EBUSY;
if (!dtl->buf) {
/* store the original allocation size for use during read */
dtl->buf_entries = n_entries;
dtl->buf = buf;
dtl->last_idx = 0;
rc = dtl_start(dtl);
if (rc)
dtl->buf = NULL;
}
spin_unlock(&dtl->lock);
if (rc)
kmem_cache_free(dtl_cache, buf);
return rc;
}
static struct cfq_io_context *bfq_alloc_io_context(struct bfq_data *bfqd,
gfp_t gfp_mask)
{
struct cfq_io_context *cic;
cic = kmem_cache_alloc_node(bfq_ioc_pool, gfp_mask | __GFP_ZERO,
bfqd->queue->node);
if (cic != NULL) {
cic->last_end_request = jiffies;
/*
* A newly created cic indicates that the process has just
* started doing I/O, and is probably mapping into memory its
* executable and libraries: it definitely needs weight raising.
* There is however the possibility that the process performs,
* for a while, I/O close to some other process. EQM intercepts
* this behavior and may merge the queue corresponding to the
* process with some other queue, BEFORE the weight of the queue
* is raised. Merged queues are not weight-raised (they are assumed
* to belong to processes that benefit only from high throughput).
* If the merge is basically the consequence of an accident, then
* the queue will be split soon and will get back its old weight.
* It is then important to write down somewhere that this queue
* does need weight raising, even if it did not make it to get its
* weight raised before being merged. To this purpose, we overload
* the field raising_time_left and assign 1 to it, to mark the queue
* as needing weight raising.
*/
cic->wr_time_left = 1;
INIT_LIST_HEAD(&cic->queue_list);
INIT_HLIST_NODE(&cic->cic_list);
cic->dtor = bfq_free_io_context;
cic->exit = bfq_exit_io_context;
elv_ioc_count_inc(bfq_ioc_count);
}
return cic;
}
static struct thread_info *alloc_thread_info_node(struct task_struct *tsk,
int node)
{
return kmem_cache_alloc_node(thread_info_cache, THREADINFO_GFP, node);
}
static inline struct task_struct *alloc_task_struct_node(int node)
{
return kmem_cache_alloc_node(task_struct_cachep, GFP_KERNEL, node);
}
void ovs_flow_stats_update(struct sw_flow *flow, struct sk_buff *skb)
{
struct flow_stats *stats;
__be16 tcp_flags = 0;
int node = numa_node_id();
stats = rcu_dereference(flow->stats[node]);
if ((flow->key.eth.type == htons(ETH_P_IP) ||
flow->key.eth.type == htons(ETH_P_IPV6)) &&
flow->key.ip.frag != OVS_FRAG_TYPE_LATER &&
flow->key.ip.proto == IPPROTO_TCP &&
likely(skb->len >= skb_transport_offset(skb) + sizeof(struct tcphdr))) {
tcp_flags = TCP_FLAGS_BE16(tcp_hdr(skb));
}
/* Check if already have node-specific stats. */
if (likely(stats)) {
spin_lock(&stats->lock);
/* Mark if we write on the pre-allocated stats. */
if (node == 0 && unlikely(flow->stats_last_writer != node))
flow->stats_last_writer = node;
} else {
stats = rcu_dereference(flow->stats[0]); /* Pre-allocated. */
spin_lock(&stats->lock);
/* If the current NUMA-node is the only writer on the
* pre-allocated stats keep using them.
*/
if (unlikely(flow->stats_last_writer != node)) {
/* A previous locker may have already allocated the
* stats, so we need to check again. If node-specific
* stats were already allocated, we update the pre-
* allocated stats as we have already locked them.
*/
if (likely(flow->stats_last_writer != NUMA_NO_NODE)
&& likely(!rcu_dereference(flow->stats[node]))) {
/* Try to allocate node-specific stats. */
struct flow_stats *new_stats;
new_stats =
kmem_cache_alloc_node(flow_stats_cache,
GFP_THISNODE |
__GFP_NOMEMALLOC,
node);
if (likely(new_stats)) {
new_stats->used = jiffies;
new_stats->packet_count = 1;
new_stats->byte_count = skb->len;
new_stats->tcp_flags = tcp_flags;
spin_lock_init(&new_stats->lock);
rcu_assign_pointer(flow->stats[node],
new_stats);
goto unlock;
}
}
flow->stats_last_writer = node;
}
}
stats->used = jiffies;
stats->packet_count++;
stats->byte_count += skb->len;
stats->tcp_flags |= tcp_flags;
unlock:
spin_unlock(&stats->lock);
}