void ovs_flow_stats_update(struct sw_flow *flow, __be16 tcp_flags, const struct sk_buff *skb) { struct flow_stats *stats; int node = numa_node_id(); stats = rcu_dereference(flow->stats[node]); /* Check if already have node-specific stats. */ if (likely(stats)) { spin_lock(&stats->lock); /* Mark if we write on the pre-allocated stats. */ if (node == 0 && unlikely(flow->stats_last_writer != node)) flow->stats_last_writer = node; } else { stats = rcu_dereference(flow->stats[0]); /* Pre-allocated. */ spin_lock(&stats->lock); /* If the current NUMA-node is the only writer on the * pre-allocated stats keep using them. */ if (unlikely(flow->stats_last_writer != node)) { /* A previous locker may have already allocated the * stats, so we need to check again. If node-specific * stats were already allocated, we update the pre- * allocated stats as we have already locked them. */ if (likely(flow->stats_last_writer != NUMA_NO_NODE) && likely(!rcu_access_pointer(flow->stats[node]))) { /* Try to allocate node-specific stats. */ struct flow_stats *new_stats; new_stats = kmem_cache_alloc_node(flow_stats_cache, GFP_THISNODE | __GFP_NOMEMALLOC, node); if (likely(new_stats)) { new_stats->used = jiffies; new_stats->packet_count = 1; new_stats->byte_count = skb->len; new_stats->tcp_flags = tcp_flags; spin_lock_init(&new_stats->lock); rcu_assign_pointer(flow->stats[node], new_stats); goto unlock; } } flow->stats_last_writer = node; } } stats->used = jiffies; stats->packet_count++; stats->byte_count += skb->len; stats->tcp_flags |= tcp_flags; unlock: spin_unlock(&stats->lock); }
void * cfs_mem_cache_cpt_alloc(struct kmem_cache *cachep, struct cfs_cpt_table *cptab, int cpt, gfp_t flags) { return kmem_cache_alloc_node(cachep, flags, cfs_cpt_spread_node(cptab, cpt)); }
struct io_context *alloc_io_context(gfp_t gfp_flags, int node) { struct io_context *ioc; ioc = kmem_cache_alloc_node(iocontext_cachep, gfp_flags, node); if (ioc) { atomic_long_set(&ioc->refcount, 1); atomic_set(&ioc->nr_tasks, 1); spin_lock_init(&ioc->lock); //* ioc->ioprio_changed = 0; bitmap_zero(ioc->ioprio_changed, IOC_IOPRIO_CHANGED_BITS); ioc->ioprio = 0; ioc->last_waited = 0; /* doesn't matter... */ ioc->nr_batch_requests = 0; /* because this is 0 */ INIT_RADIX_TREE(&ioc->radix_root, GFP_ATOMIC | __GFP_HIGH); INIT_HLIST_HEAD(&ioc->cic_list); INIT_RADIX_TREE(&ioc->bfq_radix_root, GFP_ATOMIC | __GFP_HIGH); INIT_HLIST_HEAD(&ioc->bfq_cic_list); ioc->ioc_data = NULL; #if defined(CONFIG_BLK_CGROUP) || defined(CONFIG_BLK_CGROUP_MODULE) ioc->cgroup_changed = 0; #endif } return ioc; }
struct sw_flow *ovs_flow_alloc(void) { struct sw_flow *flow; struct flow_stats *stats; int node; flow = kmem_cache_alloc(flow_cache, GFP_KERNEL); if (!flow) return ERR_PTR(-ENOMEM); flow->sf_acts = NULL; flow->mask = NULL; flow->stats_last_writer = NUMA_NO_NODE; /* Initialize the default stat node. */ stats = kmem_cache_alloc_node(flow_stats_cache, GFP_KERNEL | __GFP_ZERO, 0); if (!stats) goto err; spin_lock_init(&stats->lock); RCU_INIT_POINTER(flow->stats[0], stats); for_each_node(node) if (node != 0) RCU_INIT_POINTER(flow->stats[node], NULL); return flow; err: kmem_cache_free(flow_cache, flow); return ERR_PTR(-ENOMEM); }
void create_io_context_slowpath(struct task_struct *task, gfp_t gfp_flags, int node) { struct io_context *ioc; ioc = kmem_cache_alloc_node(iocontext_cachep, gfp_flags | __GFP_ZERO, node); if (unlikely(!ioc)) return; /* initialize */ atomic_long_set(&ioc->refcount, 1); atomic_set(&ioc->nr_tasks, 1); spin_lock_init(&ioc->lock); INIT_RADIX_TREE(&ioc->icq_tree, GFP_ATOMIC | __GFP_HIGH); INIT_HLIST_HEAD(&ioc->icq_list); INIT_WORK(&ioc->release_work, ioc_release_fn); /* * Try to install. ioc shouldn't be installed if someone else * already did or @task, which isn't %current, is exiting. Note * that we need to allow ioc creation on exiting %current as exit * path may issue IOs from e.g. exit_files(). The exit path is * responsible for not issuing IO after exit_io_context(). */ task_lock(task); if (!task->io_context && (task == current || !(task->flags & PF_EXITING))) task->io_context = ioc; else kmem_cache_free(iocontext_cachep, ioc); task_unlock(task); }
/** * __alloc_skb - allocate a network buffer * @size: size to allocate * @gfp_mask: allocation mask * @fclone: allocate from fclone cache instead of head cache * and allocate a cloned (child) skb * @node: numa node to allocate memory on * * Allocate a new &sk_buff. The returned buffer has no headroom and a * tail room of size bytes. The object has a reference count of one. * The return is the buffer. On a failure the return is %NULL. * * Buffers may only be allocated from interrupts using a @gfp_mask of * %GFP_ATOMIC. */ struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask, int fclone, int node) { struct kmem_cache *cache; struct skb_shared_info *shinfo; struct sk_buff *skb; u8 *data; cache = fclone ? skbuff_fclone_cache : skbuff_head_cache; /* Get the HEAD */ skb = kmem_cache_alloc_node(cache, gfp_mask & ~__GFP_DMA, node); if (!skb) goto out; /* Get the DATA. Size must match skb_add_mtu(). */ size = SKB_DATA_ALIGN(size); data = kmalloc_node_track_caller(size + sizeof(struct skb_shared_info), gfp_mask, node); if (!data) goto nodata; memset(skb, 0, offsetof(struct sk_buff, truesize)); skb->truesize = size + sizeof(struct sk_buff); atomic_set(&skb->users, 1); skb->head = data; skb->data = data; skb->tail = data; skb->end = data + size; /* make sure we initialize shinfo sequentially */ shinfo = skb_shinfo(skb); atomic_set(&shinfo->dataref, 1); shinfo->nr_frags = 0; shinfo->gso_size = 0; shinfo->gso_segs = 0; shinfo->gso_type = 0; shinfo->ip6_frag_id = 0; shinfo->frag_list = NULL; if (fclone) { struct sk_buff *child = skb + 1; atomic_t *fclone_ref = (atomic_t *) (child + 1); skb->fclone = SKB_FCLONE_ORIG; atomic_set(fclone_ref, 1); child->fclone = SKB_FCLONE_UNAVAILABLE; } out: return skb; nodata: kmem_cache_free(cache, skb); skb = NULL; goto out; }
struct thread_info *alloc_thread_info_node(struct task_struct *tsk, int node) { struct thread_info *ti; #ifdef CONFIG_DEBUG_STACK_USAGE gfp_t mask = GFP_KERNEL | __GFP_ZERO; #else gfp_t mask = GFP_KERNEL; #endif ti = kmem_cache_alloc_node(thread_info_cache, mask, node); return ti; }
struct thread_info *alloc_thread_info_node(struct task_struct *tsk, int node) { struct thread_info *ti; ti = kmem_cache_alloc_node(thread_info_cache, GFP_KERNEL, node); if (unlikely(ti == NULL)) return NULL; #ifdef CONFIG_DEBUG_STACK_USAGE memset(ti, 0, THREAD_SIZE); #endif return ti; }
/** * ioc_create_icq - create and link io_cq * @q: request_queue of interest * @gfp_mask: allocation mask * * Make sure io_cq linking %current->io_context and @q exists. If either * io_context and/or icq don't exist, they will be created using @gfp_mask. * * The caller is responsible for ensuring @ioc won't go away and @q is * alive and will stay alive until this function returns. */ struct io_cq *ioc_create_icq(struct request_queue *q, gfp_t gfp_mask) { struct elevator_type *et = q->elevator->type; struct io_context *ioc; struct io_cq *icq; /* allocate stuff */ ioc = create_io_context(current, gfp_mask, q->node); if (!ioc) return NULL; icq = kmem_cache_alloc_node(et->icq_cache, gfp_mask | __GFP_ZERO, q->node); if (!icq) return NULL; if (radix_tree_preload(gfp_mask) < 0) { kmem_cache_free(et->icq_cache, icq); return NULL; } icq->ioc = ioc; icq->q = q; INIT_LIST_HEAD(&icq->q_node); INIT_HLIST_NODE(&icq->ioc_node); /* lock both q and ioc and try to link @icq */ spin_lock_irq(q->queue_lock); spin_lock(&ioc->lock); if (likely(!radix_tree_insert(&ioc->icq_tree, q->id, icq))) { hlist_add_head(&icq->ioc_node, &ioc->icq_list); list_add(&icq->q_node, &q->icq_list); if (et->ops.elevator_init_icq_fn) et->ops.elevator_init_icq_fn(icq); } else { kmem_cache_free(et->icq_cache, icq); icq = ioc_lookup_icq(ioc, q); if (!icq) printk(KERN_ERR "cfq: icq link failed!\n"); } spin_unlock(&ioc->lock); spin_unlock_irq(q->queue_lock); radix_tree_preload_end(); return icq; }
static struct cfq_io_context *bfq_alloc_io_context(struct bfq_data *bfqd, gfp_t gfp_mask) { struct cfq_io_context *cic; cic = kmem_cache_alloc_node(bfq_ioc_pool, gfp_mask | __GFP_ZERO, bfqd->queue->node); if (cic != NULL) { cic->last_end_request = jiffies; INIT_LIST_HEAD(&cic->queue_list); INIT_HLIST_NODE(&cic->cic_list); cic->dtor = bfq_free_io_context; cic->exit = bfq_exit_io_context; elv_ioc_count_inc(bfq_ioc_count); } return cic; }
struct io_context *alloc_io_context(gfp_t gfp_flags, int node) { struct io_context *ret; ret = kmem_cache_alloc_node(iocontext_cachep, gfp_flags, node); if (ret) { atomic_long_set(&ret->refcount, 1); atomic_set(&ret->nr_tasks, 1); spin_lock_init(&ret->lock); ret->ioprio_changed = 0; ret->ioprio = 0; ret->last_waited = 0; /* doesn't matter... */ ret->nr_batch_requests = 0; /* because this is 0 */ INIT_RADIX_TREE(&ret->radix_root, GFP_ATOMIC | __GFP_HIGH); INIT_HLIST_HEAD(&ret->cic_list); ret->ioc_data = NULL; } return ret; }
static int dtl_enable(struct dtl *dtl) { long int n_entries; long int rc; struct dtl_entry *buf = NULL; if (!dtl_cache) return -ENOMEM; /* only allow one reader */ if (dtl->buf) return -EBUSY; n_entries = dtl_buf_entries; buf = kmem_cache_alloc_node(dtl_cache, GFP_KERNEL, cpu_to_node(dtl->cpu)); if (!buf) { printk(KERN_WARNING "%s: buffer alloc failed for cpu %d\n", __func__, dtl->cpu); return -ENOMEM; } spin_lock(&dtl->lock); rc = -EBUSY; if (!dtl->buf) { /* store the original allocation size for use during read */ dtl->buf_entries = n_entries; dtl->buf = buf; dtl->last_idx = 0; rc = dtl_start(dtl); if (rc) dtl->buf = NULL; } spin_unlock(&dtl->lock); if (rc) kmem_cache_free(dtl_cache, buf); return rc; }
static struct cfq_io_context *bfq_alloc_io_context(struct bfq_data *bfqd, gfp_t gfp_mask) { struct cfq_io_context *cic; cic = kmem_cache_alloc_node(bfq_ioc_pool, gfp_mask | __GFP_ZERO, bfqd->queue->node); if (cic != NULL) { cic->last_end_request = jiffies; /* * A newly created cic indicates that the process has just * started doing I/O, and is probably mapping into memory its * executable and libraries: it definitely needs weight raising. * There is however the possibility that the process performs, * for a while, I/O close to some other process. EQM intercepts * this behavior and may merge the queue corresponding to the * process with some other queue, BEFORE the weight of the queue * is raised. Merged queues are not weight-raised (they are assumed * to belong to processes that benefit only from high throughput). * If the merge is basically the consequence of an accident, then * the queue will be split soon and will get back its old weight. * It is then important to write down somewhere that this queue * does need weight raising, even if it did not make it to get its * weight raised before being merged. To this purpose, we overload * the field raising_time_left and assign 1 to it, to mark the queue * as needing weight raising. */ cic->wr_time_left = 1; INIT_LIST_HEAD(&cic->queue_list); INIT_HLIST_NODE(&cic->cic_list); cic->dtor = bfq_free_io_context; cic->exit = bfq_exit_io_context; elv_ioc_count_inc(bfq_ioc_count); } return cic; }
static struct thread_info *alloc_thread_info_node(struct task_struct *tsk, int node) { return kmem_cache_alloc_node(thread_info_cache, THREADINFO_GFP, node); }
static inline struct task_struct *alloc_task_struct_node(int node) { return kmem_cache_alloc_node(task_struct_cachep, GFP_KERNEL, node); }
void ovs_flow_stats_update(struct sw_flow *flow, struct sk_buff *skb) { struct flow_stats *stats; __be16 tcp_flags = 0; int node = numa_node_id(); stats = rcu_dereference(flow->stats[node]); if ((flow->key.eth.type == htons(ETH_P_IP) || flow->key.eth.type == htons(ETH_P_IPV6)) && flow->key.ip.frag != OVS_FRAG_TYPE_LATER && flow->key.ip.proto == IPPROTO_TCP && likely(skb->len >= skb_transport_offset(skb) + sizeof(struct tcphdr))) { tcp_flags = TCP_FLAGS_BE16(tcp_hdr(skb)); } /* Check if already have node-specific stats. */ if (likely(stats)) { spin_lock(&stats->lock); /* Mark if we write on the pre-allocated stats. */ if (node == 0 && unlikely(flow->stats_last_writer != node)) flow->stats_last_writer = node; } else { stats = rcu_dereference(flow->stats[0]); /* Pre-allocated. */ spin_lock(&stats->lock); /* If the current NUMA-node is the only writer on the * pre-allocated stats keep using them. */ if (unlikely(flow->stats_last_writer != node)) { /* A previous locker may have already allocated the * stats, so we need to check again. If node-specific * stats were already allocated, we update the pre- * allocated stats as we have already locked them. */ if (likely(flow->stats_last_writer != NUMA_NO_NODE) && likely(!rcu_dereference(flow->stats[node]))) { /* Try to allocate node-specific stats. */ struct flow_stats *new_stats; new_stats = kmem_cache_alloc_node(flow_stats_cache, GFP_THISNODE | __GFP_NOMEMALLOC, node); if (likely(new_stats)) { new_stats->used = jiffies; new_stats->packet_count = 1; new_stats->byte_count = skb->len; new_stats->tcp_flags = tcp_flags; spin_lock_init(&new_stats->lock); rcu_assign_pointer(flow->stats[node], new_stats); goto unlock; } } flow->stats_last_writer = node; } } stats->used = jiffies; stats->packet_count++; stats->byte_count += skb->len; stats->tcp_flags |= tcp_flags; unlock: spin_unlock(&stats->lock); }