/** * kernfs_remove_self - remove a kernfs_node from its own method * @kn: the self kernfs_node to remove * * The caller must be running off of a kernfs operation which is invoked * with an active reference - e.g. one of kernfs_ops. This can be used to * implement a file operation which deletes itself. * * For example, the "delete" file for a sysfs device directory can be * implemented by invoking kernfs_remove_self() on the "delete" file * itself. This function breaks the circular dependency of trying to * deactivate self while holding an active ref itself. It isn't necessary * to modify the usual removal path to use kernfs_remove_self(). The * "delete" implementation can simply invoke kernfs_remove_self() on self * before proceeding with the usual removal path. kernfs will ignore later * kernfs_remove() on self. * * kernfs_remove_self() can be called multiple times concurrently on the * same kernfs_node. Only the first one actually performs removal and * returns %true. All others will wait until the kernfs operation which * won self-removal finishes and return %false. Note that the losers wait * for the completion of not only the winning kernfs_remove_self() but also * the whole kernfs_ops which won the arbitration. This can be used to * guarantee, for example, all concurrent writes to a "delete" file to * finish only after the whole operation is complete. */ bool kernfs_remove_self(struct kernfs_node *kn) { bool ret; mutex_lock(&kernfs_mutex); kernfs_break_active_protection(kn); /* * SUICIDAL is used to arbitrate among competing invocations. Only * the first one will actually perform removal. When the removal * is complete, SUICIDED is set and the active ref is restored * while holding kernfs_mutex. The ones which lost arbitration * waits for SUICDED && drained which can happen only after the * enclosing kernfs operation which executed the winning instance * of kernfs_remove_self() finished. */ if (!(kn->flags & KERNFS_SUICIDAL)) { kn->flags |= KERNFS_SUICIDAL; __kernfs_remove(kn); kn->flags |= KERNFS_SUICIDED; ret = true; } else { wait_queue_head_t *waitq = &kernfs_root(kn)->deactivate_waitq; DEFINE_WAIT(wait); while (true) { prepare_to_wait(waitq, &wait, TASK_UNINTERRUPTIBLE); if ((kn->flags & KERNFS_SUICIDED) && atomic_read(&kn->active) == KN_DEACTIVATED_BIAS) break; mutex_unlock(&kernfs_mutex); schedule(); mutex_lock(&kernfs_mutex); } finish_wait(waitq, &wait); WARN_ON_ONCE(!RB_EMPTY_NODE(&kn->rb)); ret = false; } /* * This must be done while holding kernfs_mutex; otherwise, waiting * for SUICIDED && deactivated could finish prematurely. */ kernfs_unbreak_active_protection(kn); mutex_unlock(&kernfs_mutex); return ret; }
/** * timerqueue_del - Removes a timer from the timerqueue. * * @head: head of timerqueue * @node: timer node to be removed * * Removes the timer node from the timerqueue. */ void timerqueue_del(struct timerqueue_head *head, struct timerqueue_node *node) { WARN_ON_ONCE(RB_EMPTY_NODE(&node->node)); /* update next pointer */ if (head->next == node) { struct rb_node *rbn = rb_next(&node->node); head->next = rbn ? rb_entry(rbn, struct timerqueue_node, node) : NULL; } rb_erase(&node->node, &head->head); RB_CLEAR_NODE(&node->node); }
static void ion_handle_destroy(struct kref *kref) { struct ion_handle *handle = container_of(kref, struct ion_handle, ref); /* XXX Can a handle be destroyed while it's map count is non-zero?: if (handle->map_cnt) unmap */ WARN_ON(handle->kmap_cnt || handle->dmap_cnt || handle->usermap_cnt); ion_buffer_put(handle->buffer); //120817 [email protected] gpu: ion: Fix race condition with ion_import // mutex_lock(&handle->client->lock); if (!RB_EMPTY_NODE(&handle->node)) rb_erase(&handle->node, &handle->client->handles); // mutex_unlock(&handle->client->lock); kfree(handle); }
static void ion_handle_destroy(struct kref *kref) { struct ion_handle *handle = container_of(kref, struct ion_handle, ref); struct ion_client *client = handle->client; struct ion_buffer *buffer = handle->buffer; mutex_lock(&buffer->lock); while (handle->kmap_cnt) ion_handle_kmap_put(handle); mutex_unlock(&buffer->lock); if (!RB_EMPTY_NODE(&handle->node)) rb_erase(&handle->node, &client->handles); ion_buffer_put(buffer); kfree(handle); }
/** * bus1_queue_link() - link entry into sorted queue * @queue: queue to link into * @entry: entry to link * @seq: sequence number to use * * This links @entry into the message queue @queue. The caller must guarantee * that the entry is unlinked and was never marked as ready. * * The caller must hold the write-side peer-lock of the parent peer. * * Return: True if the queue became readable with this call. */ bool bus1_queue_link(struct bus1_queue *queue, struct bus1_queue_entry *entry, u64 seq) { struct bus1_queue_entry *iter; struct rb_node *prev, **slot; bool is_leftmost = true; if (WARN_ON(seq == 0 || !RB_EMPTY_NODE(&entry->rb) || (entry->seq > 0 && !(entry->seq & 1)))) return false; bus1_queue_assert_held(queue); slot = &queue->messages.rb_node; prev = NULL; while (*slot) { prev = *slot; iter = bus1_queue_entry(prev); if (seq < iter->seq) { slot = &prev->rb_left; } else /* if (seq >= iter->seq) */ { slot = &prev->rb_right; is_leftmost = false; } } entry->seq = seq; rb_link_node(&entry->rb, prev, slot); rb_insert_color(&entry->rb, &queue->messages); if (is_leftmost) { WARN_ON(rcu_access_pointer(queue->front)); if (!(seq & 1)) rcu_assign_pointer(queue->front, &entry->rb); } /* * If we're linked leftmost, we're the new front. If we're ready to be * dequeued, the list has become readable via this entry. Note that the * previous front cannot be ready in this case, as we *never* order * ready entries in front of other ready entries. */ return is_leftmost && !(seq & 1); }
/** * kernfs_activate - activate a node which started deactivated * @kn: kernfs_node whose subtree is to be activated * * If the root has KERNFS_ROOT_CREATE_DEACTIVATED set, a newly created node * needs to be explicitly activated. A node which hasn't been activated * isn't visible to userland and deactivation is skipped during its * removal. This is useful to construct atomic init sequences where * creation of multiple nodes should either succeed or fail atomically. * * The caller is responsible for ensuring that this function is not called * after kernfs_remove*() is invoked on @kn. */ void kernfs_activate(struct kernfs_node *kn) { struct kernfs_node *pos; mutex_lock(&kernfs_mutex); pos = NULL; while ((pos = kernfs_next_descendant_post(pos, kn))) { if (!pos || (pos->flags & KERNFS_ACTIVATED)) continue; WARN_ON_ONCE(pos->parent && RB_EMPTY_NODE(&pos->rb)); WARN_ON_ONCE(atomic_read(&pos->active) != KN_DEACTIVATED_BIAS); atomic_sub(KN_DEACTIVATED_BIAS, &pos->active); pos->flags |= KERNFS_ACTIVATED; } mutex_unlock(&kernfs_mutex); }
void timerqueue_add(struct timerqueue_head *head, struct timerqueue_node *node) { struct rb_node **p = &head->head.rb_node; struct rb_node *parent = NULL; struct timerqueue_node *ptr; /* Make sure we don't add nodes that are already added */ WARN_ON_ONCE(!RB_EMPTY_NODE(&node->node)); while (*p) { parent = *p; ptr = rb_entry(parent, struct timerqueue_node, node); if (node->expires.tv64 < ptr->expires.tv64) p = &(*p)->rb_left; else p = &(*p)->rb_right; } rb_link_node(&node->node, parent, p); rb_insert_color(&node->node, &head->head); if (!head->next || node->expires.tv64 < head->next->expires.tv64) head->next = node; }
/* * The list of stealable -rtws pjobs is a rb-tree with pjobs ordered by deadline, * excluding current pjob. Pjobs with equal deadline obey to a FIFO order. */ static void enqueue_stealable_pjob_rtws(struct rtws_rq *rtws_rq, struct sched_rtws_entity *rtws_se) { struct rq *rq = rq_of_rtws_rq(rtws_rq); struct task_struct *p = task_of_rtws_se(rtws_se); struct rb_node **link = &rtws_rq->stealable_pjobs.rb_node; struct rb_node *parent = NULL; struct sched_rtws_entity *entry; int edf = 1; BUG_ON(!RB_EMPTY_NODE(&rtws_se->stealable_pjob_node)); printk(KERN_INFO "****stealable pjob enqueue, task %d ****\n", p->pid); /* TODO: Optimization in case enqueueing task is next edf */ while (*link) { parent = *link; entry = rb_entry(parent, struct sched_rtws_entity, stealable_pjob_node); if (entity_preempt_rtws(rtws_se, entry)) link = &parent->rb_left; else { link = &parent->rb_right; edf = 0; } } if (edf) { rtws_rq->leftmost_stealable_pjob = &rtws_se->stealable_pjob_node; cpudl_set(&rq->rd->rtwss_cpudl, rq->cpu, rtws_se->job.deadline, 0, 1); smp_wmb(); } rb_link_node(&rtws_se->stealable_pjob_node, parent, link); rb_insert_color(&rtws_se->stealable_pjob_node, &rtws_rq->stealable_pjobs); }
static inline int on_rtws_rq(struct sched_rtws_entity *rtws_se) { return !RB_EMPTY_NODE(&rtws_se->pjob_node); }
static inline int on_global_rq(struct sched_rtws_entity *rtws_se) { return !RB_EMPTY_NODE(&rtws_se->task_node); }
static void dec_pjobs_rtws(struct rtws_rq *rtws_rq, struct sched_rtws_entity *rtws_se) { struct rq *rq = rq_of_rtws_rq(rtws_rq); struct task_struct *p = task_of_rtws_se(rtws_se); struct sched_rtws_entity *stealable; struct global_rq * global_rq = rtws_rq->global_rq; int ret = 0; WARN_ON(!rtws_prio(p->prio)); WARN_ON(!rtws_rq->nr_running); rtws_rq->nr_running--; if (!rtws_rq->nr_running) { /* If there are no more pjobs to run, we declare this rq idle */ rtws_rq->earliest_dl = 0; cpudl_set(&rq->rd->rtwsc_cpudl, rq->cpu, 0, 0, 0); smp_wmb(); } else { if (!RB_EMPTY_NODE(&rtws_se->stealable_pjob_node)) return; if (rtws_rq->earliest_dl != rtws_se->job.deadline) return; if (!has_stealable_pjobs(rtws_rq)) return; /* The leftmost stealable pjob and our next pjob share the same deadline */ stealable = rb_entry(rtws_rq->leftmost_stealable_pjob, struct sched_rtws_entity, stealable_pjob_node); if (stealable->job.deadline > rtws_rq->earliest_dl) { /* * If the next pjob has lower priority than the highest priority task on * global rq, we try to pull that task. * Clearing the earlieast deadline value (earliest_dl=0) on the rq is * a trick to allow next's rq statistics update, keeping the current ones. */ if (priority_inversion_rtws(rtws_rq, stealable)) { rtws_rq->earliest_dl = 0; raw_spin_lock(&global_rq->lock); ret = pull_task_rtws(rq); raw_spin_unlock(&global_rq->lock); if (ret) return; } } /* * We update statistics about this rq when next * pjob has a different deadline than the dequeueing one. */ if (rtws_rq->earliest_dl != stealable->job.deadline) { rtws_rq->earliest_dl = stealable->job.deadline; cpudl_set(&rq->rd->rtwsc_cpudl, rq->cpu, rtws_rq->earliest_dl, 0, 1); smp_wmb(); } } printk(KERN_INFO "dequeing task %d on cpu %d, current deadline %llu, remaining tasks %lu\n", p->pid, rq->cpu, rtws_rq->earliest_dl, rtws_rq->nr_running); }
static void __kernfs_remove(struct kernfs_node *kn) { struct kernfs_node *pos; lockdep_assert_held(&kernfs_mutex); /* * Short-circuit if non-root @kn has already finished removal. * This is for kernfs_remove_self() which plays with active ref * after removal. */ if (!kn || (kn->parent && RB_EMPTY_NODE(&kn->rb))) return; pr_debug("kernfs %s: removing\n", kn->name); /* prevent any new usage under @kn by deactivating all nodes */ pos = NULL; while ((pos = kernfs_next_descendant_post(pos, kn))) if (kernfs_active(pos)) atomic_add(KN_DEACTIVATED_BIAS, &pos->active); /* deactivate and unlink the subtree node-by-node */ do { pos = kernfs_leftmost_descendant(kn); /* * kernfs_drain() drops kernfs_mutex temporarily and @pos's * base ref could have been put by someone else by the time * the function returns. Make sure it doesn't go away * underneath us. */ kernfs_get(pos); /* * Drain iff @kn was activated. This avoids draining and * its lockdep annotations for nodes which have never been * activated and allows embedding kernfs_remove() in create * error paths without worrying about draining. */ if (kn->flags & KERNFS_ACTIVATED) kernfs_drain(pos); else WARN_ON_ONCE(atomic_read(&kn->active) != KN_DEACTIVATED_BIAS); /* * kernfs_unlink_sibling() succeeds once per node. Use it * to decide who's responsible for cleanups. */ if (!pos->parent || kernfs_unlink_sibling(pos)) { struct kernfs_iattrs *ps_iattr = pos->parent ? pos->parent->iattr : NULL; /* update timestamps on the parent */ if (ps_iattr) { ps_iattr->ia_iattr.ia_ctime = CURRENT_TIME; ps_iattr->ia_iattr.ia_mtime = CURRENT_TIME; } kernfs_put(pos); } kernfs_put(pos); } while (pos != kn); }
/** * kdbus_node_deactivate() - deactivate a node * @node: The node to deactivate. * * This function recursively deactivates this node and all its children. It * returns only once all children and the node itself were recursively disabled * (even if you call this function multiple times in parallel). * * It is safe to call this function on _any_ node that was initialized _any_ * number of times. * * This call may sleep, as it waits for all active references to be dropped. */ void kdbus_node_deactivate(struct kdbus_node *node) { struct kdbus_node *pos, *child; struct rb_node *rb; int v_pre, v_post; pos = node; /* * To avoid recursion, we perform back-tracking while deactivating * nodes. For each node we enter, we first mark the active-counter as * deactivated by adding BIAS. If the node as children, we set the first * child as current position and start over. If the node has no * children, we drain the node by waiting for all active refs to be * dropped and then releasing the node. * * After the node is released, we set its parent as current position * and start over. If the current position was the initial node, we're * done. * * Note that this function can be called in parallel by multiple * callers. We make sure that each node is only released once, and any * racing caller will wait until the other thread fully released that * node. */ for (;;) { /* * Add BIAS to node->active to mark it as inactive. If it was * never active before, immediately mark it as RELEASE_INACTIVE * so we remember this state. * We cannot remember v_pre as we might iterate into the * children, overwriting v_pre, before we can release our node. */ mutex_lock(&pos->lock); v_pre = atomic_read(&pos->active); if (v_pre >= 0) atomic_add_return(KDBUS_NODE_BIAS, &pos->active); else if (v_pre == KDBUS_NODE_NEW) atomic_set(&pos->active, KDBUS_NODE_RELEASE_DIRECT); mutex_unlock(&pos->lock); /* wait until all active references were dropped */ wait_event(pos->waitq, atomic_read(&pos->active) <= KDBUS_NODE_BIAS); mutex_lock(&pos->lock); /* recurse into first child if any */ rb = rb_first(&pos->children); if (rb) { child = kdbus_node_ref(kdbus_node_from_rb(rb)); mutex_unlock(&pos->lock); pos = child; continue; } /* mark object as RELEASE */ v_post = atomic_read(&pos->active); if (v_post == KDBUS_NODE_BIAS || v_post == KDBUS_NODE_RELEASE_DIRECT) atomic_set(&pos->active, KDBUS_NODE_RELEASE); mutex_unlock(&pos->lock); /* * If this is the thread that marked the object as RELEASE, we * perform the actual release. Otherwise, we wait until the * release is done and the node is marked as DRAINED. */ if (v_post == KDBUS_NODE_BIAS || v_post == KDBUS_NODE_RELEASE_DIRECT) { if (pos->release_cb) pos->release_cb(pos, v_post == KDBUS_NODE_BIAS); if (pos->parent) { mutex_lock(&pos->parent->lock); if (!RB_EMPTY_NODE(&pos->rb)) { rb_erase(&pos->rb, &pos->parent->children); RB_CLEAR_NODE(&pos->rb); } mutex_unlock(&pos->parent->lock); } /* mark as DRAINED */ atomic_set(&pos->active, KDBUS_NODE_DRAINED); wake_up_all(&pos->waitq); /* drop VFS cache */ kdbus_fs_flush(pos); /* * If the node was activated and someone subtracted BIAS * from it to deactivate it, we, and only us, are * responsible to release the extra ref-count that was * taken once in kdbus_node_activate(). * If the node was never activated, no-one ever * subtracted BIAS, but instead skipped that state and * immediately went to NODE_RELEASE_DIRECT. In that case * we must not drop the reference. */ if (v_post == KDBUS_NODE_BIAS) kdbus_node_unref(pos); } else { /* wait until object is DRAINED */ wait_event(pos->waitq, atomic_read(&pos->active) == KDBUS_NODE_DRAINED); } /* * We're done with the current node. Continue on its parent * again, which will try deactivating its next child, or itself * if no child is left. * If we've reached our initial node again, we are done and * can safely return. */ if (pos == node) break; child = pos; pos = pos->parent; kdbus_node_unref(child); } }
static void add_object(struct i915_mmu_object *mo) { GEM_BUG_ON(!RB_EMPTY_NODE(&mo->it.rb)); interval_tree_insert(&mo->it, &mo->mn->objects); }
/* Build a new IP datagram from all its fragments. */ static int ip_frag_reasm(struct ipq *qp, struct sk_buff *skb, struct sk_buff *prev_tail, struct net_device *dev) { struct net *net = container_of(qp->q.net, struct net, ipv4.frags); struct iphdr *iph; struct sk_buff *fp, *head = skb_rb_first(&qp->q.rb_fragments); struct sk_buff **nextp; /* To build frag_list. */ struct rb_node *rbn; int len; int ihlen; int err; u8 ecn; ipq_kill(qp); ecn = ip_frag_ecn_table[qp->ecn]; if (unlikely(ecn == 0xff)) { err = -EINVAL; goto out_fail; } /* Make the one we just received the head. */ if (head != skb) { fp = skb_clone(skb, GFP_ATOMIC); if (!fp) goto out_nomem; FRAG_CB(fp)->next_frag = FRAG_CB(skb)->next_frag; if (RB_EMPTY_NODE(&skb->rbnode)) FRAG_CB(prev_tail)->next_frag = fp; else rb_replace_node(&skb->rbnode, &fp->rbnode, &qp->q.rb_fragments); if (qp->q.fragments_tail == skb) qp->q.fragments_tail = fp; skb_morph(skb, head); FRAG_CB(skb)->next_frag = FRAG_CB(head)->next_frag; rb_replace_node(&head->rbnode, &skb->rbnode, &qp->q.rb_fragments); consume_skb(head); head = skb; } WARN_ON(head->ip_defrag_offset != 0); /* Allocate a new buffer for the datagram. */ ihlen = ip_hdrlen(head); len = ihlen + qp->q.len; err = -E2BIG; if (len > 65535) goto out_oversize; /* Head of list must not be cloned. */ if (skb_unclone(head, GFP_ATOMIC)) goto out_nomem; /* If the first fragment is fragmented itself, we split * it to two chunks: the first with data and paged part * and the second, holding only fragments. */ if (skb_has_frag_list(head)) { struct sk_buff *clone; int i, plen = 0; clone = alloc_skb(0, GFP_ATOMIC); if (!clone) goto out_nomem; skb_shinfo(clone)->frag_list = skb_shinfo(head)->frag_list; skb_frag_list_init(head); for (i = 0; i < skb_shinfo(head)->nr_frags; i++) plen += skb_frag_size(&skb_shinfo(head)->frags[i]); clone->len = clone->data_len = head->data_len - plen; head->truesize += clone->truesize; clone->csum = 0; clone->ip_summed = head->ip_summed; add_frag_mem_limit(qp->q.net, clone->truesize); skb_shinfo(head)->frag_list = clone; nextp = &clone->next; } else { nextp = &skb_shinfo(head)->frag_list; } skb_push(head, head->data - skb_network_header(head)); /* Traverse the tree in order, to build frag_list. */ fp = FRAG_CB(head)->next_frag; rbn = rb_next(&head->rbnode); rb_erase(&head->rbnode, &qp->q.rb_fragments); while (rbn || fp) { /* fp points to the next sk_buff in the current run; * rbn points to the next run. */ /* Go through the current run. */ while (fp) { *nextp = fp; nextp = &fp->next; fp->prev = NULL; memset(&fp->rbnode, 0, sizeof(fp->rbnode)); head->data_len += fp->len; head->len += fp->len; if (head->ip_summed != fp->ip_summed) head->ip_summed = CHECKSUM_NONE; else if (head->ip_summed == CHECKSUM_COMPLETE) head->csum = csum_add(head->csum, fp->csum); head->truesize += fp->truesize; fp = FRAG_CB(fp)->next_frag; } /* Move to the next run. */ if (rbn) { struct rb_node *rbnext = rb_next(rbn); fp = rb_to_skb(rbn); rb_erase(rbn, &qp->q.rb_fragments); rbn = rbnext; } } sub_frag_mem_limit(qp->q.net, head->truesize); *nextp = NULL; head->next = NULL; head->prev = NULL; head->dev = dev; head->tstamp = qp->q.stamp; IPCB(head)->frag_max_size = max(qp->max_df_size, qp->q.max_size); iph = ip_hdr(head); iph->tot_len = htons(len); iph->tos |= ecn; /* When we set IP_DF on a refragmented skb we must also force a * call to ip_fragment to avoid forwarding a DF-skb of size s while * original sender only sent fragments of size f (where f < s). * * We only set DF/IPSKB_FRAG_PMTU if such DF fragment was the largest * frag seen to avoid sending tiny DF-fragments in case skb was built * from one very small df-fragment and one large non-df frag. */ if (qp->max_df_size == qp->q.max_size) { IPCB(head)->flags |= IPSKB_FRAG_PMTU; iph->frag_off = htons(IP_DF); } else { iph->frag_off = 0; } ip_send_check(iph); __IP_INC_STATS(net, IPSTATS_MIB_REASMOKS); qp->q.fragments = NULL; qp->q.rb_fragments = RB_ROOT; qp->q.fragments_tail = NULL; qp->q.last_run_head = NULL; return 0; out_nomem: net_dbg_ratelimited("queue_glue: no memory for gluing queue %p\n", qp); err = -ENOMEM; goto out_fail; out_oversize: net_info_ratelimited("Oversized IP packet from %pI4\n", &qp->q.key.v4.saddr); out_fail: __IP_INC_STATS(net, IPSTATS_MIB_REASMFAILS); return err; }