/**
* kernfs_remove_self - remove a kernfs_node from its own method
* @kn: the self kernfs_node to remove
*
* The caller must be running off of a kernfs operation which is invoked
* with an active reference - e.g. one of kernfs_ops. This can be used to
* implement a file operation which deletes itself.
*
* For example, the "delete" file for a sysfs device directory can be
* implemented by invoking kernfs_remove_self() on the "delete" file
* itself. This function breaks the circular dependency of trying to
* deactivate self while holding an active ref itself. It isn't necessary
* to modify the usual removal path to use kernfs_remove_self(). The
* "delete" implementation can simply invoke kernfs_remove_self() on self
* before proceeding with the usual removal path. kernfs will ignore later
* kernfs_remove() on self.
*
* kernfs_remove_self() can be called multiple times concurrently on the
* same kernfs_node. Only the first one actually performs removal and
* returns %true. All others will wait until the kernfs operation which
* won self-removal finishes and return %false. Note that the losers wait
* for the completion of not only the winning kernfs_remove_self() but also
* the whole kernfs_ops which won the arbitration. This can be used to
* guarantee, for example, all concurrent writes to a "delete" file to
* finish only after the whole operation is complete.
*/
bool kernfs_remove_self(struct kernfs_node *kn)
{
bool ret;
mutex_lock(&kernfs_mutex);
kernfs_break_active_protection(kn);
/*
* SUICIDAL is used to arbitrate among competing invocations. Only
* the first one will actually perform removal. When the removal
* is complete, SUICIDED is set and the active ref is restored
* while holding kernfs_mutex. The ones which lost arbitration
* waits for SUICDED && drained which can happen only after the
* enclosing kernfs operation which executed the winning instance
* of kernfs_remove_self() finished.
*/
if (!(kn->flags & KERNFS_SUICIDAL)) {
kn->flags |= KERNFS_SUICIDAL;
__kernfs_remove(kn);
kn->flags |= KERNFS_SUICIDED;
ret = true;
} else {
wait_queue_head_t *waitq = &kernfs_root(kn)->deactivate_waitq;
DEFINE_WAIT(wait);
while (true) {
prepare_to_wait(waitq, &wait, TASK_UNINTERRUPTIBLE);
if ((kn->flags & KERNFS_SUICIDED) &&
atomic_read(&kn->active) == KN_DEACTIVATED_BIAS)
break;
mutex_unlock(&kernfs_mutex);
schedule();
mutex_lock(&kernfs_mutex);
}
finish_wait(waitq, &wait);
WARN_ON_ONCE(!RB_EMPTY_NODE(&kn->rb));
ret = false;
}
/*
* This must be done while holding kernfs_mutex; otherwise, waiting
* for SUICIDED && deactivated could finish prematurely.
*/
kernfs_unbreak_active_protection(kn);
mutex_unlock(&kernfs_mutex);
return ret;
}
/**
* timerqueue_del - Removes a timer from the timerqueue.
*
* @head: head of timerqueue
* @node: timer node to be removed
*
* Removes the timer node from the timerqueue.
*/
void timerqueue_del(struct timerqueue_head *head, struct timerqueue_node *node)
{
WARN_ON_ONCE(RB_EMPTY_NODE(&node->node));
/* update next pointer */
if (head->next == node) {
struct rb_node *rbn = rb_next(&node->node);
head->next = rbn ?
rb_entry(rbn, struct timerqueue_node, node) : NULL;
}
rb_erase(&node->node, &head->head);
RB_CLEAR_NODE(&node->node);
}
static void ion_handle_destroy(struct kref *kref)
{
struct ion_handle *handle = container_of(kref, struct ion_handle, ref);
/* XXX Can a handle be destroyed while it's map count is non-zero?:
if (handle->map_cnt) unmap
*/
WARN_ON(handle->kmap_cnt || handle->dmap_cnt || handle->usermap_cnt);
ion_buffer_put(handle->buffer);
//120817 [email protected] gpu: ion: Fix race condition with ion_import
// mutex_lock(&handle->client->lock);
if (!RB_EMPTY_NODE(&handle->node))
rb_erase(&handle->node, &handle->client->handles);
// mutex_unlock(&handle->client->lock);
kfree(handle);
}
static void ion_handle_destroy(struct kref *kref)
{
struct ion_handle *handle = container_of(kref, struct ion_handle, ref);
struct ion_client *client = handle->client;
struct ion_buffer *buffer = handle->buffer;
mutex_lock(&buffer->lock);
while (handle->kmap_cnt)
ion_handle_kmap_put(handle);
mutex_unlock(&buffer->lock);
if (!RB_EMPTY_NODE(&handle->node))
rb_erase(&handle->node, &client->handles);
ion_buffer_put(buffer);
kfree(handle);
}
/**
* bus1_queue_link() - link entry into sorted queue
* @queue: queue to link into
* @entry: entry to link
* @seq: sequence number to use
*
* This links @entry into the message queue @queue. The caller must guarantee
* that the entry is unlinked and was never marked as ready.
*
* The caller must hold the write-side peer-lock of the parent peer.
*
* Return: True if the queue became readable with this call.
*/
bool bus1_queue_link(struct bus1_queue *queue,
struct bus1_queue_entry *entry,
u64 seq)
{
struct bus1_queue_entry *iter;
struct rb_node *prev, **slot;
bool is_leftmost = true;
if (WARN_ON(seq == 0 ||
!RB_EMPTY_NODE(&entry->rb) ||
(entry->seq > 0 && !(entry->seq & 1))))
return false;
bus1_queue_assert_held(queue);
slot = &queue->messages.rb_node;
prev = NULL;
while (*slot) {
prev = *slot;
iter = bus1_queue_entry(prev);
if (seq < iter->seq) {
slot = &prev->rb_left;
} else /* if (seq >= iter->seq) */ {
slot = &prev->rb_right;
is_leftmost = false;
}
}
entry->seq = seq;
rb_link_node(&entry->rb, prev, slot);
rb_insert_color(&entry->rb, &queue->messages);
if (is_leftmost) {
WARN_ON(rcu_access_pointer(queue->front));
if (!(seq & 1))
rcu_assign_pointer(queue->front, &entry->rb);
}
/*
* If we're linked leftmost, we're the new front. If we're ready to be
* dequeued, the list has become readable via this entry. Note that the
* previous front cannot be ready in this case, as we *never* order
* ready entries in front of other ready entries.
*/
return is_leftmost && !(seq & 1);
}
/**
* kernfs_activate - activate a node which started deactivated
* @kn: kernfs_node whose subtree is to be activated
*
* If the root has KERNFS_ROOT_CREATE_DEACTIVATED set, a newly created node
* needs to be explicitly activated. A node which hasn't been activated
* isn't visible to userland and deactivation is skipped during its
* removal. This is useful to construct atomic init sequences where
* creation of multiple nodes should either succeed or fail atomically.
*
* The caller is responsible for ensuring that this function is not called
* after kernfs_remove*() is invoked on @kn.
*/
void kernfs_activate(struct kernfs_node *kn)
{
struct kernfs_node *pos;
mutex_lock(&kernfs_mutex);
pos = NULL;
while ((pos = kernfs_next_descendant_post(pos, kn))) {
if (!pos || (pos->flags & KERNFS_ACTIVATED))
continue;
WARN_ON_ONCE(pos->parent && RB_EMPTY_NODE(&pos->rb));
WARN_ON_ONCE(atomic_read(&pos->active) != KN_DEACTIVATED_BIAS);
atomic_sub(KN_DEACTIVATED_BIAS, &pos->active);
pos->flags |= KERNFS_ACTIVATED;
}
mutex_unlock(&kernfs_mutex);
}
void timerqueue_add(struct timerqueue_head *head, struct timerqueue_node *node)
{
struct rb_node **p = &head->head.rb_node;
struct rb_node *parent = NULL;
struct timerqueue_node *ptr;
/* Make sure we don't add nodes that are already added */
WARN_ON_ONCE(!RB_EMPTY_NODE(&node->node));
while (*p) {
parent = *p;
ptr = rb_entry(parent, struct timerqueue_node, node);
if (node->expires.tv64 < ptr->expires.tv64)
p = &(*p)->rb_left;
else
p = &(*p)->rb_right;
}
rb_link_node(&node->node, parent, p);
rb_insert_color(&node->node, &head->head);
if (!head->next || node->expires.tv64 < head->next->expires.tv64)
head->next = node;
}
/*
* The list of stealable -rtws pjobs is a rb-tree with pjobs ordered by deadline,
* excluding current pjob. Pjobs with equal deadline obey to a FIFO order.
*/
static void enqueue_stealable_pjob_rtws(struct rtws_rq *rtws_rq, struct sched_rtws_entity *rtws_se)
{
struct rq *rq = rq_of_rtws_rq(rtws_rq);
struct task_struct *p = task_of_rtws_se(rtws_se);
struct rb_node **link = &rtws_rq->stealable_pjobs.rb_node;
struct rb_node *parent = NULL;
struct sched_rtws_entity *entry;
int edf = 1;
BUG_ON(!RB_EMPTY_NODE(&rtws_se->stealable_pjob_node));
printk(KERN_INFO "****stealable pjob enqueue, task %d ****\n", p->pid);
/* TODO: Optimization in case enqueueing task is next edf */
while (*link) {
parent = *link;
entry = rb_entry(parent, struct sched_rtws_entity, stealable_pjob_node);
if (entity_preempt_rtws(rtws_se, entry))
link = &parent->rb_left;
else {
link = &parent->rb_right;
edf = 0;
}
}
if (edf) {
rtws_rq->leftmost_stealable_pjob = &rtws_se->stealable_pjob_node;
cpudl_set(&rq->rd->rtwss_cpudl, rq->cpu, rtws_se->job.deadline, 0, 1);
smp_wmb();
}
rb_link_node(&rtws_se->stealable_pjob_node, parent, link);
rb_insert_color(&rtws_se->stealable_pjob_node, &rtws_rq->stealable_pjobs);
}
static inline int on_rtws_rq(struct sched_rtws_entity *rtws_se)
{
return !RB_EMPTY_NODE(&rtws_se->pjob_node);
}
static inline int on_global_rq(struct sched_rtws_entity *rtws_se)
{
return !RB_EMPTY_NODE(&rtws_se->task_node);
}
static
void dec_pjobs_rtws(struct rtws_rq *rtws_rq, struct sched_rtws_entity *rtws_se)
{
struct rq *rq = rq_of_rtws_rq(rtws_rq);
struct task_struct *p = task_of_rtws_se(rtws_se);
struct sched_rtws_entity *stealable;
struct global_rq * global_rq = rtws_rq->global_rq;
int ret = 0;
WARN_ON(!rtws_prio(p->prio));
WARN_ON(!rtws_rq->nr_running);
rtws_rq->nr_running--;
if (!rtws_rq->nr_running) {
/* If there are no more pjobs to run, we declare this rq idle */
rtws_rq->earliest_dl = 0;
cpudl_set(&rq->rd->rtwsc_cpudl, rq->cpu, 0, 0, 0);
smp_wmb();
} else {
if (!RB_EMPTY_NODE(&rtws_se->stealable_pjob_node))
return;
if (rtws_rq->earliest_dl != rtws_se->job.deadline)
return;
if (!has_stealable_pjobs(rtws_rq))
return;
/* The leftmost stealable pjob and our next pjob share the same deadline */
stealable = rb_entry(rtws_rq->leftmost_stealable_pjob, struct sched_rtws_entity, stealable_pjob_node);
if (stealable->job.deadline > rtws_rq->earliest_dl) {
/*
* If the next pjob has lower priority than the highest priority task on
* global rq, we try to pull that task.
* Clearing the earlieast deadline value (earliest_dl=0) on the rq is
* a trick to allow next's rq statistics update, keeping the current ones.
*/
if (priority_inversion_rtws(rtws_rq, stealable)) {
rtws_rq->earliest_dl = 0;
raw_spin_lock(&global_rq->lock);
ret = pull_task_rtws(rq);
raw_spin_unlock(&global_rq->lock);
if (ret)
return;
}
}
/*
* We update statistics about this rq when next
* pjob has a different deadline than the dequeueing one.
*/
if (rtws_rq->earliest_dl != stealable->job.deadline) {
rtws_rq->earliest_dl = stealable->job.deadline;
cpudl_set(&rq->rd->rtwsc_cpudl, rq->cpu, rtws_rq->earliest_dl, 0, 1);
smp_wmb();
}
}
printk(KERN_INFO "dequeing task %d on cpu %d, current deadline %llu, remaining tasks %lu\n", p->pid, rq->cpu, rtws_rq->earliest_dl, rtws_rq->nr_running);
}
static void __kernfs_remove(struct kernfs_node *kn)
{
struct kernfs_node *pos;
lockdep_assert_held(&kernfs_mutex);
/*
* Short-circuit if non-root @kn has already finished removal.
* This is for kernfs_remove_self() which plays with active ref
* after removal.
*/
if (!kn || (kn->parent && RB_EMPTY_NODE(&kn->rb)))
return;
pr_debug("kernfs %s: removing\n", kn->name);
/* prevent any new usage under @kn by deactivating all nodes */
pos = NULL;
while ((pos = kernfs_next_descendant_post(pos, kn)))
if (kernfs_active(pos))
atomic_add(KN_DEACTIVATED_BIAS, &pos->active);
/* deactivate and unlink the subtree node-by-node */
do {
pos = kernfs_leftmost_descendant(kn);
/*
* kernfs_drain() drops kernfs_mutex temporarily and @pos's
* base ref could have been put by someone else by the time
* the function returns. Make sure it doesn't go away
* underneath us.
*/
kernfs_get(pos);
/*
* Drain iff @kn was activated. This avoids draining and
* its lockdep annotations for nodes which have never been
* activated and allows embedding kernfs_remove() in create
* error paths without worrying about draining.
*/
if (kn->flags & KERNFS_ACTIVATED)
kernfs_drain(pos);
else
WARN_ON_ONCE(atomic_read(&kn->active) != KN_DEACTIVATED_BIAS);
/*
* kernfs_unlink_sibling() succeeds once per node. Use it
* to decide who's responsible for cleanups.
*/
if (!pos->parent || kernfs_unlink_sibling(pos)) {
struct kernfs_iattrs *ps_iattr =
pos->parent ? pos->parent->iattr : NULL;
/* update timestamps on the parent */
if (ps_iattr) {
ps_iattr->ia_iattr.ia_ctime = CURRENT_TIME;
ps_iattr->ia_iattr.ia_mtime = CURRENT_TIME;
}
kernfs_put(pos);
}
kernfs_put(pos);
} while (pos != kn);
}
/**
* kdbus_node_deactivate() - deactivate a node
* @node: The node to deactivate.
*
* This function recursively deactivates this node and all its children. It
* returns only once all children and the node itself were recursively disabled
* (even if you call this function multiple times in parallel).
*
* It is safe to call this function on _any_ node that was initialized _any_
* number of times.
*
* This call may sleep, as it waits for all active references to be dropped.
*/
void kdbus_node_deactivate(struct kdbus_node *node)
{
struct kdbus_node *pos, *child;
struct rb_node *rb;
int v_pre, v_post;
pos = node;
/*
* To avoid recursion, we perform back-tracking while deactivating
* nodes. For each node we enter, we first mark the active-counter as
* deactivated by adding BIAS. If the node as children, we set the first
* child as current position and start over. If the node has no
* children, we drain the node by waiting for all active refs to be
* dropped and then releasing the node.
*
* After the node is released, we set its parent as current position
* and start over. If the current position was the initial node, we're
* done.
*
* Note that this function can be called in parallel by multiple
* callers. We make sure that each node is only released once, and any
* racing caller will wait until the other thread fully released that
* node.
*/
for (;;) {
/*
* Add BIAS to node->active to mark it as inactive. If it was
* never active before, immediately mark it as RELEASE_INACTIVE
* so we remember this state.
* We cannot remember v_pre as we might iterate into the
* children, overwriting v_pre, before we can release our node.
*/
mutex_lock(&pos->lock);
v_pre = atomic_read(&pos->active);
if (v_pre >= 0)
atomic_add_return(KDBUS_NODE_BIAS, &pos->active);
else if (v_pre == KDBUS_NODE_NEW)
atomic_set(&pos->active, KDBUS_NODE_RELEASE_DIRECT);
mutex_unlock(&pos->lock);
/* wait until all active references were dropped */
wait_event(pos->waitq,
atomic_read(&pos->active) <= KDBUS_NODE_BIAS);
mutex_lock(&pos->lock);
/* recurse into first child if any */
rb = rb_first(&pos->children);
if (rb) {
child = kdbus_node_ref(kdbus_node_from_rb(rb));
mutex_unlock(&pos->lock);
pos = child;
continue;
}
/* mark object as RELEASE */
v_post = atomic_read(&pos->active);
if (v_post == KDBUS_NODE_BIAS ||
v_post == KDBUS_NODE_RELEASE_DIRECT)
atomic_set(&pos->active, KDBUS_NODE_RELEASE);
mutex_unlock(&pos->lock);
/*
* If this is the thread that marked the object as RELEASE, we
* perform the actual release. Otherwise, we wait until the
* release is done and the node is marked as DRAINED.
*/
if (v_post == KDBUS_NODE_BIAS ||
v_post == KDBUS_NODE_RELEASE_DIRECT) {
if (pos->release_cb)
pos->release_cb(pos, v_post == KDBUS_NODE_BIAS);
if (pos->parent) {
mutex_lock(&pos->parent->lock);
if (!RB_EMPTY_NODE(&pos->rb)) {
rb_erase(&pos->rb,
&pos->parent->children);
RB_CLEAR_NODE(&pos->rb);
}
mutex_unlock(&pos->parent->lock);
}
/* mark as DRAINED */
atomic_set(&pos->active, KDBUS_NODE_DRAINED);
wake_up_all(&pos->waitq);
/* drop VFS cache */
kdbus_fs_flush(pos);
/*
* If the node was activated and someone subtracted BIAS
* from it to deactivate it, we, and only us, are
* responsible to release the extra ref-count that was
* taken once in kdbus_node_activate().
* If the node was never activated, no-one ever
* subtracted BIAS, but instead skipped that state and
* immediately went to NODE_RELEASE_DIRECT. In that case
* we must not drop the reference.
*/
if (v_post == KDBUS_NODE_BIAS)
kdbus_node_unref(pos);
} else {
/* wait until object is DRAINED */
wait_event(pos->waitq,
atomic_read(&pos->active) == KDBUS_NODE_DRAINED);
}
/*
* We're done with the current node. Continue on its parent
* again, which will try deactivating its next child, or itself
* if no child is left.
* If we've reached our initial node again, we are done and
* can safely return.
*/
if (pos == node)
break;
child = pos;
pos = pos->parent;
kdbus_node_unref(child);
}
}
static void add_object(struct i915_mmu_object *mo)
{
GEM_BUG_ON(!RB_EMPTY_NODE(&mo->it.rb));
interval_tree_insert(&mo->it, &mo->mn->objects);
}
/* Build a new IP datagram from all its fragments. */
static int ip_frag_reasm(struct ipq *qp, struct sk_buff *skb,
struct sk_buff *prev_tail, struct net_device *dev)
{
struct net *net = container_of(qp->q.net, struct net, ipv4.frags);
struct iphdr *iph;
struct sk_buff *fp, *head = skb_rb_first(&qp->q.rb_fragments);
struct sk_buff **nextp; /* To build frag_list. */
struct rb_node *rbn;
int len;
int ihlen;
int err;
u8 ecn;
ipq_kill(qp);
ecn = ip_frag_ecn_table[qp->ecn];
if (unlikely(ecn == 0xff)) {
err = -EINVAL;
goto out_fail;
}
/* Make the one we just received the head. */
if (head != skb) {
fp = skb_clone(skb, GFP_ATOMIC);
if (!fp)
goto out_nomem;
FRAG_CB(fp)->next_frag = FRAG_CB(skb)->next_frag;
if (RB_EMPTY_NODE(&skb->rbnode))
FRAG_CB(prev_tail)->next_frag = fp;
else
rb_replace_node(&skb->rbnode, &fp->rbnode,
&qp->q.rb_fragments);
if (qp->q.fragments_tail == skb)
qp->q.fragments_tail = fp;
skb_morph(skb, head);
FRAG_CB(skb)->next_frag = FRAG_CB(head)->next_frag;
rb_replace_node(&head->rbnode, &skb->rbnode,
&qp->q.rb_fragments);
consume_skb(head);
head = skb;
}
WARN_ON(head->ip_defrag_offset != 0);
/* Allocate a new buffer for the datagram. */
ihlen = ip_hdrlen(head);
len = ihlen + qp->q.len;
err = -E2BIG;
if (len > 65535)
goto out_oversize;
/* Head of list must not be cloned. */
if (skb_unclone(head, GFP_ATOMIC))
goto out_nomem;
/* If the first fragment is fragmented itself, we split
* it to two chunks: the first with data and paged part
* and the second, holding only fragments. */
if (skb_has_frag_list(head)) {
struct sk_buff *clone;
int i, plen = 0;
clone = alloc_skb(0, GFP_ATOMIC);
if (!clone)
goto out_nomem;
skb_shinfo(clone)->frag_list = skb_shinfo(head)->frag_list;
skb_frag_list_init(head);
for (i = 0; i < skb_shinfo(head)->nr_frags; i++)
plen += skb_frag_size(&skb_shinfo(head)->frags[i]);
clone->len = clone->data_len = head->data_len - plen;
head->truesize += clone->truesize;
clone->csum = 0;
clone->ip_summed = head->ip_summed;
add_frag_mem_limit(qp->q.net, clone->truesize);
skb_shinfo(head)->frag_list = clone;
nextp = &clone->next;
} else {
nextp = &skb_shinfo(head)->frag_list;
}
skb_push(head, head->data - skb_network_header(head));
/* Traverse the tree in order, to build frag_list. */
fp = FRAG_CB(head)->next_frag;
rbn = rb_next(&head->rbnode);
rb_erase(&head->rbnode, &qp->q.rb_fragments);
while (rbn || fp) {
/* fp points to the next sk_buff in the current run;
* rbn points to the next run.
*/
/* Go through the current run. */
while (fp) {
*nextp = fp;
nextp = &fp->next;
fp->prev = NULL;
memset(&fp->rbnode, 0, sizeof(fp->rbnode));
head->data_len += fp->len;
head->len += fp->len;
if (head->ip_summed != fp->ip_summed)
head->ip_summed = CHECKSUM_NONE;
else if (head->ip_summed == CHECKSUM_COMPLETE)
head->csum = csum_add(head->csum, fp->csum);
head->truesize += fp->truesize;
fp = FRAG_CB(fp)->next_frag;
}
/* Move to the next run. */
if (rbn) {
struct rb_node *rbnext = rb_next(rbn);
fp = rb_to_skb(rbn);
rb_erase(rbn, &qp->q.rb_fragments);
rbn = rbnext;
}
}
sub_frag_mem_limit(qp->q.net, head->truesize);
*nextp = NULL;
head->next = NULL;
head->prev = NULL;
head->dev = dev;
head->tstamp = qp->q.stamp;
IPCB(head)->frag_max_size = max(qp->max_df_size, qp->q.max_size);
iph = ip_hdr(head);
iph->tot_len = htons(len);
iph->tos |= ecn;
/* When we set IP_DF on a refragmented skb we must also force a
* call to ip_fragment to avoid forwarding a DF-skb of size s while
* original sender only sent fragments of size f (where f < s).
*
* We only set DF/IPSKB_FRAG_PMTU if such DF fragment was the largest
* frag seen to avoid sending tiny DF-fragments in case skb was built
* from one very small df-fragment and one large non-df frag.
*/
if (qp->max_df_size == qp->q.max_size) {
IPCB(head)->flags |= IPSKB_FRAG_PMTU;
iph->frag_off = htons(IP_DF);
} else {
iph->frag_off = 0;
}
ip_send_check(iph);
__IP_INC_STATS(net, IPSTATS_MIB_REASMOKS);
qp->q.fragments = NULL;
qp->q.rb_fragments = RB_ROOT;
qp->q.fragments_tail = NULL;
qp->q.last_run_head = NULL;
return 0;
out_nomem:
net_dbg_ratelimited("queue_glue: no memory for gluing queue %p\n", qp);
err = -ENOMEM;
goto out_fail;
out_oversize:
net_info_ratelimited("Oversized IP packet from %pI4\n", &qp->q.key.v4.saddr);
out_fail:
__IP_INC_STATS(net, IPSTATS_MIB_REASMFAILS);
return err;
}
