void sched_put_rd(struct root_domain *rd)
{
if (!atomic_dec_and_test(&rd->refcount))
return;
call_rcu_sched(&rd->rcu, free_rootdomain);
}
void rcu_barrier_sched(void)
{
struct rcu_synchronize rcu;
init_completion(&rcu.completion);
/* Will wake me after RCU finished. */
call_rcu_sched(&rcu.head, wakeme_after_rcu);
/* Wait for it. */
wait_for_completion(&rcu.completion);
}
/**
* synchronize_sched - wait until an rcu-sched grace period has elapsed.
*
* Control will return to the caller some time after a full rcu-sched
* grace period has elapsed, in other words after all currently executing
* rcu-sched read-side critical sections have completed. These read-side
* critical sections are delimited by rcu_read_lock_sched() and
* rcu_read_unlock_sched(), and may be nested. Note that preempt_disable(),
* local_irq_disable(), and so on may be used in place of
* rcu_read_lock_sched().
*
* This means that all preempt_disable code sequences, including NMI and
* hardware-interrupt handlers, in progress on entry will have completed
* before this primitive returns. However, this does not guarantee that
* softirq handlers will have completed, since in some kernels, these
* handlers can run in process context, and can block.
*
* This primitive provides the guarantees made by the (now removed)
* synchronize_kernel() API. In contrast, synchronize_rcu() only
* guarantees that rcu_read_lock() sections will have completed.
* In "classic RCU", these two guarantees happen to be one and
* the same, but can differ in realtime RCU implementations.
*/
void synchronize_sched(void)
{
struct rcu_synchronize rcu;
if (rcu_blocking_is_gp())
return;
init_completion(&rcu.completion);
/* Will wake me after RCU finished. */
call_rcu_sched(&rcu.head, wakeme_after_rcu);
/* Wait for it. */
wait_for_completion(&rcu.completion);
}
/*
* Called with preemption disabled, and from cross-cpu IRQ context.
*/
static void rcu_barrier_func(void *type)
{
int cpu = smp_processor_id();
struct rcu_head *head = &per_cpu(rcu_barrier_head, cpu);
atomic_inc(&rcu_barrier_cpu_count);
switch ((enum rcu_barrier)type) {
case RCU_BARRIER_STD:
call_rcu(head, rcu_barrier_callback);
break;
case RCU_BARRIER_BH:
call_rcu_bh(head, rcu_barrier_callback);
break;
case RCU_BARRIER_SCHED:
call_rcu_sched(head, rcu_barrier_callback);
break;
}
}
void rq_attach_root(struct rq *rq, struct root_domain *rd)
{
struct root_domain *old_rd = NULL;
unsigned long flags;
raw_spin_lock_irqsave(&rq->lock, flags);
if (rq->rd) {
old_rd = rq->rd;
if (cpumask_test_cpu(rq->cpu, old_rd->online))
set_rq_offline(rq);
cpumask_clear_cpu(rq->cpu, old_rd->span);
/*
* If we dont want to free the old_rd yet then
* set old_rd to NULL to skip the freeing later
* in this function:
*/
if (!atomic_dec_and_test(&old_rd->refcount))
old_rd = NULL;
}
atomic_inc(&rd->refcount);
rq->rd = rd;
cpumask_set_cpu(rq->cpu, rd->span);
if (cpumask_test_cpu(rq->cpu, cpu_active_mask))
set_rq_online(rq);
raw_spin_unlock_irqrestore(&rq->lock, flags);
if (old_rd)
call_rcu_sched(&old_rd->rcu, free_rootdomain);
}
static
void discard_event(struct latency_tracker *tracker,
struct latency_tracker_event *s)
{
#if defined(BASEHT) && !defined(LLFREELIST)
__wrapper_freelist_put_event(tracker, s);
#else
#if (LINUX_VERSION_CODE < KERNEL_VERSION(3,17,0))
/*
* Our own call_rcu because the mainline one causes sched_wakeups
* that we might want to instrument causing deadlocks.
*/
int was_empty;
was_empty = llist_add(&s->release_llist, &tracker->to_release);
if (was_empty)
queue_delayed_work(tracker->tracker_call_rcu_q,
&tracker->tracker_call_rcu_w, 100);
#else
call_rcu_sched(&s->urcuhead,
deferred_latency_tracker_put_event);
#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(3,17,0) */
#endif
}
static
void discard_event(struct latency_tracker *tracker,
struct latency_tracker_event *s)
{
#if (LINUX_VERSION_CODE < KERNEL_VERSION(3,17,0))
/*
* Our own call_rcu because the mainline one causes sched_wakeups
* that we might want to instrument causing deadlocks.
*/
int was_empty;
/*
* We can reuse llist node because it is not used anymore
* by the parent list.
*/
was_empty = llist_add(&s->llist, &tracker->to_release);
if (was_empty)
queue_delayed_work(tracker->tracker_call_rcu_q,
&tracker->tracker_call_rcu_w, 100);
#else
call_rcu_sched(&s->u.urcuhead,
deferred_latency_tracker_put_event);
#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(3,17,0) */
}
