/**
* cgroup_cpu_stat_pop_updated - iterate and dismantle cpu_stat updated tree
* @pos: current position
* @root: root of the tree to traversal
* @cpu: target cpu
*
* Walks the udpated cpu_stat tree on @cpu from @root. %NULL @pos starts
* the traversal and %NULL return indicates the end. During traversal,
* each returned cgroup is unlinked from the tree. Must be called with the
* matching cgroup_cpu_stat_lock held.
*
* The only ordering guarantee is that, for a parent and a child pair
* covered by a given traversal, if a child is visited, its parent is
* guaranteed to be visited afterwards.
*/
static struct cgroup *cgroup_cpu_stat_pop_updated(struct cgroup *pos,
struct cgroup *root, int cpu)
{
struct cgroup_cpu_stat *cstat;
struct cgroup *parent;
if (pos == root)
return NULL;
/*
* We're gonna walk down to the first leaf and visit/remove it. We
* can pick whatever unvisited node as the starting point.
*/
if (!pos)
pos = root;
else
pos = cgroup_parent(pos);
/* walk down to the first leaf */
while (true) {
cstat = cgroup_cpu_stat(pos, cpu);
if (cstat->updated_children == pos)
break;
pos = cstat->updated_children;
}
/*
* Unlink @pos from the tree. As the updated_children list is
* singly linked, we have to walk it to find the removal point.
* However, due to the way we traverse, @pos will be the first
* child in most cases. The only exception is @root.
*/
parent = cgroup_parent(pos);
if (parent && cstat->updated_next) {
struct cgroup_cpu_stat *pcstat = cgroup_cpu_stat(parent, cpu);
struct cgroup_cpu_stat *ncstat;
struct cgroup **nextp;
nextp = &pcstat->updated_children;
while (true) {
ncstat = cgroup_cpu_stat(*nextp, cpu);
if (*nextp == pos)
break;
WARN_ON_ONCE(*nextp == parent);
nextp = &ncstat->updated_next;
}
*nextp = cstat->updated_next;
cstat->updated_next = NULL;
}
return pos;
}
void cgroup_stat_show_cputime(struct seq_file *seq)
{
struct cgroup *cgrp = seq_css(seq)->cgroup;
u64 usage, utime, stime;
if (!cgroup_parent(cgrp))
return;
mutex_lock(&cgroup_stat_mutex);
cgroup_stat_flush_locked(cgrp);
usage = cgrp->stat.cputime.sum_exec_runtime;
cputime_adjust(&cgrp->stat.cputime, &cgrp->stat.prev_cputime,
&utime, &stime);
mutex_unlock(&cgroup_stat_mutex);
do_div(usage, NSEC_PER_USEC);
do_div(utime, NSEC_PER_USEC);
do_div(stime, NSEC_PER_USEC);
seq_printf(seq, "usage_usec %llu\n"
"user_usec %llu\n"
"system_usec %llu\n",
usage, utime, stime);
}
static void cgroup_cpu_stat_flush_one(struct cgroup *cgrp, int cpu)
{
struct cgroup *parent = cgroup_parent(cgrp);
struct cgroup_cpu_stat *cstat = cgroup_cpu_stat(cgrp, cpu);
struct task_cputime *last_cputime = &cstat->last_cputime;
struct task_cputime cputime;
struct cgroup_stat delta;
unsigned seq;
lockdep_assert_held(&cgroup_stat_mutex);
/* fetch the current per-cpu values */
do {
seq = __u64_stats_fetch_begin(&cstat->sync);
cputime = cstat->cputime;
} while (__u64_stats_fetch_retry(&cstat->sync, seq));
/* accumulate the deltas to propgate */
delta.cputime.utime = cputime.utime - last_cputime->utime;
delta.cputime.stime = cputime.stime - last_cputime->stime;
delta.cputime.sum_exec_runtime = cputime.sum_exec_runtime -
last_cputime->sum_exec_runtime;
*last_cputime = cputime;
/* transfer the pending stat into delta */
cgroup_stat_accumulate(&delta, &cgrp->pending_stat);
memset(&cgrp->pending_stat, 0, sizeof(cgrp->pending_stat));
/* propagate delta into the global stat and the parent's pending */
cgroup_stat_accumulate(&cgrp->stat, &delta);
if (parent)
cgroup_stat_accumulate(&parent->pending_stat, &delta);
}
/**
* cgroup_rstat_updated - keep track of updated rstat_cpu
* @cgrp: target cgroup
* @cpu: cpu on which rstat_cpu was updated
*
* @cgrp's rstat_cpu on @cpu was updated. Put it on the parent's matching
* rstat_cpu->updated_children list. See the comment on top of
* cgroup_rstat_cpu definition for details.
*/
void cgroup_rstat_updated(struct cgroup *cgrp, int cpu)
{
raw_spinlock_t *cpu_lock = per_cpu_ptr(&cgroup_rstat_cpu_lock, cpu);
struct cgroup *parent;
unsigned long flags;
/* nothing to do for root */
if (!cgroup_parent(cgrp))
return;
/*
* Paired with the one in cgroup_rstat_cpu_pop_upated(). Either we
* see NULL updated_next or they see our updated stat.
*/
smp_mb();
/*
* Because @parent's updated_children is terminated with @parent
* instead of NULL, we can tell whether @cgrp is on the list by
* testing the next pointer for NULL.
*/
if (cgroup_rstat_cpu(cgrp, cpu)->updated_next)
return;
raw_spin_lock_irqsave(cpu_lock, flags);
/* put @cgrp and all ancestors on the corresponding updated lists */
for (parent = cgroup_parent(cgrp); parent;
cgrp = parent, parent = cgroup_parent(cgrp)) {
struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu);
struct cgroup_rstat_cpu *prstatc = cgroup_rstat_cpu(parent, cpu);
/*
* Both additions and removals are bottom-up. If a cgroup
* is already in the tree, all ancestors are.
*/
if (rstatc->updated_next)
break;
rstatc->updated_next = prstatc->updated_children;
prstatc->updated_children = cgrp;
}
raw_spin_unlock_irqrestore(cpu_lock, flags);
}
/**
* cgroup_cpu_stat_updated - keep track of updated cpu_stat
* @cgrp: target cgroup
* @cpu: cpu on which cpu_stat was updated
*
* @cgrp's cpu_stat on @cpu was updated. Put it on the parent's matching
* cpu_stat->updated_children list. See the comment on top of
* cgroup_cpu_stat definition for details.
*/
static void cgroup_cpu_stat_updated(struct cgroup *cgrp, int cpu)
{
raw_spinlock_t *cpu_lock = per_cpu_ptr(&cgroup_cpu_stat_lock, cpu);
struct cgroup *parent;
unsigned long flags;
/*
* Speculative already-on-list test. This may race leading to
* temporary inaccuracies, which is fine.
*
* Because @parent's updated_children is terminated with @parent
* instead of NULL, we can tell whether @cgrp is on the list by
* testing the next pointer for NULL.
*/
if (cgroup_cpu_stat(cgrp, cpu)->updated_next)
return;
raw_spin_lock_irqsave(cpu_lock, flags);
/* put @cgrp and all ancestors on the corresponding updated lists */
for (parent = cgroup_parent(cgrp); parent;
cgrp = parent, parent = cgroup_parent(cgrp)) {
struct cgroup_cpu_stat *cstat = cgroup_cpu_stat(cgrp, cpu);
struct cgroup_cpu_stat *pcstat = cgroup_cpu_stat(parent, cpu);
/*
* Both additions and removals are bottom-up. If a cgroup
* is already in the tree, all ancestors are.
*/
if (cstat->updated_next)
break;
cstat->updated_next = pcstat->updated_children;
pcstat->updated_children = cgrp;
}
raw_spin_unlock_irqrestore(cpu_lock, flags);
}
static struct psi_group *iterate_groups(struct task_struct *task, void **iter)
{
#ifdef CONFIG_CGROUPS
struct cgroup *cgroup = NULL;
if (!*iter)
cgroup = task->cgroups->dfl_cgrp;
else if (*iter == &psi_system)
return NULL;
else
cgroup = cgroup_parent(*iter);
if (cgroup && cgroup_parent(cgroup)) {
*iter = cgroup;
return cgroup_psi(cgroup);
}
#else
if (*iter)
return NULL;
#endif
*iter = &psi_system;
return &psi_system;
}
/**
* cgroup_rstat_cpu_pop_updated - iterate and dismantle rstat_cpu updated tree
* @pos: current position
* @root: root of the tree to traversal
* @cpu: target cpu
*
* Walks the udpated rstat_cpu tree on @cpu from @root. %NULL @pos starts
* the traversal and %NULL return indicates the end. During traversal,
* each returned cgroup is unlinked from the tree. Must be called with the
* matching cgroup_rstat_cpu_lock held.
*
* The only ordering guarantee is that, for a parent and a child pair
* covered by a given traversal, if a child is visited, its parent is
* guaranteed to be visited afterwards.
*/
static struct cgroup *cgroup_rstat_cpu_pop_updated(struct cgroup *pos,
struct cgroup *root, int cpu)
{
struct cgroup_rstat_cpu *rstatc;
if (pos == root)
return NULL;
/*
* We're gonna walk down to the first leaf and visit/remove it. We
* can pick whatever unvisited node as the starting point.
*/
if (!pos)
pos = root;
else
pos = cgroup_parent(pos);
/* walk down to the first leaf */
while (true) {
rstatc = cgroup_rstat_cpu(pos, cpu);
if (rstatc->updated_children == pos)
break;
pos = rstatc->updated_children;
}
/*
* Unlink @pos from the tree. As the updated_children list is
* singly linked, we have to walk it to find the removal point.
* However, due to the way we traverse, @pos will be the first
* child in most cases. The only exception is @root.
*/
if (rstatc->updated_next) {
struct cgroup *parent = cgroup_parent(pos);
struct cgroup_rstat_cpu *prstatc = cgroup_rstat_cpu(parent, cpu);
struct cgroup_rstat_cpu *nrstatc;
struct cgroup **nextp;
nextp = &prstatc->updated_children;
while (true) {
nrstatc = cgroup_rstat_cpu(*nextp, cpu);
if (*nextp == pos)
break;
WARN_ON_ONCE(*nextp == parent);
nextp = &nrstatc->updated_next;
}
*nextp = rstatc->updated_next;
rstatc->updated_next = NULL;
/*
* Paired with the one in cgroup_rstat_cpu_updated().
* Either they see NULL updated_next or we see their
* updated stat.
*/
smp_mb();
return pos;
}
/* only happens for @root */
return NULL;
}
