static void mspin_unlock(struct mspin_node **lock, struct mspin_node *node) { struct mspin_node *next = ACCESS_ONCE(node->next); if (likely(!next)) { /* * Release the lock by setting it to NULL */ if (cmpxchg(lock, node, NULL) == node) return; /* Wait until the next pointer is set */ while (!(next = cpu_relaxed_read_long(&(node->next)))) cpu_read_relax(); } ACCESS_ONCE(next->locked) = 1; smp_wmb(); }
static noinline void mspin_lock(struct mspin_node **lock, struct mspin_node *node) { struct mspin_node *prev; /* Init node */ node->locked = 0; node->next = NULL; prev = xchg(lock, node); if (likely(prev == NULL)) { /* Lock acquired */ node->locked = 1; return; } ACCESS_ONCE(prev->next) = node; smp_wmb(); /* Wait until the lock holder passes the lock down */ while (!cpu_relaxed_read(&(node->locked))) cpu_read_relax(); }
/* Requires cpu_add_remove_lock to be held */ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen) { int err, nr_calls = 0; void *hcpu = (void *)(long)cpu; unsigned long mod = tasks_frozen ? CPU_TASKS_FROZEN : 0; struct take_cpu_down_param tcd_param = { .mod = mod, .hcpu = hcpu, }; if (num_online_cpus() == 1) return -EBUSY; if (!cpu_online(cpu)) return -EINVAL; cpu_hotplug_begin(); err = __cpu_notify(CPU_DOWN_PREPARE | mod, hcpu, -1, &nr_calls); if (err) { nr_calls--; __cpu_notify(CPU_DOWN_FAILED | mod, hcpu, nr_calls, NULL); printk("%s: attempt to take down CPU %u failed\n", __func__, cpu); goto out_release; } err = __stop_machine(take_cpu_down, &tcd_param, cpumask_of(cpu)); if (err) { /* CPU didn't die: tell everyone. Can't complain. */ cpu_notify_nofail(CPU_DOWN_FAILED | mod, hcpu); goto out_release; } BUG_ON(cpu_online(cpu)); /* * The migration_call() CPU_DYING callback will have removed all * runnable tasks from the cpu, there's only the idle task left now * that the migration thread is done doing the stop_machine thing. * * Wait for the stop thread to go away. */ while (!idle_cpu_relaxed(cpu)) cpu_read_relax(); /* This actually kills the CPU. */ __cpu_die(cpu); /* CPU is completely dead: tell everyone. Too late to complain. */ cpu_notify_nofail(CPU_DEAD | mod, hcpu); check_for_tasks(cpu); out_release: cpu_hotplug_done(); trace_sched_cpu_hotplug(cpu, err, 0); if (!err) cpu_notify_nofail(CPU_POST_DEAD | mod, hcpu); return err; }