/*
* Detect available CPUs, populate cpu_possible_map before smp_init
*
* We don't want to start the secondary CPU yet nor do we have a nice probing
* feature in PMON so we just assume presence of the secondary core.
*/
static void __init yos_smp_setup(void)
{
int i;
cpus_clear(cpu_possible_map);
for (i = 0; i < 2; i++) {
cpu_set(i, cpu_possible_map);
__cpu_number_map[i] = i;
__cpu_logical_map[i] = i;
}
}
/*
* Cycle through the APs sending Wakeup IPIs to boot each.
*/
void __init
smp_prepare_cpus (unsigned int max_cpus)
{
int boot_cpu_id = hard_smp_processor_id();
/*
* Initialize the per-CPU profiling counter/multiplier
*/
smp_setup_percpu_timer();
/*
* We have the boot CPU online for sure.
*/
cpu_set(0, cpu_online_map);
cpu_set(0, cpu_callin_map);
local_cpu_data->loops_per_jiffy = loops_per_jiffy;
ia64_cpu_to_sapicid[0] = boot_cpu_id;
printk(KERN_INFO "Boot processor id 0x%x/0x%x\n", 0, boot_cpu_id);
current_thread_info()->cpu = 0;
/*
* If SMP should be disabled, then really disable it!
*/
if (!max_cpus) {
printk(KERN_INFO "SMP mode deactivated.\n");
cpus_clear(cpu_online_map);
cpus_clear(cpu_present_map);
cpus_clear(cpu_possible_map);
cpu_set(0, cpu_online_map);
cpu_set(0, cpu_present_map);
cpu_set(0, cpu_possible_map);
return;
}
}
static void __init jzsoc_smp_setup(void)
{
int i, num;
scpu_pwc = cpm_pwc_get(PWC_SCPU);
cpus_clear(cpu_possible_map);
cpus_clear(cpu_present_map);
cpu_set(0, cpu_possible_map);
cpu_set(0, cpu_present_map);
__cpu_number_map[0] = 0;
__cpu_logical_map[0] = 0;
for (i = 1, num = 0; i < NR_CPUS; i++) {
cpu_set(i, cpu_possible_map);
cpu_set(i, cpu_present_map);
__cpu_number_map[i] = ++num;
__cpu_logical_map[num] = i;
}
pr_info("[SMP] Slave CPU(s) %i available.\n", num);
}
/**
* percpu_populate_mask - populate per-cpu data for more cpu's
* @__pdata: per-cpu data to populate further
* @size: size of per-cpu object
* @gfp: may sleep or not etc.
* @mask: populate per-cpu data for cpu's selected through mask bits
*
* Per-cpu objects are populated with zeroed buffers.
*/
int __percpu_populate_mask(void *__pdata, size_t size, gfp_t gfp,
cpumask_t *mask)
{
cpumask_t populated;
int cpu;
cpus_clear(populated);
for_each_cpu_mask(cpu, *mask)
if (unlikely(!percpu_populate(__pdata, size, gfp, cpu))) {
__percpu_depopulate_mask(__pdata, &populated);
return -ENOMEM;
} else
cpu_set(cpu, populated);
return 0;
}
/**
* @fn int xnsched_run(void)
* @brief The rescheduling procedure.
*
* This is the central rescheduling routine which should be called to
* validate and apply changes which have previously been made to the
* nucleus scheduling state, such as suspending, resuming or changing
* the priority of threads. This call performs context switches as
* needed. xnsched_run() schedules out the current thread if:
*
* - the current thread is about to block.
* - a runnable thread from a higher priority scheduling class is
* waiting for the CPU.
* - the current thread does not lead the runnable threads from its
* own scheduling class (i.e. round-robin).
*
* The Cobalt core implements a lazy rescheduling scheme so that most
* of the services affecting the threads state MUST be followed by a
* call to the rescheduling procedure for the new scheduling state to
* be applied.
*
* In other words, multiple changes on the scheduler state can be done
* in a row, waking threads up, blocking others, without being
* immediately translated into the corresponding context switches.
* When all changes have been applied, xnsched_run() should be called
* for considering those changes, and possibly switching context.
*
* As a notable exception to the previous principle however, every
* action which ends up suspending the current thread begets an
* implicit call to the rescheduling procedure on behalf of the
* blocking service.
*
* Typically, self-suspension or sleeping on a synchronization object
* automatically leads to a call to the rescheduling procedure,
* therefore the caller does not need to explicitly issue
* xnsched_run() after such operations.
*
* The rescheduling procedure always leads to a null-effect if it is
* called on behalf of an interrupt service routine. Any outstanding
* scheduler lock held by the outgoing thread will be restored when
* the thread is scheduled back in.
*
* Calling this procedure with no applicable context switch pending is
* harmless and simply leads to a null-effect.
*
* @return Non-zero is returned if a context switch actually happened,
* otherwise zero if the current thread was left running.
*
* @coretags{unrestricted}
*/
static inline int test_resched(struct xnsched *sched)
{
int resched = xnsched_resched_p(sched);
#ifdef CONFIG_SMP
/* Send resched IPI to remote CPU(s). */
if (unlikely(!cpus_empty(sched->resched))) {
smp_mb();
ipipe_send_ipi(IPIPE_RESCHEDULE_IPI, sched->resched);
cpus_clear(sched->resched);
}
#endif
sched->status &= ~XNRESCHED;
return resched;
}
/**
* build_cpu_to_node_map - setup cpu to node and node to cpumask arrays
*
* Build cpu to node mapping and initialize the per node cpu masks using
* info from the node_cpuid array handed to us by ACPI.
*/
void __init build_cpu_to_node_map(void)
{
int cpu, i, node;
for(node=0; node < MAX_NUMNODES; node++)
cpus_clear(node_to_cpu_mask[node]);
for_each_possible_early_cpu(cpu) {
node = -1;
for (i = 0; i < NR_CPUS; ++i)
if (cpu_physical_id(cpu) == node_cpuid[i].phys_id) {
node = node_cpuid[i].nid;
break;
}
map_cpu_to_node(cpu, node);
}
}
void enable_nonboot_cpus(void)
{
int cpu, error;
printk("Enabling non-boot CPUs ...\n");
for_each_cpu_mask ( cpu, frozen_cpus )
{
if ( (error = cpu_up(cpu)) )
{
BUG_ON(error == -EBUSY);
printk("Error taking CPU%d up: %d\n", cpu, error);
}
}
cpus_clear(frozen_cpus);
}
void move_masked_irq(int irq)
{
struct irq_desc *desc = irq_desc + irq;
cpumask_t tmp;
if (likely(!(desc->status & IRQ_MOVE_PENDING)))
return;
/*
* Paranoia: cpu-local interrupts shouldn't be calling in here anyway.
*/
if (CHECK_IRQ_PER_CPU(desc->status)) {
WARN_ON(1);
return;
}
desc->status &= ~IRQ_MOVE_PENDING;
if (unlikely(cpus_empty(irq_desc[irq].pending_mask)))
return;
if (!desc->chip->set_affinity)
return;
assert_spin_locked(&desc->lock);
cpus_and(tmp, irq_desc[irq].pending_mask, cpu_online_map);
/*
* If there was a valid mask to work with, please
* do the disable, re-program, enable sequence.
* This is *not* particularly important for level triggered
* but in a edge trigger case, we might be setting rte
* when an active trigger is comming in. This could
* cause some ioapics to mal-function.
* Being paranoid i guess!
*
* For correct operation this depends on the caller
* masking the irqs.
*/
if (likely(!cpus_empty(tmp))) {
desc->chip->set_affinity(irq,tmp);
}
cpus_clear(irq_desc[irq].pending_mask);
}
/**
* Runs a function on the specified CPU.
*/
static void
palacios_xcall(
int cpu_id,
void (*fn)(void *arg),
void * arg
)
{
cpumask_t cpu_mask;
cpus_clear(cpu_mask);
cpu_set(cpu_id, cpu_mask);
printk(KERN_DEBUG
"Palacios making xcall to cpu %d from cpu %d.\n",
cpu_id, current->cpu_id);
xcall_function(cpu_mask, fn, arg, 1);
}
/*
* Use CFE to find out how many CPUs are available, setting up
* phys_cpu_present_map and the logical/physical mappings.
* XXXKW will the boot CPU ever not be physical 0?
*
* Common setup before any secondaries are started
*/
void __init plat_smp_setup(void)
{
int i, num;
cpus_clear(phys_cpu_present_map);
cpu_set(0, phys_cpu_present_map);
__cpu_number_map[0] = 0;
__cpu_logical_map[0] = 0;
for (i = 1, num = 0; i < NR_CPUS; i++) {
if (cfe_cpu_stop(i) == 0) {
cpu_set(i, phys_cpu_present_map);
__cpu_number_map[i] = ++num;
__cpu_logical_map[num] = i;
}
}
printk(KERN_INFO "Detected %i available secondary CPU(s)\n", num);
}
/*
* Detect available CPUs, populate phys_cpu_present_map before smp_init
*
* We don't want to start the secondary CPU yet nor do we have a nice probing
* feature in PMON so we just assume presence of the secondary core.
*/
void prom_prepare_cpus(unsigned int max_cpus)
{
cpus_clear(phys_cpu_present_map);
/*
* The boot CPU
*/
cpu_set(0, phys_cpu_present_map);
__cpu_number_map[0] = 0;
__cpu_logical_map[0] = 0;
/*
* The secondary core
*/
cpu_set(1, phys_cpu_present_map);
__cpu_number_map[1] = 1;
__cpu_logical_map[1] = 1;
}
void __init xen_smp_prepare_cpus(unsigned int max_cpus)
{
unsigned cpu;
for_each_possible_cpu(cpu) {
cpus_clear(per_cpu(cpu_sibling_map, cpu));
/*
* cpu_core_ map will be zeroed when the per
* cpu area is allocated.
*
* cpus_clear(per_cpu(cpu_core_map, cpu));
*/
}
smp_store_cpu_info(0);
set_cpu_sibling_map(0);
if (xen_smp_intr_init(0))
BUG();
cpu_initialized_map = cpumask_of_cpu(0);
/* Restrict the possible_map according to max_cpus. */
while ((num_possible_cpus() > 1) && (num_possible_cpus() > max_cpus)) {
for (cpu = NR_CPUS-1; !cpu_isset(cpu, cpu_possible_map); cpu--)
continue;
cpu_clear(cpu, cpu_possible_map);
}
for_each_possible_cpu (cpu) {
struct task_struct *idle;
if (cpu == 0)
continue;
idle = fork_idle(cpu);
if (IS_ERR(idle))
panic("failed fork for CPU %d", cpu);
cpu_set(cpu, cpu_present_map);
}
//init_xenbus_allowed_cpumask();
}
/**
* build_cpu_to_node_map - setup cpu to node and node to cpumask arrays
*
* Build cpu to node mapping and initialize the per node cpu masks using
* info from the node_cpuid array handed to us by ACPI.
*/
void __init build_cpu_to_node_map(void)
{
int cpu, i, node;
for(node=0; node < MAX_NUMNODES; node++)
cpus_clear(node_to_cpu_mask[node]);
for(cpu = 0; cpu < NR_CPUS; ++cpu) {
node = -1;
for (i = 0; i < NR_CPUS; ++i)
if (cpu_physical_id(cpu) == node_cpuid[i].phys_id) {
node = node_cpuid[i].nid;
break;
}
cpu_to_node_map[cpu] = (node >= 0) ? node : 0;
if (node >= 0)
cpu_set(cpu, node_to_cpu_mask[node]);
}
}
static int __init mips_smp_init(void)
{
cpumask_t forbidden;
unsigned int cpu;
cpus_clear(forbidden);
#ifdef CONFIG_SMP
smp_call_function(mips_setup_cpu_mask, &forbidden, 1);
#endif
mips_setup_cpu_mask(&forbidden);
if (cpus_empty(forbidden))
return 0;
perfctr_cpus_forbidden_mask = forbidden;
for(cpu = 0; cpu < NR_CPUS; ++cpu)
if (cpu_isset(cpu, forbidden))
printk(" %u", cpu);
printk("\n");
return 0;
}
void disable_nonboot_cpus(void)
{
int cpu, error;
error = 0;
cpus_clear(frozen_cpus);
printk("Freezing cpus ...\n");
for_each_online_cpu(cpu) {
if (cpu == 0)
continue;
error = cpu_down(cpu);
if (!error) {
cpu_set(cpu, frozen_cpus);
printk("CPU%d is down\n", cpu);
continue;
}
printk("Error taking cpu %d down: %d\n", cpu, error);
}
BUG_ON(raw_smp_processor_id() != 0);
if (error)
panic("cpus not sleeping");
}
void __init xen_smp_prepare_boot_cpu(void)
{
int cpu;
BUG_ON(smp_processor_id() != 0);
native_smp_prepare_boot_cpu();
/* We've switched to the "real" per-cpu gdt, so make sure the
old memory can be recycled */
make_lowmem_page_readwrite(&per_cpu__gdt_page);
for_each_possible_cpu(cpu) {
cpus_clear(per_cpu(cpu_sibling_map, cpu));
/*
* cpu_core_map lives in a per cpu area that is cleared
* when the per cpu array is allocated.
*
* cpus_clear(per_cpu(cpu_core_map, cpu));
*/
}
xen_setup_vcpu_info_placement();
}
/*
* Initialize the logical CPU number to SAPICID mapping
*/
void __init
smp_build_cpu_map (void)
{
int sapicid, cpu, i;
int boot_cpu_id = hard_smp_processor_id();
for (cpu = 0; cpu < NR_CPUS; cpu++) {
ia64_cpu_to_sapicid[cpu] = -1;
}
ia64_cpu_to_sapicid[0] = boot_cpu_id;
cpus_clear(cpu_present_map);
cpu_set(0, cpu_present_map);
cpu_set(0, cpu_possible_map);
for (cpu = 1, i = 0; i < smp_boot_data.cpu_count; i++) {
sapicid = smp_boot_data.cpu_phys_id[i];
if (sapicid == boot_cpu_id)
continue;
cpu_set(cpu, cpu_present_map);
cpu_set(cpu, cpu_possible_map);
ia64_cpu_to_sapicid[cpu] = sapicid;
cpu++;
}
}
void __init setup_replication_mask(void)
{
/* Set only the master cnode's bit. The master cnode is always 0. */
cpus_clear(ktext_repmask);
cpu_set(0, ktext_repmask);
#ifdef CONFIG_REPLICATE_KTEXT
#ifndef CONFIG_MAPPED_KERNEL
#error Kernel replication works with mapped kernel support. No calias support.
#endif
{
cnodeid_t cnode;
for_each_online_node(cnode) {
if (cnode == 0)
continue;
/* Advertise that we have a copy of the kernel */
cpu_set(cnode, ktext_repmask);
}
}
#endif
/* Set up a GDA pointer to the replication mask. */
GDA->g_ktext_repmask = &ktext_repmask;
}
static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
{
struct vm_area_struct *mpnt, *tmp, **pprev;
struct rb_node **rb_link, *rb_parent;
int retval;
unsigned long charge;
struct mempolicy *pol;
down_write(&oldmm->mmap_sem);
flush_cache_dup_mm(oldmm);
/*
* Not linked in yet - no deadlock potential:
*/
down_write_nested(&mm->mmap_sem, SINGLE_DEPTH_NESTING);
mm->locked_vm = 0;
mm->mmap = NULL;
mm->mmap_cache = NULL;
mm->free_area_cache = oldmm->mmap_base;
mm->cached_hole_size = ~0UL;
mm->map_count = 0;
cpus_clear(mm->cpu_vm_mask);
mm->mm_rb = RB_ROOT;
rb_link = &mm->mm_rb.rb_node;
rb_parent = NULL;
pprev = &mm->mmap;
for (mpnt = oldmm->mmap; mpnt; mpnt = mpnt->vm_next) {
struct file *file;
if (mpnt->vm_flags & VM_DONTCOPY) {
long pages = vma_pages(mpnt);
mm->total_vm -= pages;
vm_stat_account(mm, mpnt->vm_flags, mpnt->vm_file,
-pages);
continue;
}
charge = 0;
if (mpnt->vm_flags & VM_ACCOUNT) {
unsigned int len = (mpnt->vm_end - mpnt->vm_start) >> PAGE_SHIFT;
if (security_vm_enough_memory(len))
goto fail_nomem;
charge = len;
}
tmp = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
if (!tmp)
goto fail_nomem;
*tmp = *mpnt;
pol = mpol_dup(vma_policy(mpnt));
retval = PTR_ERR(pol);
if (IS_ERR(pol))
goto fail_nomem_policy;
vma_set_policy(tmp, pol);
tmp->vm_flags &= ~VM_LOCKED;
tmp->vm_mm = mm;
tmp->vm_next = NULL;
anon_vma_link(tmp);
file = tmp->vm_file;
if (file) {
struct inode *inode = file->f_path.dentry->d_inode;
struct address_space *mapping = file->f_mapping;
get_file(file);
if (tmp->vm_flags & VM_DENYWRITE)
atomic_dec(&inode->i_writecount);
spin_lock(&mapping->i_mmap_lock);
if (tmp->vm_flags & VM_SHARED)
mapping->i_mmap_writable++;
tmp->vm_truncate_count = mpnt->vm_truncate_count;
flush_dcache_mmap_lock(mapping);
/* insert tmp into the share list, just after mpnt */
vma_prio_tree_add(tmp, mpnt);
flush_dcache_mmap_unlock(mapping);
spin_unlock(&mapping->i_mmap_lock);
}
/*
* Clear hugetlb-related page reserves for children. This only
* affects MAP_PRIVATE mappings. Faults generated by the child
* are not guaranteed to succeed, even if read-only
*/
if (is_vm_hugetlb_page(tmp))
reset_vma_resv_huge_pages(tmp);
/*
* Link in the new vma and copy the page table entries.
*/
*pprev = tmp;
pprev = &tmp->vm_next;
__vma_link_rb(mm, tmp, rb_link, rb_parent);
rb_link = &tmp->vm_rb.rb_right;
rb_parent = &tmp->vm_rb;
mm->map_count++;
retval = copy_page_range(mm, oldmm, mpnt);
if (tmp->vm_ops && tmp->vm_ops->open)
tmp->vm_ops->open(tmp);
if (retval)
goto out;
}
void
dtrace_xcall2(processorid_t cpu, dtrace_xcall_t func, void *arg)
{ int c;
int cpu_id = smp_processor_id();
int cpus_todo = 0;
# if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 24)
typedef struct cpumask cpumask_t;
//#define cpu_set(c, mask) cpumask_set_cpu(c, &(mask))
//#define cpus_clear(mask) cpumask_clear(&mask)
# endif
cpumask_t mask;
/***********************************************/
/* If we had an internal panic, stop doing */
/* xcalls. Shouldnt happen, but useful */
/* during debugging so we can diagnose what */
/* caused the panic. */
/***********************************************/
if (dtrace_shutdown)
return;
/***********************************************/
/* Special case - just 'us'. */
/***********************************************/
cnt_xcall1++;
if (cpu_id == cpu) {
local_irq_disable();
//dtrace_printf("[%d] sole cnt=%lu\n", smp_processor_id(), cnt_xcall1);
func(arg);
local_irq_enable();
return;
}
/***********************************************/
/* Set up the cpu mask to do just the */
/* relevant cpu. */
/***********************************************/
if (cpu != DTRACE_CPUALL) {
//dtrace_printf("just me %d %d\n", cpu_id, cpu);
cpu = 1 << cpu;
}
//dtrace_printf("xcall %d f=%p\n", cpu_id, func);
cnt_xcall2++;
if (xcall_levels[cpu_id]++)
cnt_xcall3++;
/***********************************************/
/* Set up the rendezvous with the other */
/* targetted cpus. We use a nearly square */
/* NCPU*NCPU matrix to allow for any cpu to */
/* wait for any other. We have two slots */
/* per cpu - because we may be in an */
/* interrupt. */
/* */
/* The interrupt slave will service all */
/* queued calls - sometimes it will be */
/* lucky and see multiple, especially if we */
/* are heavily loaded. */
/***********************************************/
cpus_clear(mask);
for (c = 0; c < nr_cpus; c++) {
struct xcalls *xc = &xcalls[cpu_id][c];
unsigned int cnt;
/***********************************************/
/* Dont set ourselves - we dont want our */
/* cpu to be taking an IPI interrupt and */
/* doing the work twice. We inline */
/* ourselves below. */
/***********************************************/
if ((cpu & (1 << c)) == 0 || c == cpu_id) {
continue;
}
/***********************************************/
/* Is this safe? We want to avoid an IPI */
/* call if the other cpu is idle/not doing */
/* dtrace work. If thats the case and we */
/* are calling dtrace_sync, then we can */
/* avoid the xcall. */
/***********************************************/
if ((void *) func == (void *) dtrace_sync_func &&
cpu_core[c].cpuc_probe_level == 0) {
cpu &= ~(1 << c);
cnt_xcall7++;
continue;
}
//dtrace_printf("xcall %p\n", func);
xc->xc_func = func;
xc->xc_arg = arg;
/***********************************************/
/* Spinlock in case the interrupt hasnt */
/* fired. This should be very rare, and */
/* when it happens, we would be hanging for */
/* 100m iterations (about 1s). We reduce */
/* the chance of a hit by using the */
/* NCPU*NCPU*2 array approach. These things */
/* happen when buffers are full or user is */
/* ^C-ing dtrace. */
/***********************************************/
for (cnt = 0; dtrace_cas32((void *) &xc->xc_state, XC_WORKING, XC_WORKING) == XC_WORKING; cnt++) {
/***********************************************/
/* Avoid noise for tiny windows. */
/***********************************************/
if ((cnt == 0 && xcall_debug) || !(xcall_debug && cnt == 50)) {
dtrace_printf("[%d] cpu%d in wrong state (state=%d)\n",
smp_processor_id(), c, xc->xc_state);
}
// xcall_slave2();
if (cnt == 100 * 1000 * 1000) {
dtrace_printf("[%d] cpu%d - busting lock\n",
smp_processor_id(), c);
break;
}
}
if ((cnt && xcall_debug) || (!xcall_debug && cnt > 50)) {
dtrace_printf("[%d] cpu%d in wrong state (state=%d) %u cycles\n",
smp_processor_id(), c, xc->xc_state, cnt);
}
/***********************************************/
/* As soon as we set xc_state and BEFORE */
/* the apic call, the cpu may see the */
/* change since it may be taking an IPI */
/* interrupt for someone else. We need to */
/* be careful with barriers (I think - */
/* although the clflush/wmb may be */
/* redundant). */
/***********************************************/
xc->xc_state = XC_WORKING;
// clflush(&xc->xc_state);
// smp_wmb();
cpu_set(c, mask);
cpus_todo++;
}
smp_mb();
/***********************************************/
/* Now tell the other cpus to do some work. */
/***********************************************/
if (cpus_todo)
send_ipi_interrupt(&mask, ipi_vector);
/***********************************************/
/* Check for ourselves. */
/***********************************************/
if (cpu & (1 << cpu_id)) {
func(arg);
}
if (xcall_debug)
dtrace_printf("[%d] getting ready.... (%ld) mask=%x func=%p\n", smp_processor_id(), cnt_xcall1, *(int *) &mask, func);
/***********************************************/
/* Wait for the cpus we invoked the IPI on. */
/* Cycle thru the cpus, to avoid mutual */
/* deadlock between one cpu trying to call */
/* us whilst we are calling them. */
/***********************************************/
while (cpus_todo > 0) {
for (c = 0; c < nr_cpus && cpus_todo > 0; c++) {
xcall_slave2();
if (c == cpu_id || (cpu & (1 << c)) == 0)
continue;
/***********************************************/
/* Wait a little while for this cpu to */
/* respond before going on to the next one. */
/***********************************************/
if (ack_wait(c, 100)) {
cpus_todo--;
cpu &= ~(1 << c);
}
}
}
// smp_mb();
xcall_levels[cpu_id]--;
}
int
ack_wait(int c, int attempts)
{
unsigned long cnt = 0;
int cnt1 = 0;
volatile struct xcalls *xc = &xcalls[smp_processor_id()][c];
/***********************************************/
/* Avoid holding on to a stale cache line. */
/***********************************************/
while (dtrace_cas32((void *) &xc->xc_state, XC_WORKING, XC_WORKING) != XC_IDLE) {
if (attempts-- <= 0)
return 0;
barrier();
/***********************************************/
/* Be HT friendly. */
/***********************************************/
// smt_pause();
cnt_xcall6++;
/***********************************************/
/* Keep track of the max. */
/***********************************************/
if (cnt > cnt_xcall5)
cnt_xcall5 = cnt;
/***********************************************/
/* On my Dual Core 2.26GHz system, we are */
/* seeing counters in the range of hundreds */
/* to maybe 2,000,000 for more extreme */
/* cases. (This is inside a VM). During */
/* debugging, we found problems with the */
/* two cores not seeing each other -- */
/* possibly because I wasnt doing the right */
/* things to ensure memory barriers were in */
/* place. */
/* */
/* We dont want to wait forever because */
/* that will crash/hang your machine, but */
/* we do need to give up if its taken far */
/* too long. */
/***********************************************/
// if (cnt++ == 50 * 1000 * 1000UL) {
if (cnt++ == 1 * 1000 * 1000UL) {
cnt = 0;
cnt_xcall4++;
if (cnt1 == 0) {
/***********************************************/
/* Looks like we are having trouble getting */
/* the interrupt, so try for an NMI. */
/***********************************************/
cpumask_t mask;
cpus_clear(mask);
cpu_set(c, mask);
// nmi_masks[c] = 1;
// send_ipi_interrupt(&mask, 2); //NMI_VECTOR);
}
if (1) {
// set_console_on(1);
dtrace_printf("ack_wait cpu=%d xcall %staking too long! c=%d [xcall1=%lu]\n",
smp_processor_id(),
cnt1 ? "STILL " : "",
c, cnt_xcall1);
//dump_stack();
// set_console_on(0);
}
if (cnt1++ > 3) {
dump_xcalls();
dtrace_linux_panic("xcall taking too long");
break;
}
}
}
if (xcall_debug) {
dtrace_printf("[%d] ack_wait finished c=%d cnt=%lu (max=%lu)\n", smp_processor_id(), c, cnt, cnt_xcall5);
}
return 1;
}
int acpi_processor_preregister_performance(
struct acpi_processor_performance **performance)
{
int count, count_target;
int retval = 0;
unsigned int i, j;
cpumask_t covered_cpus;
struct acpi_processor *pr;
struct acpi_psd_package *pdomain;
struct acpi_processor *match_pr;
struct acpi_psd_package *match_pdomain;
mutex_lock(&performance_mutex);
retval = 0;
/* Call _PSD for all CPUs */
for_each_possible_cpu(i) {
pr = processors[i];
if (!pr) {
/* Look only at processors in ACPI namespace */
continue;
}
if (pr->performance) {
retval = -EBUSY;
continue;
}
if (!performance || !performance[i]) {
retval = -EINVAL;
continue;
}
pr->performance = performance[i];
cpu_set(i, pr->performance->shared_cpu_map);
if (acpi_processor_get_psd(pr)) {
retval = -EINVAL;
continue;
}
}
if (retval)
goto err_ret;
/*
* Now that we have _PSD data from all CPUs, lets setup P-state
* domain info.
*/
for_each_possible_cpu(i) {
pr = processors[i];
if (!pr)
continue;
/* Basic validity check for domain info */
pdomain = &(pr->performance->domain_info);
if ((pdomain->revision != ACPI_PSD_REV0_REVISION) ||
(pdomain->num_entries != ACPI_PSD_REV0_ENTRIES)) {
retval = -EINVAL;
goto err_ret;
}
if (pdomain->coord_type != DOMAIN_COORD_TYPE_SW_ALL &&
pdomain->coord_type != DOMAIN_COORD_TYPE_SW_ANY &&
pdomain->coord_type != DOMAIN_COORD_TYPE_HW_ALL) {
retval = -EINVAL;
goto err_ret;
}
}
cpus_clear(covered_cpus);
for_each_possible_cpu(i) {
pr = processors[i];
if (!pr)
continue;
if (cpu_isset(i, covered_cpus))
continue;
pdomain = &(pr->performance->domain_info);
cpu_set(i, pr->performance->shared_cpu_map);
cpu_set(i, covered_cpus);
if (pdomain->num_processors <= 1)
continue;
/* Validate the Domain info */
count_target = pdomain->num_processors;
count = 1;
if (pdomain->coord_type == DOMAIN_COORD_TYPE_SW_ALL)
pr->performance->shared_type = CPUFREQ_SHARED_TYPE_ALL;
else if (pdomain->coord_type == DOMAIN_COORD_TYPE_HW_ALL)
pr->performance->shared_type = CPUFREQ_SHARED_TYPE_HW;
else if (pdomain->coord_type == DOMAIN_COORD_TYPE_SW_ANY)
pr->performance->shared_type = CPUFREQ_SHARED_TYPE_ANY;
for_each_possible_cpu(j) {
if (i == j)
continue;
match_pr = processors[j];
if (!match_pr)
continue;
match_pdomain = &(match_pr->performance->domain_info);
if (match_pdomain->domain != pdomain->domain)
continue;
/* Here i and j are in the same domain */
if (match_pdomain->num_processors != count_target) {
retval = -EINVAL;
goto err_ret;
}
if (pdomain->coord_type != match_pdomain->coord_type) {
retval = -EINVAL;
goto err_ret;
}
cpu_set(j, covered_cpus);
cpu_set(j, pr->performance->shared_cpu_map);
count++;
}
for_each_possible_cpu(j) {
if (i == j)
continue;
match_pr = processors[j];
if (!match_pr)
continue;
match_pdomain = &(match_pr->performance->domain_info);
if (match_pdomain->domain != pdomain->domain)
continue;
match_pr->performance->shared_type =
pr->performance->shared_type;
match_pr->performance->shared_cpu_map =
pr->performance->shared_cpu_map;
}
}
err_ret:
for_each_possible_cpu(i) {
pr = processors[i];
if (!pr || !pr->performance)
continue;
/* Assume no coordination on any error parsing domain info */
if (retval) {
cpus_clear(pr->performance->shared_cpu_map);
cpu_set(i, pr->performance->shared_cpu_map);
pr->performance->shared_type = CPUFREQ_SHARED_TYPE_ALL;
}
pr->performance = NULL; /* Will be set for real in register */
}
mutex_unlock(&performance_mutex);
return retval;
}
static int cachemiss_thread (void *data)
{
tux_req_t *req;
struct k_sigaction *ka;
DECLARE_WAITQUEUE(wait, current);
iothread_t *iot = data;
int nr = iot->ti->cpu, wake_up;
Dprintk("iot %p/%p got started.\n", iot, current);
drop_permissions();
spin_lock(&iot->async_lock);
iot->threads++;
sprintf(current->comm, "async IO %d/%d", nr, iot->threads);
spin_lock_irq(¤t->sighand->siglock);
ka = current->sighand->action + SIGCHLD-1;
ka->sa.sa_handler = SIG_IGN;
siginitsetinv(¤t->blocked, sigmask(SIGCHLD));
recalc_sigpending();
spin_unlock_irq(¤t->sighand->siglock);
spin_unlock(&iot->async_lock);
#ifdef CONFIG_SMP
{
cpumask_t mask;
if (cpu_isset(nr, cpu_online_map)) {
cpus_clear(mask);
cpu_set(nr, mask);
set_cpus_allowed(current, mask);
}
}
#endif
add_wait_queue_exclusive(&iot->async_sleep, &wait);
for (;;) {
while (!list_empty(&iot->async_queue) &&
(req = get_cachemiss(iot))) {
if (!req->atom_idx) {
add_tux_atom(req, flush_request);
add_req_to_workqueue(req);
continue;
}
tux_schedule_atom(req, 1);
if (signal_pending(current))
flush_all_signals();
}
if (signal_pending(current))
flush_all_signals();
if (!list_empty(&iot->async_queue))
continue;
if (iot->shutdown) {
Dprintk("iot %p/%p got shutdown!\n", iot, current);
break;
}
__set_current_state(TASK_INTERRUPTIBLE);
if (list_empty(&iot->async_queue)) {
Dprintk("iot %p/%p going to sleep.\n", iot, current);
schedule();
Dprintk("iot %p/%p got woken up.\n", iot, current);
}
__set_current_state(TASK_RUNNING);
}
remove_wait_queue(&iot->async_sleep, &wait);
wake_up = 0;
spin_lock(&iot->async_lock);
if (!--iot->threads)
wake_up = 1;
spin_unlock(&iot->async_lock);
Dprintk("iot %p/%p has finished shutdown!\n", iot, current);
if (wake_up) {
Dprintk("iot %p/%p waking up master.\n", iot, current);
wake_up(&iot->wait_shutdown);
}
return 0;
}
RTDECL(int) RTMpOnPair(RTCPUID idCpu1, RTCPUID idCpu2, uint32_t fFlags, PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
{
IPRT_LINUX_SAVE_EFL_AC();
int rc;
RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
AssertReturn(idCpu1 != idCpu2, VERR_INVALID_PARAMETER);
AssertReturn(!(fFlags & RTMPON_F_VALID_MASK), VERR_INVALID_FLAGS);
/*
* Check that both CPUs are online before doing the broadcast call.
*/
RTThreadPreemptDisable(&PreemptState);
if ( RTMpIsCpuOnline(idCpu1)
&& RTMpIsCpuOnline(idCpu2))
{
/*
* Use the smp_call_function variant taking a cpu mask where available,
* falling back on broadcast with filter. Slight snag if one of the
* CPUs is the one we're running on, we must do the call and the post
* call wait ourselves.
*/
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
cpumask_t DstCpuMask;
#endif
RTCPUID idCpuSelf = RTMpCpuId();
bool const fCallSelf = idCpuSelf == idCpu1 || idCpuSelf == idCpu2;
RTMPARGS Args;
Args.pfnWorker = pfnWorker;
Args.pvUser1 = pvUser1;
Args.pvUser2 = pvUser2;
Args.idCpu = idCpu1;
Args.idCpu2 = idCpu2;
Args.cHits = 0;
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 28)
cpumask_clear(&DstCpuMask);
cpumask_set_cpu(idCpu1, &DstCpuMask);
cpumask_set_cpu(idCpu2, &DstCpuMask);
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
cpus_clear(DstCpuMask);
cpu_set(idCpu1, DstCpuMask);
cpu_set(idCpu2, DstCpuMask);
#endif
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29)
smp_call_function_many(&DstCpuMask, rtmpLinuxWrapperPostInc, &Args, !fCallSelf /* wait */);
rc = 0;
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 28)
rc = smp_call_function_many(&DstCpuMask, rtmpLinuxWrapperPostInc, &Args, !fCallSelf /* wait */);
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
rc = smp_call_function_mask(DstCpuMask, rtmpLinuxWrapperPostInc, &Args, !fCallSelf /* wait */);
#else /* older kernels */
rc = smp_call_function(rtMpLinuxOnPairWrapper, &Args, 0 /* retry */, !fCallSelf /* wait */);
#endif /* older kernels */
Assert(rc == 0);
/* Call ourselves if necessary and wait for the other party to be done. */
if (fCallSelf)
{
uint32_t cLoops = 0;
rtmpLinuxWrapper(&Args);
while (ASMAtomicReadU32(&Args.cHits) < 2)
{
if ((cLoops & 0x1ff) == 0 && !RTMpIsCpuOnline(idCpuSelf == idCpu1 ? idCpu2 : idCpu1))
break;
cLoops++;
ASMNopPause();
}
}
Assert(Args.cHits <= 2);
if (Args.cHits == 2)
rc = VINF_SUCCESS;
else if (Args.cHits == 1)
rc = VERR_NOT_ALL_CPUS_SHOWED;
else if (Args.cHits == 0)
rc = VERR_CPU_OFFLINE;
else
rc = VERR_CPU_IPE_1;
}
/*
* A CPU must be present to be considered just offline.
*/
else if ( RTMpIsCpuPresent(idCpu1)
&& RTMpIsCpuPresent(idCpu2))
rc = VERR_CPU_OFFLINE;
else
rc = VERR_CPU_NOT_FOUND;
RTThreadPreemptRestore(&PreemptState);;
IPRT_LINUX_RESTORE_EFL_AC();
return rc;
}
unsigned long ipipe_critical_enter(void (*syncfn)(void))
{
int cpu __maybe_unused, n __maybe_unused;
unsigned long flags, loops __maybe_unused;
cpumask_t allbutself __maybe_unused;
flags = hard_local_irq_save();
if (num_online_cpus() == 1)
return flags;
#ifdef CONFIG_SMP
cpu = ipipe_processor_id();
if (!cpu_test_and_set(cpu, __ipipe_cpu_lock_map)) {
while (test_and_set_bit(0, &__ipipe_critical_lock)) {
n = 0;
hard_local_irq_enable();
do
cpu_relax();
while (++n < cpu);
hard_local_irq_disable();
}
restart:
spin_lock(&__ipipe_cpu_barrier);
__ipipe_cpu_sync = syncfn;
cpus_clear(__ipipe_cpu_pass_map);
cpu_set(cpu, __ipipe_cpu_pass_map);
/*
* Send the sync IPI to all processors but the current
* one.
*/
cpus_andnot(allbutself, cpu_online_map, __ipipe_cpu_pass_map);
ipipe_send_ipi(IPIPE_CRITICAL_IPI, allbutself);
loops = IPIPE_CRITICAL_TIMEOUT;
while (!cpus_equal(__ipipe_cpu_sync_map, allbutself)) {
if (--loops > 0) {
cpu_relax();
continue;
}
/*
* We ran into a deadlock due to a contended
* rwlock. Cancel this round and retry.
*/
__ipipe_cpu_sync = NULL;
spin_unlock(&__ipipe_cpu_barrier);
/*
* Ensure all CPUs consumed the IPI to avoid
* running __ipipe_cpu_sync prematurely. This
* usually resolves the deadlock reason too.
*/
while (!cpus_equal(cpu_online_map, __ipipe_cpu_pass_map))
cpu_relax();
goto restart;
}
}
atomic_inc(&__ipipe_critical_count);
#endif /* CONFIG_SMP */
return flags;
}
static void __init smp_boot_cpus(unsigned int max_cpus)
{
int apicid, cpu, bit, kicked;
unsigned long bogosum = 0;
/*
* Setup boot CPU information
*/
smp_store_cpu_info(0); /* Final full version of the data */
printk("CPU%d: ", 0);
print_cpu_info(&cpu_data[0]);
boot_cpu_physical_apicid = GET_APIC_ID(apic_read(APIC_ID));
boot_cpu_logical_apicid = logical_smp_processor_id();
x86_cpu_to_apicid[0] = boot_cpu_physical_apicid;
current_thread_info()->cpu = 0;
smp_tune_scheduling();
cpus_clear(cpu_sibling_map[0]);
cpu_set(0, cpu_sibling_map[0]);
/*
* If we couldn't find an SMP configuration at boot time,
* get out of here now!
*/
if (!smp_found_config && !acpi_lapic) {
printk(KERN_NOTICE "SMP motherboard not detected.\n");
smpboot_clear_io_apic_irqs();
phys_cpu_present_map = physid_mask_of_physid(0);
if (APIC_init_uniprocessor())
printk(KERN_NOTICE "Local APIC not detected."
" Using dummy APIC emulation.\n");
map_cpu_to_logical_apicid();
return;
}
/*
* Should not be necessary because the MP table should list the boot
* CPU too, but we do it for the sake of robustness anyway.
* Makes no sense to do this check in clustered apic mode, so skip it
*/
if (!check_phys_apicid_present(boot_cpu_physical_apicid)) {
printk("weird, boot CPU (#%d) not listed by the BIOS.\n",
boot_cpu_physical_apicid);
physid_set(hard_smp_processor_id(), phys_cpu_present_map);
}
/*
* If we couldn't find a local APIC, then get out of here now!
*/
if (APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid]) && !cpu_has_apic) {
printk(KERN_ERR "BIOS bug, local APIC #%d not detected!...\n",
boot_cpu_physical_apicid);
printk(KERN_ERR "... forcing use of dummy APIC emulation. (tell your hw vendor)\n");
smpboot_clear_io_apic_irqs();
phys_cpu_present_map = physid_mask_of_physid(0);
return;
}
verify_local_APIC();
/*
* If SMP should be disabled, then really disable it!
*/
if (!max_cpus) {
smp_found_config = 0;
printk(KERN_INFO "SMP mode deactivated, forcing use of dummy APIC emulation.\n");
smpboot_clear_io_apic_irqs();
phys_cpu_present_map = physid_mask_of_physid(0);
return;
}
connect_bsp_APIC();
setup_local_APIC();
map_cpu_to_logical_apicid();
setup_portio_remap();
/*
* Scan the CPU present map and fire up the other CPUs via do_boot_cpu
*
* In clustered apic mode, phys_cpu_present_map is a constructed thus:
* bits 0-3 are quad0, 4-7 are quad1, etc. A perverse twist on the
* clustered apic ID.
*/
Dprintk("CPU present map: %lx\n", physids_coerce(phys_cpu_present_map));
kicked = 1;
for (bit = 0; kicked < NR_CPUS && bit < MAX_APICS; bit++) {
apicid = cpu_present_to_apicid(bit);
/*
* Don't even attempt to start the boot CPU!
*/
if ((apicid == boot_cpu_apicid) || (apicid == BAD_APICID))
continue;
if (!check_apicid_present(bit))
continue;
if (max_cpus <= cpucount+1)
continue;
if (do_boot_cpu(apicid))
printk("CPU #%d not responding - cannot use it.\n",
apicid);
else
++kicked;
}
/*
* Cleanup possible dangling ends...
*/
smpboot_restore_warm_reset_vector();
/*
* Allow the user to impress friends.
*/
Dprintk("Before bogomips.\n");
for (cpu = 0; cpu < NR_CPUS; cpu++)
if (cpu_isset(cpu, cpu_callout_map))
bogosum += cpu_data[cpu].loops_per_jiffy;
printk(KERN_INFO
"Total of %d processors activated (%lu.%02lu BogoMIPS).\n",
cpucount+1,
bogosum/(500000/HZ),
(bogosum/(5000/HZ))%100);
Dprintk("Before bogocount - setting activated=1.\n");
if (smp_b_stepping)
printk(KERN_WARNING "WARNING: SMP operation may be unreliable with B stepping processors.\n");
/*
* Don't taint if we are running SMP kernel on a single non-MP
* approved Athlon
*/
if (tainted & TAINT_UNSAFE_SMP) {
if (cpucount)
printk (KERN_INFO "WARNING: This combination of AMD processors is not suitable for SMP.\n");
else
tainted &= ~TAINT_UNSAFE_SMP;
}
Dprintk("Boot done.\n");
/*
* construct cpu_sibling_map[], so that we can tell sibling CPUs
* efficiently.
*/
for (cpu = 0; cpu < NR_CPUS; cpu++)
cpus_clear(cpu_sibling_map[cpu]);
for (cpu = 0; cpu < NR_CPUS; cpu++) {
int siblings = 0;
int i;
if (!cpu_isset(cpu, cpu_callout_map))
continue;
if (smp_num_siblings > 1) {
for (i = 0; i < NR_CPUS; i++) {
if (!cpu_isset(i, cpu_callout_map))
continue;
if (phys_proc_id[cpu] == phys_proc_id[i]) {
siblings++;
cpu_set(i, cpu_sibling_map[cpu]);
}
}
} else {
siblings++;
cpu_set(cpu, cpu_sibling_map[cpu]);
}
if (siblings != smp_num_siblings)
printk(KERN_WARNING "WARNING: %d siblings found for CPU%d, should be %d\n", siblings, cpu, smp_num_siblings);
}
if (nmi_watchdog == NMI_LOCAL_APIC)
check_nmi_watchdog();
smpboot_setup_io_apic();
setup_boot_APIC_clock();
/*
* Synchronize the TSC with the AP
*/
if (cpu_has_tsc && cpucount && cpu_khz)
synchronize_tsc_bp();
}
static int
start_thread(int id, id_t cpu)
{
int status;
id_t aspace_id;
vaddr_t stack_ptr;
user_cpumask_t cpumask;
aspace_get_myid(&aspace_id);
stack_ptr = (vaddr_t)malloc(THREAD_STACK_SIZE);
cpus_clear(cpumask);
cpu_set(cpu, cpumask);
start_state_t start_state = {
.task_id = id,
.user_id = 1,
.group_id = 1,
.aspace_id = aspace_id,
.entry_point = (vaddr_t)&simple_task,
.stack_ptr = stack_ptr + THREAD_STACK_SIZE,
.cpu_id = cpu,
.edf.slice = SLICE,
.edf.period = PERIOD,
};
sprintf(start_state.task_name, "cpu%u-task%d", cpu ,id);
printf(" *** Creating Task %s (S: %llu, T: %llu) on cpu %d\n",
start_state.task_name,
(unsigned long long)start_state.edf.slice,
(unsigned long long)start_state.edf.period,
cpu);
status = task_create(&start_state, NULL);
if (status) {
printf("ERROR: failed to create thread (status=%d) on cpu %d.\n",
status,
cpu);
return -1;
}
return 0;
}
static int
edf_sched_test(void)
{
int status;
unsigned i;
id_t my_id;
cpu_set_t cpu_mask;
pthread_attr_t attr;
printf("\n EDF TEST BEGIN (Running for %d seconds)\n",TEST_TIME);
status = aspace_get_myid(&my_id);
if (status) {
printf("ERROR: task_get_myid() status=%d\n", status);
return -1;
}
pthread_attr_init(&attr);
pthread_attr_setstacksize(&attr, 1024 * 32);
printf(" My Linux process id (PID) is: %d\n", getpid());
status = pthread_getaffinity_np(pthread_self(), sizeof(cpu_mask), &cpu_mask);
if (status) {
printf(" ERROR: pthread_getaffinity_np() status=%d\n", status);
return -1;
}
printf(" My task ID is %u\n", my_id);
printf(" The following CPUs are in the cpumask:\n ");
for (i = CPU_MIN_ID; i <= CPU_MAX_ID; i++) {
if (CPU_ISSET(i, &cpu_mask))
printf("%d ", i);
}
printf("\n");
printf("Creating two EDF threads on each CPU:\n");
for (i = CPU_MIN_ID; i <= CPU_MAX_ID; i++) {
if (CPU_ISSET(i, &cpu_mask)) {
printf("CPU %d\n", i);
start_thread(0x1000+i,i);
start_thread(0x1100+i,i);
}
}
printf("\n");
return 0;
}
void __init prom_prepare_cpus(unsigned int max_cpus)
{
cpus_clear(phys_cpu_present_map);
}
static inline int dup_mmap(struct mm_struct * mm, struct mm_struct * oldmm)
{
struct vm_area_struct * mpnt, *tmp, **pprev;
struct rb_node **rb_link, *rb_parent;
int retval;
unsigned long charge;
struct mempolicy *pol;
down_write(&oldmm->mmap_sem);
flush_cache_mm(current->mm);
mm->locked_vm = 0;
mm->mmap = NULL;
mm->mmap_cache = NULL;
mm->free_area_cache = oldmm->mmap_base;
mm->map_count = 0;
mm->rss = 0;
mm->anon_rss = 0;
cpus_clear(mm->cpu_vm_mask);
mm->mm_rb = RB_ROOT;
rb_link = &mm->mm_rb.rb_node;
rb_parent = NULL;
pprev = &mm->mmap;
for (mpnt = current->mm->mmap ; mpnt ; mpnt = mpnt->vm_next) {
struct file *file;
if (mpnt->vm_flags & VM_DONTCOPY) {
__vm_stat_account(mm, mpnt->vm_flags, mpnt->vm_file,
-vma_pages(mpnt));
continue;
}
charge = 0;
if (mpnt->vm_flags & VM_ACCOUNT) {
unsigned int len = (mpnt->vm_end - mpnt->vm_start) >> PAGE_SHIFT;
if (security_vm_enough_memory(len))
goto fail_nomem;
charge = len;
}
tmp = kmem_cache_alloc(vm_area_cachep, SLAB_KERNEL);
if (!tmp)
goto fail_nomem;
*tmp = *mpnt;
pol = mpol_copy(vma_policy(mpnt));
retval = PTR_ERR(pol);
if (IS_ERR(pol))
goto fail_nomem_policy;
vma_set_policy(tmp, pol);
tmp->vm_flags &= ~VM_LOCKED;
tmp->vm_mm = mm;
tmp->vm_next = NULL;
anon_vma_link(tmp);
file = tmp->vm_file;
if (file) {
struct inode *inode = file->f_dentry->d_inode;
get_file(file);
if (tmp->vm_flags & VM_DENYWRITE)
atomic_dec(&inode->i_writecount);
/* insert tmp into the share list, just after mpnt */
spin_lock(&file->f_mapping->i_mmap_lock);
tmp->vm_truncate_count = mpnt->vm_truncate_count;
flush_dcache_mmap_lock(file->f_mapping);
vma_prio_tree_add(tmp, mpnt);
flush_dcache_mmap_unlock(file->f_mapping);
spin_unlock(&file->f_mapping->i_mmap_lock);
}
/*
* Link in the new vma and copy the page table entries:
* link in first so that swapoff can see swap entries,
* and try_to_unmap_one's find_vma find the new vma.
*/
spin_lock(&mm->page_table_lock);
*pprev = tmp;
pprev = &tmp->vm_next;
__vma_link_rb(mm, tmp, rb_link, rb_parent);
rb_link = &tmp->vm_rb.rb_right;
rb_parent = &tmp->vm_rb;
mm->map_count++;
retval = copy_page_range(mm, current->mm, tmp);
spin_unlock(&mm->page_table_lock);
if (tmp->vm_ops && tmp->vm_ops->open)
tmp->vm_ops->open(tmp);
if (retval)
goto out;
}
static void vector_allocation_domain(int cpu, cpumask_t *retmask)
{
cpus_clear(*retmask);
cpu_set(cpu, *retmask);
}
