static void smp_send_nmi_allbutself(void) { cpumask_t mask = cpu_online_map; cpu_clear(safe_smp_processor_id(), mask); if (!cpus_empty(mask)) send_IPI_mask(mask, NMI_VECTOR); }
static void __smp_call_function_many(cpumask_t *mask, void (*func) (void *info), void *info, int nonatomic, int wait) { struct call_data_struct data; int cpus; int cpu = smp_processor_id(); if (cpu_isset(cpu, *mask)) cpu_clear(cpu, *mask); cpus = cpus_weight(*mask); if (!cpus) return; data.func = func; data.info = info; atomic_set(&data.started, 0); data.wait = wait; if (wait) atomic_set(&data.finished, 0); call_data = &data; wmb(); /* Send a message to all other CPUs and wait for them to respond */ send_IPI_mask(*mask, CALL_FUNCTION_VECTOR); /* Wait for response */ while (atomic_read(&data.started) != cpus) cpu_relax(); if (!wait) return; while (atomic_read(&data.finished) != cpus) cpu_relax(); }
static void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm, unsigned long va) { int sender; union smp_flush_state *f; /* Caller has disabled preemption */ sender = smp_processor_id() % NUM_INVALIDATE_TLB_VECTORS; f = &per_cpu(flush_state, sender); /* Could avoid this lock when num_online_cpus() <= NUM_INVALIDATE_TLB_VECTORS, but it is probably not worth checking this for a cache-hot lock. */ spin_lock(&f->tlbstate_lock); f->flush_mm = mm; f->flush_va = va; cpus_or(f->flush_cpumask, cpumask, f->flush_cpumask); /* * We have to send the IPI only to * CPUs affected. */ send_IPI_mask(cpumask, INVALIDATE_TLB_VECTOR_START + sender); while (!cpus_empty(f->flush_cpumask)) cpu_relax(); f->flush_mm = NULL; f->flush_va = 0; spin_unlock(&f->tlbstate_lock); }
/* * this function sends a 'generic call function' IPI to one other CPU * in the system. * * cpu is a standard Linux logical CPU number. */ static void __smp_call_function_single(int cpu, void (*func) (void *info), void *info, int nonatomic, int wait) { struct call_data_struct data; int cpus = 1; data.func = func; data.info = info; atomic_set(&data.started, 0); data.wait = wait; if (wait) atomic_set(&data.finished, 0); call_data = &data; wmb(); /* Send a message to all other CPUs and wait for them to respond */ send_IPI_mask(cpumask_of_cpu(cpu), CALL_FUNCTION_VECTOR); /* Wait for response */ while (atomic_read(&data.started) != cpus) cpu_relax(); if (!wait) return; while (atomic_read(&data.finished) != cpus) cpu_relax(); }
static inline void send_IPI_allbutself(int vector) { /* * if there are no other CPUs in the system then * we get an APIC send error if we try to broadcast. * thus we have to avoid sending IPIs in this case. */ if (!(smp_num_cpus > 1)) return; if (clustered_apic_mode) { // Pointless. Use send_IPI_mask to do this instead int cpu; if (smp_num_cpus > 1) { for (cpu = 0; cpu < smp_num_cpus; ++cpu) { if (cpu != smp_processor_id()) send_IPI_mask(1 << cpu, vector); } } } else { __send_IPI_shortcut(APIC_DEST_ALLBUT, vector); return; } }
static void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm, unsigned long va) { /* * A couple of (to be removed) sanity checks: * * - current CPU must not be in mask * - mask must exist :) */ BUG_ON(cpus_empty(cpumask)); BUG_ON(cpu_isset(smp_processor_id(), cpumask)); BUG_ON(!mm); /* If a CPU which we ran on has gone down, OK. */ cpus_and(cpumask, cpumask, cpu_online_map); if (cpus_empty(cpumask)) return; /* * i'm not happy about this global shared spinlock in the * MM hot path, but we'll see how contended it is. * Temporarily this turns IRQs off, so that lockups are * detected by the NMI watchdog. */ spin_lock(&tlbstate_lock); flush_mm = mm; flush_va = va; #if NR_CPUS <= BITS_PER_LONG atomic_set_mask(cpumask, &flush_cpumask); #else { int k; unsigned long *flush_mask = (unsigned long *)&flush_cpumask; unsigned long *cpu_mask = (unsigned long *)&cpumask; for (k = 0; k < BITS_TO_LONGS(NR_CPUS); ++k) atomic_set_mask(cpu_mask[k], &flush_mask[k]); } #endif /* * Make the above memory operations globally visible before * sending the IPI. */ smp_mb(); /* * We have to send the IPI only to * CPUs affected. */ send_IPI_mask(flush_cpumask, INVALIDATE_TLB_VECTOR); while (!cpus_empty(flush_cpumask)) /* nothing. lockup detection does not belong here */ mb(); flush_mm = NULL; flush_va = 0; spin_unlock(&tlbstate_lock); }
/*==========================================================================* * Name: flush_tlb_others * * Description: This routine requests other CPU to execute flush TLB. * 1.Setup parameters. * 2.Send 'INVALIDATE_TLB_IPI' to other CPU. * Request other CPU to execute 'smp_invalidate_interrupt()'. * 3.Wait for other CPUs operation finished. * * Born on Date: 2002.02.05 * * Arguments: cpumask - bitmap of target CPUs * *mm - a pointer to the mm struct for flush TLB * *vma - a pointer to the vma struct include va * va - virtual address for flush TLB * * Returns: void (cannot fail) * * Modification log: * Date Who Description * ---------- --- -------------------------------------------------------- * *==========================================================================*/ static void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm, struct vm_area_struct *vma, unsigned long va) { unsigned long *mask; #ifdef DEBUG_SMP unsigned long flags; __save_flags(flags); if (!(flags & 0x0040)) /* Interrupt Disable NONONO */ BUG(); #endif /* DEBUG_SMP */ /* * A couple of (to be removed) sanity checks: * * - we do not send IPIs to not-yet booted CPUs. * - current CPU must not be in mask * - mask must exist :) */ BUG_ON(cpumask_empty(&cpumask)); BUG_ON(cpumask_test_cpu(smp_processor_id(), &cpumask)); BUG_ON(!mm); /* If a CPU which we ran on has gone down, OK. */ cpumask_and(&cpumask, &cpumask, cpu_online_mask); if (cpumask_empty(&cpumask)) return; /* * i'm not happy about this global shared spinlock in the * MM hot path, but we'll see how contended it is. * Temporarily this turns IRQs off, so that lockups are * detected by the NMI watchdog. */ spin_lock(&tlbstate_lock); flush_mm = mm; flush_vma = vma; flush_va = va; mask=cpumask_bits(&cpumask); atomic_set_mask(*mask, (atomic_t *)&flush_cpumask); /* * We have to send the IPI only to * CPUs affected. */ send_IPI_mask(&cpumask, INVALIDATE_TLB_IPI, 0); while (!cpumask_empty((cpumask_t*)&flush_cpumask)) { /* nothing. lockup detection does not belong here */ mb(); } flush_mm = NULL; flush_vma = NULL; flush_va = 0; spin_unlock(&tlbstate_lock); }
void smp_send_timer_broadcast_ipi(void) { cpumask_t mask; cpus_and(mask, cpu_online_map, timer_interrupt_broadcast_ipi_mask); if (!cpus_empty(mask)) { send_IPI_mask(mask, LOCAL_TIMER_VECTOR); } }
static void send_ipi_interrupt(cpumask_t *mask, int vector) { # if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 18) /***********************************************/ /* Theres 'flat' and theres 'cluster'. The */ /* cluster functions handle more than 8 */ /* cpus. The flat does not - since the APIC */ /* only has room for an 8-bit cpu mask. */ /***********************************************/ static void (*send_IPI_mask)(cpumask_t, int); if (send_IPI_mask == NULL) send_IPI_mask = get_proc_addr("cluster_send_IPI_mask"); if (send_IPI_mask == NULL) dtrace_printf("HELP ON send_ipi_interrupt!\n"); else send_IPI_mask(*mask, vector); # elif LINUX_VERSION_CODE == KERNEL_VERSION(2, 6, 28) /***********************************************/ /* Issue with GPL/inlined function. */ /***********************************************/ { void send_IPI_mask_sequence(cpumask_t mask, int vector); static void (*send_IPI_mask_sequence_ptr)(cpumask_t, int); if (send_IPI_mask_sequence_ptr == NULL) send_IPI_mask_sequence_ptr = get_proc_addr("send_IPI_mask_sequence"); send_IPI_mask_sequence_ptr(*mask, vector); } # elif LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 28) send_IPI_mask(*mask, vector); # else if (x_apic == NULL) { static void (*flat_send_IPI_mask)(cpumask_t *, int); if (flat_send_IPI_mask == NULL) flat_send_IPI_mask = get_proc_addr("flat_send_IPI_mask"); if (flat_send_IPI_mask) { flat_send_IPI_mask(mask, vector); return; } dtrace_linux_panic("x_apic is null - giving up\n"); return; } x_apic->send_IPI_mask(mask, vector); # endif }
void smp_send_call_function_mask(const cpumask_t *mask) { send_IPI_mask(mask, CALL_FUNCTION_VECTOR); if ( cpumask_test_cpu(smp_processor_id(), mask) ) { local_irq_disable(); smp_call_function_interrupt(); local_irq_enable(); } }
/** * smp_call_function_mask(): Run a function on a set of other CPUs. * @mask: The set of cpus to run on. Must not include the current cpu. * @func: The function to run. This must be fast and non-blocking. * @info: An arbitrary pointer to pass to the function. * @wait: If true, wait (atomically) until function has completed on other CPUs. * * Returns 0 on success, else a negative status code. * * If @wait is true, then returns once @func has returned; otherwise * it returns just before the target cpu calls @func. * * You must not call this function with disabled interrupts or from a * hardware interrupt handler or from a bottom half handler. */ static int native_smp_call_function_mask(cpumask_t mask, void (*func)(void *), void *info, int wait) { struct call_data_struct data; cpumask_t allbutself; int cpus; /* Can deadlock when called with interrupts disabled */ WARN_ON(irqs_disabled()); /* Holding any lock stops cpus from going down. */ spin_lock(&call_lock); allbutself = cpu_online_map; cpu_clear(smp_processor_id(), allbutself); cpus_and(mask, mask, allbutself); cpus = cpus_weight(mask); if (!cpus) { spin_unlock(&call_lock); return 0; } data.func = func; data.info = info; atomic_set(&data.started, 0); data.wait = wait; if (wait) atomic_set(&data.finished, 0); call_data = &data; mb(); /* Send a message to other CPUs */ if (cpus_equal(mask, allbutself)) send_IPI_allbutself(CALL_FUNCTION_VECTOR); else send_IPI_mask(mask, CALL_FUNCTION_VECTOR); /* Wait for response */ while (atomic_read(&data.started) != cpus) cpu_relax(); if (wait) while (atomic_read(&data.finished) != cpus) cpu_relax(); spin_unlock(&call_lock); return 0; }
void native_flush_tlb_others(const cpumask_t *cpumaskp, struct mm_struct *mm, unsigned long va) { cpumask_t cpumask = *cpumaskp; /* * A couple of (to be removed) sanity checks: * * - current CPU must not be in mask * - mask must exist :) */ BUG_ON(cpus_empty(cpumask)); BUG_ON(cpu_isset(smp_processor_id(), cpumask)); BUG_ON(!mm); #ifdef CONFIG_HOTPLUG_CPU /* If a CPU which we ran on has gone down, OK. */ cpus_and(cpumask, cpumask, cpu_online_map); if (unlikely(cpus_empty(cpumask))) return; #endif /* * i'm not happy about this global shared spinlock in the * MM hot path, but we'll see how contended it is. * AK: x86-64 has a faster method that could be ported. */ spin_lock(&tlbstate_lock); flush_mm = mm; flush_va = va; cpus_or(flush_cpumask, cpumask, flush_cpumask); /* * Make the above memory operations globally visible before * sending the IPI. */ smp_mb(); /* * We have to send the IPI only to * CPUs affected. */ send_IPI_mask(&cpumask, INVALIDATE_TLB_VECTOR); while (!cpus_empty(flush_cpumask)) /* nothing. lockup detection does not belong here */ cpu_relax(); flush_mm = NULL; flush_va = 0; spin_unlock(&tlbstate_lock); }
static inline void send_IPI_all(int vector) { if (clustered_apic_mode) { // Pointless. Use send_IPI_mask to do this instead int cpu; for (cpu = 0; cpu < smp_num_cpus; ++cpu) { send_IPI_mask(1 << cpu, vector); } } else { __send_IPI_shortcut(APIC_DEST_ALLINC, vector); } }
static void __smp_ipi_test_interrupt(void) { cpumask_t mask; if (smp_processor_id() == test_cpu_x) { if (nr_trips == INITIAL_DISCARD) { start_time = NOW(); send_ipi_time = 0; } if (nr_trips == NR_TRIPS + INITIAL_DISCARD) { finish_time = NOW(); tasklet_schedule(&ipi_test_tasklet); return; } nr_trips++; mask = cpumask_of_cpu(test_cpu_y); send_ipi_time -= NOW(); send_IPI_mask(&mask, IPI_TEST_VECTOR); send_ipi_time += NOW(); } else { mask = cpumask_of_cpu(test_cpu_x); send_IPI_mask(&mask, IPI_TEST_VECTOR); } }
static void send_ipi_interrupt(cpumask_t *mask, int vector) { # if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 18) /***********************************************/ /* Theres 'flat' and theres 'cluster'. The */ /* cluster functions handle more than 8 */ /* cpus. The flat does not - since the APIC */ /* only has room for an 8-bit cpu mask. */ /***********************************************/ static void (*send_IPI_mask)(cpumask_t, int); if (send_IPI_mask == NULL) send_IPI_mask = get_proc_addr("cluster_send_IPI_mask"); if (send_IPI_mask == NULL) dtrace_printf("HELP ON send_ipi_interrupt!\n"); else send_IPI_mask(*mask, vector); # elif LINUX_VERSION_CODE == KERNEL_VERSION(2, 6, 28) send_IPI_mask_sequence(*mask, vector); # elif LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 28) send_IPI_mask(*mask, vector); # else x_apic->send_IPI_mask(mask, vector); # endif }
static void run_ipi_test_tasklet(unsigned long ignore) { cpumask_t mask; BUG_ON(!local_irq_is_enabled()); if (!done_initialisation) { printk("Running initialisation; x2 apic enabled %d\n", x2apic_enabled); set_intr_gate(IPI_TEST_VECTOR, ipi_test_interrupt); test_cpu_x = 0; test_cpu_y = 1; done_initialisation = 1; } else { unsigned long time_taken = finish_time - start_time; printk("CPUs %d -> %d took %ld nanoseconds to perform %ld round trips; RTT %ldns\n", test_cpu_x, test_cpu_y, time_taken, nr_trips - INITIAL_DISCARD, time_taken / (nr_trips - INITIAL_DISCARD)); printk("%d -> %d send IPI time %ld nanoseconds (%ld each)\n", test_cpu_x, test_cpu_y, send_ipi_time, send_ipi_time / (nr_trips - INITIAL_DISCARD)); nr_trips = 0; test_cpu_y = next_cpu(test_cpu_y, cpu_online_map); if (test_cpu_y == test_cpu_x) test_cpu_y = next_cpu(test_cpu_y, cpu_online_map); if (test_cpu_y == NR_CPUS) { test_cpu_x = next_cpu(test_cpu_x, cpu_online_map); if (test_cpu_x == NR_CPUS) { printk("Finished test\n"); machine_restart(0); } test_cpu_y = 0; } } BUG_ON(test_cpu_x == test_cpu_y); if (test_cpu_x == smp_processor_id()) { local_irq_disable(); __smp_ipi_test_interrupt(); local_irq_enable(); } else { mask = cpumask_of_cpu(test_cpu_x); send_IPI_mask(&mask, IPI_TEST_VECTOR); } }
void smp_send_timer_broadcast_ipi(struct pt_regs *regs) { cpumask_t mask; cpus_and(mask, cpu_online_map, timer_bcast_ipi); if (!cpus_empty(mask)) { #ifdef CONFIG_SMP send_IPI_mask(mask, LOCAL_TIMER_VECTOR); #else /* * We can directly call the apic timer interrupt handler * in UP case. Minus all irq related functions */ up_apic_timer_interrupt_call(regs); #endif } }
void smp_send_timer_broadcast_ipi(void) { int cpu = smp_processor_id(); cpumask_t mask; cpus_and(mask, cpu_online_map, timer_interrupt_broadcast_ipi_mask); if (cpu_isset(cpu, mask)) { cpu_clear(cpu, mask); add_pda(apic_timer_irqs, 1); smp_local_timer_interrupt(); } if (!cpus_empty(mask)) { send_IPI_mask(mask, LOCAL_TIMER_VECTOR); } }
/*==========================================================================* * Name: smp_flush_cache_all * * Description: This routine sends a 'INVALIDATE_CACHE_IPI' to all other * CPUs in the system. * * Born on Date: 2003-05-28 * * Arguments: NONE * * Returns: void (cannot fail) * * Modification log: * Date Who Description * ---------- --- -------------------------------------------------------- * *==========================================================================*/ void smp_flush_cache_all(void) { cpumask_t cpumask; unsigned long *mask; preempt_disable(); cpumask_copy(&cpumask, cpu_online_mask); cpumask_clear_cpu(smp_processor_id(), &cpumask); spin_lock(&flushcache_lock); mask=cpumask_bits(&cpumask); atomic_set_mask(*mask, (atomic_t *)&flushcache_cpumask); send_IPI_mask(&cpumask, INVALIDATE_CACHE_IPI, 0); _flush_cache_copyback_all(); while (flushcache_cpumask) mb(); spin_unlock(&flushcache_lock); preempt_enable(); }
void native_flush_tlb_others(const cpumask_t *cpumaskp, struct mm_struct *mm, unsigned long va) { int sender; union smp_flush_state *f; cpumask_t cpumask = *cpumaskp; if (is_uv_system() && uv_flush_tlb_others(&cpumask, mm, va)) return; /* Caller has disabled preemption */ sender = smp_processor_id() % NUM_INVALIDATE_TLB_VECTORS; f = &per_cpu(flush_state, sender); /* * Could avoid this lock when * num_online_cpus() <= NUM_INVALIDATE_TLB_VECTORS, but it is * probably not worth checking this for a cache-hot lock. */ spin_lock(&f->tlbstate_lock); f->flush_mm = mm; f->flush_va = va; cpus_or(f->flush_cpumask, cpumask, f->flush_cpumask); /* * Make the above memory operations globally visible before * sending the IPI. */ smp_mb(); /* * We have to send the IPI only to * CPUs affected. */ send_IPI_mask(cpumask, INVALIDATE_TLB_VECTOR_START + sender); while (!cpus_empty(f->flush_cpumask)) cpu_relax(); f->flush_mm = NULL; f->flush_va = 0; spin_unlock(&f->tlbstate_lock); }
static void flush_tlb_others (unsigned long cpumask, struct mm_struct *mm, unsigned long va) { /* * A couple of (to be removed) sanity checks: * * - we do not send IPIs to not-yet booted CPUs. * - current CPU must not be in mask * - mask must exist :) */ if (!cpumask) BUG(); if ((cpumask & cpu_online_map) != cpumask) BUG(); if (cpumask & (1 << smp_processor_id())) BUG(); if (!mm) BUG(); /* * i'm not happy about this global shared spinlock in the * MM hot path, but we'll see how contended it is. * Temporarily this turns IRQs off, so that lockups are * detected by the NMI watchdog. */ spin_lock(&tlbstate_lock); flush_mm = mm; flush_va = va; atomic_set_mask(cpumask, &flush_cpumask); /* * We have to send the IPI only to * CPUs affected. */ send_IPI_mask(cpumask, INVALIDATE_TLB_VECTOR); while (flush_cpumask) /* nothing. lockup detection does not belong here */; flush_mm = NULL; flush_va = 0; spin_unlock(&tlbstate_lock); }
void flush_area_mask(const cpumask_t *mask, const void *va, unsigned int flags) { ASSERT(local_irq_is_enabled()); if ( cpumask_test_cpu(smp_processor_id(), mask) ) flush_area_local(va, flags); if ( !cpumask_subset(mask, cpumask_of(smp_processor_id())) ) { spin_lock(&flush_lock); cpumask_and(&flush_cpumask, mask, &cpu_online_map); cpumask_clear_cpu(smp_processor_id(), &flush_cpumask); flush_va = va; flush_flags = flags; send_IPI_mask(&flush_cpumask, INVALIDATE_TLB_VECTOR); while ( !cpumask_empty(&flush_cpumask) ) cpu_relax(); spin_unlock(&flush_lock); } }
int fastcall __ipipe_send_ipi(unsigned ipi, cpumask_t cpumask) { unsigned long flags; ipipe_declare_cpuid; int self; ipipe_lock_cpu(flags); self = cpu_isset(cpuid,cpumask); cpu_clear(cpuid,cpumask); if (!cpus_empty(cpumask)) send_IPI_mask(cpumask,ipi + FIRST_EXTERNAL_VECTOR); if (self) ipipe_trigger_irq(ipi); ipipe_unlock_cpu(flags); return 0; }
void smp_send_state_dump(unsigned int cpu) { /* We overload the spurious interrupt handler to handle the dump. */ per_cpu(state_dump_pending, cpu) = 1; send_IPI_mask(cpumask_of(cpu), SPURIOUS_APIC_VECTOR); }
void arch_send_call_function_ipi_mask(const struct cpumask *mask) { send_IPI_mask(mask, IPI_CALL_FUNC); }
/* * this function sends a 'reschedule' IPI to another CPU. * it goes straight through and wastes no time serializing * anything. Worst case is that we lose a reschedule ... */ void smp_send_reschedule(int cpu) { WARN_ON(cpu_is_offline(cpu)); send_IPI_mask(cpumask_of_cpu(cpu), RESCHEDULE_VECTOR); }
void arch_send_call_function_single_ipi(int cpu) { send_IPI_mask(cpumask_of(cpu), CALL_FUNC_SINGLE_IPI, 0); }
/* * this function sends a 'reschedule' IPI to another CPU. * it goes straight through and wastes no time serializing * anything. Worst case is that we lose a reschedule ... */ void smp_send_reschedule(int cpu) { send_IPI_mask(cpumask_of_cpu(cpu), RESCHEDULE_VECTOR); }
void smp_send_reschedule(int cpu) { send_IPI_mask(1 << cpu, RESCHEDULE_VECTOR); }
/* * Local APIC timer broadcast function */ static void lapic_timer_broadcast(cpumask_t mask) { #ifdef CONFIG_SMP send_IPI_mask(mask, LOCAL_TIMER_VECTOR); #endif }