/* * Wait specified idle threads to switch once. This ensures that even * preempted threads have cycled through the switch function once, * exiting their codepaths. This allows us to change global pointers * with no other synchronization. */ int quiesce_cpus(cpuset_t map, const char *wmesg, int prio) { struct pcpu *pcpu; u_int gen[MAXCPU]; int error; int cpu; error = 0; for (cpu = 0; cpu <= mp_maxid; cpu++) { if (!CPU_ISSET(cpu, &map) || CPU_ABSENT(cpu)) continue; pcpu = pcpu_find(cpu); gen[cpu] = pcpu->pc_idlethread->td_generation; } for (cpu = 0; cpu <= mp_maxid; cpu++) { if (!CPU_ISSET(cpu, &map) || CPU_ABSENT(cpu)) continue; pcpu = pcpu_find(cpu); thread_lock(curthread); sched_bind(curthread, cpu); thread_unlock(curthread); while (gen[cpu] == pcpu->pc_idlethread->td_generation) { error = tsleep(quiesce_cpus, prio, wmesg, 1); if (error) goto out; } } out: thread_lock(curthread); sched_unbind(curthread); thread_unlock(curthread); return (error); }
/* * Find the nth present CPU and return its pc_cpuid as well as set the * pc_acpi_id from the most reliable source. */ static int acpi_pcpu_get_id(uint32_t idx, uint32_t *acpi_id, uint32_t *cpu_id) { struct pcpu *pcpu_data; uint32_t i; KASSERT(acpi_id != NULL, ("Null acpi_id")); KASSERT(cpu_id != NULL, ("Null cpu_id")); for (i = 0; i <= mp_maxid; i++) { if (CPU_ABSENT(i)) continue; pcpu_data = pcpu_find(i); KASSERT(pcpu_data != NULL, ("no pcpu data for %d", i)); if (idx-- == 0) { /* * If pc_acpi_id was not initialized (e.g., a non-APIC UP box) * override it with the value from the ASL. Otherwise, if the * two don't match, prefer the MADT-derived value. Finally, * return the pc_cpuid to reference this processor. */ if (pcpu_data->pc_acpi_id == 0xffffffff) pcpu_data->pc_acpi_id = *acpi_id; else if (pcpu_data->pc_acpi_id != *acpi_id) *acpi_id = pcpu_data->pc_acpi_id; *cpu_id = pcpu_data->pc_cpuid; return (0); } } return (ESRCH); }
/* * Map the given inp to a CPU id. * * This queries RSS if it's compiled in, else it defaults to the current * CPU ID. */ static inline int inp_to_cpuid(struct inpcb *inp) { u_int cpuid; #ifdef RSS if (per_cpu_timers) { cpuid = rss_hash2cpuid(inp->inp_flowid, inp->inp_flowtype); if (cpuid == NETISR_CPUID_NONE) return (curcpu); /* XXX */ else return (cpuid); } #else /* Legacy, pre-RSS behaviour */ if (per_cpu_timers) { /* * We don't have a flowid -> cpuid mapping, so cheat and * just map unknown cpuids to curcpu. Not the best, but * apparently better than defaulting to swi 0. */ cpuid = inp->inp_flowid % (mp_maxid + 1); if (! CPU_ABSENT(cpuid)) return (cpuid); return (curcpu); } #endif /* Default for RSS and non-RSS - cpuid 0 */ else { return (0); } }
/* * Setup per-CPU domain IDs. */ static void srat_set_cpus(void *dummy) { struct cpu_info *cpu; struct pcpu *pc; u_int i; if (srat_physaddr == 0) return; for (i = 0; i < MAXCPU; i++) { if (CPU_ABSENT(i)) continue; pc = pcpu_find(i); KASSERT(pc != NULL, ("no pcpu data for CPU %u", i)); cpu = &cpus[pc->pc_apic_id]; if (!cpu->enabled) panic("SRAT: CPU with APIC ID %u is not known", pc->pc_apic_id); pc->pc_domain = cpu->domain; CPU_SET(i, &cpuset_domain[cpu->domain]); if (bootverbose) printf("SRAT: CPU %u has memory domain %d\n", i, cpu->domain); } }
static void start_softclock(void *dummy) { struct callout_cpu *cc; #ifdef SMP int cpu; #endif cc = CC_CPU(timeout_cpu); if (swi_add(&clk_intr_event, "clock", softclock, cc, SWI_CLOCK, INTR_MPSAFE, &softclock_ih)) panic("died while creating standard software ithreads"); cc->cc_cookie = softclock_ih; #ifdef SMP for (cpu = 0; cpu <= mp_maxid; cpu++) { if (cpu == timeout_cpu) continue; if (CPU_ABSENT(cpu)) continue; cc = CC_CPU(cpu); if (swi_add(NULL, "clock", softclock, cc, SWI_CLOCK, INTR_MPSAFE, &cc->cc_cookie)) panic("died while creating standard software ithreads"); cc->cc_callout = NULL; /* Only cpu0 handles timeout(). */ cc->cc_callwheel = malloc( sizeof(struct callout_tailq) * callwheelsize, M_CALLOUT, M_WAITOK); callout_cpu_init(cc); } #endif }
int pmc_cpu_is_present(int cpu) { #ifdef SMP return (!CPU_ABSENT(cpu)); #else return (1); #endif }
void smp_rendezvous_cpus(cpumask_t map, void (* setup_func)(void *), void (* action_func)(void *), void (* teardown_func)(void *), void *arg) { int i, ncpus = 0; if (!smp_started) { if (setup_func != NULL) setup_func(arg); if (action_func != NULL) action_func(arg); if (teardown_func != NULL) teardown_func(arg); return; } for (i = 0; i <= mp_maxid; i++) if (((1 << i) & map) != 0 && !CPU_ABSENT(i)) ncpus++; if (ncpus == 0) panic("ncpus is 0 with map=0x%x", map); /* obtain rendezvous lock */ mtx_lock_spin(&smp_ipi_mtx); /* set static function pointers */ smp_rv_ncpus = ncpus; smp_rv_setup_func = setup_func; smp_rv_action_func = action_func; smp_rv_teardown_func = teardown_func; smp_rv_func_arg = arg; smp_rv_waiters[1] = 0; smp_rv_waiters[2] = 0; atomic_store_rel_int(&smp_rv_waiters[0], 0); /* signal other processors, which will enter the IPI with interrupts off */ ipi_selected(map & ~(1 << curcpu), IPI_RENDEZVOUS); /* Check if the current CPU is in the map */ if ((map & (1 << curcpu)) != 0) smp_rendezvous_action(); if (teardown_func == smp_no_rendevous_barrier) while (atomic_load_acq_int(&smp_rv_waiters[2]) < ncpus) cpu_spinwait(); /* release lock */ mtx_unlock_spin(&smp_ipi_mtx); }
/* * Setup per-CPU domain IDs from information saved in 'cpus'. */ void acpi_pxm_set_cpu_locality(void) { struct cpu_info *cpu; struct pcpu *pc; u_int i; if (srat_physaddr == 0) return; for (i = 0; i < MAXCPU; i++) { if (CPU_ABSENT(i)) continue; pc = pcpu_find(i); KASSERT(pc != NULL, ("no pcpu data for CPU %u", i)); cpu = cpu_get_info(pc); pc->pc_domain = vm_ndomains > 1 ? cpu->domain : 0; CPU_SET(i, &cpuset_domain[pc->pc_domain]); if (bootverbose) printf("SRAT: CPU %u has memory domain %d\n", i, pc->pc_domain); } }
static int sysctl_kern_cp_times(SYSCTL_HANDLER_ARGS) { struct pcpu *pcpu; int error; int c; long *cp_time; #ifdef SCTL_MASK32 unsigned int cp_time32[CPUSTATES]; int i; #endif if (!req->oldptr) { #ifdef SCTL_MASK32 if (req->flags & SCTL_MASK32) return SYSCTL_OUT(req, 0, sizeof(cp_time32) * (mp_maxid + 1)); else #endif return SYSCTL_OUT(req, 0, sizeof(long) * CPUSTATES * (mp_maxid + 1)); } for (error = 0, c = 0; error == 0 && c <= mp_maxid; c++) { if (!CPU_ABSENT(c)) { pcpu = pcpu_find(c); cp_time = pcpu->pc_cp_time; } else { cp_time = empty; } #ifdef SCTL_MASK32 if (req->flags & SCTL_MASK32) { for (i = 0; i < CPUSTATES; i++) cp_time32[i] = (unsigned int)cp_time[i]; error = SYSCTL_OUT(req, cp_time32, sizeof(cp_time32)); } else #endif error = SYSCTL_OUT(req, cp_time, sizeof(long) * CPUSTATES); } return error; }
RTDECL(bool) RTMpIsCpuOnline(RTCPUID idCpu) { return idCpu <= mp_maxid && !CPU_ABSENT(idCpu); }
static void rss_init(__unused void *arg) { u_int i; u_int cpuid; /* * Validate tunables, coerce to sensible values. */ switch (rss_hashalgo) { case RSS_HASH_TOEPLITZ: case RSS_HASH_NAIVE: break; default: RSS_DEBUG("invalid RSS hashalgo %u, coercing to %u\n", rss_hashalgo, RSS_HASH_TOEPLITZ); rss_hashalgo = RSS_HASH_TOEPLITZ; } /* * Count available CPUs. * * XXXRW: Note incorrect assumptions regarding contiguity of this set * elsewhere. */ rss_ncpus = 0; for (i = 0; i <= mp_maxid; i++) { if (CPU_ABSENT(i)) continue; rss_ncpus++; } if (rss_ncpus > RSS_MAXCPUS) rss_ncpus = RSS_MAXCPUS; /* * Tune RSS table entries to be no less than 2x the number of CPUs * -- unless we're running uniprocessor, in which case there's not * much point in having buckets to rearrange for load-balancing! */ if (rss_ncpus > 1) { if (rss_bits == 0) rss_bits = fls(rss_ncpus - 1) + 1; /* * Microsoft limits RSS table entries to 128, so apply that * limit to both auto-detected CPU counts and user-configured * ones. */ if (rss_bits == 0 || rss_bits > RSS_MAXBITS) { RSS_DEBUG("RSS bits %u not valid, coercing to %u\n", rss_bits, RSS_MAXBITS); rss_bits = RSS_MAXBITS; } /* * Figure out how many buckets to use; warn if less than the * number of configured CPUs, although this is not a fatal * problem. */ rss_buckets = (1 << rss_bits); if (rss_buckets < rss_ncpus) RSS_DEBUG("WARNING: rss_buckets (%u) less than " "rss_ncpus (%u)\n", rss_buckets, rss_ncpus); rss_mask = rss_buckets - 1; } else { rss_bits = 0; rss_buckets = 1; rss_mask = 0; } /* * Set up initial CPU assignments: round-robin by default. */ cpuid = CPU_FIRST(); for (i = 0; i < rss_buckets; i++) { rss_table[i].rte_cpu = cpuid; cpuid = CPU_NEXT(cpuid); } /* * Randomize rrs_key. * * XXXRW: Not yet. If nothing else, will require an rss_isbadkey() * loop to check for "bad" RSS keys. */ }