unsigned long gnttab_end_foreign_transfer_ref(grant_ref_t ref) { unsigned long frame; uint16_t flags; /* * If a transfer is not even yet started, try to reclaim the grant * reference and return failure (== 0). */ while (!((flags = shared[ref].flags) & GTF_transfer_committed)) { if ( synch_cmpxchg(&shared[ref].flags, flags, 0) == flags ) return (0); cpu_spinwait(); } /* If a transfer is in progress then wait until it is completed. */ while (!(flags & GTF_transfer_completed)) { flags = shared[ref].flags; cpu_spinwait(); } /* Read the frame number /after/ reading completion status. */ rmb(); frame = shared[ref].frame; KASSERT(frame != 0, ("grant table inconsistent")); return (frame); }
RTDECL(void) RTSpinlockAcquire(RTSPINLOCK Spinlock) { PRTSPINLOCKINTERNAL pThis = (PRTSPINLOCKINTERNAL)Spinlock; RT_ASSERT_PREEMPT_CPUID_VAR(); AssertPtr(pThis); Assert(pThis->u32Magic == RTSPINLOCK_MAGIC); if (pThis->fFlags & RTSPINLOCK_FLAGS_INTERRUPT_SAFE) { for (;;) { uint32_t fIntSaved = ASMIntDisableFlags(); critical_enter(); int c = 50; for (;;) { if (ASMAtomicCmpXchgU32(&pThis->fLocked, 1, 0)) { RT_ASSERT_PREEMPT_CPUID_SPIN_ACQUIRED(pThis); pThis->fIntSaved = fIntSaved; return; } if (--c <= 0) break; cpu_spinwait(); } /* Enable interrupts while we sleep. */ ASMSetFlags(fIntSaved); critical_exit(); DELAY(1); } } else { for (;;) { critical_enter(); int c = 50; for (;;) { if (ASMAtomicCmpXchgU32(&pThis->fLocked, 1, 0)) { RT_ASSERT_PREEMPT_CPUID_SPIN_ACQUIRED(pThis); return; } if (--c <= 0) break; cpu_spinwait(); } critical_exit(); DELAY(1); } } }
/* * When called the executing CPU will send an IPI to all other CPUs * requesting that they halt execution. * * Usually (but not necessarily) called with 'other_cpus' as its arg. * * - Signals all CPUs in map to stop. * - Waits for each to stop. * * Returns: * -1: error * 0: NA * 1: ok * */ static int generic_stop_cpus(cpuset_t map, u_int type) { #ifdef KTR char cpusetbuf[CPUSETBUFSIZ]; #endif static volatile u_int stopping_cpu = NOCPU; int i; volatile cpuset_t *cpus; KASSERT( #if defined(__amd64__) || defined(__i386__) type == IPI_STOP || type == IPI_STOP_HARD || type == IPI_SUSPEND, #else type == IPI_STOP || type == IPI_STOP_HARD, #endif ("%s: invalid stop type", __func__)); if (!smp_started) return (0); CTR2(KTR_SMP, "stop_cpus(%s) with %u type", cpusetobj_strprint(cpusetbuf, &map), type); if (stopping_cpu != PCPU_GET(cpuid)) while (atomic_cmpset_int(&stopping_cpu, NOCPU, PCPU_GET(cpuid)) == 0) while (stopping_cpu != NOCPU) cpu_spinwait(); /* spin */ /* send the stop IPI to all CPUs in map */ ipi_selected(map, type); #if defined(__amd64__) || defined(__i386__) if (type == IPI_SUSPEND) cpus = &suspended_cpus; else #endif cpus = &stopped_cpus; i = 0; while (!CPU_SUBSET(cpus, &map)) { /* spin */ cpu_spinwait(); i++; if (i == 100000000) { printf("timeout stopping cpus\n"); break; } } stopping_cpu = NOCPU; return (1); }
void execute_the_co_test(struct callout_run *rn) { int i, ret, cpu; uint32_t tk_s, tk_e, tk_d; mtx_lock(&rn->lock); rn->callout_waiting = 0; for(i=0; i<rn->co_number_callouts; i++) { if (rn->co_test == 1) { /* start all on spread out cpu's */ cpu = i % mp_ncpus; callout_reset_sbt_on(&rn->co_array[i], 3, 0, test_callout, rn, cpu, 0); } else { /* Start all on the same CPU */ callout_reset_sbt_on(&rn->co_array[i], 3, 0, test_callout, rn, rn->index, 0); } } tk_s = ticks; while (rn->callout_waiting != rn->co_number_callouts) { cpu_spinwait(); tk_e = ticks; tk_d = tk_e - tk_s; if (tk_d > 100) { break; } } /* OK everyone is waiting and we have the lock */ for(i=0; i<rn->co_number_callouts; i++) { ret = callout_async_drain(&rn->co_array[i], drainit); if (ret) { rn->cnt_one++; } else { rn->cnt_zero++; } } rn->callout_waiting -= rn->cnt_one; mtx_unlock(&rn->lock); /* Now wait until all are done */ tk_s = ticks; while (rn->callout_waiting > 0) { cpu_spinwait(); tk_e = ticks; tk_d = tk_e - tk_s; if (tk_d > 100) { break; } } co_saydone((void *)rn); }
/* * When called the executing CPU will send an IPI to all other CPUs * requesting that they halt execution. * * Usually (but not necessarily) called with 'other_cpus' as its arg. * * - Signals all CPUs in map to suspend. * - Waits for each to suspend. * * Returns: * -1: error * 0: NA * 1: ok * * XXX FIXME: this is not MP-safe, needs a lock to prevent multiple CPUs * from executing at same time. */ int suspend_cpus(cpumask_t map) { int i; if (!smp_started) return (0); CTR1(KTR_SMP, "suspend_cpus(%x)", map); /* send the suspend IPI to all CPUs in map */ ipi_selected(map, IPI_SUSPEND); i = 0; while ((stopped_cpus & map) != map) { /* spin */ cpu_spinwait(); i++; #ifdef DIAGNOSTIC if (i == 100000) { printf("timeout suspending cpus\n"); break; } #endif } return (1); }
/* * When called the executing CPU will send an IPI to all other CPUs * requesting that they halt execution. * * Usually (but not necessarily) called with 'other_cpus' as its arg. * * - Signals all CPUs in map to stop. * - Waits for each to stop. * * Returns: * -1: error * 0: NA * 1: ok * * XXX FIXME: this is not MP-safe, needs a lock to prevent multiple CPUs * from executing at same time. */ static int generic_stop_cpus(cpumask_t map, u_int type) { int i; KASSERT(type == IPI_STOP || type == IPI_STOP_HARD, ("%s: invalid stop type", __func__)); if (!smp_started) return 0; CTR2(KTR_SMP, "stop_cpus(%x) with %u type", map, type); /* send the stop IPI to all CPUs in map */ ipi_selected(map, type); i = 0; while ((stopped_cpus & map) != map) { /* spin */ cpu_spinwait(); i++; #ifdef DIAGNOSTIC if (i == 100000) { printf("timeout stopping cpus\n"); break; } #endif } return 1; }
static void dmar_qi_ensure(struct dmar_unit *unit, int descr_count) { uint32_t head; int bytes; DMAR_ASSERT_LOCKED(unit); bytes = descr_count << DMAR_IQ_DESCR_SZ_SHIFT; for (;;) { if (bytes <= unit->inv_queue_avail) break; /* refill */ head = dmar_read4(unit, DMAR_IQH_REG); head &= DMAR_IQH_MASK; unit->inv_queue_avail = head - unit->inv_queue_tail - DMAR_IQ_DESCR_SZ; if (head <= unit->inv_queue_tail) unit->inv_queue_avail += unit->inv_queue_size; if (bytes <= unit->inv_queue_avail) break; /* * No space in the queue, do busy wait. Hardware must * make a progress. But first advance the tail to * inform the descriptor streamer about entries we * might have already filled, otherwise they could * clog the whole queue.. */ dmar_qi_advance_tail(unit); unit->inv_queue_full++; cpu_spinwait(); } unit->inv_queue_avail -= bytes; }
static void dmar_qi_emit_wait_seq(struct dmar_unit *unit, struct dmar_qi_genseq *pseq, bool emit_wait) { struct dmar_qi_genseq gsec; uint32_t seq; KASSERT(pseq != NULL, ("wait descriptor with no place for seq")); DMAR_ASSERT_LOCKED(unit); if (unit->inv_waitd_seq == 0xffffffff) { gsec.gen = unit->inv_waitd_gen; gsec.seq = unit->inv_waitd_seq; dmar_qi_ensure(unit, 1); dmar_qi_emit_wait_descr(unit, gsec.seq, false, true, false); dmar_qi_advance_tail(unit); while (!dmar_qi_seq_processed(unit, &gsec)) cpu_spinwait(); unit->inv_waitd_gen++; unit->inv_waitd_seq = 1; } seq = unit->inv_waitd_seq++; pseq->gen = unit->inv_waitd_gen; pseq->seq = seq; if (emit_wait) { dmar_qi_ensure(unit, 1); dmar_qi_emit_wait_descr(unit, seq, true, true, false); } }
/* * Set the time of day clock based on the value of the struct timespec arg. * Return 0 on success, an error number otherwise. */ static int at91_rtc_settime(device_t dev, struct timespec *ts) { struct at91_rtc_softc *sc; struct clocktime ct; int rv; sc = device_get_softc(dev); clock_ts_to_ct(ts, &ct); /* * Can't set the clock unless a second has elapsed since we last did so. */ while ((RD4(sc, RTC_SR) & RTC_SR_SECEV) == 0) cpu_spinwait(); /* * Stop the clocks for an update; wait until hardware is ready. * Clear the update-ready status after it gets asserted (the manual says * to do this before updating the value registers). */ WR4(sc, RTC_CR, RTC_CR_UPDCAL | RTC_CR_UPDTIM); while ((RD4(sc, RTC_SR) & RTC_SR_ACKUPD) == 0) cpu_spinwait(); WR4(sc, RTC_SCCR, RTC_SR_ACKUPD); /* * Set the values in the hardware, then check whether the hardware was * happy with them so we can return the correct status. */ WR4(sc, RTC_TIMR, RTC_TIMR_MK(ct.hour, ct.min, ct.sec)); WR4(sc, RTC_CALR, RTC_CALR_MK(ct.year, ct.mon, ct.day, ct.dow+1)); if (RD4(sc, RTC_VER) & (RTC_VER_NVTIM | RTC_VER_NVCAL)) rv = EINVAL; else rv = 0; /* * Restart the clocks (turn off the update bits). * Clear the second-event bit (because the manual says to). */ WR4(sc, RTC_CR, RD4(sc, RTC_CR) & ~(RTC_CR_UPDCAL | RTC_CR_UPDTIM)); WR4(sc, RTC_SCCR, RTC_SR_SECEV); return (0); }
void * virtqueue_poll(struct virtqueue *vq, uint32_t *len) { void *cookie; while ((cookie = virtqueue_dequeue(vq, len)) == NULL) cpu_spinwait(); return (cookie); }
static int dmar_disable_qi(struct dmar_unit *unit) { DMAR_ASSERT_LOCKED(unit); unit->hw_gcmd &= ~DMAR_GCMD_QIE; dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd); /* XXXKIB should have a timeout */ while ((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_QIES) != 0) cpu_spinwait(); return (0); }
void delay_boot(int usec) { u_long end; if (usec < 0) return; end = rd(tick) + (u_long)usec * clock_boot / 1000000; while (rd(tick) < end) cpu_spinwait(); }
void smp_rendezvous_cpus(cpumask_t map, void (* setup_func)(void *), void (* action_func)(void *), void (* teardown_func)(void *), void *arg) { int i, ncpus = 0; if (!smp_started) { if (setup_func != NULL) setup_func(arg); if (action_func != NULL) action_func(arg); if (teardown_func != NULL) teardown_func(arg); return; } CPU_FOREACH(i) { if (((1 << i) & map) != 0) ncpus++; } if (ncpus == 0) panic("ncpus is 0 with map=0x%x", map); /* obtain rendezvous lock */ mtx_lock_spin(&smp_ipi_mtx); /* set static function pointers */ smp_rv_ncpus = ncpus; smp_rv_setup_func = setup_func; smp_rv_action_func = action_func; smp_rv_teardown_func = teardown_func; smp_rv_func_arg = arg; smp_rv_waiters[1] = 0; smp_rv_waiters[2] = 0; atomic_store_rel_int(&smp_rv_waiters[0], 0); /* signal other processors, which will enter the IPI with interrupts off */ ipi_selected(map & ~(1 << curcpu), IPI_RENDEZVOUS); /* Check if the current CPU is in the map */ if ((map & (1 << curcpu)) != 0) smp_rendezvous_action(); if (teardown_func == smp_no_rendevous_barrier) while (atomic_load_acq_int(&smp_rv_waiters[2]) < ncpus) cpu_spinwait(); /* release lock */ mtx_unlock_spin(&smp_ipi_mtx); }
/* * _mtx_lock_spin: the tougher part of acquiring an MTX_SPIN lock. * * This is only called if we need to actually spin for the lock. Recursion * is handled inline. */ void _mtx_lock_spin(struct mtx *m, uintptr_t tid, int opts, const char *file, int line) { int i = 0; #ifdef LOCK_PROFILING int contested = 0; uint64_t waittime = 0; #endif if (LOCK_LOG_TEST(&m->lock_object, opts)) CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m); lock_profile_obtain_lock_failed(&m->lock_object, &contested, &waittime); while (!_obtain_lock(m, tid)) { /* Give interrupts a chance while we spin. */ spinlock_exit(); while (m->mtx_lock != MTX_UNOWNED) { if (i++ < 10000000) { cpu_spinwait(); continue; } if (i < 60000000 || kdb_active || panicstr != NULL) DELAY(1); else _mtx_lock_spin_failed(m); cpu_spinwait(); } spinlock_enter(); } if (LOCK_LOG_TEST(&m->lock_object, opts)) CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m); LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_MTX_SPIN_LOCK_ACQUIRE, m, contested, waittime, (file), (line)); LOCKSTAT_RECORD1(LS_MTX_SPIN_LOCK_SPIN, m, i); }
/* * All-CPU rendezvous. CPUs are signalled, all execute the setup function * (if specified), rendezvous, execute the action function (if specified), * rendezvous again, execute the teardown function (if specified), and then * resume. * * Note that the supplied external functions _must_ be reentrant and aware * that they are running in parallel and in an unknown lock context. */ void smp_rendezvous_action(void) { void* local_func_arg = smp_rv_func_arg; void (*local_setup_func)(void*) = smp_rv_setup_func; void (*local_action_func)(void*) = smp_rv_action_func; void (*local_teardown_func)(void*) = smp_rv_teardown_func; /* Ensure we have up-to-date values. */ atomic_add_acq_int(&smp_rv_waiters[0], 1); while (smp_rv_waiters[0] < smp_rv_ncpus) cpu_spinwait(); /* setup function */ if (local_setup_func != smp_no_rendevous_barrier) { if (smp_rv_setup_func != NULL) smp_rv_setup_func(smp_rv_func_arg); /* spin on entry rendezvous */ atomic_add_int(&smp_rv_waiters[1], 1); while (smp_rv_waiters[1] < smp_rv_ncpus) cpu_spinwait(); } /* action function */ if (local_action_func != NULL) local_action_func(local_func_arg); /* spin on exit rendezvous */ atomic_add_int(&smp_rv_waiters[2], 1); if (local_teardown_func == smp_no_rendevous_barrier) return; while (smp_rv_waiters[2] < smp_rv_ncpus) cpu_spinwait(); /* teardown function */ if (local_teardown_func != NULL) local_teardown_func(local_func_arg); }
/* * Take a local copy of a policy. * * The destination policy isn't write-barriered; this is used * for doing local copies into something that isn't shared. */ void vm_domain_policy_localcopy(struct vm_domain_policy *dst, const struct vm_domain_policy *src) { seq_t seq; for (;;) { seq = seq_read(&src->seq); *dst = *src; if (seq_consistent(&src->seq, seq)) return; cpu_spinwait(); } }
static void dmar_qi_wait_for_seq(struct dmar_unit *unit, const struct dmar_qi_genseq *gseq) { DMAR_ASSERT_LOCKED(unit); unit->inv_seq_waiters++; while (!dmar_qi_seq_processed(unit, gseq)) { if (cold) { cpu_spinwait(); } else { msleep(&unit->inv_seq_waiters, &unit->lock, 0, "dmarse", hz); } } unit->inv_seq_waiters--; }
void vm_domain_iterator_set_policy(struct vm_domain_iterator *vi, const struct vm_domain_policy *vt) { seq_t seq; struct vm_domain_policy vt_lcl; for (;;) { seq = seq_read(&vt->seq); vt_lcl = *vt; if (seq_consistent(&vt->seq, seq)) { _vm_domain_iterator_set_policy(vi, &vt_lcl); return; } cpu_spinwait(); } }
static void delay_tsc(int n) { uint64_t end, now; /* * Pin the current thread ensure correct behavior if the TSCs * on different CPUs are not in sync. */ sched_pin(); now = rdtsc(); end = now + tsc_freq * n / 1000000; do { cpu_spinwait(); now = rdtsc(); } while (now < end); sched_unpin(); }
/* * Called by a CPU to restart stopped CPUs. * * Usually (but not necessarily) called with 'stopped_cpus' as its arg. * * - Signals all CPUs in map to restart. * - Waits for each to restart. * * Returns: * -1: error * 0: NA * 1: ok */ int restart_cpus(cpumask_t map) { if (!smp_started) return 0; CTR1(KTR_SMP, "restart_cpus(%x)", map); /* signal other cpus to restart */ atomic_store_rel_int(&started_cpus, map); /* wait for each to clear its bit */ while ((stopped_cpus & map) != 0) cpu_spinwait(); return 1; }
/* * Take a write-barrier copy of a policy. * * The destination policy is write -barriered; this is used * for doing copies into policies that may be read by other * threads. */ void vm_domain_policy_copy(struct vm_domain_policy *dst, const struct vm_domain_policy *src) { seq_t seq; struct vm_domain_policy d; for (;;) { seq = seq_read(&src->seq); d = *src; if (seq_consistent(&src->seq, seq)) { seq_write_begin(&dst->seq); dst->p.domain = d.p.domain; dst->p.policy = d.p.policy; seq_write_end(&dst->seq); return; } cpu_spinwait(); } }
void delay_tick(int usec) { u_long end; if (usec < 0) return; /* * We avoid being migrated to another CPU with a possibly * unsynchronized TICK timer while spinning. */ sched_pin(); end = rd(tick) + (u_long)usec * PCPU_GET(clock) / 1000000; while (rd(tick) < end) cpu_spinwait(); sched_unpin(); }
/* * Called by a CPU to restart stopped CPUs. * * Usually (but not necessarily) called with 'stopped_cpus' as its arg. * * - Signals all CPUs in map to restart. * - Waits for each to restart. * * Returns: * -1: error * 0: NA * 1: ok */ int restart_cpus(cpuset_t map) { #ifdef KTR char cpusetbuf[CPUSETBUFSIZ]; #endif if (!smp_started) return 0; CTR1(KTR_SMP, "restart_cpus(%s)", cpusetobj_strprint(cpusetbuf, &map)); /* signal other cpus to restart */ CPU_COPY_STORE_REL(&map, &started_cpus); /* wait for each to clear its bit */ while (CPU_OVERLAP(&stopped_cpus, &map)) cpu_spinwait(); return 1; }
/* * Reconfigure specified timer. * For per-CPU timers use IPI to make other CPUs to reconfigure. */ static void configtimer(int i) { #ifdef SMP tc *conf; int cpu; critical_enter(); #endif /* Start/stop global timer or per-CPU timer of this CPU. */ if (i == 0 ? timer1hz : timer2hz) et_start(timer[i], NULL, &timerperiod[i]); else et_stop(timer[i]); #ifdef SMP if ((timer[i]->et_flags & ET_FLAGS_PERCPU) == 0 || !smp_started) { critical_exit(); return; } /* Set reconfigure flags for other CPUs. */ CPU_FOREACH(cpu) { conf = DPCPU_ID_PTR(cpu, configtimer); atomic_store_rel_int(*conf + i, (cpu == curcpu) ? 0 : 1); } /* Send reconfigure IPI. */ ipi_all_but_self(i == 0 ? IPI_HARDCLOCK : IPI_STATCLOCK); /* Wait for reconfiguration completed. */ restart: cpu_spinwait(); CPU_FOREACH(cpu) { if (cpu == curcpu) continue; conf = DPCPU_ID_PTR(cpu, configtimer); if (atomic_load_acq_int(*conf + i)) goto restart; } critical_exit(); #endif }
static int delay_tc(int n) { struct timecounter *tc; timecounter_get_t *func; uint64_t end, freq, now; u_int last, mask, u; /* * Only use the TSC if it is P-state invariant. If the TSC is * not P-state invariant and the CPU is not running at the * "full" P-state, then the TSC will increment at some rate * less than tsc_freq and delay_tsc() will wait too long. */ if (tsc_is_invariant && tsc_freq != 0) { delay_tsc(n); return (1); } tc = timecounter; if (tc->tc_quality <= 0) return (0); func = tc->tc_get_timecount; mask = tc->tc_counter_mask; freq = tc->tc_frequency; now = 0; end = freq * n / 1000000; last = func(tc) & mask; do { cpu_spinwait(); u = func(tc) & mask; if (u < last) now += mask - last + u + 1; else now += u - last; last = u; } while (now < end); return (1); }
static int delay_tc(int n) { struct timecounter *tc; timecounter_get_t *func; uint64_t end, freq, now; u_int last, mask, u; tc = timecounter; freq = atomic_load_acq_64(&tsc_freq); if (tsc_is_invariant && freq != 0) { func = get_tsc; mask = ~0u; } else { if (tc->tc_quality <= 0) return (0); func = tc->tc_get_timecount; mask = tc->tc_counter_mask; freq = tc->tc_frequency; } now = 0; end = freq * n / 1000000; if (func == get_tsc) sched_pin(); last = func(tc) & mask; do { cpu_spinwait(); u = func(tc) & mask; if (u < last) now += mask - last + u + 1; else now += u - last; last = u; } while (now < end); if (func == get_tsc) sched_unpin(); return (1); }
static void mpc85xx_jog_set_int(void *arg) { struct jog_rv_args *args = arg; uint32_t reg; if (PCPU_GET(cpuid) == args->cpu) { reg = ccsr_read4(GUTS_PMJCR); reg &= ~PMJCR_CORE_MULT(PMJCR_RATIO_M, args->cpu); reg |= PMJCR_CORE_MULT(args->mult, args->cpu); if (args->slow) reg &= ~(1 << (12 + args->cpu)); else reg |= (1 << (12 + args->cpu)); ccsr_write4(GUTS_PMJCR, reg); reg = ccsr_read4(GUTS_POWMGTCSR); reg |= POWMGTCSR_JOG | POWMGTCSR_INT_MASK; ccsr_write4(GUTS_POWMGTCSR, reg); /* Wait for completion */ do { DELAY(100); reg = ccsr_read4(GUTS_POWMGTCSR); } while (reg & POWMGTCSR_JOG); reg = ccsr_read4(GUTS_POWMGTCSR); ccsr_write4(GUTS_POWMGTCSR, reg & ~POWMGTCSR_INT_MASK); ccsr_read4(GUTS_POWMGTCSR); args->inprogress = 0; } else { while (args->inprogress) cpu_spinwait(); } }
/* * Called by a CPU to restart stopped CPUs. * * Usually (but not necessarily) called with 'stopped_cpus' as its arg. * * - Signals all CPUs in map to restart. * - Waits for each to restart. * * Returns: * -1: error * 0: NA * 1: ok */ static int generic_restart_cpus(cpuset_t map, u_int type) { #ifdef KTR char cpusetbuf[CPUSETBUFSIZ]; #endif volatile cpuset_t *cpus; KASSERT( #if defined(__amd64__) || defined(__i386__) type == IPI_STOP || type == IPI_STOP_HARD || type == IPI_SUSPEND, #else type == IPI_STOP || type == IPI_STOP_HARD, #endif ("%s: invalid stop type", __func__)); if (!smp_started) return 0; CTR1(KTR_SMP, "restart_cpus(%s)", cpusetobj_strprint(cpusetbuf, &map)); #if defined(__amd64__) || defined(__i386__) if (type == IPI_SUSPEND) cpus = &suspended_cpus; else #endif cpus = &stopped_cpus; /* signal other cpus to restart */ CPU_COPY_STORE_REL(&map, &started_cpus); /* wait for each to clear its bit */ while (CPU_OVERLAP(cpus, &map)) cpu_spinwait(); return 1; }
/* * Enqueue n items and maybe drain the ring for some time. * * Returns an errno. */ int mp_ring_enqueue(struct mp_ring *r, void **items, int n, int budget) { union ring_state os, ns; uint16_t pidx_start, pidx_stop; int i; MPASS(items != NULL); MPASS(n > 0); /* * Reserve room for the new items. Our reservation, if successful, is * from 'pidx_start' to 'pidx_stop'. */ for (;;) { os.state = r->state; if (n >= space_available(r, os)) { counter_u64_add(r->drops, n); MPASS(os.flags != IDLE); if (os.flags == STALLED) mp_ring_check_drainage(r, 0); return (ENOBUFS); } ns.state = os.state; ns.pidx_head = increment_idx(r, os.pidx_head, n); critical_enter(); if (atomic_cmpset_64(&r->state, os.state, ns.state)) break; critical_exit(); cpu_spinwait(); } pidx_start = os.pidx_head; pidx_stop = ns.pidx_head; /* * Wait for other producers who got in ahead of us to enqueue their * items, one producer at a time. It is our turn when the ring's * pidx_tail reaches the begining of our reservation (pidx_start). */ while (ns.pidx_tail != pidx_start) { cpu_spinwait(); ns.state = r->state; } /* Now it is our turn to fill up the area we reserved earlier. */ i = pidx_start; do { r->items[i] = *items++; if (__predict_false(++i == r->size)) i = 0; } while (i != pidx_stop); /* * Update the ring's pidx_tail. The release style atomic guarantees * that the items are visible to any thread that sees the updated pidx. */ do { os.state = ns.state = r->state; ns.pidx_tail = pidx_stop; ns.flags = BUSY; } while (atomic_cmpset_rel_64(&r->state, os.state, ns.state) == 0); critical_exit(); counter_u64_add(r->enqueues, n); /* * Turn into a consumer if some other thread isn't active as a consumer * already. */ if (os.flags != BUSY) drain_ring(r, ns, os.flags, budget); return (0); }
void _thread_lock_flags(struct thread *td, int opts, const char *file, int line) { struct mtx *m; uintptr_t tid; int i; #ifdef LOCK_PROFILING int contested = 0; uint64_t waittime = 0; #endif #ifdef KDTRACE_HOOKS uint64_t spin_cnt = 0; #endif i = 0; tid = (uintptr_t)curthread; for (;;) { retry: spinlock_enter(); m = td->td_lock; KASSERT(m->mtx_lock != MTX_DESTROYED, ("thread_lock() of destroyed mutex @ %s:%d", file, line)); KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin, ("thread_lock() of sleep mutex %s @ %s:%d", m->lock_object.lo_name, file, line)); if (mtx_owned(m)) KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0, ("thread_lock: recursed on non-recursive mutex %s @ %s:%d\n", m->lock_object.lo_name, file, line)); WITNESS_CHECKORDER(&m->lock_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE, file, line, NULL); while (!_obtain_lock(m, tid)) { #ifdef KDTRACE_HOOKS spin_cnt++; #endif if (m->mtx_lock == tid) { m->mtx_recurse++; break; } lock_profile_obtain_lock_failed(&m->lock_object, &contested, &waittime); /* Give interrupts a chance while we spin. */ spinlock_exit(); while (m->mtx_lock != MTX_UNOWNED) { if (i++ < 10000000) cpu_spinwait(); else if (i < 60000000 || kdb_active || panicstr != NULL) DELAY(1); else _mtx_lock_spin_failed(m); cpu_spinwait(); if (m != td->td_lock) goto retry; } spinlock_enter(); } if (m == td->td_lock) break; _rel_spin_lock(m); /* does spinlock_exit() */ #ifdef KDTRACE_HOOKS spin_cnt++; #endif } if (m->mtx_recurse == 0) LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_MTX_SPIN_LOCK_ACQUIRE, m, contested, waittime, (file), (line)); LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file, line); WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line); LOCKSTAT_RECORD1(LS_THREAD_LOCK_SPIN, m, spin_cnt); }