void cpu_thread_exit(void) { curthread->td_switch = cpu_exit_switch; lwkt_switch(); panic("cpu_exit"); }
/* * Terminate the current thread. The caller must have already acquired * the thread's rwlock and placed it on a reap list or otherwise notified * a reaper of its existance. We set a special assembly switch function which * releases td_rwlock after it has cleaned up the MMU state and switched * out the stack. * * Must be caller from a critical section and with the thread descheduled. */ void cpu_thread_exit(void) { curthread->td_switch = cpu_exit_switch; curthread->td_flags |= TDF_EXITING; lwkt_switch(); panic("cpu_thread_exit: lwkt_switch() unexpectedly returned"); }
/* * Use this to lwkt_switch() when the scheduler clock is not * yet running, otherwise lwkt_switch() won't do anything. * XXX needs cleaning up in lwkt_thread.c */ static void lwkt_force_switch(void) { crit_enter(); lwkt_schedulerclock(curthread); crit_exit(); lwkt_switch(); }
/* * Similar to gettoken but we acquire a shared token instead of an exclusive * token. */ void lwkt_gettoken_shared(lwkt_token_t tok) { thread_t td = curthread; lwkt_tokref_t ref; ref = td->td_toks_stop; KKASSERT(ref < &td->td_toks_end); ++td->td_toks_stop; cpu_ccfence(); _lwkt_tokref_init(ref, tok, td, TOK_EXCLREQ); #ifdef DEBUG_LOCKS /* * Taking a pool token in shared mode is a bad idea; other * addresses deeper in the call stack may hash to the same pool * token and you may end up with an exclusive-shared livelock. * Warn in this condition. */ if ((tok >= &pool_tokens[0].token) && (tok < &pool_tokens[LWKT_NUM_POOL_TOKENS].token)) kprintf("Warning! Taking pool token %p in shared mode\n", tok); #endif if (_lwkt_trytokref_spin(ref, td, TOK_EXCLREQ)) return; /* * Give up running if we can't acquire the token right now. * * Since the tokref is already active the scheduler now * takes care of acquisition, so we need only call * lwkt_switch(). * * Since we failed this was not a recursive token so upon * return tr_tok->t_ref should be assigned to this specific * ref. */ td->td_wmesg = tok->t_desc; ++tok->t_collisions; logtoken(fail, ref); td->td_toks_have = td->td_toks_stop - 1; if (tokens_debug_output > 0) { --tokens_debug_output; spin_lock(&tok_debug_spin); kprintf("Shar Token thread %p %s %s\n", td, tok->t_desc, td->td_comm); print_backtrace(6); kprintf("\n"); spin_unlock(&tok_debug_spin); } lwkt_switch(); logtoken(succ, ref); }
void cpu_idle(void) { struct thread *td = curthread; struct mdglobaldata *gd = mdcpu; int reqflags; crit_exit(); KKASSERT(td->td_critcount == 0); cpu_enable_intr(); for (;;) { /* * See if there are any LWKTs ready to go. */ lwkt_switch(); /* * The idle loop halts only if no threads are scheduleable * and no signals have occured. */ if (cpu_idle_hlt && (td->td_gd->gd_reqflags & RQF_IDLECHECK_WK_MASK) == 0) { splz(); #ifdef SMP KKASSERT(MP_LOCK_HELD() == 0); #endif if ((td->td_gd->gd_reqflags & RQF_IDLECHECK_WK_MASK) == 0) { #ifdef DEBUGIDLE struct timeval tv1, tv2; gettimeofday(&tv1, NULL); #endif reqflags = gd->mi.gd_reqflags & ~RQF_IDLECHECK_WK_MASK; umtx_sleep(&gd->mi.gd_reqflags, reqflags, 1000000); #ifdef DEBUGIDLE gettimeofday(&tv2, NULL); if (tv2.tv_usec - tv1.tv_usec + (tv2.tv_sec - tv1.tv_sec) * 1000000 > 500000) { kprintf("cpu %d idlelock %08x %08x\n", gd->mi.gd_cpuid, gd->mi.gd_reqflags, gd->gd_fpending); } #endif } ++cpu_idle_hltcnt; } else { splz(); #ifdef SMP __asm __volatile("pause"); #endif ++cpu_idle_spincnt; } } }
/* * Terminate the current thread. The caller must have already acquired * the thread's rwlock and placed it on a reap list or otherwise notified * a reaper of its existance. We set a special assembly switch function which * releases td_rwlock after it has cleaned up the MMU state and switched * out the stack. * * Must be caller from a critical section and with the thread descheduled. */ void cpu_thread_exit(void) { #if NNPX > 0 npxexit(); #endif curthread->td_switch = cpu_exit_switch; curthread->td_flags |= TDF_EXITING; lwkt_switch(); panic("cpu_exit"); }
/* * Get a serializing token. This routine can block. */ void lwkt_gettoken(lwkt_token_t tok) { thread_t td = curthread; lwkt_tokref_t ref; ref = td->td_toks_stop; KKASSERT(ref < &td->td_toks_end); ++td->td_toks_stop; cpu_ccfence(); _lwkt_tokref_init(ref, tok, td, TOK_EXCLUSIVE|TOK_EXCLREQ); #ifdef DEBUG_LOCKS /* * Taking an exclusive token after holding it shared will * livelock. Scan for that case and assert. */ lwkt_tokref_t tk; int found = 0; for (tk = &td->td_toks_base; tk < ref; tk++) { if (tk->tr_tok != tok) continue; found++; if (tk->tr_count & TOK_EXCLUSIVE) goto good; } /* We found only shared instances of this token if found >0 here */ KASSERT((found == 0), ("Token %p s/x livelock", tok)); good: #endif if (_lwkt_trytokref_spin(ref, td, TOK_EXCLUSIVE|TOK_EXCLREQ)) return; /* * Give up running if we can't acquire the token right now. * * Since the tokref is already active the scheduler now * takes care of acquisition, so we need only call * lwkt_switch(). * * Since we failed this was not a recursive token so upon * return tr_tok->t_ref should be assigned to this specific * ref. */ td->td_wmesg = tok->t_desc; ++tok->t_collisions; logtoken(fail, ref); td->td_toks_have = td->td_toks_stop - 1; lwkt_switch(); logtoken(succ, ref); KKASSERT(tok->t_ref == ref); }
static void lwkt_idleloop(void *dummy) { globaldata_t gd = mycpu; DBPRINTF(("idlestart cpu %d pri %d (should be < 32) mpcount %d (should be 0)\n", gd->gd_cpuid, curthread->td_pri, curthread->td_mpcount)); gd->gd_pid = getpid(); for (;;) { /* * If only our 'main' thread is left, schedule it. */ if (gd->gd_num_threads == gd->gd_sys_threads) { int i; globaldata_t tgd; for (i = 0; i < ncpus; ++i) { tgd = globaldata_find(i); if (tgd->gd_num_threads != tgd->gd_sys_threads) break; } if (i == ncpus && (main_td.td_flags & TDF_RUNQ) == 0) lwkt_schedule(&main_td); } /* * Wait for an interrupt, aka wait for a signal or an upcall to * occur, then switch away. */ crit_enter(); if (gd->gd_runqmask || (curthread->td_flags & TDF_IDLE_NOHLT)) { curthread->td_flags &= ~TDF_IDLE_NOHLT; } else { printf("cpu %d halting\n", gd->gd_cpuid); cpu_halt(); printf("cpu %d resuming\n", gd->gd_cpuid); } crit_exit(); lwkt_switch(); } }
int main(int ac, char **av) { thread_t td; thread_t td1; thread_t td2; uthread_init(); td = curthread; printf("mainthread %p crit_count %d nest %d mp_lock %d\n", td, td->td_pri / TDPRI_CRIT, td->td_mpcount, mp_lock ); lwkt_create(thread1, NULL, &td1, NULL, 0, -1, "thread1"); lwkt_create(thread2, NULL, &td2, NULL, 0, -1, "thread2"); printf("thread #1 %p #2 %p\n", td1, td2); printf("switching away from main (should come back before exit)\n"); lwkt_switch(); printf("Switched back to main, main Exiting\n"); exit(1); }
/* * Panic is called on unresolvable fatal errors. It prints "panic: mesg", * and then reboots. If we are called twice, then we avoid trying to sync * the disks as this often leads to recursive panics. */ void panic(const char *fmt, ...) { int bootopt, newpanic; globaldata_t gd = mycpu; thread_t td = gd->gd_curthread; __va_list ap; static char buf[256]; #ifdef SMP /* * If a panic occurs on multiple cpus before the first is able to * halt the other cpus, only one cpu is allowed to take the panic. * Attempt to be verbose about this situation but if the kprintf() * itself panics don't let us overrun the kernel stack. * * Be very nasty about descheduling our thread at the lowest * level possible in an attempt to freeze the thread without * inducing further panics. * * Bumping gd_trap_nesting_level will also bypass assertions in * lwkt_switch() and allow us to switch away even if we are a * FAST interrupt or IPI. * * The setting of panic_cpu_gd also determines how kprintf() * spin-locks itself. DDB can set panic_cpu_gd as well. */ for (;;) { globaldata_t xgd = panic_cpu_gd; /* * Someone else got the panic cpu */ if (xgd && xgd != gd) { crit_enter(); ++mycpu->gd_trap_nesting_level; if (mycpu->gd_trap_nesting_level < 25) { kprintf("SECONDARY PANIC ON CPU %d THREAD %p\n", mycpu->gd_cpuid, td); } td->td_release = NULL; /* be a grinch */ for (;;) { lwkt_deschedule_self(td); lwkt_switch(); } /* NOT REACHED */ /* --mycpu->gd_trap_nesting_level */ /* crit_exit() */ } /* * Reentrant panic */ if (xgd && xgd == gd) break; /* * We got it */ if (atomic_cmpset_ptr(&panic_cpu_gd, NULL, gd)) break; } #else panic_cpu_gd = gd; #endif /* * Try to get the system into a working state. Save information * we are about to destroy. */ kvcreinitspin(); if (panicstr == NULL) { bcopy(td->td_toks_array, panic_tokens, sizeof(panic_tokens)); panic_tokens_count = td->td_toks_stop - &td->td_toks_base; } lwkt_relalltokens(td); td->td_toks_stop = &td->td_toks_base; /* * Setup */ bootopt = RB_AUTOBOOT | RB_DUMP; if (sync_on_panic == 0) bootopt |= RB_NOSYNC; newpanic = 0; if (panicstr) { bootopt |= RB_NOSYNC; } else { panicstr = fmt; newpanic = 1; } /* * Format the panic string. */ __va_start(ap, fmt); kvsnprintf(buf, sizeof(buf), fmt, ap); if (panicstr == fmt) panicstr = buf; __va_end(ap); kprintf("panic: %s\n", buf); #ifdef SMP /* two separate prints in case of an unmapped page and trap */ kprintf("cpuid = %d\n", mycpu->gd_cpuid); #endif #if (NGPIO > 0) && defined(ERROR_LED_ON_PANIC) led_switch("error", 1); #endif #if defined(WDOG_DISABLE_ON_PANIC) && defined(WATCHDOG_ENABLE) wdog_disable(); #endif /* * Enter the debugger or fall through & dump. Entering the * debugger will stop cpus. If not entering the debugger stop * cpus here. */ #if defined(DDB) if (newpanic && trace_on_panic) print_backtrace(-1); if (debugger_on_panic) Debugger("panic"); else #endif #ifdef SMP if (newpanic) stop_cpus(mycpu->gd_other_cpus); #else ; #endif boot(bootopt); }
/* * This procedure is the main loop of our per-cpu helper thread. The * sc->isrunning flag prevents us from racing hardclock_softtick() and * a critical section is sufficient to interlock sc->curticks and protect * us from remote IPI's / list removal. * * The thread starts with the MP lock released and not in a critical * section. The loop itself is MP safe while individual callbacks * may or may not be, so we obtain or release the MP lock as appropriate. */ static void softclock_handler(void *arg) { softclock_pcpu_t sc; struct callout *c; struct callout_tailq *bucket; void (*c_func)(void *); void *c_arg; int mpsafe = 1; /* * Run the callout thread at the same priority as other kernel * threads so it can be round-robined. */ /*lwkt_setpri_self(TDPRI_SOFT_NORM);*/ sc = arg; crit_enter(); loop: while (sc->softticks != (int)(sc->curticks + 1)) { bucket = &sc->callwheel[sc->softticks & callwheelmask]; for (c = TAILQ_FIRST(bucket); c; c = sc->next) { if (c->c_time != sc->softticks) { sc->next = TAILQ_NEXT(c, c_links.tqe); continue; } if (c->c_flags & CALLOUT_MPSAFE) { if (mpsafe == 0) { mpsafe = 1; rel_mplock(); } } else { /* * The request might be removed while we * are waiting to get the MP lock. If it * was removed sc->next will point to the * next valid request or NULL, loop up. */ if (mpsafe) { mpsafe = 0; sc->next = c; get_mplock(); if (c != sc->next) continue; } } sc->next = TAILQ_NEXT(c, c_links.tqe); TAILQ_REMOVE(bucket, c, c_links.tqe); sc->running = c; c_func = c->c_func; c_arg = c->c_arg; c->c_func = NULL; KKASSERT(c->c_flags & CALLOUT_DID_INIT); c->c_flags &= ~CALLOUT_PENDING; crit_exit(); c_func(c_arg); crit_enter(); sc->running = NULL; /* NOTE: list may have changed */ } ++sc->softticks; } sc->isrunning = 0; lwkt_deschedule_self(&sc->thread); /* == curthread */ lwkt_switch(); goto loop; /* NOT REACHED */ }
/* * This procedure is the main loop of our per-cpu helper thread. The * sc->isrunning flag prevents us from racing hardclock_softtick() and * a critical section is sufficient to interlock sc->curticks and protect * us from remote IPI's / list removal. * * The thread starts with the MP lock released and not in a critical * section. The loop itself is MP safe while individual callbacks * may or may not be, so we obtain or release the MP lock as appropriate. */ static void softclock_handler(void *arg) { softclock_pcpu_t sc; struct callout *c; struct callout_tailq *bucket; struct callout slotimer; int mpsafe = 1; int flags; /* * Setup pcpu slow clocks which we want to run from the callout * thread. */ callout_init_mp(&slotimer); callout_reset(&slotimer, hz * 10, slotimer_callback, &slotimer); /* * Run the callout thread at the same priority as other kernel * threads so it can be round-robined. */ /*lwkt_setpri_self(TDPRI_SOFT_NORM);*/ /* * Loop critical section against ipi operations to this cpu. */ sc = arg; crit_enter(); loop: while (sc->softticks != (int)(sc->curticks + 1)) { bucket = &sc->callwheel[sc->softticks & cwheelmask]; for (c = TAILQ_FIRST(bucket); c; c = sc->next) { if (c->c_time != sc->softticks) { sc->next = TAILQ_NEXT(c, c_links.tqe); continue; } flags = c->c_flags; if (flags & CALLOUT_MPSAFE) { if (mpsafe == 0) { mpsafe = 1; rel_mplock(); } } else { /* * The request might be removed while we * are waiting to get the MP lock. If it * was removed sc->next will point to the * next valid request or NULL, loop up. */ if (mpsafe) { mpsafe = 0; sc->next = c; get_mplock(); if (c != sc->next) continue; } } /* * Queue protection only exists while we hold the * critical section uninterrupted. * * Adjust sc->next when removing (c) from the queue, * note that an IPI on this cpu may make further * adjustments to sc->next. */ sc->next = TAILQ_NEXT(c, c_links.tqe); TAILQ_REMOVE(bucket, c, c_links.tqe); KASSERT((c->c_flags & CALLOUT_ARMED) && (c->c_flags & CALLOUT_PENDING) && CALLOUT_FLAGS_TO_CPU(c->c_flags) == mycpu->gd_cpuid, ("callout %p: bad flags %08x", c, c->c_flags)); /* * Once CALLOUT_PENDING is cleared, sc->running * protects the callout structure's existance but * only until we call c_func(). A callout_stop() * or callout_reset() issued from within c_func() * will not block. The callout can also be kfree()d * by c_func(). * * We set EXECUTED before calling c_func() so a * callout_stop() issued from within c_func() returns * the correct status. */ if ((flags & (CALLOUT_AUTOLOCK | CALLOUT_ACTIVE)) == (CALLOUT_AUTOLOCK | CALLOUT_ACTIVE)) { void (*c_func)(void *); void *c_arg; struct lock *c_lk; int error; /* * NOTE: sc->running must be set prior to * CALLOUT_PENDING being cleared to * avoid missed CANCELs and *_stop() * races. */ sc->running = (intptr_t)c; c_func = c->c_func; c_arg = c->c_arg; c_lk = c->c_lk; c->c_func = NULL; KKASSERT(c->c_flags & CALLOUT_DID_INIT); flags = callout_unpend_disarm(c); error = lockmgr(c_lk, LK_EXCLUSIVE | LK_CANCELABLE); if (error == 0) { atomic_set_int(&c->c_flags, CALLOUT_EXECUTED); crit_exit(); c_func(c_arg); crit_enter(); lockmgr(c_lk, LK_RELEASE); } } else if (flags & CALLOUT_ACTIVE) { void (*c_func)(void *); void *c_arg; sc->running = (intptr_t)c; c_func = c->c_func; c_arg = c->c_arg; c->c_func = NULL; KKASSERT(c->c_flags & CALLOUT_DID_INIT); flags = callout_unpend_disarm(c); atomic_set_int(&c->c_flags, CALLOUT_EXECUTED); crit_exit(); c_func(c_arg); crit_enter(); } else { flags = callout_unpend_disarm(c); } /* * Read and clear sc->running. If bit 0 was set, * a callout_stop() is likely blocked waiting for * the callback to complete. * * The sigclear above also cleared CALLOUT_WAITING * and returns the contents of flags prior to clearing * any bits. * * Interlock wakeup any _stop's waiting on us. Note * that once c_func() was called, the callout * structure (c) pointer may no longer be valid. It * can only be used for the wakeup. */ if ((atomic_readandclear_ptr(&sc->running) & 1) || (flags & CALLOUT_WAITING)) { wakeup(c); } /* NOTE: list may have changed */ } ++sc->softticks; } /* * Don't leave us holding the MP lock when we deschedule ourselves. */ if (mpsafe == 0) { mpsafe = 1; rel_mplock(); } sc->isrunning = 0; lwkt_deschedule_self(&sc->thread); /* == curthread */ lwkt_switch(); goto loop; /* NOT REACHED */ }
/* * Start threading. */ void lwkt_start_threading(thread_t td) { lwkt_switch(); }