void
init0(void)
{
int i;
char buf[2*KNAMELEN];
up->nerrlab = 0;
spllo();
/*
* These are o.k. because rootinit is null.
* Then early kproc's will have a root and dot.
*/
up->slash = namec("#/", Atodir, 0, 0);
pathclose(up->slash->path);
up->slash->path = newpath("/");
up->dot = cclone(up->slash);
chandevinit();
if(!waserror()){
snprint(buf, sizeof(buf), "%s %s", arch->id, conffile);
ksetenv("terminal", buf, 0);
ksetenv("cputype", "386", 0);
if(cpuserver)
ksetenv("service", "cpu", 0);
else
ksetenv("service", "terminal", 0);
for(i = 0; i < nconf; i++){
if(confname[i][0] != '*')
ksetenv(confname[i], confval[i], 0);
ksetenv(confname[i], confval[i], 1);
}
poperror();
}
kproc("alarm", alarmkproc, 0);
touser(sp);
}
/*
* pick a process to run.
* most of this is used in AMP sched.
* (on a quad core or less, we use SMP).
* In the case of core 0 we always return nil, but
* schedule the picked process at any other available TC.
* In the case of other cores we wait until a process is given
* by core 0.
*/
Proc*
runproc(void)
{
Schedq *rq;
Proc *p;
uint32_t start, now;
if(sys->nmach <= AMPmincores)
return smprunproc();
start = perfticks();
run.preempts++;
rq = nil;
if(machp()->machno != 0){
do{
spllo();
while(m->proc == nil)
idlehands();
now = perfticks();
m->perf.inidle += now-start;
start = now;
splhi();
p = m->proc;
}while(p == nil);
p->state = Scheding;
p->mp = sys->machptr[machp()->machno];
if(edflock(p)){
edfrun(p, rq == &run.runq[PriEdf]); /* start deadline timer and do admin */
edfunlock();
}
if(p->trace)
proctrace(p, SRun, 0);
return p;
}
mach0sched();
return nil; /* not reached */
}
void
init0(void)
{
Osenv *o;
char buf[2*KNAMELEN];
up->nerrlab = 0;
//print("Starting init0()\n");
spllo();
if(waserror())
panic("init0 %r");
/* These are o.k. because rootinit is null.
* Then early kproc's will have a root and dot. */
o = up->env;
o->pgrp->slash = namec("#/", Atodir, 0, 0);
cnameclose(o->pgrp->slash->name);
o->pgrp->slash->name = newcname("/");
o->pgrp->dot = cclone(o->pgrp->slash);
chandevinit();
if(!waserror()){
ksetenv("cputype", "arm", 0);
snprint(buf, sizeof(buf), "arm %s", conffile);
ksetenv("terminal", buf, 0);
snprint(buf, sizeof(buf), "%s", getethermac());
ksetenv("ethermac", buf, 0);
poperror();
}
poperror();
disinit("/osinit.dis");
}
static void
shutdown(int ispanic)
{
Mach *m = machp();
int ms, once;
lock(&active);
if(ispanic)
active.ispanic = ispanic;
else if(m->machno == 0 && m->online == 0)
active.ispanic = 0;
once = m->online;
m->online = 0;
adec(&active.nonline);
active.exiting = 1;
unlock(&active);
if(once)
iprint("cpu%d: exiting\n", m->machno);
spllo();
for(ms = 5*1000; ms > 0; ms -= TK2MS(2)){
delay(TK2MS(2));
if(active.nonline == 0)
break;
}
if(active.ispanic && m->machno == 0){
if(cpuserver)
delay(30000);
else
for(;;)
halt();
}
else
delay(1000);
}
void
init0(void)
{
char buf[2*KNAMELEN];
up->nerrlab = 0;
// if(consuart == nil)
// i8250console("0");
spllo();
/*
* These are o.k. because rootinit is null.
* Then early kproc's will have a root and dot.
*/
up->slash = namec("#/", Atodir, 0, 0);
pathclose(up->slash->path);
up->slash->path = newpath("/");
up->dot = cclone(up->slash);
devtabinit();
if(!waserror()){
snprint(buf, sizeof(buf), "%s %s", "AMD64", conffile);
ksetenv("terminal", buf, 0);
ksetenv("cputype", "amd64", 0);
if(cpuserver)
ksetenv("service", "cpu", 0);
else
ksetenv("service", "terminal", 0);
ksetenv("pgsz", "2097152", 0);
confsetenv();
poperror();
}
kproc("alarm", alarmkproc, 0);
touser(sp);
}
static void
shutdown(int ispanic)
{
int ms, once;
lock(&active);
if(ispanic)
active.ispanic = ispanic;
else if(m->machno == 0 && (active.machs & (1<<m->machno)) == 0)
active.ispanic = 0;
once = active.machs & (1<<m->machno);
/*
* setting exiting will make hzclock() on each processor call exit(0),
* which calls shutdown(0) and idles non-bootstrap cpus and returns
* on bootstrap processors (to permit a reboot). clearing our bit
* in machs avoids calling exit(0) from hzclock() on this processor.
*/
active.machs &= ~(1<<m->machno);
active.exiting = 1;
unlock(&active);
if(once) {
delay(m->machno*1000); /* stagger them */
iprint("cpu%d: exiting\n", m->machno);
}
spllo();
if (m->machno == 0)
ms = 5*1000;
else
ms = 2*1000;
for(; ms > 0; ms -= TK2MS(2)){
delay(TK2MS(2));
if(active.machs == 0 && consactive() == 0)
break;
}
delay(500);
}
/*
* if the controller or a specific drive is in a confused state,
* reset it and get back to a kown state
*/
static void
floppyrevive(void)
{
FDrive *dp;
/*
* reset the controller if it's confused
*/
if(fl.confused) {
DPRINT("floppyrevive in\n");
fldump();
/* reset controller and turn all motors off */
splhi();
fl.ncmd = 1;
fl.cmd[0] = 0;
outb(Pdor, 0);
delay(10);
outb(Pdor, Fintena|Fena);
delay(10);
spllo();
fl.motor = 0;
fl.confused = 0;
floppywait(0);
/* mark all drives in an unknown state */
for(dp = fl.d; dp < &fl.d[fl.ndrive]; dp++)
dp->confused = 1;
/* set rate to a known value */
outb(Pdsr, 0);
fl.rate = 0;
DPRINT("floppyrevive out\n");
fldump();
}
}
int
fault(uintptr addr, int read)
{
Segment *s;
char *sps;
if(up == nil)
panic("fault: nil up");
if(up->nlocks.ref)
print("fault: addr %#p: nlocks %ld\n", addr, up->nlocks.ref);
sps = up->psstate;
up->psstate = "Fault";
spllo();
m->pfault++;
for(;;) {
s = seg(up, addr, 1); /* leaves s->lk qlocked if seg != nil */
if(s == 0) {
up->psstate = sps;
return -1;
}
if(!read && (s->type&SG_RONLY)) {
qunlock(&s->lk);
up->psstate = sps;
return -1;
}
if(fixfault(s, addr, read, 1) == 0) /* qunlocks s->lk */
break;
}
up->psstate = sps;
return 0;
}
void
shutdown(int ispanic)
{
int ms, once;
lock(&active);
if(ispanic)
active.ispanic = ispanic;
else if(m->machno == 0 && (active.machs & (1<<m->machno)) == 0)
active.ispanic = 0;
once = active.machs & (1<<m->machno);
active.machs &= ~(1<<m->machno);
active.exiting = 1;
unlock(&active);
if(once)
print("cpu%d: exiting\n", m->machno);
spllo();
for(ms = 5*1000; ms > 0; ms -= TK2MS(2)){
delay(TK2MS(2));
if(active.machs == 0 && consactive() == 0)
break;
}
#ifdef notdef
if(active.ispanic && m->machno == 0){
if(cpuserver)
delay(30000);
else
for(;;)
halt();
}
else
#endif /* notdef */
delay(1000);
}
/*
* sleep if a condition is not true. Another process will
* awaken us after it sets the condition. When we awaken
* the condition may no longer be true.
*
* we lock both the process and the rendezvous to keep r->p
* and p->r synchronized.
*/
void
sleep(Rendez *r, int (*f)(void*), void *arg)
{
int s;
void (*pt)(Proc*, int, vlong);
s = splhi();
if(up->nlocks)
print("process %lud sleeps with %d locks held, last lock %#p locked at pc %#p, sleep called from %#p\n",
up->pid, up->nlocks, up->lastlock, up->lastlock->pc, getcallerpc(&r));
lock(r);
lock(&up->rlock);
if(r->p != nil){
print("double sleep called from %#p, %lud %lud\n", getcallerpc(&r), r->p->pid, up->pid);
dumpstack();
}
/*
* Wakeup only knows there may be something to do by testing
* r->p in order to get something to lock on.
* Flush that information out to memory in case the sleep is
* committed.
*/
r->p = up;
if((*f)(arg) || up->notepending){
/*
* if condition happened or a note is pending
* never mind
*/
r->p = nil;
unlock(&up->rlock);
unlock(r);
} else {
/*
* now we are committed to
* change state and call scheduler
*/
pt = proctrace;
if(pt != nil)
pt(up, SSleep, 0);
up->state = Wakeme;
up->r = r;
/* statistics */
m->cs++;
procsave(up);
if(setlabel(&up->sched)) {
/*
* here when the process is awakened
*/
procrestore(up);
spllo();
} else {
/*
* here to go to sleep (i.e. stop Running)
*/
unlock(&up->rlock);
unlock(r);
gotolabel(&m->sched);
}
}
if(up->notepending) {
up->notepending = 0;
splx(s);
interrupted();
}
splx(s);
}
/*
* pick a process to run
*/
Proc*
runproc(void)
{
Schedq *rq;
Proc *p;
ulong start, now;
int i;
void (*pt)(Proc*, int, vlong);
start = perfticks();
/* cooperative scheduling until the clock ticks */
if((p = m->readied) != nil && p->mach == nil && p->state == Ready
&& (p->wired == nil || p->wired == MACHP(m->machno))
&& runq[Nrq-1].head == nil && runq[Nrq-2].head == nil){
skipscheds++;
rq = &runq[p->priority];
goto found;
}
preempts++;
loop:
/*
* find a process that last ran on this processor (affinity),
* or one that hasn't moved in a while (load balancing). Every
* time around the loop affinity goes down.
*/
spllo();
for(i = 0;; i++){
/*
* find the highest priority target process that this
* processor can run given affinity constraints.
*
*/
for(rq = &runq[Nrq-1]; rq >= runq; rq--){
for(p = rq->head; p != nil; p = p->rnext){
if(p->mp == nil || p->mp == MACHP(m->machno)
|| (p->wired == nil && i > 0))
goto found;
}
}
/* waste time or halt the CPU */
idlehands();
/* remember how much time we're here */
now = perfticks();
m->perf.inidle += now-start;
start = now;
}
found:
splhi();
p = dequeueproc(rq, p);
if(p == nil)
goto loop;
p->state = Scheding;
p->mp = MACHP(m->machno);
if(edflock(p)){
edfrun(p, rq == &runq[PriEdf]); /* start deadline timer and do admin */
edfunlock();
}
pt = proctrace;
if(pt != nil)
pt(p, SRun, 0);
return p;
}
/*
* If changing this routine, look also at sleep(). It
* contains a copy of the guts of sched().
*/
void
sched(void)
{
Proc *p;
if(m->ilockdepth)
panic("cpu%d: ilockdepth %d, last lock %#p at %#p, sched called from %#p",
m->machno,
m->ilockdepth,
up != nil ? up->lastilock: nil,
(up != nil && up->lastilock != nil) ? up->lastilock->pc: 0,
getcallerpc(&p+2));
if(up != nil) {
/*
* Delay the sched until the process gives up the locks
* it is holding. This avoids dumb lock loops.
* Don't delay if the process is Moribund.
* It called sched to die.
* But do sched eventually. This avoids a missing unlock
* from hanging the entire kernel.
* But don't reschedule procs holding palloc or procalloc.
* Those are far too important to be holding while asleep.
*
* This test is not exact. There can still be a few instructions
* in the middle of taslock when a process holds a lock
* but Lock.p has not yet been initialized.
*/
if(up->nlocks)
if(up->state != Moribund)
if(up->delaysched < 20
|| palloc.Lock.p == up
|| procalloc.Lock.p == up){
up->delaysched++;
delayedscheds++;
return;
}
up->delaysched = 0;
splhi();
/* statistics */
m->cs++;
procsave(up);
if(setlabel(&up->sched)){
procrestore(up);
spllo();
return;
}
gotolabel(&m->sched);
}
p = runproc();
if(p->edf == nil){
updatecpu(p);
p->priority = reprioritize(p);
}
if(p != m->readied)
m->schedticks = m->ticks + HZ/10;
m->readied = nil;
up = p;
up->state = Running;
up->mach = MACHP(m->machno);
m->proc = up;
mmuswitch(up);
gotolabel(&up->sched);
}
/*
* All traps come here. It is slower to have all traps call trap()
* rather than directly vectoring the handler. However, this avoids a
* lot of code duplication and possible bugs. The only exception is
* VectorSYSCALL.
* Trap is called with interrupts disabled via interrupt-gates.
*/
void
trap(Ureg* ureg)
{
int clockintr, vno, user;
// cache the previous vno to see what might be causing
// trouble
static int lastvno;
vno = ureg->type;
uint64_t gsbase = rdmsr(GSbase);
//if (sce > scx) iprint("====================");
if (vno == 8) {
iprint("Lstar is %p\n", (void *)rdmsr(Lstar));
iprint("GSbase is %p\n", (void *)gsbase);
iprint("ire %d irx %d sce %d scx %d lastvno %d\n",
ire, irx, sce, scx, lastvno);
iprint("irxe %d \n",
irxe);
die("8");
}
lastvno = vno;
if (gsbase < 1ULL<<63)
die("bogus gsbase");
Mach *m = machp();
char buf[ERRMAX];
Vctl *ctl, *v;
if (0 && m && m->externup && m->externup->pid == 6) {
//iprint("type %x\n", ureg->type);
if (ureg->type != 0x49)
die("6\n");
}
m->perf.intrts = perfticks();
user = userureg(ureg);
if(user && (m->nixtype == NIXTC)){
m->externup->dbgreg = ureg;
cycles(&m->externup->kentry);
}
clockintr = 0;
//_pmcupdate(m);
if(ctl = vctl[vno]){
if(ctl->isintr){
m->intr++;
if(vno >= VectorPIC && vno != VectorSYSCALL)
m->lastintr = ctl->irq;
}else
if(m->externup)
m->externup->nqtrap++;
if(ctl->isr)
ctl->isr(vno);
for(v = ctl; v != nil; v = v->next){
if(v->f)
v->f(ureg, v->a);
}
if(ctl->eoi)
ctl->eoi(vno);
intrtime(vno);
if(ctl->isintr){
if(ctl->irq == IrqCLOCK || ctl->irq == IrqTIMER)
clockintr = 1;
if(m->externup && !clockintr)
preempted();
}
}
else if(vno < nelem(excname) && user){
spllo();
snprint(buf, sizeof buf, "sys: trap: %s", excname[vno]);
postnote(m->externup, 1, buf, NDebug);
}
else if(vno >= VectorPIC && vno != VectorSYSCALL){
/*
* An unknown interrupt.
* Check for a default IRQ7. This can happen when
* the IRQ input goes away before the acknowledge.
* In this case, a 'default IRQ7' is generated, but
* the corresponding bit in the ISR isn't set.
* In fact, just ignore all such interrupts.
*/
/* clear the interrupt */
i8259isr(vno);
iprint("cpu%d: spurious interrupt %d, last %d\n",
m->machno, vno, m->lastintr);
intrtime(vno);
if(user)
kexit(ureg);
return;
}
else{
if(vno == VectorNMI){
nmienable();
if(m->machno != 0){
iprint("cpu%d: PC %#llux\n",
m->machno, ureg->ip);
for(;;);
}
}
dumpregs(ureg);
if(!user){
ureg->sp = PTR2UINT(&ureg->sp);
dumpstackwithureg(ureg);
}
if(vno < nelem(excname))
panic("%s", excname[vno]);
panic("unknown trap/intr: %d\n", vno);
}
splhi();
/* delaysched set because we held a lock or because our quantum ended */
if(m->externup && m->externup->delaysched && clockintr){
if(0)
if(user && m->externup->ac == nil && m->externup->nqtrap == 0 && m->externup->nqsyscall == 0){
if(!waserror()){
m->externup->ac = getac(m->externup, -1);
poperror();
runacore();
return;
}
}
sched();
splhi();
}
if(user){
if(m->externup && m->externup->procctl || m->externup->nnote)
notify(ureg);
kexit(ureg);
}
}
static inline void
enable_ints_and_unlock(void)
{
thread_call_unlock();
(void)spllo();
}
void
trap(Ureg *ureg)
{
int rem, itype, t;
if(up != nil)
rem = ((char*)ureg)-up->kstack;
else rem = ((char*)ureg)-(char*)m->stack;
if(ureg->type != PsrMfiq && rem < 256) {
dumpregs(ureg);
panic("trap %d stack bytes remaining (%s), "
"up=#%8.8lux ureg=#%8.8lux pc=#%8.8ux"
,rem, up?up->text:"", up, ureg, ureg->pc);
for(;;);
}
itype = ureg->type;
/* All interrupts/exceptions should be resumed at ureg->pc-4,
except for Data Abort which resumes at ureg->pc-8. */
if(itype == PsrMabt+1)
ureg->pc -= 8;
else ureg->pc -= 4;
if(up){
up->pc = ureg->pc;
up->dbgreg = ureg;
}
switch(itype) {
case PsrMirq:
t = m->ticks; /* CPU time per proc */
up = nil; /* no process at interrupt level */
irq(ureg);
up = m->proc;
preemption(m->ticks - t);
m->intr++;
break;
case PsrMund:
if(*(ulong*)ureg->pc == BREAK && breakhandler) {
int s;
Proc *p;
p = up;
s = breakhandler(ureg, p);
if(s == BrkSched) {
p->preempted = 0;
sched();
} else if(s == BrkNoSched) {
/* stop it being preempted until next instruction */
p->preempted = 1;
if(up)
up->dbgreg = 0;
return;
}
break;
}
if(up == nil) goto faultpanic;
spllo();
if(waserror()) {
if(waslo(ureg->psr) && up->type == Interp)
disfault(ureg, up->env->errstr);
setpanic();
dumpregs(ureg);
panic("%s", up->env->errstr);
}
if(!fpiarm(ureg)) {
dumpregs(ureg);
sys_trap_error(ureg->type);
}
poperror();
break;
case PsrMsvc: /* Jump through 0 or SWI */
if(waslo(ureg->psr) && up && up->type == Interp) {
spllo();
dumpregs(ureg);
sys_trap_error(ureg->type);
}
setpanic();
dumpregs(ureg);
panic("SVC/SWI exception");
break;
case PsrMabt: /* Prefetch abort */
if(catchdbg && catchdbg(ureg, 0))
break;
/* FALL THROUGH */
case PsrMabt+1: /* Data abort */
if(waslo(ureg->psr) && up && up->type == Interp) {
spllo();
faultarm(ureg);
}
print("Data Abort\n");
/* FALL THROUGH */
default:
faultpanic:
setpanic();
dumpregs(ureg);
panic("exception %uX %s\n", ureg->type, trapname(ureg->type));
break;
}
splhi();
if(up)
up->dbgreg = 0; /* becomes invalid after return from trap */
}
Proc*
runproc(void)
{
Proc *p, *l;
Schedq *rq, *erq;
erq = runq + Nrq - 1;
loop:
splhi();
for(rq = runq; rq->head == 0; rq++)
if(rq >= erq) {
idlehands();
spllo();
goto loop;
}
if(!canlock(runq))
goto loop;
/* choose first one we last ran on this processor at this level or hasn't moved recently */
l = nil;
for(p = rq->head; p != nil; p = p->rnext)
if(p->mp == nil || p->mp == MACHP(m->machno) || p->movetime < MACHP(0)->ticks)
break;
if(p == nil)
p = rq->head;
/* p->mach==0 only when process state is saved */
if(p == 0 || p->mach) {
unlock(runq);
goto loop;
}
if(p->rnext == nil)
rq->tail = l;
if(l)
l->rnext = p->rnext;
else
rq->head = p->rnext;
if(rq->head == nil){
rq->tail = nil;
occupied &= ~(1<<p->pri);
}
nrdy--;
if(p->dbgstop){
p->state = Stopped;
unlock(runq);
goto loop;
}
if(p->state != Ready)
print("runproc %s %lud %s\n", p->text, p->pid, statename[p->state]);
unlock(runq);
p->state = Scheding;
if(p->mp != MACHP(m->machno))
p->movetime = MACHP(0)->ticks + HZ/10;
p->mp = MACHP(m->machno);
/*
if(edflock(p)){
edfrun(p, rq == &runq[PriEdf]); // start deadline timer and do admin
edfunlock();
}
*/
return p;
}
/*
* thread_terminate_daemon:
*
* Perform final clean up for terminating threads.
*/
static void
thread_terminate_daemon(void)
{
thread_t self, thread;
task_t task;
self = current_thread();
self->options |= TH_OPT_SYSTEM_CRITICAL;
(void)splsched();
simple_lock(&thread_terminate_lock);
while ((thread = (thread_t)dequeue_head(&thread_terminate_queue)) != THREAD_NULL) {
simple_unlock(&thread_terminate_lock);
(void)spllo();
task = thread->task;
task_lock(task);
task->total_user_time += timer_grab(&thread->user_timer);
if (thread->precise_user_kernel_time) {
task->total_system_time += timer_grab(&thread->system_timer);
} else {
task->total_user_time += timer_grab(&thread->system_timer);
}
task->c_switch += thread->c_switch;
task->p_switch += thread->p_switch;
task->ps_switch += thread->ps_switch;
task->syscalls_unix += thread->syscalls_unix;
task->syscalls_mach += thread->syscalls_mach;
queue_remove(&task->threads, thread, thread_t, task_threads);
task->thread_count--;
/*
* If the task is being halted, and there is only one thread
* left in the task after this one, then wakeup that thread.
*/
if (task->thread_count == 1 && task->halting)
thread_wakeup((event_t)&task->halting);
task_unlock(task);
lck_mtx_lock(&tasks_threads_lock);
queue_remove(&threads, thread, thread_t, threads);
threads_count--;
lck_mtx_unlock(&tasks_threads_lock);
thread_deallocate(thread);
(void)splsched();
simple_lock(&thread_terminate_lock);
}
assert_wait((event_t)&thread_terminate_queue, THREAD_UNINT);
simple_unlock(&thread_terminate_lock);
/* splsched */
self->options &= ~TH_OPT_SYSTEM_CRITICAL;
thread_block((thread_continue_t)thread_terminate_daemon);
/*NOTREACHED*/
}
/*
* Now running in a thread. Kick off other services,
* invoke user bootstrap, enter pageout loop.
*/
static void
kernel_bootstrap_thread(void)
{
processor_t processor = current_processor();
#define kernel_bootstrap_thread_kprintf(x...) /* kprintf("kernel_bootstrap_thread: " x) */
kernel_bootstrap_thread_kprintf("calling idle_thread_create\n");
/*
* Create the idle processor thread.
*/
idle_thread_create(processor);
/*
* N.B. Do not stick anything else
* before this point.
*
* Start up the scheduler services.
*/
kernel_bootstrap_thread_kprintf("calling sched_startup\n");
sched_startup();
/*
* Thread lifecycle maintenance (teardown, stack allocation)
*/
kernel_bootstrap_thread_kprintf("calling thread_daemon_init\n");
thread_daemon_init();
/*
* Thread callout service.
*/
kernel_bootstrap_thread_kprintf("calling thread_call_initialize\n");
thread_call_initialize();
/*
* Remain on current processor as
* additional processors come online.
*/
kernel_bootstrap_thread_kprintf("calling thread_bind\n");
thread_bind(processor);
/*
* Kick off memory mapping adjustments.
*/
kernel_bootstrap_thread_kprintf("calling mapping_adjust\n");
mapping_adjust();
/*
* Create the clock service.
*/
kernel_bootstrap_thread_kprintf("calling clock_service_create\n");
clock_service_create();
/*
* Create the device service.
*/
device_service_create();
kth_started = 1;
#if (defined(__i386__) || defined(__x86_64__)) && NCOPY_WINDOWS > 0
/*
* Create and initialize the physical copy window for processor 0
* This is required before starting kicking off IOKit.
*/
cpu_physwindow_init(0);
#endif
vm_kernel_reserved_entry_init();
#if MACH_KDP
kernel_bootstrap_kprintf("calling kdp_init\n");
kdp_init();
#endif
#if CONFIG_COUNTERS
pmc_bootstrap();
#endif
#if (defined(__i386__) || defined(__x86_64__))
if (turn_on_log_leaks && !new_nkdbufs)
new_nkdbufs = 200000;
start_kern_tracing(new_nkdbufs);
if (turn_on_log_leaks)
log_leaks = 1;
#endif
#ifdef IOKIT
PE_init_iokit();
#endif
(void) spllo(); /* Allow interruptions */
#if (defined(__i386__) || defined(__x86_64__)) && NCOPY_WINDOWS > 0
/*
* Create and initialize the copy window for processor 0
* This also allocates window space for all other processors.
* However, this is dependent on the number of processors - so this call
* must be after IOKit has been started because IOKit performs processor
* discovery.
*/
cpu_userwindow_init(0);
#endif
#if (!defined(__i386__) && !defined(__x86_64__))
if (turn_on_log_leaks && !new_nkdbufs)
new_nkdbufs = 200000;
start_kern_tracing(new_nkdbufs);
if (turn_on_log_leaks)
log_leaks = 1;
#endif
/*
* Initialize the shared region module.
*/
vm_shared_region_init();
vm_commpage_init();
vm_commpage_text_init();
#if CONFIG_MACF
mac_policy_initmach();
#endif
/*
* Initialize the global used for permuting kernel
* addresses that may be exported to userland as tokens
* using VM_KERNEL_ADDRPERM(). Force the random number
* to be odd to avoid mapping a non-zero
* word-aligned address to zero via addition.
*/
vm_kernel_addrperm = (vm_offset_t)early_random() | 1;
/*
* Start the user bootstrap.
*/
#ifdef MACH_BSD
bsd_init();
#endif
/*
* Get rid of segments used to bootstrap kext loading. This removes
* the KLD, PRELINK symtab, LINKEDIT, and symtab segments/load commands.
*/
#if 0
OSKextRemoveKextBootstrap();
#endif
serial_keyboard_init(); /* Start serial keyboard if wanted */
vm_page_init_local_q();
thread_bind(PROCESSOR_NULL);
/*
* Become the pageout daemon.
*/
vm_pageout();
/*NOTREACHED*/
}
/*
* If changing this routine, look also at sleep(). It
* contains a copy of the guts of sched().
*/
void
sched(void)
{
Mach *m = machp();
Proc *p;
if(m->ilockdepth)
panic("cpu%d: ilockdepth %d, last lock %#p at %#p, sched called from %#p",
m->machno,
m->ilockdepth,
m->externup? m->externup->lastilock: nil,
(m->externup && m->externup->lastilock)? m->externup->lastilock->_pc: 0,
getcallerpc(&p+2));
kstackok();
if(m->externup){
/*
* Delay the sched until the process gives up the locks
* it is holding. This avoids dumb lock loops.
* Don't delay if the process is Moribund.
* It called sched to die.
* But do sched eventually. This avoids a missing unlock
* from hanging the entire kernel.
* But don't reschedule procs holding palloc or procalloc.
* Those are far too important to be holding while asleep.
*
* This test is not exact. There can still be a few
* instructions in the middle of taslock when a process
* holds a lock but Lock.p has not yet been initialized.
*/
if(m->externup->nlocks)
if(m->externup->state != Moribund)
if(m->externup->delaysched < 20
|| pga.Lock.p == m->externup
|| procalloc.Lock.p == m->externup){
m->externup->delaysched++;
run.delayedscheds++;
return;
}
m->externup->delaysched = 0;
splhi();
/* statistics */
if(m->externup->nqtrap == 0 && m->externup->nqsyscall == 0)
m->externup->nfullq++;
m->cs++;
procsave(m->externup);
mmuflushtlb(m->pml4->pa);
if(setlabel(&m->externup->sched)){
procrestore(m->externup);
spllo();
return;
}
/*debug*/gotolabel(&m->sched);
}
m->inidle = 1;
p = runproc(); /* core 0 never returns */
m->inidle = 0;
if(!p->edf){
updatecpu(p);
p->priority = reprioritize(p);
}
if(nosmp){
if(p != m->readied)
m->schedticks = m->ticks + HZ/10;
m->readied = 0;
}
m->externup = p;
m->qstart = m->ticks;
m->externup->nqtrap = 0;
m->externup->nqsyscall = 0;
m->externup->state = Running;
//m->externup->mach = m;
m->externup->mach = sys->machptr[m->machno];
m->proc = m->externup;
// iprint("m->externup->sched.sp %p * %p\n", up->sched.sp,
// *(void **) m->externup->sched.sp);
mmuswitch(m->externup);
assert(!m->externup->wired || m->externup->wired == m);
if (0) hi("gotolabel\n");
/*debug*/gotolabel(&m->externup->sched);
}
/*
* sleep if a condition is not true. Another process will
* awaken us after it sets the condition. When we awaken
* the condition may no longer be true.
*
* we lock both the process and the rendezvous to keep r->p
* and p->r synchronized.
*/
void
sleep(Rendez *r, int (*f)(void*), void *arg)
{
Mach *m = machp();
Mpl pl;
pl = splhi();
if(m->externup->nlocks)
print("process %d sleeps with %d locks held, last lock %#p locked at pc %#p, sleep called from %#p\n",
m->externup->pid, m->externup->nlocks, m->externup->lastlock, m->externup->lastlock->_pc, getcallerpc(&r));
lock(r);
lock(&m->externup->rlock);
if(r->_p){
print("double sleep called from %#p, %d %d\n",
getcallerpc(&r), r->_p->pid, m->externup->pid);
dumpstack();
}
/*
* Wakeup only knows there may be something to do by testing
* r->p in order to get something to lock on.
* Flush that information out to memory in case the sleep is
* committed.
*/
r->_p = m->externup;
if((*f)(arg) || m->externup->notepending){
/*
* if condition happened or a note is pending
* never mind
*/
r->_p = nil;
unlock(&m->externup->rlock);
unlock(r);
} else {
/*
* now we are committed to
* change state and call scheduler
*/
if(m->externup->trace)
proctrace(m->externup, SSleep, 0);
m->externup->state = Wakeme;
m->externup->r = r;
/* statistics */
m->cs++;
procsave(m->externup);
mmuflushtlb(m->pml4->pa);
if(setlabel(&m->externup->sched)) {
/*
* here when the process is awakened
*/
procrestore(m->externup);
spllo();
} else {
/*
* here to go to sleep (i.e. stop Running)
*/
unlock(&m->externup->rlock);
unlock(r);
/*debug*/gotolabel(&m->sched);
}
}
if(m->externup->notepending) {
m->externup->notepending = 0;
splx(pl);
if(m->externup->procctl == Proc_exitme && m->externup->closingfgrp)
forceclosefgrp();
error(Eintr);
}
splx(pl);
}
/* it should be unsigned. FIXME */
void
syscall(int badscallnr, Ureg* ureg)
{
unsigned int scallnr = (unsigned int) badscallnr;
char *e;
uintptr sp;
int s;
vlong startns, stopns;
Ar0 ar0;
static Ar0 zar0;
if(!userureg(ureg))
panic("syscall: cs %#llux\n", ureg->cs);
cycles(&up->kentry);
m->syscall++;
up->nsyscall++;
up->nqsyscall++;
up->insyscall = 1;
up->pc = ureg->ip;
up->dbgreg = ureg;
sp = ureg->sp;
startns = 0;
if(up->procctl == Proc_tracesyscall){
/*
* Redundant validaddr. Do we care?
* Tracing syscalls is not exactly a fast path...
* Beware, validaddr currently does a pexit rather
* than an error if there's a problem; that might
* change in the future.
*/
if(sp < (USTKTOP-BIGPGSZ) || sp > (USTKTOP-sizeof(up->arg)-BY2SE))
validaddr(UINT2PTR(sp), sizeof(up->arg)+BY2SE, 0);
syscallfmt(scallnr, (va_list)(sp+BY2SE));
up->procctl = Proc_stopme;
procctl(up);
if(up->syscalltrace)
free(up->syscalltrace);
up->syscalltrace = nil;
startns = todget(nil);
}
up->scallnr = scallnr;
if(scallnr == RFORK)
fpusysrfork(ureg);
spllo();
sp = ureg->sp;
up->nerrlab = 0;
ar0 = zar0;
if(!waserror()){
if(scallnr >= nsyscall || systab[scallnr].f == nil){
pprint("bad sys call number %d pc %#llux\n",
scallnr, ureg->ip);
postnote(up, 1, "sys: bad sys call", NDebug);
error(Ebadarg);
}
if(sp < (USTKTOP-BIGPGSZ) || sp > (USTKTOP-sizeof(up->arg)-BY2SE))
validaddr(UINT2PTR(sp), sizeof(up->arg)+BY2SE, 0);
memmove(up->arg, UINT2PTR(sp+BY2SE), sizeof(up->arg));
up->psstate = systab[scallnr].n;
systab[scallnr].f(&ar0, (va_list)up->arg);
if(scallnr == SYSR1){
/*
* BUG: must go when ron binaries go.
* NIX: Returning from execac().
* This means that the process is back to the
* time sharing core. However, the process did
* already return from the system call, when dispatching
* the user code to the AC. The only thing left is to
* return. The user registers should be ok, because
* up->dbgreg has been the user context for the process.
*/
return;
}
poperror();
}
else{
/* failure: save the error buffer for errstr */
e = up->syserrstr;
up->syserrstr = up->errstr;
up->errstr = e;
if(DBGFLG && up->pid == 1)
iprint("%s: syscall %s error %s\n",
up->text, systab[scallnr].n, up->syserrstr);
ar0 = systab[scallnr].r;
}
/*
* NIX: for the execac() syscall, what follows is done within
* the system call, because it never returns.
* See acore.c:/^retfromsyscall
*/
noerrorsleft();
/*
* Put return value in frame.
*/
ureg->ax = ar0.p;
if(up->procctl == Proc_tracesyscall){
stopns = todget(nil);
up->procctl = Proc_stopme;
sysretfmt(scallnr, (va_list)(sp+BY2SE), &ar0, startns, stopns);
s = splhi();
procctl(up);
splx(s);
if(up->syscalltrace)
free(up->syscalltrace);
up->syscalltrace = nil;
}else if(up->procctl == Proc_totc || up->procctl == Proc_toac)
procctl(up);
up->insyscall = 0;
up->psstate = 0;
if(scallnr == NOTED)
noted(ureg, *(uintptr*)(sp+BY2SE));
splhi();
if(scallnr != RFORK && (up->procctl || up->nnote))
notify(ureg);
/* if we delayed sched because we held a lock, sched now */
if(up->delaysched){
sched();
splhi();
}
kexit(ureg);
}
/*
* Call user, if necessary, with note.
* Pass user the Ureg struct and the note on his stack.
*/
int
notify(Ureg* ureg)
{
int l;
Mpl pl;
Note note;
uintptr sp;
NFrame *nf;
/*
* Calls procctl splhi, see comment in procctl for the reasoning.
*/
if(up->procctl)
procctl(up);
if(up->nnote == 0)
return 0;
fpunotify(ureg);
pl = spllo();
qlock(&up->debug);
up->notepending = 0;
memmove(¬e, &up->note[0], sizeof(Note));
if(strncmp(note.msg, "sys:", 4) == 0){
l = strlen(note.msg);
if(l > ERRMAX-sizeof(" pc=0x0123456789abcdef"))
l = ERRMAX-sizeof(" pc=0x0123456789abcdef");
sprint(note.msg+l, " pc=%#p", ureg->ip);
}
if(note.flag != NUser && (up->notified || up->notify == nil)){
qunlock(&up->debug);
if(note.flag == NDebug)
pprint("suicide: %s\n", note.msg);
pexit(note.msg, note.flag != NDebug);
}
if(up->notified){
qunlock(&up->debug);
splhi();
return 0;
}
if(up->notify == nil){
qunlock(&up->debug);
pexit(note.msg, note.flag != NDebug);
}
if(!okaddr(PTR2UINT(up->notify), sizeof(ureg->ip), 0)){
qunlock(&up->debug);
pprint("suicide: bad function address %#p in notify\n",
up->notify);
pexit("Suicide", 0);
}
sp = ureg->sp - sizeof(NFrame);
if(!okaddr(sp, sizeof(NFrame), 1)){
qunlock(&up->debug);
pprint("suicide: bad stack address %#p in notify\n", sp);
pexit("Suicide", 0);
}
nf = UINT2PTR(sp);
memmove(&nf->ureg, ureg, sizeof(Ureg));
nf->old = up->ureg;
up->ureg = nf; /* actually the NFrame, for noted */
memmove(nf->msg, note.msg, ERRMAX);
nf->arg1 = nf->msg;
nf->arg0 = &nf->ureg;
ureg->bp = PTR2UINT(nf->arg0);
nf->ip = 0;
ureg->sp = sp;
ureg->ip = PTR2UINT(up->notify);
up->notified = 1;
up->nnote--;
memmove(&up->lastnote, ¬e, sizeof(Note));
memmove(&up->note[0], &up->note[1], up->nnote*sizeof(Note));
qunlock(&up->debug);
splx(pl);
return 1;
}
void halt(void) { spllo(); for(;;); }
void
clockinit(void)
{
int i, s;
Timerregs *tn;
clockshutdown();
/* turn cycle counter on */
cpwrsc(0, CpCLD, CpCLDena, CpCLDenacyc, 1<<31);
/* turn all counters on and clear the cycle counter */
cpwrsc(0, CpCLD, CpCLDena, CpCLDenapmnc, 1<<2 | 1);
/* let users read the cycle counter directly */
cpwrsc(0, CpCLD, CpCLDena, CpCLDenapmnc, 1);
ilock(&clklck);
m->fastclock = 1;
m->ticks = ticks = 0;
/*
* T0 is a freerunning timer (cycle counter); it wraps,
* automatically reloads, and does not dispatch interrupts.
*/
tn = (Timerregs *)Tn0;
tn->tcrr = Freebase; /* count up to 0 */
tn->tldr = Freebase;
coherence();
tn->tclr = Ar | St;
iunlock(&clklck);
/*
* T1 is the interrupting timer and does not participate
* in measuring time. It is initially set to HZ.
*/
tn = (Timerregs *)Tn1;
irqenable(Tn0irq+1, clockintr, tn, "clock");
ilock(&clklck);
tn->tcrr = -Tcycles; /* approx.; count up to 0 */
tn->tldr = -Tcycles;
coherence();
tn->tclr = Ar | St;
coherence();
tn->tier = Ovf_it;
coherence();
iunlock(&clklck);
/*
* verify sanity of timer1
*/
s = spllo(); /* risky */
for (i = 0; i < 5 && ticks == 0; i++) {
delay(10);
cachedwbinvse(&ticks, sizeof ticks);
}
splx(s);
if (ticks == 0) {
if (tn->tcrr == 0)
panic("clock not interrupting");
else if (tn->tcrr == tn->tldr)
panic("clock not ticking at all");
#ifdef PARANOID
else
panic("clock running very slowly");
#endif
}
guessmips(issue1loop, "single");
if (Debug)
iprint(", ");
guessmips(issue2loop, "dual");
if (Debug)
iprint("\n");
/*
* m->delayloop should be the number of delay loop iterations
* needed to consume 1 ms. 2 is min. instructions in the delay loop.
*/
m->delayloop = m->cpuhz / (1000 * 2);
// iprint("m->delayloop = %lud\n", m->delayloop);
/*
* desynchronize the processor clocks so that they all don't
* try to resched at the same time.
*/
delay(m->machno*2);
}
/*
* Call user, if necessary, with note.
* Pass user the Ureg struct and the note on his stack.
*/
int
notify(Ureg* ureg)
{
int l;
ulong s, sp;
Note *n;
if(up->procctl)
procctl(up);
if(up->nnote == 0)
return 0;
if(up->fpstate == FPactive){
fpsave(&up->fpsave);
up->fpstate = FPinactive;
}
up->fpstate |= FPillegal;
s = spllo();
qlock(&up->debug);
up->notepending = 0;
n = &up->note[0];
if(strncmp(n->msg, "sys:", 4) == 0){
l = strlen(n->msg);
if(l > ERRMAX-15) /* " pc=0x12345678\0" */
l = ERRMAX-15;
sprint(n->msg+l, " pc=0x%.8lux", ureg->pc);
}
if(n->flag!=NUser && (up->notified || up->notify==0)){
if(n->flag == NDebug)
pprint("suicide: %s\n", n->msg);
qunlock(&up->debug);
pexit(n->msg, n->flag!=NDebug);
}
if(up->notified){
qunlock(&up->debug);
splhi();
return 0;
}
if(!up->notify){
qunlock(&up->debug);
pexit(n->msg, n->flag!=NDebug);
}
sp = ureg->usp;
sp -= 256; /* debugging: preserve context causing problem */
sp -= sizeof(Ureg);
if(0) print("%s %lud: notify %.8lux %.8lux %.8lux %s\n",
up->text, up->pid, ureg->pc, ureg->usp, sp, n->msg);
if(!okaddr((ulong)up->notify, 1, 0)
|| !okaddr(sp-ERRMAX-4*BY2WD, sizeof(Ureg)+ERRMAX+4*BY2WD, 1)){
pprint("suicide: bad address in notify\n");
qunlock(&up->debug);
pexit("Suicide", 0);
}
memmove((Ureg*)sp, ureg, sizeof(Ureg));
*(Ureg**)(sp-BY2WD) = up->ureg; /* word under Ureg is old up->ureg */
up->ureg = (void*)sp;
sp -= BY2WD+ERRMAX;
memmove((char*)sp, up->note[0].msg, ERRMAX);
sp -= 3*BY2WD;
*(ulong*)(sp+2*BY2WD) = sp+3*BY2WD; /* arg 2 is string */
*(ulong*)(sp+1*BY2WD) = (ulong)up->ureg; /* arg 1 is ureg* */
*(ulong*)(sp+0*BY2WD) = 0; /* arg 0 is pc */
ureg->usp = sp;
ureg->pc = (ulong)up->notify;
up->notified = 1;
up->nnote--;
memmove(&up->lastnote, &up->note[0], sizeof(Note));
memmove(&up->note[0], &up->note[1], up->nnote*sizeof(Note));
qunlock(&up->debug);
splx(s);
return 1;
}
/*
* Now running in a thread. Kick off other services,
* invoke user bootstrap, enter pageout loop.
*/
static void
kernel_bootstrap_thread(void)
{
processor_t processor = current_processor();
#define kernel_bootstrap_thread_kprintf(x...) /* kprintf("kernel_bootstrap_thread: " x) */
kernel_bootstrap_thread_log("idle_thread_create");
/*
* Create the idle processor thread.
*/
idle_thread_create(processor);
/*
* N.B. Do not stick anything else
* before this point.
*
* Start up the scheduler services.
*/
kernel_bootstrap_thread_log("sched_startup");
sched_startup();
/*
* Thread lifecycle maintenance (teardown, stack allocation)
*/
kernel_bootstrap_thread_log("thread_daemon_init");
thread_daemon_init();
/* Create kernel map entry reserve */
vm_kernel_reserved_entry_init();
/*
* Thread callout service.
*/
kernel_bootstrap_thread_log("thread_call_initialize");
thread_call_initialize();
/*
* Remain on current processor as
* additional processors come online.
*/
kernel_bootstrap_thread_log("thread_bind");
thread_bind(processor);
/*
* Initialize ipc thread call support.
*/
kernel_bootstrap_thread_log("ipc_thread_call_init");
ipc_thread_call_init();
/*
* Kick off memory mapping adjustments.
*/
kernel_bootstrap_thread_log("mapping_adjust");
mapping_adjust();
/*
* Create the clock service.
*/
kernel_bootstrap_thread_log("clock_service_create");
clock_service_create();
/*
* Create the device service.
*/
device_service_create();
kth_started = 1;
#if (defined(__i386__) || defined(__x86_64__)) && NCOPY_WINDOWS > 0
/*
* Create and initialize the physical copy window for processor 0
* This is required before starting kicking off IOKit.
*/
cpu_physwindow_init(0);
#endif
#if MACH_KDP
kernel_bootstrap_log("kdp_init");
kdp_init();
#endif
#if ALTERNATE_DEBUGGER
alternate_debugger_init();
#endif
#if KPC
kpc_init();
#endif
#if CONFIG_ECC_LOGGING
ecc_log_init();
#endif
#if KPERF
kperf_bootstrap();
#endif
#if HYPERVISOR
hv_support_init();
#endif
#if CONFIG_TELEMETRY
kernel_bootstrap_log("bootprofile_init");
bootprofile_init();
#endif
#if (defined(__i386__) || defined(__x86_64__)) && CONFIG_VMX
vmx_init();
#endif
#if (defined(__i386__) || defined(__x86_64__))
if (kdebug_serial) {
new_nkdbufs = 1;
if (trace_typefilter == 0)
trace_typefilter = 1;
}
if (turn_on_log_leaks && !new_nkdbufs)
new_nkdbufs = 200000;
if (trace_typefilter)
start_kern_tracing_with_typefilter(new_nkdbufs,
FALSE,
trace_typefilter);
else
start_kern_tracing(new_nkdbufs, FALSE);
if (turn_on_log_leaks)
log_leaks = 1;
#endif
kernel_bootstrap_log("prng_init");
prng_cpu_init(master_cpu);
#ifdef IOKIT
PE_init_iokit();
#endif
assert(ml_get_interrupts_enabled() == FALSE);
(void) spllo(); /* Allow interruptions */
#if (defined(__i386__) || defined(__x86_64__)) && NCOPY_WINDOWS > 0
/*
* Create and initialize the copy window for processor 0
* This also allocates window space for all other processors.
* However, this is dependent on the number of processors - so this call
* must be after IOKit has been started because IOKit performs processor
* discovery.
*/
cpu_userwindow_init(0);
#endif
#if (!defined(__i386__) && !defined(__x86_64__))
if (turn_on_log_leaks && !new_nkdbufs)
new_nkdbufs = 200000;
if (trace_typefilter)
start_kern_tracing_with_typefilter(new_nkdbufs, FALSE, trace_typefilter);
else
start_kern_tracing(new_nkdbufs, FALSE);
if (turn_on_log_leaks)
log_leaks = 1;
#endif
/*
* Initialize the shared region module.
*/
vm_shared_region_init();
vm_commpage_init();
vm_commpage_text_init();
#if CONFIG_MACF
kernel_bootstrap_log("mac_policy_initmach");
mac_policy_initmach();
#endif
#if CONFIG_SCHED_SFI
kernel_bootstrap_log("sfi_init");
sfi_init();
#endif
/*
* Initialize the globals used for permuting kernel
* addresses that may be exported to userland as tokens
* using VM_KERNEL_ADDRPERM()/VM_KERNEL_ADDRPERM_EXTERNAL().
* Force the random number to be odd to avoid mapping a non-zero
* word-aligned address to zero via addition.
* Note: at this stage we can use the cryptographically secure PRNG
* rather than early_random().
*/
read_random(&vm_kernel_addrperm, sizeof(vm_kernel_addrperm));
vm_kernel_addrperm |= 1;
read_random(&buf_kernel_addrperm, sizeof(buf_kernel_addrperm));
buf_kernel_addrperm |= 1;
read_random(&vm_kernel_addrperm_ext, sizeof(vm_kernel_addrperm_ext));
vm_kernel_addrperm_ext |= 1;
vm_set_restrictions();
/*
* Start the user bootstrap.
*/
#ifdef MACH_BSD
bsd_init();
#endif
/*
* Get rid of segments used to bootstrap kext loading. This removes
* the KLD, PRELINK symtab, LINKEDIT, and symtab segments/load commands.
*/
OSKextRemoveKextBootstrap();
serial_keyboard_init(); /* Start serial keyboard if wanted */
vm_page_init_local_q();
thread_bind(PROCESSOR_NULL);
/*
* Become the pageout daemon.
*/
vm_pageout();
/*NOTREACHED*/
}
static Proc*
singlerunproc(void)
{
Mach *m = machp();
Schedq *rq;
Proc *p;
uint32_t start, now, skipscheds;
int i;
start = perfticks();
/* cooperative scheduling until the clock ticks */
if((p=m->readied) && p->mach==0 && p->state==Ready
&& &run.runq[Nrq-1].head == nil && &run.runq[Nrq-2].head == nil){
skipscheds++;
rq = &run.runq[p->priority];
if(0)hi("runproc going to found before loop...\n");
goto found;
}
run.preempts++;
loop:
/*
* find a process that last ran on this processor (affinity),
* or one that hasn't moved in a while (load balancing). Every
* time around the loop affinity goes down.
*/
spllo();
for(i = 0;; i++){
/*
* find the highest priority target process that this
* processor can run given affinity constraints.
*
*/
for(rq = &run.runq[Nrq-1]; rq >= run.runq; rq--){
for(p = rq->head; p; p = p->rnext){
if(p->mp == nil || p->mp == sys->machptr[m->machno]
|| (!p->wired && i > 0))
{
if(0)hi("runproc going to found inside loop...\n");
goto found;
}
}
}
/* waste time or halt the CPU */
idlehands();
/* remember how much time we're here */
now = perfticks();
m->perf.inidle += now-start;
start = now;
}
found:
splhi();
if(0)hi("runproc into found...\n");
p = dequeueproc(&run, rq, p);
if(p == nil)
{
if(0)hi("runproc p=nil :(\n");
goto loop;
}
p->state = Scheding;
if(0)hi("runproc, pm->mp = sys->machptr[m->machno]\n");
p->mp = sys->machptr[m->machno];
if(0){hi("runproc, sys->machptr[m->machno] = "); put64((uint64_t)p->mp); hi("\n");}
if(edflock(p)){
edfrun(p, rq == &run.runq[PriEdf]); /* start deadline timer and do admin */
edfunlock();
}
if(p->trace)
proctrace(p, SRun, 0);
/* avoiding warnings, this will be removed */
USED(mach0sched); USED(smprunproc);
if(0){hi("runproc, returning p ");
put64((uint64_t)p);
hi("\n");}
return p;
}
void
setpanic(void)
{
spllo();
consoleprint = 1;
}
/*
* Scheduling thread run as the main loop of cpu 0
* Used in AMP sched.
*/
static void
mach0sched(void)
{
Mach *m = machp();
Schedq *rq;
Proc *p;
Mach *mp;
uint32_t start, now;
int n, i; //, j;
assert(m->machno == 0);
acmodeset(NIXKC); /* we don't time share any more */
n = 0;
start = perfticks();
loop:
/*
* find a ready process that we might run.
*/
spllo();
for(rq = &run.runq[Nrq-1]; rq >= run.runq; rq--)
for(p = rq->head; p; p = p->rnext){
/*
* wired processes may only run when their core is available.
*/
if(p->wired != nil){
if(p->wired->proc == nil)
goto found;
continue;
}
/*
* find a ready process that did run at an available core
* or one that has not moved for some time.
*/
if(p->mp == nil || p->mp->proc == nil || n>0){
goto found;
}
}
/* waste time or halt the CPU */
idlehands();
/* remember how much time we're here */
now = perfticks();
m->perf.inidle += now-start;
start = now;
n++;
goto loop;
found:
assert(m->machno == 0);
splhi();
/*
* find a core for this process, but honor wiring.
*/
mp = p->wired;
if(mp != nil){
if(mp->proc != nil)
goto loop;
}else{
for(i = 0; i < MACHMAX; i++){
/*j = pickcore(p->color, i);
if((mp = sys->machptr[j]) != nil && mp->online && mp->nixtype == NIXTC){*/
if((mp = sys->machptr[i]) != nil){ // && mp->online && mp->nixtype == NIXTC){
if(mp != m && mp->proc == nil)
break;
}
}
if(i == MACHMAX){
preemptfor(p);
goto loop;
}
}
p = dequeueproc(&run, rq, p);
mp->proc = p;
if(p != nil){
p->state = Scheding;
p->mp = mp;
}
n = 0;
goto loop;
}
/*
* Now running in a thread. Create the rest of the kernel threads
* and the bootstrap task.
*/
void
start_kernel_threads(void)
{
register int i;
/*
* Create the idle threads and the other
* service threads.
*/
for (i = 0; i < NCPUS; i++) {
if (machine_slot[i].is_cpu) {
thread_t th;
spl_t s;
processor_t processor = cpu_to_processor(i);
(void) thread_create_at(kernel_task, &th, idle_thread);
s=splsched();
thread_lock(th);
thread_bind_locked(th, processor);
processor->idle_thread = th;
/*(void) thread_resume(th->top_act);*/
th->state |= TH_RUN;
thread_setrun( th, TRUE, TAIL_Q);
thread_unlock( th );
splx(s);
}
}
(void) kernel_thread(kernel_task, reaper_thread, (char *) 0);
#if THREAD_SWAPPER
(void) kernel_thread(kernel_task, swapin_thread, (char *) 0);
(void) kernel_thread(kernel_task, swapout_thread, (char *) 0);
#endif /* THREAD_SWAPPER */
#if TASK_SWAPPER
if (task_swap_on) {
(void) kernel_thread(kernel_task, task_swapper, (char *) 0);
(void) kernel_thread(kernel_task, task_swap_swapout_thread,
(char *) 0);
}
#endif /* TASK_SWAPPER */
(void) kernel_thread(kernel_task, sched_thread, (char *) 0);
(void) kernel_thread(kernel_task, timeout_thread, (char *) 0);
#if NORMA_VM
(void) kernel_thread(kernel_task, vm_object_thread, (char *) 0);
#endif /* NORMA_VM */
/*
* Create the clock service.
*/
clock_service_create();
/*
* Create the device service.
*/
device_service_create();
/*
* Initialize distributed services, starting
* with distributed IPC and progressing to any
* services layered on top of that.
*
* This stub exists even in non-NORMA systems.
*/
norma_bootstrap();
/*
* Initialize any testing services blocking the main kernel
* thread so that the in-kernel tests run without interference
* from other boot time activities. We will resume this thread
* in kernel_test_thread().
*/
#if KERNEL_TEST
/*
* Initialize the lock that will be used to guard
* variables that will be used in the test synchronization
* scheme.
*/
simple_lock_init(&kernel_test_lock, ETAP_MISC_KERNEL_TEST);
#if PARAGON860
{
char *s;
unsigned int firstnode;
/*
* Only start up loopback tests on boot node.
*/
if ((s = (char *) getbootenv("BOOT_FIRST_NODE")) == 0)
panic("startup");
firstnode = atoi(s);
(void) kernel_thread(kernel_task, kernel_test_thread,
(char * )(dipc_node_self() ==
(node_name) firstnode));
}
#else /* PARAGON860 */
(void) kernel_thread(kernel_task, kernel_test_thread, (char *) 0);
#endif /* PARAGON860 */
{
/*
* The synchronization scheme uses a simple lock, two
* booleans and the wakeup event. The wakeup event will
* be posted by kernel_test_thread().
*/
spl_t s;
s = splsched();
simple_lock(&kernel_test_lock);
while(!kernel_test_thread_sync_done){
assert_wait((event_t) &start_kernel_threads, FALSE);
start_kernel_threads_blocked = TRUE;
simple_unlock(&kernel_test_lock);
splx(s);
thread_block((void (*)(void)) 0);
s = splsched();
simple_lock(&kernel_test_lock);
start_kernel_threads_blocked = FALSE;
}
kernel_test_thread_sync_done = FALSE; /* Reset for next use */
simple_unlock(&kernel_test_lock);
splx(s);
}
#endif /* KERNEL_TEST */
/*
* Start the user bootstrap.
*/
bootstrap_create();
#if XPR_DEBUG
xprinit(); /* XXX */
#endif /* XPR_DEBUG */
#if NCPUS > 1
/*
* Create the shutdown thread.
*/
(void) kernel_thread(kernel_task, action_thread, (char *) 0);
/*
* Allow other CPUs to run.
*
* (this must be last, to allow bootstrap_create to fiddle with
* its child thread before some cpu tries to run it)
*/
start_other_cpus();
#endif /* NCPUS > 1 */
/*
* Become the pageout daemon.
*/
(void) spllo();
vm_pageout();
/*NOTREACHED*/
}
