/*
* SMP performs better than AMP with few cores.
* So, leave this here by now. We should probably
* write a unified version of runproc good enough for
* both SMP and AMP.
*/
static Proc*
smprunproc(void)
{
Mach *m = machp();
Schedq *rq;
Proc *p;
uint32_t start, now;
int i;
start = perfticks();
run.preempts++;
loop:
/*
* find a process that last ran on this processor (affinity),
* or one that hasn't moved in a while (load balancing). Every
* time around the loop affinity goes down.
*/
spllo();
for(i = 0;; i++){
/*
* find the highest priority target process that this
* processor can run given affinity constraints.
*
*/
for(rq = &run.runq[Nrq-1]; rq >= run.runq; rq--){
for(p = rq->head; p; p = p->rnext){
if(p->mp == nil || p->mp == sys->machptr[m->machno]
|| (!p->wired && i > 0))
goto found;
}
}
/* waste time or halt the CPU */
idlehands();
/* remember how much time we're here */
now = perfticks();
m->perf.inidle += now-start;
start = now;
}
found:
splhi();
p = dequeueproc(&run, rq, p);
if(p == nil)
goto loop;
p->state = Scheding;
p->mp = sys->machptr[m->machno];
if(edflock(p)){
edfrun(p, rq == &run.runq[PriEdf]); /* start deadline timer and do admin */
edfunlock();
}
if(p->trace)
proctrace(p, SRun, 0);
return p;
}
Proc*
runproc(void)
{
Mach *m = machp();
Schedq *rq;
Proc *p;
uint32_t start, now;
if(nosmp)
return singlerunproc();
//NIX modeset cannot work without halt every cpu at boot
//if(sys->nmach <= AMPmincores)
else
return smprunproc();
start = perfticks();
run.preempts++;
rq = nil;
if(m->machno != 0){
do{
spllo();
while(m->proc == nil)
idlehands();
now = perfticks();
m->perf.inidle += now-start;
start = now;
splhi();
p = m->proc;
}while(p == nil);
p->state = Scheding;
p->mp = sys->machptr[m->machno];
if(edflock(p)){
edfrun(p, rq == &run.runq[PriEdf]); /* start deadline timer and do admin */
edfunlock();
}
if(p->trace)
proctrace(p, SRun, 0);
return p;
}
mach0sched();
return nil; /* not reached */
}
static long
issue1loop(void)
{
register int i;
long st;
i = Instrs;
st = perfticks();
do {
--i; --i; --i; --i; --i; --i; --i; --i; --i; --i;
--i; --i; --i; --i; --i; --i; --i; --i; --i; --i;
--i; --i; --i; --i; --i; --i; --i; --i; --i; --i;
--i; --i; --i; --i; --i; --i; --i; --i; --i; --i;
--i; --i; --i; --i; --i; --i; --i; --i; --i; --i;
--i; --i; --i; --i; --i; --i; --i; --i; --i; --i;
--i; --i; --i; --i; --i; --i; --i; --i; --i; --i;
--i; --i; --i; --i; --i; --i; --i; --i; --i; --i;
--i; --i; --i; --i; --i; --i; --i; --i; --i; --i;
--i; --i; --i; --i; --i;
/* omit 3 (--i) to account for conditional branch, nop & jump */
i -= 1+3; /* --i plus 3 omitted (--i) instructions */
} while(--i >= 0);
return perfticks() - st;
}
/*
* keep interrupt service times and counts
*/
void
intrtime(int vno)
{
Mach *m = machp();
uint32_t diff, x; /* should be uint64_t */
x = perfticks();
diff = x - m->perf.intrts;
m->perf.intrts = x;
m->perf.inintr += diff;
if(m->externup == nil && m->perf.inidle > diff)
m->perf.inidle -= diff;
intrtimes[vno].cycles += diff;
intrtimes[vno].count++;
}
/*
* keep interrupt service times and counts
*/
void
intrtime(int vno)
{
Proc *up = externup();
uint64_t diff, x;
x = perfticks();
diff = x - machp()->perf.intrts;
machp()->perf.intrts = x;
machp()->perf.inintr += diff;
if(up == nil && machp()->perf.inidle > diff)
machp()->perf.inidle -= diff;
intrtimes[vno].cycles += diff;
intrtimes[vno].count++;
}
/*
* keep histogram of interrupt service times
*/
void
intrtime(Mach*, int vno)
{
ulong diff;
ulong x;
x = perfticks();
diff = x - m->perf.intrts;
m->perf.intrts = x;
m->perf.inintr += diff;
if(up == nil && m->perf.inidle > diff)
m->perf.inidle -= diff;
diff /= m->cpumhz*100; /* quantum = 100µsec */
if(diff >= Ntimevec)
diff = Ntimevec-1;
intrtimes[vno][diff]++;
}
long
lcycles(void)
{
return perfticks();
}
/*
* pick a process to run
*/
Proc*
runproc(void)
{
Schedq *rq;
Proc *p;
ulong start, now;
int i;
void (*pt)(Proc*, int, vlong);
start = perfticks();
/* cooperative scheduling until the clock ticks */
if((p = m->readied) != nil && p->mach == nil && p->state == Ready
&& (p->wired == nil || p->wired == MACHP(m->machno))
&& runq[Nrq-1].head == nil && runq[Nrq-2].head == nil){
skipscheds++;
rq = &runq[p->priority];
goto found;
}
preempts++;
loop:
/*
* find a process that last ran on this processor (affinity),
* or one that hasn't moved in a while (load balancing). Every
* time around the loop affinity goes down.
*/
spllo();
for(i = 0;; i++){
/*
* find the highest priority target process that this
* processor can run given affinity constraints.
*
*/
for(rq = &runq[Nrq-1]; rq >= runq; rq--){
for(p = rq->head; p != nil; p = p->rnext){
if(p->mp == nil || p->mp == MACHP(m->machno)
|| (p->wired == nil && i > 0))
goto found;
}
}
/* waste time or halt the CPU */
idlehands();
/* remember how much time we're here */
now = perfticks();
m->perf.inidle += now-start;
start = now;
}
found:
splhi();
p = dequeueproc(rq, p);
if(p == nil)
goto loop;
p->state = Scheding;
p->mp = MACHP(m->machno);
if(edflock(p)){
edfrun(p, rq == &runq[PriEdf]); /* start deadline timer and do admin */
edfunlock();
}
pt = proctrace;
if(pt != nil)
pt(p, SRun, 0);
return p;
}
/*
* All traps come here. It is slower to have all traps call trap()
* rather than directly vectoring the handler. However, this avoids a
* lot of code duplication and possible bugs. The only exception is
* VectorSYSCALL.
* Trap is called with interrupts disabled via interrupt-gates.
*/
void
trap(Ureg* ureg)
{
int clockintr, vno, user;
// cache the previous vno to see what might be causing
// trouble
static int lastvno;
vno = ureg->type;
uint64_t gsbase = rdmsr(GSbase);
//if (sce > scx) iprint("====================");
if (vno == 8) {
iprint("Lstar is %p\n", (void *)rdmsr(Lstar));
iprint("GSbase is %p\n", (void *)gsbase);
iprint("ire %d irx %d sce %d scx %d lastvno %d\n",
ire, irx, sce, scx, lastvno);
iprint("irxe %d \n",
irxe);
die("8");
}
lastvno = vno;
if (gsbase < 1ULL<<63)
die("bogus gsbase");
Mach *m = machp();
char buf[ERRMAX];
Vctl *ctl, *v;
if (0 && m && m->externup && m->externup->pid == 6) {
//iprint("type %x\n", ureg->type);
if (ureg->type != 0x49)
die("6\n");
}
m->perf.intrts = perfticks();
user = userureg(ureg);
if(user && (m->nixtype == NIXTC)){
m->externup->dbgreg = ureg;
cycles(&m->externup->kentry);
}
clockintr = 0;
//_pmcupdate(m);
if(ctl = vctl[vno]){
if(ctl->isintr){
m->intr++;
if(vno >= VectorPIC && vno != VectorSYSCALL)
m->lastintr = ctl->irq;
}else
if(m->externup)
m->externup->nqtrap++;
if(ctl->isr)
ctl->isr(vno);
for(v = ctl; v != nil; v = v->next){
if(v->f)
v->f(ureg, v->a);
}
if(ctl->eoi)
ctl->eoi(vno);
intrtime(vno);
if(ctl->isintr){
if(ctl->irq == IrqCLOCK || ctl->irq == IrqTIMER)
clockintr = 1;
if(m->externup && !clockintr)
preempted();
}
}
else if(vno < nelem(excname) && user){
spllo();
snprint(buf, sizeof buf, "sys: trap: %s", excname[vno]);
postnote(m->externup, 1, buf, NDebug);
}
else if(vno >= VectorPIC && vno != VectorSYSCALL){
/*
* An unknown interrupt.
* Check for a default IRQ7. This can happen when
* the IRQ input goes away before the acknowledge.
* In this case, a 'default IRQ7' is generated, but
* the corresponding bit in the ISR isn't set.
* In fact, just ignore all such interrupts.
*/
/* clear the interrupt */
i8259isr(vno);
iprint("cpu%d: spurious interrupt %d, last %d\n",
m->machno, vno, m->lastintr);
intrtime(vno);
if(user)
kexit(ureg);
return;
}
else{
if(vno == VectorNMI){
nmienable();
if(m->machno != 0){
iprint("cpu%d: PC %#llux\n",
m->machno, ureg->ip);
for(;;);
}
}
dumpregs(ureg);
if(!user){
ureg->sp = PTR2UINT(&ureg->sp);
dumpstackwithureg(ureg);
}
if(vno < nelem(excname))
panic("%s", excname[vno]);
panic("unknown trap/intr: %d\n", vno);
}
splhi();
/* delaysched set because we held a lock or because our quantum ended */
if(m->externup && m->externup->delaysched && clockintr){
if(0)
if(user && m->externup->ac == nil && m->externup->nqtrap == 0 && m->externup->nqsyscall == 0){
if(!waserror()){
m->externup->ac = getac(m->externup, -1);
poperror();
runacore();
return;
}
}
sched();
splhi();
}
if(user){
if(m->externup && m->externup->procctl || m->externup->nnote)
notify(ureg);
kexit(ureg);
}
}
static Proc*
singlerunproc(void)
{
Mach *m = machp();
Schedq *rq;
Proc *p;
uint32_t start, now, skipscheds;
int i;
start = perfticks();
/* cooperative scheduling until the clock ticks */
if((p=m->readied) && p->mach==0 && p->state==Ready
&& &run.runq[Nrq-1].head == nil && &run.runq[Nrq-2].head == nil){
skipscheds++;
rq = &run.runq[p->priority];
if(0)hi("runproc going to found before loop...\n");
goto found;
}
run.preempts++;
loop:
/*
* find a process that last ran on this processor (affinity),
* or one that hasn't moved in a while (load balancing). Every
* time around the loop affinity goes down.
*/
spllo();
for(i = 0;; i++){
/*
* find the highest priority target process that this
* processor can run given affinity constraints.
*
*/
for(rq = &run.runq[Nrq-1]; rq >= run.runq; rq--){
for(p = rq->head; p; p = p->rnext){
if(p->mp == nil || p->mp == sys->machptr[m->machno]
|| (!p->wired && i > 0))
{
if(0)hi("runproc going to found inside loop...\n");
goto found;
}
}
}
/* waste time or halt the CPU */
idlehands();
/* remember how much time we're here */
now = perfticks();
m->perf.inidle += now-start;
start = now;
}
found:
splhi();
if(0)hi("runproc into found...\n");
p = dequeueproc(&run, rq, p);
if(p == nil)
{
if(0)hi("runproc p=nil :(\n");
goto loop;
}
p->state = Scheding;
if(0)hi("runproc, pm->mp = sys->machptr[m->machno]\n");
p->mp = sys->machptr[m->machno];
if(0){hi("runproc, sys->machptr[m->machno] = "); put64((uint64_t)p->mp); hi("\n");}
if(edflock(p)){
edfrun(p, rq == &run.runq[PriEdf]); /* start deadline timer and do admin */
edfunlock();
}
if(p->trace)
proctrace(p, SRun, 0);
/* avoiding warnings, this will be removed */
USED(mach0sched); USED(smprunproc);
if(0){hi("runproc, returning p ");
put64((uint64_t)p);
hi("\n");}
return p;
}
/*
* Scheduling thread run as the main loop of cpu 0
* Used in AMP sched.
*/
static void
mach0sched(void)
{
Mach *m = machp();
Schedq *rq;
Proc *p;
Mach *mp;
uint32_t start, now;
int n, i; //, j;
assert(m->machno == 0);
acmodeset(NIXKC); /* we don't time share any more */
n = 0;
start = perfticks();
loop:
/*
* find a ready process that we might run.
*/
spllo();
for(rq = &run.runq[Nrq-1]; rq >= run.runq; rq--)
for(p = rq->head; p; p = p->rnext){
/*
* wired processes may only run when their core is available.
*/
if(p->wired != nil){
if(p->wired->proc == nil)
goto found;
continue;
}
/*
* find a ready process that did run at an available core
* or one that has not moved for some time.
*/
if(p->mp == nil || p->mp->proc == nil || n>0){
goto found;
}
}
/* waste time or halt the CPU */
idlehands();
/* remember how much time we're here */
now = perfticks();
m->perf.inidle += now-start;
start = now;
n++;
goto loop;
found:
assert(m->machno == 0);
splhi();
/*
* find a core for this process, but honor wiring.
*/
mp = p->wired;
if(mp != nil){
if(mp->proc != nil)
goto loop;
}else{
for(i = 0; i < MACHMAX; i++){
/*j = pickcore(p->color, i);
if((mp = sys->machptr[j]) != nil && mp->online && mp->nixtype == NIXTC){*/
if((mp = sys->machptr[i]) != nil){ // && mp->online && mp->nixtype == NIXTC){
if(mp != m && mp->proc == nil)
break;
}
}
if(i == MACHMAX){
preemptfor(p);
goto loop;
}
}
p = dequeueproc(&run, rq, p);
mp->proc = p;
if(p != nil){
p->state = Scheding;
p->mp = mp;
}
n = 0;
goto loop;
}
/*
* All traps come here. It is slower to have all traps call trap()
* rather than directly vectoring the handler. However, this avoids a
* lot of code duplication and possible bugs. The only exception is
* VectorSYSCALL.
* Trap is called with interrupts disabled via interrupt-gates.
*/
void
trap(Ureg* ureg)
{
int clockintr, i, vno, user;
char buf[ERRMAX];
Vctl *ctl, *v;
Mach *mach;
if(!trapinited){
/* fault386 can give a better error message */
if(ureg->trap == VectorPF)
fault386(ureg, nil);
panic("trap %lud: not ready", ureg->trap);
}
m->perf.intrts = perfticks();
user = (ureg->cs & 0xFFFF) == UESEL;
if(user){
up->dbgreg = ureg;
cycles(&up->kentry);
}
clockintr = 0;
vno = ureg->trap;
if(ctl = vctl[vno]){
if(ctl->isintr){
m->intr++;
if(vno >= VectorPIC && vno != VectorSYSCALL)
m->lastintr = ctl->irq;
}
if(ctl->isr)
ctl->isr(vno);
for(v = ctl; v != nil; v = v->next){
if(v->f)
v->f(ureg, v->a);
}
if(ctl->eoi)
ctl->eoi(vno);
if(ctl->isintr){
intrtime(m, vno);
if(ctl->irq == IrqCLOCK || ctl->irq == IrqTIMER)
clockintr = 1;
if(up && !clockintr)
preempted();
}
}
else if(vno < nelem(excname) && user){
spllo();
sprint(buf, "sys: trap: %s", excname[vno]);
postnote(up, 1, buf, NDebug);
}
else if(vno >= VectorPIC && vno != VectorSYSCALL){
/*
* An unknown interrupt.
* Check for a default IRQ7. This can happen when
* the IRQ input goes away before the acknowledge.
* In this case, a 'default IRQ7' is generated, but
* the corresponding bit in the ISR isn't set.
* In fact, just ignore all such interrupts.
*/
/* call all interrupt routines, just in case */
for(i = VectorPIC; i <= MaxIrqLAPIC; i++){
ctl = vctl[i];
if(ctl == nil)
continue;
if(!ctl->isintr)
continue;
for(v = ctl; v != nil; v = v->next){
if(v->f)
v->f(ureg, v->a);
}
/* should we do this? */
if(ctl->eoi)
ctl->eoi(i);
}
/* clear the interrupt */
i8259isr(vno);
if(0)print("cpu%d: spurious interrupt %d, last %d\n",
m->machno, vno, m->lastintr);
if(0)if(conf.nmach > 1){
for(i = 0; i < 32; i++){
if(!(active.machs & (1<<i)))
continue;
mach = MACHP(i);
if(m->machno == mach->machno)
continue;
print(" cpu%d: last %d",
mach->machno, mach->lastintr);
}
print("\n");
}
m->spuriousintr++;
if(user)
kexit(ureg);
return;
}
else{
if(vno == VectorNMI){
/*
* Don't re-enable, it confuses the crash dumps.
nmienable();
*/
iprint("cpu%d: PC %#8.8lux\n", m->machno, ureg->pc);
while(m->machno != 0)
;
}
dumpregs(ureg);
if(!user){
ureg->sp = (ulong)&ureg->sp;
_dumpstack(ureg);
}
if(vno < nelem(excname))
panic("%s", excname[vno]);
panic("unknown trap/intr: %d", vno);
}
splhi();
/* delaysched set because we held a lock or because our quantum ended */
if(up && up->delaysched && clockintr){
sched();
splhi();
}
if(user){
if(up->procctl || up->nnote)
notify(ureg);
kexit(ureg);
}
}
/*
* All traps come here. It is slower to have all traps call trap()
* rather than directly vectoring the handler. However, this avoids a
* lot of code duplication and possible bugs. The only exception is
* VectorSYSCALL.
* Trap is called with interrupts disabled via interrupt-gates.
*/
void
trap(Ureg* ureg)
{
int clockintr, vno, user;
// cache the previous vno to see what might be causing
// trouble
vno = ureg->type;
uint64_t gsbase = rdmsr(GSbase);
//if (sce > scx) iprint("====================");
lastvno = vno;
if (gsbase < 1ULL<<63)
die("bogus gsbase");
Proc *up = externup();
char buf[ERRMAX];
Vctl *ctl, *v;
machp()->perf.intrts = perfticks();
user = userureg(ureg);
if(user && (machp()->NIX.nixtype == NIXTC)){
up->dbgreg = ureg;
cycles(&up->kentry);
}
clockintr = 0;
//_pmcupdate(machp());
if((ctl = vctl[vno]) != nil){
if(ctl->isintr){
machp()->intr++;
if(vno >= VectorPIC && vno != VectorSYSCALL)
machp()->lastintr = ctl->Vkey.irq;
}else
if(up)
up->nqtrap++;
if(ctl->isr){
ctl->isr(vno);
if(islo())print("trap %d: isr %p enabled interrupts\n", vno, ctl->isr);
}
for(v = ctl; v != nil; v = v->next){
if(v->f){
v->f(ureg, v->a);
if(islo())print("trap %d: ctlf %p enabled interrupts\n", vno, v->f);
}
}
if(ctl->eoi){
ctl->eoi(vno);
if(islo())print("trap %d: eoi %p enabled interrupts\n", vno, ctl->eoi);
}
intrtime(vno);
if(ctl->isintr){
if(ctl->Vkey.irq == IrqCLOCK || ctl->Vkey.irq == IrqTIMER)
clockintr = 1;
if (ctl->Vkey.irq == IrqTIMER)
oprof_alarm_handler(ureg);
if(up && !clockintr)
preempted();
}
}
else if(vno < nelem(excname) && user){
spllo();
snprint(buf, sizeof buf, "sys: trap: %s", excname[vno]);
postnote(up, 1, buf, NDebug);
}
else if(vno >= VectorPIC && vno != VectorSYSCALL){
/*
* An unknown interrupt.
* Check for a default IRQ7. This can happen when
* the IRQ input goes away before the acknowledge.
* In this case, a 'default IRQ7' is generated, but
* the corresponding bit in the ISR isn't set.
* In fact, just ignore all such interrupts.
*/
/* clear the interrupt */
i8259isr(vno);
iprint("cpu%d: spurious interrupt %d, last %d\n",
machp()->machno, vno, machp()->lastintr);
intrtime(vno);
if(user)
kexit(ureg);
return;
}
else{
if(vno == VectorNMI){
nmienable();
if(machp()->machno != 0){
iprint("cpu%d: PC %#llx\n",
machp()->machno, ureg->ip);
for(;;);
}
}
dumpregs(ureg);
if(!user){
ureg->sp = PTR2UINT(&ureg->sp);
dumpstackwithureg(ureg);
}
if(vno < nelem(excname))
panic("%s", excname[vno]);
panic("unknown trap/intr: %d\n", vno);
}
splhi();
/* delaysched set because we held a lock or because our quantum ended */
if(up && up->delaysched && clockintr){
if(0)
if(user && up->ac == nil && up->nqtrap == 0 && up->nqsyscall == 0){
if(!waserror()){
up->ac = getac(up, -1);
poperror();
runacore();
return;
}
}
sched();
splhi();
}
if(user){
if(up != nil && (up->procctl || up->nnote))
notify(ureg);
kexit(ureg);
}
}
