static void
pass1(int pass, volatile Diag *dp)
{
int i;
if(m->machno == 0)
iprint(" %d", pass);
for (i = 1000*1000; --i > 0; ) {
ainc(&dp->cnt);
adec(&dp->cnt);
}
synccpus(&dp->sync, navailcpus);
/* all cpus are now here */
ilock(dp);
if(dp->cnt != 0)
panic("cpu%d: diag: failed w count %ld", m->machno, dp->cnt);
iunlock(dp);
synccpus(&dp->sync, 2 * navailcpus);
/* all cpus are now here */
adec(&dp->sync);
adec(&dp->sync);
}
/*
* Rendezvous with other cores. Set roles for those that came
* up online, and wait until they are initialized.
* Sync TSC with them.
* We assume other processors that could boot had time to
* set online to 1 by now.
*/
static void
nixsquids(void)
{
Mach *mp;
int i;
uvlong now, start;
for(i = 1; i < MACHMAX; i++)
if((mp = sys->machptr[i]) != nil && mp->online != 0){
/*
* Inter-core calls. A ensure *mp->iccall and mp->icargs
* go into different cache lines.
*/
mp->icc = mallocalign(sizeof *m->icc, ICCLNSZ, 0, 0);
mp->icc->fn = nil;
if(i < initialTCs){
conf.nmach++;
mp->nixtype = NIXTC;
}
ainc(&active.nbooting);
}
sys->epoch = rdtsc();
mfence();
wrmsr(0x10, sys->epoch);
m->rdtsc = rdtsc();
active.thunderbirdsarego = 1;
start = fastticks2us(fastticks(nil));
do{
now = fastticks2us(fastticks(nil));
}while(active.nbooting > 0 && now - start < 1000000)
;
if(active.nbooting > 0)
print("cpu0: %d cores couldn't start\n", active.nbooting);
active.nbooting = 0;
}
void
addwaitstat(uintptr_t pc, uint64_t t0, int type)
{
uint i;
uint64_t w;
if(waitstats.on == 0)
return;
cycles(&w);
w -= t0;
mfence();
for(i = 0; i < NWstats; i++)
if(waitstats.pcs[i] == pc){
ainc(&waitstats.ns[i]);
if(w > waitstats.wait[i])
waitstats.wait[i] = w; /* race but ok */
waitstats.total[i] += w; /* race but ok */
return;
}
if(!canlock(&waitstatslk))
return;
for(i = 0; i < NWstats; i++)
if(waitstats.pcs[i] == pc){
ainc(&waitstats.ns[i]);
if(w > waitstats.wait[i])
waitstats.wait[i] = w; /* race but ok */
waitstats.total[i] += w;
unlock(&waitstatslk);
return;
}
for(i = 0; i < NWstats; i++)
if(waitstats.pcs[i] == 0){
waitstats.ns[i] = 1;
waitstats.type[i] = type;
waitstats.wait[i] = w;
waitstats.total[i] = w;
mfence();
waitstats.pcs[i] = pc;
waitstats.npcs++;
break;
}
unlock(&waitstatslk);
}
/*
* Always splhi()'ed.
*/
void
schedinit(void) /* never returns */
{
Edf *e;
machp()->inidle = 1;
machp()->proc = nil;
ainc(&run.nmach);
setlabel(&machp()->sched);
Proc *up = externup();
if(infected_with_std()){
print("mach %d got an std from %s (pid %d)!\n",
machp()->machno,
up ? up->text : "*notext",
up ? up->pid : -1
);
disinfect_std();
}
if(up) {
if((e = up->edf) && (e->flags & Admitted))
edfrecord(up);
machp()->qstart = 0;
machp()->qexpired = 0;
coherence();
machp()->proc = 0;
switch(up->state) {
case Running:
ready(up);
break;
case Moribund:
up->state = Dead;
stopac();
edfstop(up);
if (up->edf)
free(up->edf);
up->edf = nil;
/*
* Holding locks from pexit:
* procalloc
* pga
*/
mmurelease(up);
unlock(&pga.l);
psrelease(up);
unlock(&procalloc.l);
break;
}
up->mach = nil;
updatecpu(up);
machp()->externup = nil;
}
sched();
}
static void
synccpus(volatile long *cntp, int n)
{
ainc(cntp);
while (*cntp < n)
;
/* all cpus should now be here */
}
void
lock(Lock *l)
{
if(ainc(&l->key) == 1)
return; /* changed from 0 -> 1: we hold lock */
/* otherwise wait in kernel */
while(semacquire(&l->sem, 1) < 0){
/* interrupted; try again */
}
}
int
canlock(Lock *l)
{
if(ainc(&l->key) == 1)
return 1; /* changed from 0 -> 1: success */
/* Undo increment (but don't miss wakeup) */
if(adec(&l->key) == 0)
return 0; /* changed from 1 -> 0: no contention */
semrelease(&l->sem, 1);
return 0;
}
/*
* Always splhi()'ed.
*/
void
schedinit(void) /* never returns */
{
Mach *m = machp();
Edf *e;
m->inidle = 1;
m->proc = nil;
ainc(&run.nmach);
setlabel(&m->sched);
if(m->externup) {
if((e = m->externup->edf) && (e->flags & Admitted))
edfrecord(m->externup);
m->qstart = 0;
m->qexpired = 0;
coherence();
m->proc = 0;
switch(m->externup->state) {
case Running:
ready(m->externup);
break;
case Moribund:
m->externup->state = Dead;
stopac();
edfstop(m->externup);
if (m->externup->edf)
free(m->externup->edf);
m->externup->edf = nil;
/*
* Holding locks from pexit:
* procalloc
* pga
*/
mmurelease(m->externup);
unlock(&pga);
psrelease(m->externup);
unlock(&procalloc);
break;
}
m->externup->mach = nil;
updatecpu(m->externup);
m->externup = nil;
}
sched();
}
/*
* Rendezvous with other cores. Set roles for those that came
* up online, and wait until they are initialized.
* Sync TSC with them.
* We assume other processors that could boot had time to
* set online to 1 by now.
*/
static void
nixsquids(void)
{
Mach *m = machp();
Mach *mp;
int i;
uint64_t now, start;
/* Not AC for now :-) */
for(i = 1; i <= MACHMAX; i++)
//for(i = 1; i < MACHMAX; i++)
if((mp = sys->machptr[i]) != nil && mp->online){
/*
* Inter-core calls. A ensure *mp->iccall and mp->icargs
* go into different cache lines.
*/
mp->icc = mallocalign(sizeof *m->icc, ICCLNSZ, 0, 0);
mp->icc->fn = nil;
if(i < numtcs){
sys->nmach++;
mp->nixtype = NIXTC;
sys->nc[NIXTC]++;
}//else
//sys->nc[NIXAC]++;
ainc(&active.nbooting);
}
sys->epoch = rdtsc();
mfence();
wrmsr(0x10, sys->epoch);
m->rdtsc = rdtsc();
active.thunderbirdsarego = 1;
start = fastticks2us(fastticks(nil));
do{
now = fastticks2us(fastticks(nil));
}while(active.nbooting > 0 && now - start < 1000000)
;
if(active.nbooting > 0)
print("cpu0: %d cores couldn't start\n", active.nbooting);
active.nbooting = 0;
}
int
canlock(Lock *l)
{
Proc *up = externup();
if(up)
ainc(&up->nlocks);
if(TAS(&l->key)){
if(up)
adec(&up->nlocks);
return 0;
}
if(up)
up->lastlock = l;
l->_pc = getcallerpc();
l->p = up;
l->isilock = 0;
if(LOCKCYCLES)
cycles(&l->lockcycles);
return 1;
}
void
squidboy(int apicno)
{
char *n[] = {
[NIXAC] "AC",
[NIXTC] "TC",
[NIXKC] "KC"
};
vlong hz;
sys->machptr[m->machno] = m;
setmachsched(m);
/*
* Need something for initial delays
* until a timebase is worked out.
*/
m->cpuhz = 2000000000ll;
m->cpumhz = 2000;
m->perf.period = 1;
m->nixtype = NIXAC;
DBG("Hello Squidboy %d %d\n", apicno, m->machno);
vsvminit(MACHSTKSZ, m->nixtype);
/*
* Beware the Curse of The Non-Interruptable Were-Temporary.
*/
hz = archhz();
if(hz == 0)
ndnr();
m->cpuhz = hz;
m->cyclefreq = hz;
m->cpumhz = hz/1000000ll;
mmuinit();
if(!apiconline())
ndnr();
fpuinit();
acmodeset(m->nixtype);
m->splpc = 0;
m->online = 1;
/*
* CAUTION: no time sync done, etc.
*/
DBG("Wait for the thunderbirds!\n");
while(!active.thunderbirdsarego)
;
wrmsr(0x10, sys->epoch);
m->rdtsc = rdtsc();
print("cpu%d color %d role %s tsc %lld\n",
m->machno, corecolor(m->machno), n[m->nixtype], m->rdtsc);
switch(m->nixtype){
case NIXAC:
acmmuswitch();
acinit();
adec(&active.nbooting);
ainc(&active.nonline); /* this was commented out */
acsched();
panic("squidboy");
break;
case NIXTC:
/*
* We only need the idt and syscall entry point actually.
* At boot time the boot processor might set our role after
* we have decided to become an AC.
*/
vsvminit(MACHSTKSZ, NIXTC);
/*
* Enable the timer interrupt.
*/
apicpri(0);
timersinit();
adec(&active.nbooting);
ainc(&active.nonline); /* this was commented out */
schedinit();
break;
}
panic("squidboy returns (type %d)", m->nixtype);
}
int
lock(Lock *l)
{
Proc *up = externup();
int i;
uintptr_t pc;
uint64_t t0;
pc = getcallerpc();
lockstats.locks++;
if(up)
ainc(&up->nlocks); /* prevent being scheded */
if(TAS(&l->key) == 0){
if(up)
up->lastlock = l;
l->_pc = pc;
l->p = up;
l->isilock = 0;
if(LOCKCYCLES)
cycles(&l->lockcycles);
return 0;
}
if(up)
adec(&up->nlocks);
cycles(&t0);
lockstats.glare++;
for(;;){
lockstats.inglare++;
i = 0;
while(l->key){
if(sys->nmach < 2 && up && up->edf && (up->edf->flags & Admitted)){
/*
* Priority inversion, yield on a uniprocessor; on a
* multiprocessor, the other processor will unlock
*/
print("inversion %#p pc %#p proc %d held by pc %#p proc %d\n",
l, pc, up ? up->pid : 0, l->_pc, l->p ? l->p->pid : 0);
up->edf->d = todget(nil); /* yield to process with lock */
}
if(i++ > 100000000){
i = 0;
lockloop(l, pc);
}
}
if(up)
ainc(&up->nlocks);
if(TAS(&l->key) == 0){
if(up)
up->lastlock = l;
l->_pc = pc;
l->p = up;
l->isilock = 0;
if(LOCKCYCLES)
cycles(&l->lockcycles);
if(l != &waitstatslk)
addwaitstat(pc, t0, WSlock);
return 1;
}
if(up)
adec(&up->nlocks);
}
}
/*
* Get a file data block, perhaps allocating it on demand
* if mkit. The file must be r/wlocked and melted if mkit.
*
* Adds disk refs for dir entries copied during melts and
* considers that /archive is always melted.
*
* Read-ahead is not considered here. The file only records
* the last accessed block number, to help the caller do RA.
*
*/
static Memblk*
dfblk(Memblk *f, ulong bno, int mkit)
{
ulong prev, nblks;
int i, idx, nindir, type, isdir, chg;
Memblk *b, *pb;
daddrt *addrp;
if(mkit)
ismelted(f);
isdir = (f->d.mode&DMDIR);
if(bno != f->mf->lastbno){
f->mf->sequential = (!mkit && bno == f->mf->lastbno + 1);
f->mf->lastbno = bno;
}
/*
* bno: block # relative to the the block we are looking at.
* prev: # of blocks before the current one.
*/
prev = 0;
chg = 0;
/*
* Direct block?
*/
if(bno < nelem(f->d.dptr)){
if(mkit)
b = getmelted(isdir, DBdata, &f->d.dptr[bno], &chg);
else
b = dbget(DBdata, f->d.dptr[bno]);
if(chg)
changed(f);
return b;
}
bno -= nelem(f->d.dptr);
prev += nelem(f->d.dptr);
/*
* Indirect block
* nblks: # of data blocks addressed by the block we look at.
*/
nblks = Dptrperblk;
for(i = 0; i < nelem(f->d.iptr); i++){
if(bno < nblks)
break;
bno -= nblks;
prev += nblks;
nblks *= Dptrperblk;
}
if(i == nelem(f->d.iptr))
error("offset exceeds file capacity");
ainc(&fs->nindirs[i]);
type = DBptr0+i;
dFprint("dfblk: indirect %s nblks %uld (ppb %ud) bno %uld\n",
tname(type), nblks, Dptrperblk, bno);
addrp = &f->d.iptr[i];
if(mkit)
b = getmelted(isdir, type, addrp, &chg);
else
b = dbget(type, *addrp);
if(chg)
changed(f);
pb = b;
if(catcherror()){
mbput(pb);
error(nil);
}
/* at the loop header:
* pb: parent of b
* b: DBptr block we are looking at.
* addrp: ptr to b within fb.
* nblks: # of data blocks addressed by b
*/
for(nindir = i+1; nindir >= 0; nindir--){
chg = 0;
dFprint("indir %s d%#ullx nblks %uld ptrperblk %d bno %uld\n\n",
tname(DBdata+nindir), *addrp, nblks, Dptrperblk, bno);
idx = 0;
if(nindir > 0){
nblks /= Dptrperblk;
idx = bno/nblks;
}
if(*addrp == 0 && !mkit){
/* hole */
fprint(2, "HOLE\n");
b = nil;
}else{
assert(type >= DBdata);
if(mkit)
b = getmelted(isdir, type, addrp, &chg);
else
b = dbget(type, *addrp);
if(chg)
changed(pb);
addrp = &b->d.ptr[idx];
}
mbput(pb);
pb = b;
USED(&b); /* force to memory in case of error */
USED(&pb); /* force to memory in case of error */
bno -= idx * nblks;
prev += idx * nblks;
type--;
}
noerror();
return b;
}
/*
* Caller walked down p, and now requires the nth element to be
* melted, and wlocked for writing. (nth count starts at 1);
*
* Return the path with the version of f that we must use,
* locked for writing and melted.
* References kept in the path are traded for the ones returned.
*/
Path*
dfmelt(Path **pp, int nth)
{
int i;
Memblk *f, **fp, *nf;
Path *p;
ownpath(pp);
p = *pp;
assert(nth >= 1 && p->nf >= nth && p->nf >= 2);
assert(p->f[0] == fs->root);
fp = &p->f[nth-1];
/*
* 1. Optimistic: Try to get a loaded melted version for f.
*/
followmelted(fp, Wr);
f = *fp;
if(!f->frozen)
return p;
ainc(&fs->nmelts);
rwunlock(f, Wr);
/*
* 2. Realistic:
* walk down the path, melting every frozen thing until we
* reach f. Keep wlocks so melted files are not frozen while we walk.
* /active is special, because it's only frozen temporarily while
* creating a frozen version of the tree. Instead of melting it,
* we should just wait for it.
* p[0] is /
* p[1] is /active
*/
for(;;){
followmelted(&p->f[1], Wr);
if(p->f[1]->frozen == 0)
break;
rwunlock(p->f[1], Wr);
yield();
}
/*
* At loop header, parent is p->f[i-1], melted and wlocked.
* At the end of the loop, p->f[i] is melted and wlocked.
*/
for(i = 2; i < nth; i++){
followmelted(&p->f[i], Wr);
if(!p->f[i]->frozen){
rwunlock(p->f[i-1], Wr);
continue;
}
if(catcherror()){
rwunlock(p->f[i-1], Wr);
rwunlock(p->f[i], Wr);
error(nil);
}
nf = dbdup(p->f[i]);
rwlock(nf, Wr);
if(catcherror()){
rwunlock(nf, Wr);
mbput(nf);
error(nil);
}
dfchdentry(p->f[i-1], p->f[i]->addr, nf->addr, Mkit);
noerror();
noerror();
/* committed */
rwunlock(p->f[i-1], Wr); /* parent */
rwunlock(p->f[i], Wr); /* old frozen version */
f = p->f[i];
p->f[i] = nf;
assert(f->ref > 1);
mbput(f); /* ref from path */
if(!catcherror()){
dbput(f, f->type, f->addr); /* p->f[i] ref from disk */
noerror();
}
}
return p;
}
void
incref(Ref *r)
{
ainc(&r->ref);
}
void
squidboy(int apicno, Mach *m)
{
// FIX QEMU. extern int64_t hz;
int64_t hz;
sys->machptr[m->machno] = m;
/*
* Need something for initial delays
* until a timebase is worked out.
*/
m->cpuhz = 2000000000ll;
m->cpumhz = 2000;
m->perf.period = 1;
m->nixtype = NIXAC;
// no NIXAC for now.
m->nixtype = NIXTC;
// NOTE: you can't do ANYTHING here before vsvminit.
// PRINT WILL PANIC. So wait.
vsvminit(MACHSTKSZ, m->nixtype, m);
//DBG("Hello squidboy %d %d\n", apicno, m->machno);
/*
* Beware the Curse of The Non-Interruptable Were-Temporary.
*/
hz = archhz();
/* Intel cpu's in archk10 must be reviewed */
if(hz == 0)
hz = 2000000000ll;
m->cpuhz = hz;
m->cyclefreq = hz;
m->cpumhz = hz/1000000ll;
mmuinit();
if(!apiconline())
ndnr();
fpuinit();
acmodeset(m->nixtype);
m->splpc = 0;
m->online = 1;
/*
* CAUTION: no time sync done, etc.
* Stupid print to avoid up = nil or
* last cpu couldn't start in nixquids.
*/
DBG("Wait for the thunderbirds!\n");
while(!active.thunderbirdsarego)
;
wrmsr(0x10, sys->epoch);
m->rdtsc = rdtsc();
print("cpu%d color %d role %s tsc %lld\n",
m->machno, corecolor(m->machno), rolename[m->nixtype], m->rdtsc);
switch(m->nixtype){
case NIXAC:
acmmuswitch();
acinit();
adec(&active.nbooting);
ainc(&active.nonline); /* this was commented out */
acsched();
panic("squidboy");
break;
case NIXTC:
/*
* We only need the idt and syscall entry point actually.
* At boot time the boot processor might set our role after
* we have decided to become an AC.
*/
vsvminit(MACHSTKSZ, NIXTC, m);
/*
* Enable the timer interrupt.
*/
apictimerenab();
apicpri(0);
timersinit();
adec(&active.nbooting);
ainc(&active.nonline);
/* Ready? steady? going to timer */
ndnr();
schedinit();
break;
}
panic("squidboy returns (type %d)", m->nixtype);
}