/*
* Called from imagereclaim, to try to release Images.
* The argument shows the preferred image to release pages from.
* All images will be tried, from lru to mru.
*/
uint64_t
pagereclaim(Image *i)
{
Page *p;
uint64_t ticks;
lock(&pga.l);
ticks = fastticks(nil);
/*
* All the pages with images backing them are at the
* end of the list (see putpage) so start there and work
* backward.
*/
for(p = pga.pgsza[0].tail; p && p->image == i; p = p->prev){
if(p->ref == 0 && canlock(&p->l)){
if(p->ref == 0) {
uncachepage(p);
}
unlock(&p->l);
}
}
ticks = fastticks(nil) - ticks;
unlock(&pga.l);
return ticks;
}
static int64_t
tadd(Timers *tt, Timer *nt)
{
int64_t when;
Timer *t, **last;
/* Called with tt locked */
assert(nt->tt == nil);
switch(nt->tmode){
default:
panic("timer");
break;
case Trelative:
if(nt->tns <= 0)
nt->tns = 1;
nt->twhen = fastticks(nil) + ns2fastticks(nt->tns);
break;
case Tperiodic:
/*
* Periodic timers must have a period of at least 100µs.
*/
assert(nt->tns >= 100000);
if(nt->twhen == 0){
/*
* Look for another timer at the
* same frequency for combining.
*/
for(t = tt->head; t; t = t->tnext){
if(t->tmode == Tperiodic && t->tns == nt->tns)
break;
}
if(t)
nt->twhen = t->twhen;
else
nt->twhen = fastticks(nil);
}
/*
* The new time must be in the future.
* ns2fastticks() can return 0 if the tod clock
* has been adjusted by, e.g. timesync.
*/
when = ns2fastticks(nt->tns);
if(when == 0)
when = 1;
nt->twhen += when;
break;
}
for(last = &tt->head; t = *last; last = &t->tnext){
if(t->twhen > nt->twhen)
break;
}
nt->tnext = *last;
*last = nt;
nt->tt = tt;
if(last == &tt->head)
return nt->twhen;
return 0;
}
ulong
µs(void)
{
if(SystimerFreq != 1*Mhz)
return fastticks2us(fastticks(nil));
return fastticks(nil);
}
/*
* Rendezvous with other cores. Set roles for those that came
* up online, and wait until they are initialized.
* Sync TSC with them.
* We assume other processors that could boot had time to
* set online to 1 by now.
*/
static void
nixsquids(void)
{
Mach *mp;
int i;
uvlong now, start;
for(i = 1; i < MACHMAX; i++)
if((mp = sys->machptr[i]) != nil && mp->online != 0){
/*
* Inter-core calls. A ensure *mp->iccall and mp->icargs
* go into different cache lines.
*/
mp->icc = mallocalign(sizeof *m->icc, ICCLNSZ, 0, 0);
mp->icc->fn = nil;
if(i < initialTCs){
conf.nmach++;
mp->nixtype = NIXTC;
}
ainc(&active.nbooting);
}
sys->epoch = rdtsc();
mfence();
wrmsr(0x10, sys->epoch);
m->rdtsc = rdtsc();
active.thunderbirdsarego = 1;
start = fastticks2us(fastticks(nil));
do{
now = fastticks2us(fastticks(nil));
}while(active.nbooting > 0 && now - start < 1000000)
;
if(active.nbooting > 0)
print("cpu0: %d cores couldn't start\n", active.nbooting);
active.nbooting = 0;
}
static int
chanwait(Ep *ep, Ctlr *ctlr, Hostchan *hc, int mask)
{
int intr, n, ointr;
ulong start, now;
Dwcregs *r;
r = ctlr->regs;
n = hc - r->hchan;
for(;;){
restart:
filock(ctlr);
r->haintmsk |= 1<<n;
hc->hcintmsk = mask;
fiunlock(ctlr);
tsleep(&ctlr->chanintr[n], chandone, hc, 1000);
if((intr = hc->hcint) == 0)
goto restart;
hc->hcintmsk = 0;
if(intr & Chhltd)
return intr;
start = fastticks(0);
ointr = intr;
now = start;
do{
intr = hc->hcint;
if(intr & Chhltd){
if((ointr != Ack && ointr != (Ack|Xfercomp)) ||
intr != (Ack|Chhltd|Xfercomp) ||
(now - start) > 60)
dprint("await %x after %ldµs %x -> %x\n",
mask, now - start, ointr, intr);
return intr;
}
if((intr & mask) == 0){
if(intr != Nak)
dprint("ep%d.%d await %x after %ldµs intr %x -> %x\n",
ep->dev->nb, ep->nb, mask, now - start, ointr, intr);
goto restart;
}
now = fastticks(0);
}while(now - start < 100);
dprint("ep%d.%d halting channel %8.8ux hcchar %8.8ux "
"grxstsr %8.8ux gnptxsts %8.8ux hptxsts %8.8ux\n",
ep->dev->nb, ep->nb, intr, hc->hcchar, r->grxstsr,
r->gnptxsts, r->hptxsts);
mask = Chhltd;
hc->hcchar |= Chdis;
start = m->ticks;
while(hc->hcchar & Chen){
if(m->ticks - start >= 100){
print("ep%d.%d channel won't halt hcchar %8.8ux\n",
ep->dev->nb, ep->nb, hc->hcchar);
break;
}
}
logdump(ep);
}
}
static void
imagereclaim(void)
{
Image *i;
uvlong ticks0, ticks;
irstats.calls++;
/* Somebody is already cleaning the page cache */
if(!canqlock(&imagealloc.ireclaim))
return;
DBG("imagereclaim maxt %ulld noluck %d nolock %d\n",
irstats.maxt, irstats.noluck, irstats.nolock);
ticks0 = fastticks(nil);
if(!canlock(&imagealloc)){
/* never happen in the experiments I made */
qunlock(&imagealloc.ireclaim);
return;
}
for(i = imagealloc.lru; i != nil; i = i->prev){
if(canlock(i)){
i->ref++; /* make sure it does not go away */
unlock(i);
pagereclaim(i);
lock(i);
DBG("imagereclaim: image %p(c%p, r%d)\n", i, i->c, i->ref);
if(i->ref == 1){ /* no pages referring to it, it's ours */
unlock(i);
unlock(&imagealloc);
putimage(i);
break;
}else
--i->ref;
unlock(i);
}
}
if(i == nil){
irstats.noluck++;
unlock(&imagealloc);
}
irstats.loops++;
ticks = fastticks(nil) - ticks0;
irstats.ticks += ticks;
if(ticks > irstats.maxt)
irstats.maxt = ticks;
//print("T%llud+", ticks);
qunlock(&imagealloc.ireclaim);
}
void
timerset(uvlong next)
{
Systimers *tn;
vlong now, period;
tn = (Systimers*)SYSTIMERS;
now = fastticks(nil);
period = next - fastticks(nil);
if(period < MinPeriod)
next = now + MinPeriod;
else if(period > MaxPeriod)
next = now + MaxPeriod;
tn->c3 = (ulong)next;
}
/*
* Set the time of day struct
*/
void
todset(vlong t, vlong delta, int n)
{
if(!tod.init)
todinit();
ilock(&tod);
if(t >= 0){
tod.off = t;
tod.last = fastticks(nil);
tod.lasttime = 0;
tod.delta = 0;
tod.sstart = tod.send;
} else {
if(n <= 0)
n = 1;
n *= HZ;
if(delta < 0 && n > -delta)
n = -delta;
if(delta > 0 && n > delta)
n = delta;
if (n == 0) {
iprint("todset: n == 0, delta == %lld\n", delta);
delta = 0;
} else
delta /= n;
tod.sstart = MACHP(0)->ticks;
tod.send = tod.sstart + n;
tod.delta = delta;
}
iunlock(&tod);
}
ulong
µs(void)
{
if(SystimerFreq != 1*Mhz)
return fastticks2us(fastticks(nil));
return ((Systimers*)SYSTIMERS)->clo;
}
/*
* called regularly to avoid calculation overflows
*/
static void
todfix(void)
{
vlong ticks, diff;
uvlong x;
ticks = fastticks(nil);
diff = ticks - tod.last;
if(diff <= tod.hz)
return;
ilock(&tod);
diff = ticks - tod.last;
if(diff > tod.hz){
/* convert to epoch */
mul64fract(&x, diff, tod.multiplier);
if(x > 30000000000ULL) iprint("todfix %llud\n", x);
x += tod.off;
/* protect against overflows */
tod.last = ticks;
tod.off = x;
}
iunlock(&tod);
}
/*
* read binary time info. all numbers are little endian.
* ticks and nsec are syncronized.
*/
static int
readbintime(char *buf, int n)
{
int i;
vlong nsec, ticks;
uchar *b = (uchar*)buf;
i = 0;
if(fasthz == 0LL)
fastticks((uvlong*)&fasthz);
nsec = todget(&ticks);
if(n >= 3*sizeof(uvlong)){
vlong2le(b+2*sizeof(uvlong), fasthz);
i += sizeof(uvlong);
}
if(n >= 2*sizeof(uvlong)){
vlong2le(b+sizeof(uvlong), ticks);
i += sizeof(uvlong);
}
if(n >= 8){
vlong2le(b, nsec);
i += sizeof(vlong);
}
return i;
}
/*
* read binary time info. all numbers are little endian.
* ticks and nsec are syncronized.
*/
static int
readbintime(char *buf, int n)
{
int i;
int64_t nsec, ticks;
uint8_t *b = (uint8_t*)buf;
i = 0;
if(fasthz == 0LL)
fastticks((uint64_t*)&fasthz);
nsec = todget(&ticks);
if(n >= 3*sizeof(uint64_t)){
int64_t2le(b+2*sizeof(uint64_t), fasthz);
i += sizeof(uint64_t);
}
if(n >= 2*sizeof(uint64_t)){
int64_t2le(b+sizeof(uint64_t), ticks);
i += sizeof(uint64_t);
}
if(n >= 8){
int64_t2le(b, nsec);
i += sizeof(int64_t);
}
return i;
}
/*
* Set the time of day struct
*/
void
todset(int64_t t, int64_t delta, int n)
{
if(!tod.init)
todinit();
ilock(&tod.Lock);
if(t >= 0){
tod.off = t;
tod.last = fastticks(nil);
tod.lasttime = 0;
tod.delta = 0;
tod.sstart = tod.send;
} else {
if(n <= 0)
n = 1;
n *= HZ;
if(delta < 0 && n > -delta)
n = -delta;
if(delta > 0 && n > delta)
n = delta;
delta = delta/n;
tod.sstart = sys->ticks;
tod.send = tod.sstart + n;
tod.delta = delta;
}
iunlock(&tod.Lock);
}
void
acsyscall(void)
{
panic("acsyscall");
#if 0
Proc *p;
/*
* If we saved the Ureg into m->proc->dbgregs,
* There's nothing else we have to do.
* Otherwise, we should m->proc->dbgregs = u;
*/
DBG("acsyscall: cpu%d\n", machp()->machno);
_pmcupdate(m);
p = m->proc;
p->actime1 = fastticks(nil);
m->syscall++; /* would also count it in the TS core */
m->icc->rc = ICCSYSCALL;
m->cr2 = cr2get();
fpuprocsave(p);
_pmcupdate(m);
mfence();
m->icc->fn = nil;
ready(p);
/*
* The next call is probably going to make us jmp
* into user code, forgetting all our state in this
* stack, upon the next syscall.
* We don't nest calls in the current stack for too long.
*/
acsched();
#endif
}
/*
* read binary time info. all numbers are little endian.
* ticks and nsec are syncronized.
*/
static int readbintime(char *buf, int n)
{
int i;
int64_t nsec, ticks;
uint8_t *b = (uint8_t *) buf;
i = 0;
if (fasthz == 0LL)
fasthz = system_timing.tsc_freq;
#if 0
fastticks((uint64_t *) & fasthz);
nsec = todget(&ticks);
#endif
ticks = read_tsc();
nsec = tsc2nsec(ticks);
if (n >= 3 * sizeof(uint64_t)) {
int64_t2le(b + 2 * sizeof(uint64_t), fasthz);
i += sizeof(uint64_t);
}
if (n >= 2 * sizeof(uint64_t)) {
int64_t2le(b + sizeof(uint64_t), ticks);
i += sizeof(uint64_t);
}
if (n >= 8) {
int64_t2le(b, nsec);
i += sizeof(int64_t);
}
return i;
}
static vlong
tadd(Timers *tt, Timer *nt)
{
Timer *t, **last;
/* Called with tt locked */
assert(nt->tt == nil);
switch(nt->tmode){
default:
panic("timer");
break;
case Trelative:
if(nt->tns <= 0)
nt->tns = 1;
nt->twhen = fastticks(nil) + ns2fastticks(nt->tns);
break;
case Tperiodic:
assert(nt->tns >= 100000); /* At least 100 µs period */
if(nt->twhen == 0){
/* look for another timer at same frequency for combining */
for(t = tt->head; t; t = t->tnext){
if(t->tmode == Tperiodic && t->tns == nt->tns)
break;
}
if (t)
nt->twhen = t->twhen;
else
nt->twhen = fastticks(nil);
}
nt->twhen += ns2fastticks(nt->tns);
break;
}
for(last = &tt->head; t = *last; last = &t->tnext){
if(t->twhen > nt->twhen)
break;
}
nt->tnext = *last;
*last = nt;
nt->tt = tt;
if(last == &tt->head)
return nt->twhen;
return 0;
}
void
timerkproc(void *v)
{
sigset_t sigs;
Timers *tt;
Timer *t;
uvlong when, now;
struct timespec ts;
Ureg u;
int signo;
memset(&u, 0, sizeof u);
timer_pid = pthread_self();
tt = &timers;
ilock(&tt->lk);
for(;;){
if((t = tt->head) == nil){
iunlock(&tt->lk);
sigemptyset(&sigs);
sigaddset(&sigs, SIGURG);
sigwait(&sigs, &signo);
ilock(&tt->lk);
continue;
}
/*
* No need to ilock t here: any manipulation of t
* requires tdel(t) and this must be done with a
* lock to tt held. We have tt, so the tdel will
* wait until we're done
*/
now = fastticks(nil);
when = t->twhen;
if(when > now){
iunlock(&tt->lk);
when -= now;
ts.tv_sec = when/1000000000;
ts.tv_nsec = when%1000000000;
pthread_sigmask(SIG_SETMASK, nil, &sigs);
sigdelset(&sigs, SIGURG);
pselect(0, nil, nil, nil, &ts, &sigs);
ilock(&tt->lk);
continue;
}
tt->head = t->tnext;
assert(t->tt == tt);
t->tt = nil;
iunlock(&tt->lk);
(*t->tf)(&u, t);
ilock(&tt->lk);
if(t->tmode == Tperiodic)
tadd(tt, t);
}
}
/*
* Called in AP core context, to return from system call.
*/
void
acsysret(void)
{
panic("acsysret");
#if 0
DBG("acsysret\n");
if(m->proc != nil)
m->proc->actime += fastticks2us(fastticks(nil) - m->proc->actime1);
_acsysret();
#endif
}
void
randominit(void)
{
#ifdef USE_RANDOM
srandom(getpid()+fastticks(nil)+ticks());
#else
if((randfd = open("/dev/urandom", OREAD)) < 0)
if((randfd = open("/dev/random", OREAD)) < 0)
panic("open /dev/random: %r");
#endif
}
void
todinit(void)
{
if(tod.init)
return;
ilock(&tod);
tod.init = 1; /* prevent reentry via fastticks */
tod.last = fastticks((uvlong *)&tod.hz);
iunlock(&tod);
todsetfreq(tod.hz);
addclock0link(todfix, 100);
}
void
todinit(void)
{
if(tod.init)
return;
ilock(&tod.Lock);
tod.last = fastticks((uint64_t *)&tod.hz);
iunlock(&tod.Lock);
todsetfreq(tod.hz);
tod.init = 1;
addclock0link(todfix, 100);
}
/*
* Rendezvous with other cores. Set roles for those that came
* up online, and wait until they are initialized.
* Sync TSC with them.
* We assume other processors that could boot had time to
* set online to 1 by now.
*/
static void
nixsquids(void)
{
Mach *m = machp();
Mach *mp;
int i;
uint64_t now, start;
/* Not AC for now :-) */
for(i = 1; i <= MACHMAX; i++)
//for(i = 1; i < MACHMAX; i++)
if((mp = sys->machptr[i]) != nil && mp->online){
/*
* Inter-core calls. A ensure *mp->iccall and mp->icargs
* go into different cache lines.
*/
mp->icc = mallocalign(sizeof *m->icc, ICCLNSZ, 0, 0);
mp->icc->fn = nil;
if(i < numtcs){
sys->nmach++;
mp->nixtype = NIXTC;
sys->nc[NIXTC]++;
}//else
//sys->nc[NIXAC]++;
ainc(&active.nbooting);
}
sys->epoch = rdtsc();
mfence();
wrmsr(0x10, sys->epoch);
m->rdtsc = rdtsc();
active.thunderbirdsarego = 1;
start = fastticks2us(fastticks(nil));
do{
now = fastticks2us(fastticks(nil));
}while(active.nbooting > 0 && now - start < 1000000)
;
if(active.nbooting > 0)
print("cpu0: %d cores couldn't start\n", active.nbooting);
active.nbooting = 0;
}
void
timerintr(Ureg *u, Tval)
{
Timer *t;
Timers *tt;
uvlong when, now;
int count, callhzclock;
intrcount[m->machno]++;
callhzclock = 0;
tt = &timers[m->machno];
now = fastticks(nil);
if(now == 0)
panic("timerintr: zero fastticks()");
ilock(tt);
count = Maxtimerloops;
while((t = tt->head) != nil){
/*
* No need to ilock t here: any manipulation of t
* requires tdel(t) and this must be done with a
* lock to tt held. We have tt, so the tdel will
* wait until we're done
*/
when = t->twhen;
if(when > now){
timerset(when);
iunlock(tt);
if(callhzclock)
hzclock(u);
return;
}
tt->head = t->tnext;
assert(t->tt == tt);
t->tt = nil;
fcallcount[m->machno]++;
iunlock(tt);
if(t->tf)
(*t->tf)(u, t);
else
callhzclock++;
ilock(tt);
if(t->tmode == Tperiodic)
tadd(tt, t);
if (--count <= 0) {
count = Maxtimerloops;
iprint("timerintr: probably stuck in while loop; "
"scrutinise clock.c or use faster cycle "
"counter\n");
}
}
iunlock(tt);
}
static void
imagereclaim(void)
{
int n;
Page *p;
uvlong ticks;
irstats.calls++;
/* Somebody is already cleaning the page cache */
if(!canqlock(&imagealloc.ireclaim))
return;
lock(&palloc);
ticks = fastticks(nil);
n = 0;
/*
* All the pages with images backing them are at the
* end of the list (see putpage) so start there and work
* backward.
*/
for(p = palloc.tail; p && p->image && (n<1000 || !imagealloc.free); p = p->prev) {
if(p->ref == 0 && canlock(p)) {
if(p->ref == 0 && p->image && !p->image->notext) {
n++;
uncachepage(p);
}
unlock(p);
}
}
ticks = fastticks(nil) - ticks;
unlock(&palloc);
irstats.loops++;
irstats.ticks += ticks;
if(ticks > irstats.maxt)
irstats.maxt = ticks;
//print("T%llud+", ticks);
qunlock(&imagealloc.ireclaim);
}
/*
* get time of day
*/
vlong
todget(vlong *ticksp)
{
uvlong x;
vlong ticks, diff;
ulong t;
if(!tod.init)
todinit();
/*
* we don't want time to pass twixt the measuring of fastticks
* and grabbing tod.last. Also none of the vlongs are atomic so
* we have to look at them inside the lock.
*/
ilock(&tod);
tod.cnt++;
ticks = fastticks(nil);
/* add in correction */
if(tod.sstart != tod.send){
t = MACHP(0)->ticks;
if(t >= tod.send)
t = tod.send;
tod.off = tod.off + tod.delta*(t - tod.sstart);
tod.sstart = t;
}
/* convert to epoch */
diff = ticks - tod.last;
if(diff < 0)
diff = 0;
mul64fract(&x, diff, tod.multiplier);
x += tod.off;
/* time can't go backwards */
if(x < tod.lasttime)
x = tod.lasttime;
else
tod.lasttime = x;
iunlock(&tod);
if(ticksp != nil)
*ticksp = ticks;
return x;
}
/* go to user space */
void
kexit(Ureg*)
{
uvlong t;
Tos *tos;
/* precise time accounting, kernel exit */
tos = (Tos*)(USTKTOP-sizeof(Tos));
t = fastticks(nil);
tos->kcycles += t - up->kentry;
tos->pcycles = up->pcycles;
tos->cyclefreq = Frequency;
tos->pid = up->pid;
/* make visible immediately to user proc */
cachedwbinvse(tos, sizeof *tos);
}
void
timerset(Tval next)
{
long offset;
Timerregs *tn = (Timerregs *)Tn1;
static Lock setlck;
ilock(&setlck);
offset = next - fastticks(nil);
if(offset < MinPeriod)
offset = MinPeriod;
else if(offset > MaxPeriod)
offset = MaxPeriod;
tn->tcrr = -offset;
coherence();
iunlock(&setlck);
}
/*
* Tval is supposed to be in fastticks units.
* One fasttick unit is 1/Basetickfreq seconds.
*/
void
timerset(Tval next)
{
int x;
long period;
if(next == 0)
return;
x = splhi(); /* don't let us get scheduled */
period = next - fastticks(nil);
if(period > m->maxperiod - m->minperiod)
period = m->maxperiod;
else if(period < m->minperiod)
period = m->minperiod;
wrcompare(rdcount()+period);
splx(x);
}
/*
* like the old #c/time but with added info. Return
*
* secs nanosecs fastticks fasthz
*/
static int
readtime(uint32_t off, char *buf, int n)
{
int64_t nsec, ticks;
int32_t sec;
char str[7*NUMSIZE];
nsec = todget(&ticks);
if(fasthz == 0LL)
fastticks((uint64_t*)&fasthz);
sec = nsec/1000000000ULL;
snprint(str, sizeof(str), "%*lu %*llu %*llu %*llu ",
NUMSIZE-1, sec,
VLNUMSIZE-1, nsec,
VLNUMSIZE-1, ticks,
VLNUMSIZE-1, fasthz);
return readstr(off, buf, n, str);
}
/*
* like the old #c/time but with added info. Return
*
* secs nanosecs fastticks fasthz
*/
static int
readtime(ulong off, char *buf, int n)
{
vlong nsec, ticks;
long sec;
char str[7*NUMSIZE];
nsec = todget(&ticks);
if(fasthz == 0LL)
fastticks((uvlong*)&fasthz);
sec = nsec/1000000000ULL;
snprint(str, sizeof(str), "%*lud %*llud %*llud %*llud ",
NUMSIZE-1, sec,
VLNUMSIZE-1, nsec,
VLNUMSIZE-1, ticks,
VLNUMSIZE-1, fasthz);
return readstr(off, buf, n, str);
}