DRAMBufferBlock* FairAllocator::allocate_one_page( unsigned process_id )
{
futex_lock(&pool_lock);
assert(!global_clean_pool.empty()|| !global_dirty_pool.empty())
Address alloc_block_id = INVALID_PAGE_ADDR;
if( !global_clean_pool.empty())
{
alloc_block_id = global_clean_pool.front();
global_clean_pool.pop_front();
}
else if( !global_dirty_pool.empty())
{
alloc_block_id = global_dirty_pool.front();
global_dirty_pool.pop_front();
}
if( alloc_block_id != INVALID_PAGE_ADDR )
{
clean_pools[process_id].insert( alloc_block_id);
busy_pages++;
proc_busy_pages[process_id]++;
futex_unlock(&pool_lock);
return buffer_array[alloc_block_id];
}
futex_unlock(&pool_lock);
return NULL;
}
/*
* @function:
* @param:
*/
void FairAllocator::convert_to_dirty( unsigned process_id , Address block_id )
{
futex_lock(&pool_lock);
if( clean_pools[process_id].find(block_id) == clean_pools[process_id].end())
{
futex_unlock(&pool_lock);
return;
}
else
{
clean_pools[process_id].erase(block_id);
dirty_pools[process_id].insert(block_id);
}
futex_unlock(&pool_lock);
}
static void
futex_put(struct futex *f, struct waiting_proc *wp)
{
LIN_SDT_PROBE2(futex, futex_put, entry, f, wp);
if (wp != NULL) {
if ((wp->wp_flags & FUTEX_WP_REMOVED) == 0)
TAILQ_REMOVE(&f->f_waiting_proc, wp, wp_list);
free(wp, M_FUTEX_WP);
}
FUTEXES_LOCK;
if (--f->f_refcount == 0) {
LIST_REMOVE(f, f_list);
FUTEXES_UNLOCK;
if (FUTEX_LOCKED(f))
futex_unlock(f);
LIN_SDT_PROBE3(futex, futex_put, destroy, f->f_uaddr,
f->f_refcount, f->f_key.shared);
LINUX_CTR3(sys_futex, "futex_put destroy uaddr %p ref %d "
"shared %d", f->f_uaddr, f->f_refcount, f->f_key.shared);
umtx_key_release(&f->f_key);
FUTEX_DESTROY(f);
free(f, M_FUTEX);
LIN_SDT_PROBE0(futex, futex_put, return);
return;
}
uint64_t Scheduler::getFutexWakePhase(bool realtime, FutexInfo fi, CONTEXT* ctxt, SYSCALL_STANDARD std) {
int64_t waitNsec = 0;
uint64_t wakeupPhase = 0;
waitNsec = fi.timeout.tv_sec*1000000000L + fi.timeout.tv_nsec;
if (fi.op & FUTEX_CLOCK_REALTIME || realtime) {
uint32_t domain = zinfo->procArray[procIdx]->getClockDomain();
uint64_t simNs = cyclesToNs(zinfo->globPhaseCycles);
uint64_t offsetNs = simNs + zinfo->clockDomainInfo[domain].realtimeOffsetNs;
warn(" REALTIME FUTEX: %ld %ld %ld %ld", waitNsec, simNs, offsetNs, waitNsec-offsetNs);
waitNsec = (waitNsec > (int64_t)offsetNs)? (waitNsec - offsetNs) : 0;
}
if (waitNsec > 0) {
struct timespec fakeTimeouts = (struct timespec){0}; //Never timeout.
PIN_SetSyscallArgument(ctxt, std, 3, (ADDRINT)&fakeTimeouts);
uint64_t waitCycles = waitNsec*zinfo->freqMHz/1000;
uint64_t waitPhases = waitCycles/zinfo->phaseLength;
wakeupPhase = zinfo->numPhases + waitPhases;
}
return wakeupPhase;
}
bool Scheduler::futexSynchronized(uint32_t pid, uint32_t tid, FutexInfo fi) {
futex_lock(&schedLock);
uint32_t gid = getGid(pid, tid);
ThreadInfo* th = gidMap[gid];
futex_unlock(&schedLock);
while (true) {
int futex_res = syscall(SYS_futex, th->futexWord, FUTEX_WAIT, 1 /*a racing thread waking us up will change value to 0, and we won't block*/, nullptr, nullptr, 0);
if (futex_res == 0 || th->futexWord != 1) break;
}
join(pid, tid);
return true;
}
void gm_free(void* ptr) {
assert(GM);
assert(GM->mspace_ptr);
futex_lock(&GM->lock);
mspace_free(GM->mspace_ptr, ptr);
futex_unlock(&GM->lock);
}
unsigned FairAllocator::Release( unsigned process_id, unsigned evict_size )
{
futex_lock(&pool_lock);
//evict clean page firs<<std::endl;t
unsigned clean_pool_size = clean_pools[process_id].size();
unsigned clean_evict_size = (evict_size > clean_pool_size)? clean_pool_size:evict_size;
assert( clean_pool_size + dirty_pools[process_id].size()>= evict_size);
//reclaim clean pages
for( unsigned i=0 ; i < clean_evict_size; i++)
{
//if( *clean_pools[proc].)
Address block_id = *clean_pools[process_id].begin();
clean_pools[process_id].erase( block_id );
global_clean_pool.push_back(block_id);
busy_pages--;
}
if( clean_evict_size < evict_size)
{
//std::cout<<"evict dirty pages"<<std::endl;
//evict dirty pages
for( unsigned i = 0 ; i < (evict_size-clean_evict_size); i++ )
{
Address block_id = *dirty_pools[process_id].begin();
dirty_pools[process_id].erase(block_id);
global_dirty_pool.push_back(block_id);
busy_pages--;
}
}
proc_busy_pages[process_id] -= evict_size;
assert(should_reclaim()==false);
futex_unlock(&pool_lock);
return evict_size;
}
void Scheduler::notifyFutexWaitWoken(uint32_t pid, uint32_t tid) {
futex_lock(&schedLock);
ThreadInfo* th = gidMap[getGid(pid, tid)];
DEBUG_FUTEX("[%d/%d] waitWoken", pid, tid);
th->futexJoin = {FJA_WAIT, 0, 0};
futex_unlock(&schedLock);
}
void Scheduler::notifyFutexWakeEnd(uint32_t pid, uint32_t tid, uint32_t wokenUp) {
futex_lock(&schedLock);
ThreadInfo* th = gidMap[getGid(pid, tid)];
DEBUG_FUTEX("[%d/%d] wakeEnd woken %d", pid, tid, wokenUp);
th->futexJoin.action = FJA_WAKE;
th->futexJoin.wokenUp = wokenUp;
futex_unlock(&schedLock);
}
void* __gm_calloc(size_t num, size_t size) {
assert(GM);
assert(GM->mspace_ptr);
futex_lock(&GM->lock);
void* ptr = mspace_calloc(GM->mspace_ptr, num, size);
futex_unlock(&GM->lock);
if (!ptr) panic("gm_calloc(): Out of global heap memory, use a larger GM segment");
return ptr;
}
void* __gm_memalign(size_t blocksize, size_t bytes) {
assert(GM);
assert(GM->mspace_ptr);
futex_lock(&GM->lock);
void* ptr = mspace_memalign(GM->mspace_ptr, blocksize, bytes);
futex_unlock(&GM->lock);
if (!ptr) panic("gm_memalign(): Out of global heap memory, use a larger GM segment");
return ptr;
}
//Must be called without lock. Returns # waiters.
int Scheduler::futexWakeNWaiters(int bitmask, bool p_waiter, int *uaddr, int val) {
DEBUG_FUTEX("Scheduler: FUTEX WAKE N WAITERS called with bitmask %d pi %d uaddr %p val %d", bitmask, p_waiter, uaddr, val);
futex_lock(&schedLock);
int waitersToWake = 0;
int waitersFailed = 0;
bool piWakeInUserSpace = ((futexTable[uaddr].size() == 1) && futexTable[uaddr].front().pi_waiter);
assert(!piWakeInUserSpace)
if(p_waiter) {
for(int i = 1; (uint32_t)i < futexTable[uaddr].size(); i++) {
if(futexTable[uaddr][i].pi_waiter) {
FutexWaiter tempWaiter = futexTable[uaddr][i];
futexTable[uaddr].erase(futexTable[uaddr].begin() + waitersFailed);
futexTable[uaddr].push_front(tempWaiter);
DEBUG_FUTEX("we had a pi requestor jump to front of line");
uint32_t gid = getGid(tempWaiter.pid, tempWaiter.tid);
ThreadInfo* th = gidMap[gid];
assert(th->state == SLEEPING);
notifySleepEnd(tempWaiter.pid, tempWaiter.tid);
wakeup(th, true);
futex_unlock(&schedLock);
return 1;
}
}
}
while((uint32_t)waitersToWake < futexTable[uaddr].size() && waitersToWake < val) {
FutexWaiter tempWaiter = futexTable[uaddr][waitersFailed];
if(bitmask && tempWaiter.mask & ~bitmask) {
waitersFailed++;
} else {
waitersToWake++;
futexTable[uaddr].erase(futexTable[uaddr].begin() + waitersFailed);
}
uint32_t gid = getGid(tempWaiter.pid, tempWaiter.tid);
ThreadInfo* th = gidMap[gid];
assert(th->state == SLEEPING);
notifySleepEnd(tempWaiter.pid, tempWaiter.tid);
DEBUG_FUTEX("WAKE N finished sleep end");
wakeup(th, true);
DEBUG_FUTEX("WAKE N finished wakeup");
}
DEBUG_FUTEX("WAKE N WAITERS Looped %d times", waitersFailed + waitersToWake);
futex_unlock(&schedLock);
return waitersToWake;
}
__attribute__ ((__regparm__ (1))) ___tls_get_addr (struct tls_index *ti)
{
void *retval = (void*) vdl_tls_get_addr_fast (ti->ti_module, ti->ti_offset);
if (retval == 0)
{
futex_lock (g_vdl.futex);
retval = (void*) vdl_tls_get_addr_slow (ti->ti_module, ti->ti_offset);
futex_unlock (g_vdl.futex);
}
return retval;
}
EXPORT void *
__tls_get_addr (struct tls_index *ti)
{
void *retval = (void*) vdl_tls_get_addr_fast (ti->ti_module, ti->ti_offset);
if (retval == 0)
{
futex_lock (g_vdl.futex);
retval = (void*) vdl_tls_get_addr_slow (ti->ti_module, ti->ti_offset);
futex_unlock (g_vdl.futex);
}
return retval;
}
int main(int argc, char** argv)
{
if (futex_init(&f))
return 1;
if (futex_lock(&f))
return 1;
if (futex_unlock(&f))
return 1;
if (futex_destroy(&f))
return 1;
return 0;
}
void *
alloc_malloc (struct Alloc *alloc, uint32_t size)
{
futex_lock (&alloc->futex);
void *buffer =
alloc_do_malloc (alloc, size + sizeof (struct AllocMallocMetadata));
futex_unlock (&alloc->futex);
struct AllocMallocMetadata *metadata = (struct AllocMallocMetadata *) buffer;
metadata->alloc = alloc;
metadata->size = size;
return buffer + sizeof (struct AllocMallocMetadata);
}
void
alloc_free (void *buffer)
{
struct AllocMallocMetadata *metadata =
(struct AllocMallocMetadata *) (buffer - sizeof (struct AllocMallocMetadata));
unsigned long size = metadata->size;
struct Alloc *alloc = metadata->alloc;
//vdl_memset (buf, 0x66, size);
futex_lock (&alloc->futex);
alloc_do_free (alloc, (void *) (metadata),
size + sizeof (struct AllocMallocMetadata));
futex_unlock (&alloc->futex);
}
internal_function
_dl_allocate_tls_init (void *tcb)
{
if (tcb == 0)
{
return 0;
}
futex_lock (g_vdl.futex);
vdl_tls_dtv_initialize ((unsigned long)tcb);
futex_unlock (g_vdl.futex);
return tcb;
}
// Accurate join-leave implementation
void Scheduler::syscallLeave(uint32_t pid, uint32_t tid, uint32_t cid, uint64_t pc, int syscallNumber, uint64_t arg0, uint64_t arg1) {
futex_lock(&schedLock);
uint32_t gid = getGid(pid, tid);
ThreadInfo* th = contexts[cid].curThread;
assert(th->gid == gid);
assert_msg(th->cid == cid, "%d != %d", th->cid, cid);
assert(th->state == RUNNING);
assert_msg(pid < blockingSyscalls.size(), "%d >= %ld?", pid, blockingSyscalls.size());
bool blacklisted = blockingSyscalls[pid].find(pc) != blockingSyscalls[pid].end();
if (blacklisted || th->markedForSleep) {
DEBUG_FL("%s @ 0x%lx calling leave(), reason: %s", GetSyscallName(syscallNumber), pc, blacklisted? "blacklist" : "sleep");
futex_unlock(&schedLock);
leave(pid, tid, cid);
} else {
DEBUG_FL("%s @ 0x%lx skipping leave()", GetSyscallName(syscallNumber), pc);
FakeLeaveInfo* si = new FakeLeaveInfo(pc, th, syscallNumber, arg0, arg1);
fakeLeaves.push_back(si);
// FIXME(dsm): zsim.cpp's SyscallEnter may be checking whether we are in a syscall and not calling us.
// If that's the case, this would be stale, which may lead to some false positives/negatives
futex_unlock(&schedLock);
}
}
// External interface, must be non-blocking
void Scheduler::notifyFutexWakeStart(uint32_t pid, uint32_t tid, uint32_t maxWakes) {
futex_lock(&schedLock);
ThreadInfo* th = gidMap[getGid(pid, tid)];
DEBUG_FUTEX("[%d/%d] wakeStart max %d", pid, tid, maxWakes);
assert(th->futexJoin.action == FJA_NONE);
// Programs sometimes call FUTEX_WAIT with maxWakes = UINT_MAX to wake
// everyone waiting on it; we cap to a reasonably high number to avoid
// overflows on maxAllowedFutexWakeups
maxWakes = MIN(maxWakes, 1<<24 /*16M wakes*/);
maxAllowedFutexWakeups += maxWakes;
th->futexJoin.maxWakes = maxWakes;
futex_unlock(&schedLock);
}
internal_function
_dl_allocate_tls (void *mem)
{
futex_lock (g_vdl.futex);
unsigned long tcb = (unsigned long)mem;
if (tcb == 0)
{
tcb = vdl_tls_tcb_allocate ();
}
vdl_tls_dtv_allocate (tcb);
vdl_tls_dtv_initialize ((unsigned long)tcb);
futex_unlock (g_vdl.futex);
return (void*)tcb;
}
EXPORT
void
internal_function
_dl_deallocate_tls (void *ptcb, bool dealloc_tcb)
{
futex_lock (g_vdl.futex);
unsigned long tcb = (unsigned long) ptcb;
vdl_tls_dtv_deallocate (tcb);
if (dealloc_tcb)
{
vdl_tls_tcb_deallocate (tcb);
}
futex_unlock (g_vdl.futex);
}
// Internal, called with schedLock held
void Scheduler::futexWakeJoin(ThreadInfo* th) { // may release schedLock
assert(th->futexJoin.action == FJA_WAKE);
uint32_t maxWakes = th->futexJoin.maxWakes;
uint32_t wokenUp = th->futexJoin.wokenUp;
// Adjust allowance
assert(maxWakes <= maxAllowedFutexWakeups);
assert(wokenUp <= maxWakes);
maxAllowedFutexWakeups -= (maxWakes - wokenUp);
assert(unmatchedFutexWakeups <= maxAllowedFutexWakeups); // should panic...
DEBUG_FUTEX("Futex wake matching %d %d", unmatchedFutexWakeups, maxAllowedFutexWakeups);
while (true) {
futex_unlock(&schedLock);
uint64_t startNs = getNs();
uint32_t iters = 0;
while (wokenUp > unmatchedFutexWakeups) {
TrueSleep(10*(1 + iters)); // linear backoff, start small but avoid overwhelming the OS with short sleeps
iters++;
uint64_t curNs = getNs();
if (curNs - startNs > (2L<<31L) /* ~2s */) {
futex_lock(&schedLock);
warn("Futex wake matching failed (%d/%d) (external/ff waiters?)", unmatchedFutexWakeups, wokenUp);
unmatchedFutexWakeups = 0;
maxAllowedFutexWakeups -= wokenUp;
return;
}
}
futex_lock(&schedLock);
// Recheck after acquire, may have concurrent wakes here
if (wokenUp <= unmatchedFutexWakeups) {
unmatchedFutexWakeups -= wokenUp;
maxAllowedFutexWakeups -= wokenUp;
break;
}
}
DEBUG_FUTEX("Finished futex wake matching");
}
uint64_t MD1Memory::access(MemReq& req) {
if (zinfo->numPhases > lastPhase) {
futex_lock(&updateLock);
//Recheck, someone may have updated already
if (zinfo->numPhases > lastPhase) {
updateLatency();
}
futex_unlock(&updateLock);
}
switch (req.type) {
case PUTX:
//Dirty wback
profWrites.atomicInc();
profTotalWrLat.atomicInc(curLatency);
__sync_fetch_and_add(&curPhaseAccesses, 1);
//Note no break
case PUTS:
//Not a real access -- memory must treat clean wbacks as if they never happened.
*req.state = I;
break;
case GETS:
profReads.atomicInc();
profTotalRdLat.atomicInc(curLatency);
__sync_fetch_and_add(&curPhaseAccesses, 1);
*req.state = req.is(MemReq::NOEXCL)? S : E;
break;
case GETX:
profReads.atomicInc();
profTotalRdLat.atomicInc(curLatency);
__sync_fetch_and_add(&curPhaseAccesses, 1);
*req.state = M;
break;
default: panic("!?");
}
return req.cycle + ((req.type == PUTS)? 0 /*PUTS is not a real access*/ : curLatency);
}
void Scheduler::watchdogThreadFunc() {
info("Started scheduler watchdog thread");
uint64_t lastPhase = 0;
int multiplier = 1;
uint64_t lastMs = 0;
uint64_t fakeLeaveStalls = 0;
while (true) {
TrueSleep(multiplier*WATCHDOG_INTERVAL_USEC);
if (zinfo->terminationConditionMet) {
// Synchronize to avoid racing with EndOfPhaseActions code
// (zinfo->terminationConditionMet is set on EndOfPhaseActions,
// which has schedLock held, we must let it finish)
futex_lock(&schedLock);
info("Terminating scheduler watchdog thread");
futex_unlock(&schedLock);
SimEnd();
}
//Fastpath (unlocked, benign read races, only modifies local state)
if (lastPhase != curPhase && pendingPidCleanups.size() == 0) {
lastPhase = curPhase;
fakeLeaveStalls = 0;
if (multiplier < WATCHDOG_MAX_MULTIPLER) multiplier++;
continue;
}
//if (lastPhase == curPhase && scheduledThreads == outQueue.size() && !sleepQueue.empty()) info("Mult %d curPhase %ld", multiplier, curPhase);
futex_lock(&schedLock);
if (lastPhase == curPhase && !fakeLeaves.empty() && (fakeLeaves.front()->th->futexJoin.action != FJA_WAKE)) {
if (++fakeLeaveStalls >= WATCHDOG_STALL_THRESHOLD) {
info("Detected possible stall due to fake leaves (%ld current)", fakeLeaves.size());
// Uncomment to print all leaves
FakeLeaveInfo* pfl = fakeLeaves.front();
while (pfl) {
info(" [%d/%d] %s (%d) @ 0x%lx", getPid(pfl->th->gid), getTid(pfl->th->gid), GetSyscallName(pfl->syscallNumber), pfl->syscallNumber, pfl->pc);
pfl = pfl->next;
}
// Trigger a leave() on the first process, if the process's blacklist regex allows it
FakeLeaveInfo* fl = fakeLeaves.front();
ThreadInfo* th = fl->th;
uint32_t pid = getPid(th->gid);
uint32_t tid = getTid(th->gid);
uint32_t cid = th->cid;
const g_string& sbRegexStr = zinfo->procArray[pid]->getSyscallBlacklistRegex();
std::regex sbRegex(sbRegexStr.c_str());
if (std::regex_match(GetSyscallName(fl->syscallNumber), sbRegex)) {
// If this is the last leave we catch, it is the culprit for sure -> blacklist it
// Over time, this will blacklist every blocking syscall
// The root reason for being conservative though is that we don't have a sure-fire
// way to distinguish IO waits from truly blocking syscalls (TODO)
if (fakeLeaves.size() == 1) {
info("Blacklisting from future fake leaves: [%d] %s @ 0x%lx | arg0 0x%lx arg1 0x%lx", pid, GetSyscallName(fl->syscallNumber), fl->pc, fl->arg0, fl->arg1);
blockingSyscalls[pid].insert(fl->pc);
}
uint64_t pc = fl->pc;
do {
finishFakeLeave(th);
futex_unlock(&schedLock);
leave(pid, tid, cid);
futex_lock(&schedLock);
// also do real leave for other threads blocked at the same pc ...
fl = fakeLeaves.front();
if (fl == nullptr || getPid(th->gid) != pid || fl->pc != pc)
break;
th = fl->th;
tid = getTid(th->gid);
cid = th->cid;
// ... until a lower bound on queue size, in order to make blacklist work
} while (fakeLeaves.size() > 8);
} else {
info("Skipping, [%d] %s @ 0x%lx | arg0 0x%lx arg1 0x%lx does not match blacklist regex (%s)",
pid, GetSyscallName(fl->syscallNumber), fl->pc, fl->arg0, fl->arg1, sbRegexStr.c_str());
}
fakeLeaveStalls = 0;
}
} else {
fakeLeaveStalls = 0;
}
if (lastPhase == curPhase && scheduledThreads == outQueue.size() && !sleepQueue.empty()) {
//info("Watchdog Thread: Sleep dep detected...")
int64_t wakeupPhase = sleepQueue.front()->wakeupPhase;
int64_t wakeupCycles = (wakeupPhase - curPhase)*zinfo->phaseLength;
int64_t wakeupUsec = (wakeupCycles > 0)? wakeupCycles/zinfo->freqMHz : 0;
//info("Additional usecs of sleep %ld", wakeupUsec);
if (wakeupUsec > 10*1000*1000) warn("Watchdog sleeping for a long time due to long sleep, %ld secs", wakeupUsec/1000/1000);
futex_unlock(&schedLock);
TrueSleep(WATCHDOG_INTERVAL_USEC + wakeupUsec);
futex_lock(&schedLock);
if (lastPhase == curPhase && scheduledThreads == outQueue.size() && !sleepQueue.empty()) {
ThreadInfo* sth = sleepQueue.front();
uint64_t curMs = curPhase*zinfo->phaseLength/zinfo->freqMHz/1000;
uint64_t endMs = sth->wakeupPhase*zinfo->phaseLength/zinfo->freqMHz/1000;
(void)curMs; (void)endMs; //make gcc happy
if (curMs > lastMs + 1000) {
info("Watchdog Thread: Driving time forward to avoid deadlock on sleep (%ld -> %ld ms)", curMs, endMs);
lastMs += 1000;
}
while (sth->state == SLEEPING) {
idlePhases.inc();
callback(); //sth will eventually get woken up
if (futex_haswaiters(&schedLock)) {
//happens commonly with multiple sleepers and very contended I/O...
//info("Sched: Threads waiting on advance, startPhase %ld curPhase %ld", lastPhase, curPhase);
break;
}
if (zinfo->terminationConditionMet) {
info("Termination condition met inside watchdog thread loop, exiting");
break;
}
}
idlePeriods.inc();
multiplier = 0;
}
}
if (multiplier < WATCHDOG_MAX_MULTIPLER) {
multiplier++;
}
lastPhase = curPhase;
//Lazily clean state of processes that terminated abruptly
//NOTE: For now, we rely on the process explicitly telling us that it's going to terminate.
//We could make this self-checking by periodically checking for liveness of the processes we're supposedly running.
//The bigger problem is that if we get SIGKILL'd, we may not even leave a consistent zsim state behind.
while (pendingPidCleanups.size()) {
std::pair<uint32_t, uint32_t> p = pendingPidCleanups.back();
uint32_t pid = p.first; //the procIdx pid
uint32_t osPid = p.second;
std::stringstream ss;
ss << "/proc/" << osPid;
struct stat dummy;
if (stat(ss.str().c_str(), &dummy) == 0) {
info("[watchdog] Deferring cleanup of pid %d (%d), not finished yet", pid, osPid);
break;
}
pendingPidCleanups.pop_back(); //must happen while we have the lock
futex_unlock(&schedLock);
processCleanup(pid);
futex_lock(&schedLock);
}
if (terminateWatchdogThread) {
futex_unlock(&schedLock);
break;
} else {
futex_unlock(&schedLock);
}
}
info("Finished scheduler watchdog thread");
}
void __log_unlock() {futex_unlock(&log_printLock);}
bool Scheduler::futexWait(bool bitmask, bool pi_waiter, uint32_t pid, uint32_t tid, FutexInfo fi, CONTEXT* ctxt, SYSCALL_STANDARD std) {
DEBUG_FUTEX("Scheduler: FUTEX WAIT called with bitmask %d pi %d pid %u tid %u", bitmask, pi_waiter, pid, tid);
uint64_t wakeUpPhases = getFutexWakePhase(pi_waiter, fi, ctxt, std); //pi versions all interpret as realtime
futex_lock(&schedLock);
uint32_t cid = gidMap[getGid(pid, tid)]->cid;
futex_unlock(&schedLock);
if(wakeUpPhases == 0) {
wakeUpPhases = 0xffffffff;
}
FutexWaiter tempWaiter; tempWaiter.pid = pid; tempWaiter.tid = tid; tempWaiter.pi_waiter = pi_waiter;
tempWaiter.mask = 0xffffffff; tempWaiter.val = fi.val; tempWaiter.fi = fi; tempWaiter.allow_requeue = !pi_waiter;
if(!pi_waiter) { //Normal cases
if(fi.val != *fi.uaddr) {
DEBUG_FUTEX("Cur val didn't match val in futex wait");
return false;
}
DEBUG_FUTEX("WAIT took normal path");
if(bitmask) {
tempWaiter.mask = fi.val3;
}
futexTable[fi.uaddr].push_back(tempWaiter);
markForSleep(pid, tid, wakeUpPhases);
leave(pid, tid, cid);
} else { //if pi_waiter
switch (fi.op & FUTEX_CMD_MASK) {
case FUTEX_LOCK_PI:
DEBUG_FUTEX("WAIT took lock path");
if(futexTable[fi.uaddr].size() == 0) // Check that no one else is in line.
{
DEBUG_FUTEX("FUTEX_LOCK_PI successfully locked");
futexTable[fi.uaddr].push_back(tempWaiter); //Notice we don't deschedule
return true;
} else {
DEBUG_FUTEX("LOCK delayed");
if(bitmask) {
tempWaiter.mask = fi.val3;
}
futexTable[fi.uaddr].push_back(tempWaiter);
markForSleep(pid, tid, wakeUpPhases);
leave(pid, tid, cid);
}
break;
case FUTEX_WAIT_REQUEUE_PI:
DEBUG_FUTEX("WAIT took reque pi path");
if(fi.val != *fi.uaddr) {
DEBUG_FUTEX("Cur val didn't match val in futex wait");
return false;
}
tempWaiter.allow_requeue = true;
if(bitmask) {
tempWaiter.mask = fi.val3;
}
markForSleep(pid, tid, wakeUpPhases);
leave(pid, tid, cid);
futexTable[fi.uaddr].push_back(tempWaiter);
break;
case FUTEX_TRYLOCK_PI:
DEBUG_FUTEX("WAIT took trylock path");
if(futexTable[fi.uaddr].size() == 0) {
DEBUG_FUTEX("FUTEX_LOCK_PI successfully locked");
futexTable[fi.uaddr].push_back(tempWaiter); //Notice we don't deschedule
return true;
} else {
return false;
}
break;
default:
panic("We missed something in futex wait.");
}
}
return true;
}
