void Fpc1020Sensor::FingerprintThread::handleShutdown()
{
ALOGV("Shutting down thread %d", getTid());
mSensor->deactivate();
mSensor->stopWatchdogThread();
android::Mutex::Autolock l(mSensor->mThreadStateLock);
mSensor->mWaitingForWakeup = false;
mSensor->mThread.clear();
ALOGV("Thread %d now shut down", getTid());
}
EXP_OPTION int getLastError()
{
THREAD_ID Tid = getTid();
ThreadErrors *threadErrors = findThreadErrorsByTid(Tid);
if (threadErrors == NULL)
return 0;
return (threadErrors->readErrorIdx >= 0 ?
threadErrors->ddocLastErrors[threadErrors->readErrorIdx].code : ERR_OK);
}
EXP_OPTION void clearErrors()
{
THREAD_ID Tid = getTid();
ThreadErrors *threadErrors = findThreadErrorsByTid(Tid);
if (threadErrors == NULL)
return;
memset(threadErrors->ddocLastErrors, 0, sizeof(ErrorInfo) * ERROR_BUF_LENGTH);
threadErrors->readErrorIdx = -1;
threadErrors->currentErrorIdx = -1;
}
void Scheduler::waitUntilQueued(ThreadInfo* th) {
uint64_t startNs = getNs();
uint32_t sleepUs = 1;
while(!IsSleepingInFutex(th->linuxPid, th->linuxTid, (uintptr_t)&schedLock)) {
TrueSleep(sleepUs++); // linear backoff, start small but avoid overwhelming the OS with short sleeps
uint64_t curNs = getNs();
if (curNs - startNs > (2L<<31L) /* ~2s */) {
warn("waitUntilQueued for pid %d tid %d timed out", getPid(th->gid), getTid(th->gid));
return;
}
}
}
virtual void run(void)
{
ULong sAllocSize;
SInt i;
iduMemoryClientIndex sTestIndex = (iduMemoryClientIndex)0;
for (i = 0; i < TEST_ALLOC_COUNT / 2; i++)
{
sAllocSize = gAllocSize[i];
ACT_CHECK_DESC(iduMemMgr::malloc(sTestIndex,
sAllocSize,
(void **)&mBufPtrMalloc[i])
== IDE_SUCCESS,
("malloc() failed at %p", (void*)getTid()));
ACT_CHECK(mBufPtrMalloc[i] != NULL);
}
for (i = 0; i < TEST_ALLOC_COUNT / 2; i++)
{
sAllocSize = gAllocSize[i];
ACT_CHECK_DESC(iduMemMgr::malloc(sTestIndex,
sAllocSize,
(void **)&mBufPtrMalloc[i + TEST_ALLOC_COUNT / 2])
== IDE_SUCCESS,
("malloc() failed at %p", (void*)getTid()));
ACT_CHECK(mBufPtrMalloc[i] != NULL);
ACT_CHECK(iduMemMgr::free(mBufPtrMalloc[i]) == IDE_SUCCESS);
}
for (i = 0; i < TEST_ALLOC_COUNT / 2; i++)
{
ACT_CHECK(iduMemMgr::free(mBufPtrMalloc[i + TEST_ALLOC_COUNT / 2]) == IDE_SUCCESS);
}
}
EXP_OPTION void addError(int code, char *fileName, int line, char *assertion)
{
//no errors found yet. Set a trace-back mark to the end of array.
//printf("Error : %d at %s line %d, assertion %s\n", code, fileName, line, assertion);
ThreadErrors *threadErrors;
THREAD_ID Tid = getTid(); //Find our identity
threadErrors = findThreadErrorsByTid(Tid);
// printf("addError init: tid=%ld, threadErrors=%p\n", Tid, threadErrors);
if (threadErrors == NULL) { //This Tid has no entry in ThreadErrors table
threadErrors = addThreadErrorsByTid(Tid); // MEMLEAK: ???
if (threadErrors == NULL)
return; // What else can we do?
}
// printf("addError step 1 : tid=%ld, threadErrors=%p\n", Tid, threadErrors);
if(threadErrors->currentErrorIdx < 0)
resetError(&(threadErrors->ddocLastErrors[ERROR_BUF_LENGTH - 1]));
threadErrors->currentErrorIdx++;
//index at the end -> roll it over to the beginning
if(threadErrors->currentErrorIdx == ERROR_BUF_LENGTH)
threadErrors->currentErrorIdx = 0;
//set the information
ddocDebug(4, "addError", "Index: %d Error : %d at %s line %d, assertion %s", threadErrors->currentErrorIdx, code, fileName, line, assertion);
threadErrors->ddocLastErrors[threadErrors->currentErrorIdx].code = code;
threadErrors->ddocLastErrors[threadErrors->currentErrorIdx].fileName = fileName;
threadErrors->ddocLastErrors[threadErrors->currentErrorIdx].line = line;
threadErrors->ddocLastErrors[threadErrors->currentErrorIdx].assertion = assertion;
//index at the end? Set the traceback mark to the beginning
if(threadErrors->currentErrorIdx == ERROR_BUF_LENGTH - 1)
resetError(&(threadErrors->ddocLastErrors[0]));
else //set the traceback mark to the next position
resetError(&(threadErrors->ddocLastErrors[threadErrors->currentErrorIdx + 1]));
threadErrors->readErrorIdx = threadErrors->currentErrorIdx;
}
//returns the last
EXP_OPTION ErrorInfo* getErrorInfo()
{
ErrorInfo *pErrInfo = 0;
THREAD_ID Tid = getTid();
ThreadErrors *threadErrors = findThreadErrorsByTid(Tid);
if (threadErrors == NULL)
return 0;
if(threadErrors->readErrorIdx >= 0 &&
threadErrors->ddocLastErrors[threadErrors->readErrorIdx].code != ERR_OK) {
pErrInfo = &(threadErrors->ddocLastErrors[threadErrors->readErrorIdx]);
threadErrors->readErrorIdx--;
if(threadErrors->readErrorIdx < 0) //roll over
threadErrors->readErrorIdx = ERROR_BUF_LENGTH - 1;
}
else
pErrInfo = 0;
return pErrInfo;
}
// . ***** CHECKOFF LOOP *****
// . loop over the checkoff requests TO SEND
// . send checkoff requests out that we need to send and have not yet got
// a reply for. when we get a reply for them the "need to send checkoff
// request" key is immediately deleted
// . make a big list then call msg1 with an rdbId of RDB_SYNCDB
// . do not include dead hosts
void Syncdb::loop5 ( ) {
// make the start key
key128_t k = m_nextk;
// end key
key128_t ek;
ek = makeKey(1,0,0,0,0xffffffff,0xffffffff,0xffffffffffffffffLL,1);
// reset
m_nk = 0;
// get it
long nn = m_qt.getNextNode ( 0 , (char *)&k );
// do one tid at a time
uint32_t startTid = getTid ( &k );
// do the loop
for ( ; nn >= 0 ; nn = m_qt.getNextNode ( nn ) ) {
// breathe
QUICKPOLL ( MAX_NICENESS );
// get key
key128_t k = *(key128_t *)m_qt.getKey ( nn );
// save it
m_nextk = k;
// add one
m_nextk += 1;
// stop when we hit the end
if ( k > ek ) break;
// get twin id
uint32_t tid = getTid ( &k );
// stop if tid changed
if ( tid != startTid && m_nk >= 0 ) break;
// skip if tid is dead
if ( g_hostdb.isDead ( tid ) ) {
// skip to next tid!
tid++;
// . find the key of the FIRST meta list we need to add
// for this new senderId, "sid"
// . mdw: i reset sid to 0
key128_t nk = makeKey ( 1,0,0,0,tid,0,0,0 );
// undo the m_qt.getNextNode(nn) we call in for loop
nn = m_qt.getPrevNode ( 0 , (char *)&nk );
// bail if no good
if ( nn < 0 ) break;
// get next key from this new sid
continue;
}
// get zid
uint64_t zid = getZid ( &k );
// get sid
uint32_t sid = getSid ( &k );
// note it
log("sync: storing key for meta request list sid=%lu zid=%llu "
"from twin hostid #%li",(unsigned long)sid,
(unsigned long long)zid,(long)tid);
// make the key. make NEGATIVE "b" keys.
m_keys [ m_nk++ ] = makeKey ( 0,1,0,0,0,sid,zid,0 );
// stop if full
if ( m_nk >= MAX_CHECKOFF_KEYS ) break;
}
// all done?
if ( k <= ek && m_nk <= 0 ) { m_calledLoop5 = true; return; }
// . add the whole list at once
// . make our own msg1 request to send to just one host
if ( g_udpServer.sendRequest ( (char *)m_keys ,
m_nk * 16 ,
0x5d , // SYNCDB REQUEST
0 , // ip
0 , // port
startTid , // hostId
NULL , // retSlot
NULL , // state
addedListWrapper5 , // wrapper
60 , // timeout
-1 , // backoff
-1 , // maxWait
NULL , // replyBuf
0 , // replyBufSize
MAX_NICENESS ))
// it blocked, return now
return;
// i guess we had an error...
if ( ! addedList5() ) return;
}
// . ***** SYNC LOOP *****
// . loop over the checkoff requests we are waiting for. "b" keys.
// . send out msg0 requests for meta lists we need
// . if we got a 60+ sec old checkoff recvd but never got the meta list
// ourselves, then request it!!
bool Syncdb::loop4 ( ) {
// make the start key. a "b" key
key128_t k = m_syncKey;
// end key
key128_t ek;
ek = makeKey(0,1,0,0,0xffffffff,0xffffffff,0xffffffffffffffffLL,1);
// get next node
long nn = m_qt.getNode ( 0 , (char *)&k );
// get group we are in
//Host *group = g_hostdb.getMyShard();
// use this for determining approximate age of meta lists
long long nowms = gettimeofdayInMilliseconds();
// do the loop
for ( ; nn >= 0 ; nn = m_qt.getNextNode ( nn ) ) {
// breathe
QUICKPOLL ( MAX_NICENESS );
// give other loops some time so we can digest these
// meta lists we added
if ( m_addCount >= 100 ) break;
// get key
key128_t k = *(key128_t *)m_qt.getKey ( nn );
// stop when we hit the end
if ( k > ek ) break;
// update
m_syncKey = k;
// get twin id
uint32_t tid = getTid ( &k );
// skip if tid is dead
if ( g_hostdb.isDead ( tid ) ) {
// skip to next tid!
tid++;
// . find the key of the FIRST meta list we need to add
// for this new senderId, "sid"
// . mdw: i reset sid to 0
key128_t nk = makeKey ( 0,1,0,0,tid,0,0,0 );
// undo the m_qt.getNextNode(nn) we call in for loop
nn = m_qt.getPrevNode ( 0 , (char *)&nk );
// sanity check
if ( nn < 0 ) { char *xx=NULL;*xx=0; }
// get next key from this new sid
continue;
}
// . skip if delbit set
// . the delbit set means we got the meta list and are
// waiting to receive the checkoff request from twin #sid
// . when it comes in it will annihilate with this key!
// . right now we are only interested in *negative* keys,
// those with their delbit cleared
if ( (k.n0 & 0x01) == 0x01 ) continue;
// . ok we got checkoff request from tid for this sid/zid
// . and we have not got the meta list ourselves yet cuz
// we would have had k with the delbit set.
// . get zid
uint64_t zid = getZid ( &k );
// get sid
uint32_t sid = getSid ( &k );
// get its approximate age
long long age = nowms - zid;
// go to next sid if not 60 seconds yet for this one
if ( age < 60000 ) {
// no use banging away at this sid any more since we
// are missing another action for this one
sid++;
// find the key of the FIRST meta list we need to add
// for this new senderId, "sid". make the "b" key.
key128_t nk = makeKey ( 0,1,0,0,tid,sid,0,0 );
// undo the m_qt.getNextNode(nn) we call in for loop
nn = m_qt.getPrevNode ( 0 , (char *)&nk );
// sanity check
if ( nn < 0 ) { char *xx=NULL;*xx=0; }
// get next key from this new sid
continue;
}
// make the "d" key to see if we got the meta list just
// to make sure! we do not want to re-add a meta list!
key128_t dk = makeKey(0,0,0,1,0,sid,zid,1);
if ( m_qt.getNode ( 0 , (char *)&dk ) >= 0 ) continue;
// note it
log("sync: requesting meta list sid=%lu zid=%llu age=%llu "
"from twin hostid #%li",
(unsigned long)sid,
(unsigned long long)zid,
(unsigned long long)age,
(long)tid);
// i guess we are out of sync
g_hostdb.m_myHost->m_inSync = false;
// do not let sleep ticker call bigLoop
m_outstanding = true;
// record the sid
m_requestedSid = sid;
// make the c key range
key128_t sk2 = makeKey(0,0,1,0,0,sid,zid,0);
key128_t ek2 = makeKey(0,0,1,0,0,sid,zid,1);
// ok, ask tid for this meta list
if ( ! m_msg0.getList ( tid , // hostId
0 , // ip
0 , // port
0 , // maxCacheAge
false , // addToCache
RDB_SYNCDB ,
0 , // collnum
&m_list ,
(char *)&sk2 ,
(char *)&ek2 ,
999 , // minRecSizes
NULL ,
gotListWrapper4 ,
MAX_NICENESS ,
false ))// err correction?
return false;
// stop on error. return true with g_errno set on error
if ( ! gotList4() ) return true;
}
// we completed
m_calledLoop4 = true;
return true;
}
void Scheduler::watchdogThreadFunc() {
info("Started scheduler watchdog thread");
uint64_t lastPhase = 0;
int multiplier = 1;
uint64_t lastMs = 0;
uint64_t fakeLeaveStalls = 0;
while (true) {
TrueSleep(multiplier*WATCHDOG_INTERVAL_USEC);
if (zinfo->terminationConditionMet) {
// Synchronize to avoid racing with EndOfPhaseActions code
// (zinfo->terminationConditionMet is set on EndOfPhaseActions,
// which has schedLock held, we must let it finish)
futex_lock(&schedLock);
info("Terminating scheduler watchdog thread");
futex_unlock(&schedLock);
SimEnd();
}
//Fastpath (unlocked, benign read races, only modifies local state)
if (lastPhase != curPhase && pendingPidCleanups.size() == 0) {
lastPhase = curPhase;
fakeLeaveStalls = 0;
if (multiplier < WATCHDOG_MAX_MULTIPLER) multiplier++;
continue;
}
//if (lastPhase == curPhase && scheduledThreads == outQueue.size() && !sleepQueue.empty()) info("Mult %d curPhase %ld", multiplier, curPhase);
futex_lock(&schedLock);
if (lastPhase == curPhase && !fakeLeaves.empty() && (fakeLeaves.front()->th->futexJoin.action != FJA_WAKE)) {
if (++fakeLeaveStalls >= WATCHDOG_STALL_THRESHOLD) {
info("Detected possible stall due to fake leaves (%ld current)", fakeLeaves.size());
// Uncomment to print all leaves
FakeLeaveInfo* pfl = fakeLeaves.front();
while (pfl) {
info(" [%d/%d] %s (%d) @ 0x%lx", getPid(pfl->th->gid), getTid(pfl->th->gid), GetSyscallName(pfl->syscallNumber), pfl->syscallNumber, pfl->pc);
pfl = pfl->next;
}
// Trigger a leave() on the first process, if the process's blacklist regex allows it
FakeLeaveInfo* fl = fakeLeaves.front();
ThreadInfo* th = fl->th;
uint32_t pid = getPid(th->gid);
uint32_t tid = getTid(th->gid);
uint32_t cid = th->cid;
const g_string& sbRegexStr = zinfo->procArray[pid]->getSyscallBlacklistRegex();
std::regex sbRegex(sbRegexStr.c_str());
if (std::regex_match(GetSyscallName(fl->syscallNumber), sbRegex)) {
// If this is the last leave we catch, it is the culprit for sure -> blacklist it
// Over time, this will blacklist every blocking syscall
// The root reason for being conservative though is that we don't have a sure-fire
// way to distinguish IO waits from truly blocking syscalls (TODO)
if (fakeLeaves.size() == 1) {
info("Blacklisting from future fake leaves: [%d] %s @ 0x%lx | arg0 0x%lx arg1 0x%lx", pid, GetSyscallName(fl->syscallNumber), fl->pc, fl->arg0, fl->arg1);
blockingSyscalls[pid].insert(fl->pc);
}
uint64_t pc = fl->pc;
do {
finishFakeLeave(th);
futex_unlock(&schedLock);
leave(pid, tid, cid);
futex_lock(&schedLock);
// also do real leave for other threads blocked at the same pc ...
fl = fakeLeaves.front();
if (fl == nullptr || getPid(th->gid) != pid || fl->pc != pc)
break;
th = fl->th;
tid = getTid(th->gid);
cid = th->cid;
// ... until a lower bound on queue size, in order to make blacklist work
} while (fakeLeaves.size() > 8);
} else {
info("Skipping, [%d] %s @ 0x%lx | arg0 0x%lx arg1 0x%lx does not match blacklist regex (%s)",
pid, GetSyscallName(fl->syscallNumber), fl->pc, fl->arg0, fl->arg1, sbRegexStr.c_str());
}
fakeLeaveStalls = 0;
}
} else {
fakeLeaveStalls = 0;
}
if (lastPhase == curPhase && scheduledThreads == outQueue.size() && !sleepQueue.empty()) {
//info("Watchdog Thread: Sleep dep detected...")
int64_t wakeupPhase = sleepQueue.front()->wakeupPhase;
int64_t wakeupCycles = (wakeupPhase - curPhase)*zinfo->phaseLength;
int64_t wakeupUsec = (wakeupCycles > 0)? wakeupCycles/zinfo->freqMHz : 0;
//info("Additional usecs of sleep %ld", wakeupUsec);
if (wakeupUsec > 10*1000*1000) warn("Watchdog sleeping for a long time due to long sleep, %ld secs", wakeupUsec/1000/1000);
futex_unlock(&schedLock);
TrueSleep(WATCHDOG_INTERVAL_USEC + wakeupUsec);
futex_lock(&schedLock);
if (lastPhase == curPhase && scheduledThreads == outQueue.size() && !sleepQueue.empty()) {
ThreadInfo* sth = sleepQueue.front();
uint64_t curMs = curPhase*zinfo->phaseLength/zinfo->freqMHz/1000;
uint64_t endMs = sth->wakeupPhase*zinfo->phaseLength/zinfo->freqMHz/1000;
(void)curMs; (void)endMs; //make gcc happy
if (curMs > lastMs + 1000) {
info("Watchdog Thread: Driving time forward to avoid deadlock on sleep (%ld -> %ld ms)", curMs, endMs);
lastMs += 1000;
}
while (sth->state == SLEEPING) {
idlePhases.inc();
callback(); //sth will eventually get woken up
if (futex_haswaiters(&schedLock)) {
//happens commonly with multiple sleepers and very contended I/O...
//info("Sched: Threads waiting on advance, startPhase %ld curPhase %ld", lastPhase, curPhase);
break;
}
if (zinfo->terminationConditionMet) {
info("Termination condition met inside watchdog thread loop, exiting");
break;
}
}
idlePeriods.inc();
multiplier = 0;
}
}
if (multiplier < WATCHDOG_MAX_MULTIPLER) {
multiplier++;
}
lastPhase = curPhase;
//Lazily clean state of processes that terminated abruptly
//NOTE: For now, we rely on the process explicitly telling us that it's going to terminate.
//We could make this self-checking by periodically checking for liveness of the processes we're supposedly running.
//The bigger problem is that if we get SIGKILL'd, we may not even leave a consistent zsim state behind.
while (pendingPidCleanups.size()) {
std::pair<uint32_t, uint32_t> p = pendingPidCleanups.back();
uint32_t pid = p.first; //the procIdx pid
uint32_t osPid = p.second;
std::stringstream ss;
ss << "/proc/" << osPid;
struct stat dummy;
if (stat(ss.str().c_str(), &dummy) == 0) {
info("[watchdog] Deferring cleanup of pid %d (%d), not finished yet", pid, osPid);
break;
}
pendingPidCleanups.pop_back(); //must happen while we have the lock
futex_unlock(&schedLock);
processCleanup(pid);
futex_lock(&schedLock);
}
if (terminateWatchdogThread) {
futex_unlock(&schedLock);
break;
} else {
futex_unlock(&schedLock);
}
}
info("Finished scheduler watchdog thread");
}
void EventThread::onFirstRef() {
run("EventThread", PRIORITY_URGENT_DISPLAY + PRIORITY_MORE_FAVORABLE);
android_set_rt_ioprio(getTid(), 1);
}
