void flushStatsWrapper ( int fd , void *state ) {
g_statsdb.addDocsIndexed();
// force a statsdb tree dump if running out of room
Rdb *rdb = &g_statsdb.m_rdb;
RdbTree *tree = &rdb->m_tree;
// if we got 20% room left and 50k available mem, do not dump
if ( (float)tree->getNumUsedNodes() * 1.2 <
(float)tree->getNumAvailNodes () &&
//tree->getNumAvailNodes () > 1000 &&
rdb-> m_mem.getAvailMem() > 50000 )
return;
if ( ! isClockInSync() ) return;
// force a dump
rdb->dumpTree ( 1 );
}
// . m_key bitmap in statsdb:
// tttttttt tttttttt tttttttt tttttttt t = time in milliseconds, t1
// tttttttt tttttttt tttttttt tttttttt
// hhhhhhhh hhhhhhhh hhhhhhhh hhhhhhhh h = hash32 of m_title
// . returns false if could not add stat, true otherwise
// . do not set g_errno if we return false just to keep things simple
// . we only add the stat to our local statsdb rdb, but because
// we might be dumping statsdb to disk or something it is possible
// we get an ETRYAGAIN error, so we try to accumulate stats in a
// local buffer in that case
// . "label" is something like "queryLatency" or whatever
// . [t1,t2] are the time endpoints for the operation being measured
// . "value" is usually "numBytes", or a quantity indicator of whatever
// was processed.
// . oldVal, newVal are reflect a state change, like maybe changing the
// value of a parm. typically for such things t1 equals t2
bool Statsdb::addStat ( long niceness ,
char *label ,
long long t1Arg ,
long long t2Arg ,
float value , // y-value really, "numBytes"
long parmHash ,
float oldVal ,
float newVal ,
long userId32 ) {
if ( ! g_conf.m_useStatsdb ) return true;
// so Process.cpp can turn it off when dumping core
if ( m_disabled ) return true;
// not thread safe!
//if ( g_threads.amThread() ) {
// log("statsdb: called from thread");
// char *xx=NULL;*xx=0;
//}
// . for now we can only add stats if we are synced with host #0 clock
// . this is kinda a hack and it would be nice to not miss stats!
if ( ! isClockInSync() ) return true;
RdbTree *tree = &m_rdb.m_tree;
// do not add stats to our tree if it is loading
if ( tree->m_isLoading ) return true;
// convert into host #0 synced time
t1Arg = localToGlobalTimeMilliseconds ( t1Arg );
t2Arg = localToGlobalTimeMilliseconds ( t2Arg );
// sanity check
if ( ! label ) { char *xx=NULL;*xx=0; }
long labelHash;
if ( parmHash ) labelHash = parmHash;
else labelHash = hash32n ( label );
// fix it for parm changes, and docs_indexed stat, etc.
if ( t1Arg == t2Arg ) t2Arg++;
// how many SECONDS did the op take? (convert from ms to secs)
float dtms = (t2Arg - t1Arg);
float dtSecs = dtms / 1000.0;
// we have already flushed stats 30+ seconds old, so if this op took
// 30 seconds, discard it!
if ( dtSecs >= 30 ) {
//log("statsdb: stat is %li secs > 30 secs old, discarding.",
// (long)dtSecs);
return true;
}
long long nextup;
// loop over all "second" buckets
for ( long long tx = t1Arg ; tx < t2Arg ; tx = nextup ) {
// get next second-aligned point in milliseconds
nextup = ((tx +1000)/ 1000) * 1000;
// truncate if we need to
if ( nextup > t2Arg ) nextup = t2Arg;
// . how much of the stat is in this time interval?
// . like if operation took 3 seconds, we might cover
// 50% of the first 1-second interval. so we use this
// as a weight for the stats we keep for that particular
// second. then we can plot a point for each second
// in time which is an average of all the queries that
// were in progress at that second.
float fractionTime = ((float)(nextup - tx)) / dtms;
// . get the time point bucket in which this stat belongs
// . every "second" in time has a bucket
unsigned long t1 = tx / 1000;
StatKey sk;
sk.m_zero = 0x01; // make it a positive key
sk.m_time1 = t1;
sk.m_labelHash = labelHash;
// so we can show just the stats for a particular user...
if ( userId32 ) {
sk.m_zero = userId32;
// make it positive
sk.m_zero |= 0x01;
}
// if we already have added a bucket for this "second" then
// get it from the tree so we can add to its accumulated stats.
long node1 = tree->getNode ( 0 , (char *)&sk );
long node2;
StatData *sd;
// get that stat, see if we are accumulating it already
if ( node1 >= 0 )
sd = (StatData *)tree->getData ( node1 );
// make a new one if not there
else {
StatData tmp;
// init it
tmp.m_totalOps = 0.0;
tmp.m_totalQuantity = 0.0;
tmp.m_totalTime = 0.0;
// save this
long saved = g_errno;
// need to add using rdb so it can memcpy the data
if ( ! m_rdb.addRecord ( (collnum_t)0 ,
(char *)&sk,
(char *)&tmp,
sizeof(StatData),
niceness ) ) {
if ( g_errno != ETRYAGAIN )
log("statsdb: add rec failed: %s",
mstrerror(g_errno));
// caller does not care about g_errno
g_errno = saved;
return false;
}
// caller does not care about g_errno
g_errno = saved;
// get the node in the tree
//sd = (StatData *)tree->getData ( node1 );
// must be there!
node2 = tree->getNode ( 0 , (char *)&sk );
// must be there!
if ( node2 < 0 ) { char *xx=NULL;*xx=0; }
// point to it
sd = (StatData *)tree->getData ( node2 );
}
// use the milliseconds elapsed as the value if none given
//if ( value == 0 && ! parmHash )
// value = t2Arg - t1Arg;
// if we got it for this time, accumulate it
// convert x into pixel position
sd->m_totalOps += 1 * fractionTime;
sd->m_totalQuantity += value * fractionTime;
sd->m_totalTime += dtSecs * fractionTime;
if ( ! parmHash ) continue;
sd->m_totalOps = 0;
sd->m_totalQuantity = oldVal;
sd->m_newVal = newVal;
// no fractions for this!
break;
}
//logf(LOG_DEBUG,"statsdb: sp=0x%lx",(long)sp);
return true;
}
// . ***** META LIST ADD LOOP *****
// . add meta lists corresponding to the keys in m_addme[]
// . call msg5 in case some got dumped to disk!
// . never add zids out of order for the same sid
// . returns false if blocked, sets g_errno on error and returns true
bool Syncdb::loop2 ( ) {
// sanity check
if ( ! m_calledLoop1 ) { char *xx=NULL;*xx=0; }
loop:
// breathe just in case
QUICKPOLL ( MAX_NICENESS );
// return if done!
if ( m_ia >= m_na ) { m_calledLoop2 = true ; return true; }
// . check the tree for the next key to add first!!!
// . most of the time this should be the case!
RdbTree *stree = &m_rdb.m_tree;
// get the key
key128_t k = m_addMe[m_ia];
// is it there?
long n = stree->getNode ( 0 , (char *)&k );
// yes! easy add...
if ( n >= 0 ) {
//long reqSize = stree->getDataSize ( n );
char *req = stree->getData ( n );
// get zid from key
uint64_t zid1 = getZid ( &k );
// get zid from request
uint64_t zid2 = *(uint64_t *)(req+4);
// must match!
if ( zid1 != zid2 ) { char *xx=NULL;*xx=0; }
// . add away using Msg4.cpp
// . return with g_errno set on error
if ( ! addMetaList ( req ) ) return true;
// success, delete the key from QUICK TREE
m_qt.deleteNode ( 0 , (char *)&k , true );
// . and delete that node (freeData = true)
// . no! not until we got all the checkoff requests in!
// stree->deleteNode ( n , true );
// advance on success
m_ia++;
// go back for more
goto loop;
}
// make the key range
key128_t sk = k;
key128_t ek = k;
// make negative
sk.n0 &= 0xfffffffffffffffeLL;
// do not let sleep ticker call bigLoop
m_outstanding = true;
// get the meta list from on disk i guess, if not in tree
if ( ! m_msg5.getList ( RDB_SYNCDB ,
(collnum_t)0 , // coll
&m_list ,
(char *)&sk , // startKey
(char *)&ek , // endKey
999 , // minRecSizes
true , // includeTree?
false , // addToCache
0 , // maxCacheAge
0 , // startFielNum
-1 , // numFiles
NULL , // state
gotListWrapper ,
MAX_NICENESS ,
true )) // do error correction?
return false;
// process it. loop back up for another on success!
if ( gotList ( ) ) goto loop;
// bad. g_errno must be set
return false;
}