// . m_key bitmap in statsdb: // tttttttt tttttttt tttttttt tttttttt t = time in milliseconds, t1 // tttttttt tttttttt tttttttt tttttttt // hhhhhhhh hhhhhhhh hhhhhhhh hhhhhhhh h = hash32 of m_title // . returns false if could not add stat, true otherwise // . do not set g_errno if we return false just to keep things simple // . we only add the stat to our local statsdb rdb, but because // we might be dumping statsdb to disk or something it is possible // we get an ETRYAGAIN error, so we try to accumulate stats in a // local buffer in that case // . "label" is something like "queryLatency" or whatever // . [t1,t2] are the time endpoints for the operation being measured // . "value" is usually "numBytes", or a quantity indicator of whatever // was processed. // . oldVal, newVal are reflect a state change, like maybe changing the // value of a parm. typically for such things t1 equals t2 bool Statsdb::addStat ( long niceness , char *label , long long t1Arg , long long t2Arg , float value , // y-value really, "numBytes" long parmHash , float oldVal , float newVal , long userId32 ) { if ( ! g_conf.m_useStatsdb ) return true; // so Process.cpp can turn it off when dumping core if ( m_disabled ) return true; // not thread safe! //if ( g_threads.amThread() ) { // log("statsdb: called from thread"); // char *xx=NULL;*xx=0; //} // . for now we can only add stats if we are synced with host #0 clock // . this is kinda a hack and it would be nice to not miss stats! if ( ! isClockInSync() ) return true; RdbTree *tree = &m_rdb.m_tree; // do not add stats to our tree if it is loading if ( tree->m_isLoading ) return true; // convert into host #0 synced time t1Arg = localToGlobalTimeMilliseconds ( t1Arg ); t2Arg = localToGlobalTimeMilliseconds ( t2Arg ); // sanity check if ( ! label ) { char *xx=NULL;*xx=0; } long labelHash; if ( parmHash ) labelHash = parmHash; else labelHash = hash32n ( label ); // fix it for parm changes, and docs_indexed stat, etc. if ( t1Arg == t2Arg ) t2Arg++; // how many SECONDS did the op take? (convert from ms to secs) float dtms = (t2Arg - t1Arg); float dtSecs = dtms / 1000.0; // we have already flushed stats 30+ seconds old, so if this op took // 30 seconds, discard it! if ( dtSecs >= 30 ) { //log("statsdb: stat is %li secs > 30 secs old, discarding.", // (long)dtSecs); return true; } long long nextup; // loop over all "second" buckets for ( long long tx = t1Arg ; tx < t2Arg ; tx = nextup ) { // get next second-aligned point in milliseconds nextup = ((tx +1000)/ 1000) * 1000; // truncate if we need to if ( nextup > t2Arg ) nextup = t2Arg; // . how much of the stat is in this time interval? // . like if operation took 3 seconds, we might cover // 50% of the first 1-second interval. so we use this // as a weight for the stats we keep for that particular // second. then we can plot a point for each second // in time which is an average of all the queries that // were in progress at that second. float fractionTime = ((float)(nextup - tx)) / dtms; // . get the time point bucket in which this stat belongs // . every "second" in time has a bucket unsigned long t1 = tx / 1000; StatKey sk; sk.m_zero = 0x01; // make it a positive key sk.m_time1 = t1; sk.m_labelHash = labelHash; // so we can show just the stats for a particular user... if ( userId32 ) { sk.m_zero = userId32; // make it positive sk.m_zero |= 0x01; } // if we already have added a bucket for this "second" then // get it from the tree so we can add to its accumulated stats. long node1 = tree->getNode ( 0 , (char *)&sk ); long node2; StatData *sd; // get that stat, see if we are accumulating it already if ( node1 >= 0 ) sd = (StatData *)tree->getData ( node1 ); // make a new one if not there else { StatData tmp; // init it tmp.m_totalOps = 0.0; tmp.m_totalQuantity = 0.0; tmp.m_totalTime = 0.0; // save this long saved = g_errno; // need to add using rdb so it can memcpy the data if ( ! m_rdb.addRecord ( (collnum_t)0 , (char *)&sk, (char *)&tmp, sizeof(StatData), niceness ) ) { if ( g_errno != ETRYAGAIN ) log("statsdb: add rec failed: %s", mstrerror(g_errno)); // caller does not care about g_errno g_errno = saved; return false; } // caller does not care about g_errno g_errno = saved; // get the node in the tree //sd = (StatData *)tree->getData ( node1 ); // must be there! node2 = tree->getNode ( 0 , (char *)&sk ); // must be there! if ( node2 < 0 ) { char *xx=NULL;*xx=0; } // point to it sd = (StatData *)tree->getData ( node2 ); } // use the milliseconds elapsed as the value if none given //if ( value == 0 && ! parmHash ) // value = t2Arg - t1Arg; // if we got it for this time, accumulate it // convert x into pixel position sd->m_totalOps += 1 * fractionTime; sd->m_totalQuantity += value * fractionTime; sd->m_totalTime += dtSecs * fractionTime; if ( ! parmHash ) continue; sd->m_totalOps = 0; sd->m_totalQuantity = oldVal; sd->m_newVal = newVal; // no fractions for this! break; } //logf(LOG_DEBUG,"statsdb: sp=0x%lx",(long)sp); return true; }
// . ***** META LIST ADD LOOP ***** // . add meta lists corresponding to the keys in m_addme[] // . call msg5 in case some got dumped to disk! // . never add zids out of order for the same sid // . returns false if blocked, sets g_errno on error and returns true bool Syncdb::loop2 ( ) { // sanity check if ( ! m_calledLoop1 ) { char *xx=NULL;*xx=0; } loop: // breathe just in case QUICKPOLL ( MAX_NICENESS ); // return if done! if ( m_ia >= m_na ) { m_calledLoop2 = true ; return true; } // . check the tree for the next key to add first!!! // . most of the time this should be the case! RdbTree *stree = &m_rdb.m_tree; // get the key key128_t k = m_addMe[m_ia]; // is it there? long n = stree->getNode ( 0 , (char *)&k ); // yes! easy add... if ( n >= 0 ) { //long reqSize = stree->getDataSize ( n ); char *req = stree->getData ( n ); // get zid from key uint64_t zid1 = getZid ( &k ); // get zid from request uint64_t zid2 = *(uint64_t *)(req+4); // must match! if ( zid1 != zid2 ) { char *xx=NULL;*xx=0; } // . add away using Msg4.cpp // . return with g_errno set on error if ( ! addMetaList ( req ) ) return true; // success, delete the key from QUICK TREE m_qt.deleteNode ( 0 , (char *)&k , true ); // . and delete that node (freeData = true) // . no! not until we got all the checkoff requests in! // stree->deleteNode ( n , true ); // advance on success m_ia++; // go back for more goto loop; } // make the key range key128_t sk = k; key128_t ek = k; // make negative sk.n0 &= 0xfffffffffffffffeLL; // do not let sleep ticker call bigLoop m_outstanding = true; // get the meta list from on disk i guess, if not in tree if ( ! m_msg5.getList ( RDB_SYNCDB , (collnum_t)0 , // coll &m_list , (char *)&sk , // startKey (char *)&ek , // endKey 999 , // minRecSizes true , // includeTree? false , // addToCache 0 , // maxCacheAge 0 , // startFielNum -1 , // numFiles NULL , // state gotListWrapper , MAX_NICENESS , true )) // do error correction? return false; // process it. loop back up for another on success! if ( gotList ( ) ) goto loop; // bad. g_errno must be set return false; }