void gotReplyWrapper3a ( void *state , void *state2 ) {
Msg3a *THIS = (Msg3a *)state;
// timestamp log
if ( THIS->m_debug )
logf(LOG_DEBUG,"query: msg3a: [%lu] got reply #%li in %lli ms."
" err=%s", (long)THIS, THIS->m_numReplies ,
gettimeofdayInMilliseconds() - THIS->m_startTime ,
mstrerror(g_errno) );
else if ( g_errno )
logf(LOG_DEBUG,"msg3a: error reply. [%lu] got reply #%li "
" err=%s", (long)THIS, THIS->m_numReplies ,
mstrerror(g_errno) );
// if one split times out, ignore it!
if ( g_errno == EQUERYTRUNCATED ||
g_errno == EUDPTIMEDOUT )
g_errno = 0;
// record it
if ( g_errno && ! THIS->m_errno )
THIS->m_errno = g_errno;
// set it
Multicast *m = (Multicast *)state2;
// update time
long long endTime = gettimeofdayInMilliseconds();
// update host table
Host *h = m->m_replyingHost;
// i guess h is NULL on error?
if ( h ) {
// how long did it take from the launch of request until now
// for host "h" to give us the docids?
long long delta = (endTime - m->m_replyLaunchTime);
// . sanity check
// . ntpd can screw with our local time and make this negative
if ( delta >= 0 ) {
// count the split
h->m_splitsDone++;
// accumulate the times so we can do an average display
// in PageHosts.cpp.
h->m_splitTimes += delta;
}
}
// update count of how many replies we got
THIS->m_numReplies++;
// bail if still awaiting more replies
if ( THIS->m_numReplies < THIS->m_numHosts ) return;
// return if gotAllSplitReplies() blocked
if ( ! THIS->gotAllSplitReplies( ) ) return;
// set g_errno i guess so parent knows
if ( THIS->m_errno ) g_errno = THIS->m_errno;
// call callback if we did not block, since we're here. all done.
THIS->m_callback ( THIS->m_state );
}
void *startUp ( void *state ) {
int32_t id = (int32_t) state;
// . what this lwp's priority be?
// . can range from -20 to +20
// . the lower p, the more cpu time it gets
// . this is really the niceness, not the priority
int p = 0;
//if ( id == 1 ) p = 0;
//else p = 30;
// . set this process's priority
// . setpriority() is only used for SCHED_OTHER threads
if ( setpriority ( PRIO_PROCESS, getpid() , p ) < 0 ) {
fprintf(stderr,"Threads::startUp: setpriority: failed\n");
exit(-1);
}
// read buf
char buf [ MAX_READ_SIZE ];
// we got ourselves
s_launched++;
// msg
fprintf(stderr,"id = %"INT32" launched\n",id);
// wait for lock to be unleashed
while ( s_launched != s_numThreads ) usleep(10);
// now do a stupid loop
int32_t j, off , size;
for ( int32_t i = 0 ; i < 100000 ; i++ ) {
off = rand() % (s_filesize - s_maxReadSize );
// rand size
//size = rand() % s_maxReadSize;
size = s_maxReadSize;
//if ( size < 32*1024 ) size = 32*1024;
// time it
int64_t start = gettimeofdayInMilliseconds();
//fprintf(stderr,"%"INT32") i=%"INT32" start\n",id,i );
pread ( s_fd1 , buf , size , off );
//fprintf(stderr,"%"INT32") i=%"INT32" done\n",id,i );
int64_t now = gettimeofdayInMilliseconds();
s_count++;
float sps = (float)((float)s_count * 1000.0) /
(float)(now - s_startTime);
fprintf(stderr,"count=%"INT32" off=%"INT32" size=%"INT32" time=%"INT32"ms "
"(%.2f seeks/sec)\n",
(int32_t)s_count,
(int32_t)off,
(int32_t)size,
(int32_t)(now - start) ,
sps );
}
// dummy return
return NULL;
}
// . return ptr to the buffer we serialize into
// . return NULL and set g_errno on error
bool Msg20Reply::sendReply ( XmlDoc *xd ) {
// get it
UdpSlot *slot = (UdpSlot *)xd->m_slot;
if ( g_errno ) {
// extract titleRec ptr
log("query: Had error generating msg20 reply for d=%lli: "
"%s",m_docId, mstrerror(g_errno));
// don't forget to delete this list
haderror:
mdelete ( xd, sizeof(XmlDoc) , "Msg20" );
delete ( xd );
g_udpServer.sendErrorReply ( slot , g_errno ) ;
return true;
}
// now create a buffer to store title/summary/url/docLen and send back
long need = getStoredSize();
char *buf = (char *)mmalloc ( need , "Msg20Reply" );
if ( ! buf ) goto haderror;
// should never have an error!
long used = serialize ( buf , need );
// sanity
if ( used != need ) { char *xx=NULL;*xx=0; }
// sanity check, no, might have been banned/filtered above around
// line 956 and just called sendReply directly
//if ( st->m_memUsed == 0 ) { char *xx=NULL;*xx=0; }
// use blue for our color
long color = 0x0000ff;
// but use dark blue for niceness > 0
if ( xd->m_niceness > 0 ) color = 0x0000b0;
//Msg20Reply *tt = (Msg20Reply *)buf;
// sanity check
if ( ! xd->m_utf8ContentValid ) { char *xx=NULL;*xx=0; }
// for records
long clen = 0;
if ( xd->m_utf8ContentValid ) clen = xd->size_utf8Content - 1;
// show it in performance graph
if ( xd->m_startTimeValid )
g_stats.addStat_r ( clen ,
xd->m_startTime ,
gettimeofdayInMilliseconds() ,
color );
// . del the list at this point, we've copied all the data into reply
// . this will free a non-null State20::m_ps (ParseState) for us
mdelete ( xd , sizeof(XmlDoc) , "xd20" );
delete ( xd );
g_udpServer.sendReply_ass ( buf , need , buf , need , slot );
return true;
}
bool gotReplyWrapperxd ( void *state ) {
// grab it
XmlDoc *xd = (XmlDoc *)state;
// get it
UdpSlot *slot = (UdpSlot *)xd->m_slot;
// parse the request
Msg20Request *req = (Msg20Request *)slot->m_readBuf;
// print time
long long took = gettimeofdayInMilliseconds() - xd->m_setTime;
// if there is a baclkog of msg20 summary generation requests this
// is really not the cpu it took to make the smmary, but how long it
// took to get the reply. this request might have had to wait for the
// other summaries to finish computing before it got its turn,
// meanwhile its clock was ticking. TODO: make this better?
// only do for niceness 0 otherwise it gets interrupted by quickpoll
// and can take a long time.
if ( (req->m_isDebug || took > 100) && req->m_niceness == 0 )
log("query: Took %lli ms to compute summary for d=%lli u=%s "
"niceness=%li",
took,
xd->m_docId,xd->m_firstUrl.m_url,
xd->m_niceness );
// error?
if ( g_errno ) { xd->m_reply.sendReply ( xd ); return true; }
// this should not block now
Msg20Reply *reply = xd->getMsg20Reply ( );
// sanity check, should not block here now
if ( reply == (void *)-1 ) { char *xx=NULL;*xx=0; }
// NULL means error, -1 means blocked. on error g_errno should be set
if ( ! reply && ! g_errno ) { char *xx=NULL;*xx=0;}
// send it off. will send an error reply if g_errno is set
return reply->sendReply ( xd );
}
// just clear our points array when we're born
Stats::Stats ( ) {
//m_gotLock = false;
m_next = 0;
//m_minWindowStartTime = 0;
memset ( m_pts , 0 , sizeof(StatPoint)*MAX_POINTS );
m_slowDiskReads = 0;
m_queryTimes = 0;
m_numQueries = 0;
m_numSuccess = 0;
m_numFails = 0;
m_avgQueryTime = 0;
m_successRate = 1.0;
m_totalNumQueries = 0;
m_totalNumSuccess = 0;
m_totalNumFails = 0;
m_avgQueriesPerSec = 0;
m_lastQueryLogTime = gettimeofdayInMilliseconds();
m_startTime = m_lastQueryLogTime;
m_upTime = 0;
m_closedSockets = 0;
m_spiderSample = 0;
m_spiderErrors = 0;
m_spiderNew = 0;
m_spiderErrorsNew = 0;
m_totalSpiderSuccessNew = 0;
m_totalSpiderErrorsNew = 0;
m_totalSpiderSuccessOld = 0;
m_totalSpiderErrorsOld = 0;
m_msg3aRecallCnt = 0;
m_tierHits[0] = 0;
m_tierHits[1] = 0;
m_tierHits[2] = 0;
m_tier2Misses = 0;
m_tierTimes[0] = 0;
m_tierTimes[1] = 0;
m_tierTimes[2] = 0;
//m_totalDedupCand = 0;
//m_dedupedCand = 0;
//m_bannedDups = 0;
//m_bigHackDups = 0;
//m_summaryDups = 0;
//m_contentDups = 0;
//m_clusteredTier1 = 0;
//m_clusteredTier2 = 0;
//m_errored = 0;
m_msg3aRecalls[0] = 0;
m_msg3aRecalls[1] = 0;
m_msg3aRecalls[2] = 0;
m_msg3aRecalls[3] = 0;
m_msg3aRecalls[4] = 0;
m_msg3aRecalls[5] = 0;
memset(m_errCodes, 0, 1000*4);
memset(m_isSampleNew, 0, 1000);
memset(m_allErrorsNew, 0, 65536*8);
memset(m_allErrorsOld, 0, 65536*8);
clearMsgStats();
};
main ( int argc , char *argv[] ) {
if ( argc != 4 ) {
fprintf(stderr,"usage: seektest <bigfilename> <numThreads> "
"<maxReadSize>\n");
exit(-1);
}
s_numThreads = atoi(argv[2]);
s_maxReadSize = atoi(argv[3]);
fprintf(stderr,"threads = %"INT32" maxReadSize = %"INT32"\n",
s_numThreads, s_maxReadSize );
if ( s_maxReadSize <= 0 ) s_maxReadSize = 1;
if ( s_maxReadSize > MAX_READ_SIZE ) s_maxReadSize = MAX_READ_SIZE;
// allow the substitution of another filename
struct stat stats;
stats.st_size = 0;
int status = stat ( argv[1] , &stats );
// return the size if the status was ok
if ( status != 0 ) {
fprintf (stderr,"stats failed");
exit(-1);
}
s_filesize = stats.st_size;
fprintf(stderr,"file size = %"INT32"\n",s_filesize);
// seed rand
srand(time(NULL));
// open 2 file descriptors
//s_fd1 = open ( "/tmp/glibc-2.2.2.tar" , O_RDONLY );
s_fd1 = open ( argv[1] , O_RDONLY );
//s_fd2 = open ( "/tmp/glibc-2.2.5.tar" , O_RDONLY );
// . set up the thread attribute we use for all threads
// . fill up with the default values first
if ( pthread_attr_init( &s_attr ) )
fprintf (stderr,"Threads::init: pthread_attr_init: error\n");
// then customize
if ( pthread_attr_setdetachstate(&s_attr,PTHREAD_CREATE_DETACHED) )
fprintf ( stderr,"Threads::init: pthread_attr_setdeatchstate:\n");
if ( setpriority ( PRIO_PROCESS, getpid() , 0 ) < 0 ) {
fprintf(stderr,"Threads:: setpriority: failed\n");
exit(-1);
}
s_lock = 1;
pthread_t tid1, tid2;
// set time
s_startTime = gettimeofdayInMilliseconds();
for ( int32_t i = 0 ; i < s_numThreads ; i++ ) {
int err = pthread_create ( &tid1,&s_attr,startUp,(void *)i) ;
if ( err != 0 ) return -1;
}
// unset lock
s_lock = 0;
// sleep til done
while ( 1 == 1 ) sleep(1000);
}
void Collectiondb::updateTime() {
// get time now in milliseconds
long long newTime = gettimeofdayInMilliseconds();
// change it
if ( m_lastUpdateTime == newTime ) newTime++;
// update it
m_lastUpdateTime = newTime;
// we need a save
m_needsSave = true;
}
bool PageNetTest::netTestStart_r( bool amThread, long num ) {
long long endTime = 0;
long long calcTime = 0;
long count;
long index = 0;
m_running = true;
m_startTime = gettimeofdayInMilliseconds();
endTime = gettimeofdayInMilliseconds();
if( (m_sock[num] = openSock( num, m_type[num], &m_name[num],
m_port[num])) == -1 )
return false;
while( (endTime - m_startTime) < (m_testDuration * 1000) ) {
count = 0;
calcTime = gettimeofdayInMilliseconds();
if( m_type[num] == TEST_READ )
count = readSock( m_sock[num] );
else if( m_type[num] == TEST_SEND )
count = sendSock( m_sock[num] );
endTime = gettimeofdayInMilliseconds();
float secs = (endTime - calcTime)/1000.0;
float mb = (float)count * 8.0 / (1024.0 * 1024.0);
float mbps = mb/secs;
log( LOG_INFO, "net: nettest: took %lli ms to %s %li bytes at "
"%.2f Mbps", endTime - calcTime,
(m_type[num] == TEST_READ)?"receive":"send", count, mbps );
log( LOG_INFO, "net: nettest: run time %lli s",
(endTime-m_startTime)/1000 );
m_calcTable[num][index] = (unsigned long)mbps;
if( ++index >= AVG_TABLE_SIZE ) index = 0;
if( !m_runNetTest ) break;
}
m_sock[num] = closeSock( m_sock[num] );
return true;
}
bool SummaryCache::lookup(int64_t key, const void **data, size_t *datalen)
{
purge_step();
std::map<int64_t,Item>::iterator iter = m.find(key);
if(iter!=m.end() && iter->second.timestamp+max_age>=gettimeofdayInMilliseconds()) {
*data = iter->second.data;
*datalen = iter->second.datalen;
return true;
} else
return false;
}
// send back a reply to the originator of the msg7 injection request
void sendUdpReply7 ( void *state ) {
XmlDoc *xd = (XmlDoc *)state;
// remove from linked list
if ( xd->m_nextInject )
xd->m_nextInject->m_prevInject = xd->m_prevInject;
if ( xd->m_prevInject )
xd->m_prevInject->m_nextInject = xd->m_nextInject;
if ( s_injectHead == xd )
s_injectHead = xd->m_nextInject;
if ( s_injectTail == xd )
s_injectTail = xd->m_prevInject;
xd->m_nextInject = NULL;
xd->m_prevInject = NULL;
UdpSlot *slot = xd->m_injectionSlot;
uint32_t statColor = 0xccffcc;
if(xd->m_indexCode) {
statColor = 0xaaddaa;//0x4e99e9;
}
g_stats.addStat_r ( xd->m_rawUtf8ContentSize,
xd->m_injectStartTime,
gettimeofdayInMilliseconds(),
statColor );
// injecting a warc seems to not set m_indexCodeValid to true
// for the container doc... hmmm...
int32_t indexCode = -1;
int64_t docId = 0;
if ( xd && xd->m_indexCodeValid ) indexCode = xd->m_indexCode;
if ( xd && xd->m_docIdValid ) docId = xd->m_docId;
mdelete ( xd, sizeof(XmlDoc) , "PageInject" );
delete (xd);
if ( g_errno ) {
g_udpServer.sendErrorReply(slot,g_errno);
return;
}
// just send back the 4 byte indexcode, which is 0 on success,
// otherwise it is the errno
char *tmp = slot->m_tmpBuf;
char *p = tmp;
memcpy ( p , (char *)&indexCode , 4 );
p += 4;
memcpy ( p , (char *)&docId , 8 );
p += 8;
g_udpServer.sendReply_ass(tmp,(p-tmp),NULL,0,slot);
}
void Stats::calcQueryStats() {
long long now = gettimeofdayInMilliseconds();
m_upTime = now - m_startTime;
m_avgQueryTime = (float)m_queryTimes /
((float)m_numQueries * 1000.0);
m_successRate = (float)m_numSuccess /
(float)(m_numSuccess + m_numFails);
//(number of queries) / seconds that it took to get this many queries
m_avgQueriesPerSec = ((float)m_numQueries * 1000.0) /
(float)(now - m_lastQueryLogTime);
}
// . returns false and sets g_errno on error
// . we are responsible for freeing reply/replySize
void Msg0::gotReply ( char *reply , int32_t replySize , int32_t replyMaxSize ) {
logTrace( g_conf.m_logTraceMsg0, "BEGIN" );
// timing debug
if ( g_conf.m_logTimingNet && m_rdbId==RDB_POSDB && m_startTime > 0 )
log(LOG_TIMING,"net: msg0: Got termlist, termId=%" PRIu64". "
"Took %" PRId64" ms, replySize=%" PRId32" (niceness=%" PRId32").",
g_posdb.getTermId ( m_startKey ) ,
gettimeofdayInMilliseconds()-m_startTime,
replySize,m_niceness);
// TODO: insert some seals for security, may have to alloc
// separate space for the list then
// set the list w/ the remaining data
QUICKPOLL(m_niceness);
m_list->set ( reply ,
replySize ,
reply , // alloc buf begins here, too
replyMaxSize ,
m_startKey ,
m_endKey ,
m_fixedDataSize ,
true , // ownData?
m_useHalfKeys ,
m_ks );
// return now if we don't add to cache
//if ( ! m_addToCache ) return;
//
// add posdb list to termlist cache
//
//if ( m_rdbId != RDB_POSDB ) return;
// add to LOCAL termlist cache
//addToTermListCache(m_coll,m_startKey,m_endKey,m_list);
// ignore any error adding to cache
//g_errno = 0;
// . NO! no more network caching, we got gigabit... save space
// for our disk, no replication, man, mem is expensive
// . throw the just the list into the net cache
// . addToNetCache() will copy it for it's own
// . our current copy should be freed by the user's callback somewhere
// . grab our corresponding rdb's local cache
// . we'll use it to store this list since there's no collision chance
//RdbCache *cache = m_rdb->getCache ();
// . add the list to this cache
// . returns false and sets g_errno on error
// . will not be added if cannot copy the data
//cache->addList ( m_startKey , m_list ) ;
// reset g_errno -- we don't care if cache coulnd't add it
//g_errno = 0;
logTrace( g_conf.m_logTraceMsg0, "END" );
}
int main ( int argc , char *argv[] ) {
long long last = -1LL;
loop:
long long now = gettimeofdayInMilliseconds();
char *msg;
long long diff = now - last;
if ( last != -1LL && diff >= 2000 )
fprintf (stderr,"last=%lli now=%lli diff=%lli\n",
last,now,diff);
last = now;
sleep(1);
goto loop;
}
void SummaryCache::purge_step()
{
if(purge_iter==m.end())
purge_iter = m.begin();
else {
int64_t now = gettimeofdayInMilliseconds();
if(purge_iter->second.timestamp+max_age<now) {
std::map<int64_t,Item>::iterator iter = purge_iter;
++purge_iter;
mfree(iter->second.data,iter->second.datalen,memory_note);
memory_used -= iter->second.datalen;
m.erase(iter);
} else
++purge_iter;
}
}
// . this may be called from a signal handler
// . we call from a signal handler to keep msg21 zippy
// . this may be called twice, onece from sig handler and next time not
// from the sig handler
void doneSending_ass ( void *state , UdpSlot *slot ) {
// point to our state
State00 *st0 = (State00 *)state;
// this is nULL if we hit the cache above
if ( ! st0 ) return;
// this might be inaccurate cuz sig handler can't call it!
int64_t now = gettimeofdayInMilliseconds();
// log the stats
if ( g_conf.m_logTimingNet ) {
double mbps ;
mbps = (((double)slot->m_sendBufSize) * 8.0 / (1024.0*1024.0))/
(((double)slot->m_startTime)/1000.0);
log("net: msg0: Sent %"INT32" bytes of data in %"INT64" ms (%3.1fMbps) "
"(niceness=%"INT32").",
slot->m_sendBufSize , now - slot->m_startTime , mbps ,
st0->m_niceness );
}
// can't go any further if we're in a sig handler
//if ( g_inSigHandler ) return;
// . mark it in pinkish purple
// . BUT, do not add stats here for tagdb, we get WAY too many lookups
// and it clutters the performance graph
if ( st0->m_rdbId == RDB_TAGDB ) {
}
else if(slot->m_niceness > 0) {
g_stats.addStat_r ( slot->m_sendBufSize ,
st0->m_startTime ,
now ,
//"transmit_data_nice",
0x00aa00aa);
}
else {
g_stats.addStat_r ( slot->m_sendBufSize ,
st0->m_startTime ,
now ,
//"transmit_data",
0x00ff00ff );
}
// release st0 now
mdelete ( st0 , sizeof(State00) , "Msg0" );
delete ( st0 );
}
bool gotReplyWrapperxd(void *state_) {
Msg20State *state = static_cast<Msg20State*>(state_);
// print time
int64_t now = gettimeofdayInMilliseconds();
int64_t took = now - state->m_xmldoc.m_setTime;
int64_t took2 = 0;
if ( state->m_xmldoc.m_cpuSummaryStartTime) {
took2 = now - state->m_xmldoc.m_cpuSummaryStartTime;
}
// if there is a baclkog of msg20 summary generation requests this
// is really not the cpu it took to make the smmary, but how long it
// took to get the reply. this request might have had to wait for the
// other summaries to finish computing before it got its turn,
// meanwhile its clock was ticking. TODO: make this better?
// only do for niceness 0 otherwise it gets interrupted by quickpoll
// and can take a int32_t time.
if ( state->m_req->m_niceness == 0 && (state->m_req->m_isDebug || took > 100 || took2 > 100 ) ) {
log(LOG_TIMING, "query: Took %" PRId64" ms (total=%" PRId64" ms) to compute summary for d=%" PRId64" "
"u=%s status=%s q=%s",
took2,
took,
state->m_xmldoc.m_docId, state->m_xmldoc.m_firstUrl.getUrl(),
mstrerror(g_errno),
state->m_req->ptr_qbuf);
}
// error?
if ( g_errno ) {
state->m_xmldoc.m_reply.sendReply(state);
return true;
}
// this should not block now
Msg20Reply *reply = state->m_xmldoc.getMsg20Reply();
// sanity check, should not block here now
if ( reply == (void *)-1 ) { g_process.shutdownAbort(true); }
// NULL means error, -1 means blocked. on error g_errno should be set
if ( ! reply && ! g_errno ) { g_process.shutdownAbort(true);}
// send it off. will send an error reply if g_errno is set
return reply->sendReply(state);
}
void SummaryCache::insert(int64_t key, const void *data, size_t datalen)
{
purge_step();
if(max_age==0 || max_memory==0)
return; //cache disabled
std::map<int64_t,Item>::iterator iter = m.find(key);
if(iter!=m.end()) {
//remove the old entry first
if(purge_iter==iter)
++purge_iter;
mfree(iter->second.data,iter->second.datalen,memory_note);
memory_used -= iter->second.datalen;
m.erase(iter);
}
Item item;
item.timestamp = 0; //temporarily, for exception+memoryleak reason
item.data = 0;
item.datalen = 0;
iter = m.insert(std::make_pair(key,item)).first;
void *datacopy = mmalloc(datalen, memory_note);
if(!datacopy) {
m.erase(iter);
return;
}
memcpy(datacopy,data,datalen);
iter->second.data = datacopy;
iter->second.datalen = datalen;
iter->second.timestamp = gettimeofdayInMilliseconds();
memory_used += datalen;
if(memory_used>max_memory)
forced_purge_step();
}
long PageNetTest::readSock( int sock ) {
int n;
unsigned int fromLen;
long count = 0;
// send more than expected to make up for losses
while( count < m_testBytes ) {
if( !m_runNetTest ) return count;
fromLen = sizeof ( struct sockaddr );
n = recvfrom( sock, m_rdgram, NTDGRAM_SIZE, 0,
(sockaddr *)&m_from, &fromLen );
if ( n <= 0 ) {log( "net: nettest: recvfrom:%s",
strerror(errno) );}
else {count += n;}
if( (gettimeofdayInMilliseconds() - m_startTime) >
(m_testDuration * 1000) )
return count;
}
return count;
}
long PageNetTest::sendSock( int sock ) {
int n;
unsigned int toLen;
long count = 0;
// send more than expected to make up for losses
long nn = m_testBytes * 10;
toLen = sizeof(struct sockaddr);
while( count < nn ) {
if( !m_runNetTest ) return count;
n = sendto( sock, m_sdgram, NTDGRAM_SIZE, 0,
(struct sockaddr *)&m_to, toLen );
if ( n != NTDGRAM_SIZE ) log("net: nettest: sendto:%s",
strerror(errno));
else count += n;
if( (gettimeofdayInMilliseconds() - m_startTime) >
(m_testDuration * 1000) )
return count;
}
return count;
}
void startSpidering ( ) {
// url class for parsing/normalizing url
Url u;
// count total urls done
static int64_t s_startTime = 0;
// set startTime
if ( s_startTime == 0 ) s_startTime = gettimeofdayInMilliseconds();
// get time now
int64_t now = gettimeofdayInMilliseconds();
// elapsed time to do all urls
double took = (double)(now - s_startTime) / 1000.0 ;
// log this every 20 urls
if ( s_printIt && s_total > 0 && ( s_total % 20 ) == 0 ) {
logf(LOG_INFO,"did %"INT32" urls in %f seconds. %f urls per second."
" threads now = %"INT32".",
s_total , took , ((double)s_total) / took, s_launched);
s_printIt = false;
}
// did we wait int32_t enough?
if ( now - s_lastTime < s_wait ) return;
s_lastTime = now;
// . use HttpServer.getDoc() to fetch it
// . fetch X at a time
while ( (s_server || s_p < s_pend) && s_launched < s_maxNumThreads ) {
// clear any error
g_errno = 0;
//append s_append to the url
char url[MAX_URL_LEN];
char *p = url;
char *pend = url + MAX_URL_LEN;
char *t = NULL;
if(s_server) {
int32_t len = gbstrlen(s_server);
gbmemcpy ( p, s_server, len);
p += len;
p += getRandomWords(p, pend, s_numRandWords);
int32_t appendLen = gbstrlen(s_append);
if ( p + appendLen < pend ) {
gbmemcpy ( p, s_append, gbstrlen(s_append) );
p += gbstrlen(s_append);
}
*p++ = '\0';
u.set ( url , p - url, false, false, false, false, false, 0x7fffffff );
t = g_mem.strdup(url, "saved url");
}
else {
gbmemcpy ( p, s_p, gbstrlen(s_p));
p += gbstrlen ( s_p );
if ( gbstrlen(s_p) + gbstrlen(s_append) < MAX_URL_LEN )
gbmemcpy ( p, s_append, gbstrlen(s_append) );
p += gbstrlen(s_append);
//null end
*p ='\0';
// make into a url class
u.set ( url , gbstrlen(url), false, false, false, false, false, 0x7fffffff );
// set port if port switch is true
//if ( s_portSwitch ) {
// int32_t r = rand() % 32;
// u.setPort ( 8000 + r );
//}
// save s_p
t = s_p;
// skip to next url
s_p += gbstrlen ( s_p ) + 1;
}
// count it
s_launched++;
// get it
bool status = g_httpServer.getDoc ( u.getUrl() , // url
0, // ip
0 , // offset
-1 , // size
0 , // ifModifiedSince
(void *)t , // state
gotDocWrapper, // callback
20*1000, // timeout
0, // proxy ip
0, // proxy port
30*1024*1024, //maxLen
30*1024*1024);//maxOtherLen
// continue if it blocked
if ( ! status ) continue;
// otherwise, got it right away
s_launched--;
// log msg
log("got doc1 %s: %s", u.getUrl() , mstrerror(g_errno) );
// we gotta wait
break;
}
// bail if not done yet
//if ( s_launched > 0 ) return;
if ( s_server || s_p < s_pend ) return;
// otherwise, we're all done
logf(LOG_INFO,"blaster: did %"INT32" urls in %f seconds. %f urls per "
"second.",
s_total , took , ((double)s_total) / took );
// exit now
exit ( 0 );
}
// . now come here when we got the necessary index lists
// . returns false if blocked, true otherwise
// . sets g_errno on error
bool Msg39::intersectLists ( ) { // bool updateReadInfo ) {
// bail on error
if ( g_errno ) {
hadError:
log("msg39: Had error getting termlists: %s.",
mstrerror(g_errno));
if ( ! g_errno ) { char *xx=NULL;*xx=0; }
//sendReply (m_slot,this,NULL,0,0,true);
return true;
}
// timestamp log
if ( m_debug ) {
log(LOG_DEBUG,"query: msg39: [%"PTRFMT"] "
"Got %"INT32" lists in %"INT64" ms"
, (PTRTYPE)this,m_tmpq.getNumTerms(),
gettimeofdayInMilliseconds() - m_startTime);
m_startTime = gettimeofdayInMilliseconds();
}
// breathe
QUICKPOLL ( m_r->m_niceness );
// ensure collection not deleted from under us
CollectionRec *cr = g_collectiondb.getRec ( m_r->m_collnum );
if ( ! cr ) {
g_errno = ENOCOLLREC;
goto hadError;
}
// . set the IndexTable so it can set it's score weights from the
// termFreqs of each termId in the query
// . this now takes into account the special termIds used for sorting
// by date (0xdadadada and 0xdadadad2 & TERMID_MASK)
// . it should weight them so much so that the summation of scores
// from other query terms cannot make up for a lower date score
// . this will actually calculate the top
// . this might also change m_tmpq.m_termSigns
// . this won't do anything if it was already called
m_posdbTable.init ( &m_tmpq ,
m_debug ,
this ,
&m_tt ,
m_r->m_collnum,//ptr_coll ,
&m_msg2 , // m_lists ,
//m_tmpq.m_numTerms , // m_numLists
m_r );
// breathe
QUICKPOLL ( m_r->m_niceness );
// . we have to do this here now too
// . but if we are getting weights, we don't need m_tt!
// . actually we were using it before for rat=0/bool queries but
// i got rid of NO_RAT_SLOTS
if ( ! m_allocedTree && ! m_posdbTable.allocTopTree() ) {
if ( ! g_errno ) { char *xx=NULL;*xx=0; }
//sendReply ( m_slot , this , NULL , 0 , 0 , true);
return true;
}
// if msg2 had ALL empty lists we can cut it int16_t
if ( m_posdbTable.m_topTree->m_numNodes == 0 ) {
//estimateHitsAndSendReply ( );
return true;
}
// we have to allocate this with each call because each call can
// be a different docid range from doDocIdSplitLoop.
if ( ! m_posdbTable.allocWhiteListTable() ) {
log("msg39: Had error allocating white list table: %s.",
mstrerror(g_errno));
if ( ! g_errno ) { char *xx=NULL;*xx=0; }
//sendReply (m_slot,this,NULL,0,0,true);
return true;
}
// do not re do it if doing docid range splitting
m_allocedTree = true;
// . now we must call this separately here, not in allocTopTree()
// . we have to re-set the QueryTermInfos with each docid range split
// since it will set the list ptrs from the msg2 lists
if ( ! m_posdbTable.setQueryTermInfo () ) return true;
// print query term bit numbers here
for ( int32_t i = 0 ; m_debug && i < m_tmpq.getNumTerms() ; i++ ) {
QueryTerm *qt = &m_tmpq.m_qterms[i];
//utf16ToUtf8(bb, 256, qt->m_term, qt->m_termLen);
char *tpc = qt->m_term + qt->m_termLen;
char tmp = *tpc;
*tpc = '\0';
SafeBuf sb;
sb.safePrintf("query: msg39: BITNUM query term #%"INT32" \"%s\" "
"bitnum=%"INT32" ", i , qt->m_term, qt->m_bitNum );
// put it back
*tpc = tmp;
logf(LOG_DEBUG,"%s",sb.getBufStart());
}
// timestamp log
if ( m_debug ) {
log(LOG_DEBUG,"query: msg39: [%"PTRFMT"] "
"Preparing to intersect "
"took %"INT64" ms",
(PTRTYPE)this,
gettimeofdayInMilliseconds() - m_startTime );
m_startTime = gettimeofdayInMilliseconds();
}
// time it
int64_t start = gettimeofdayInMilliseconds();
int64_t diff;
// . don't bother making a thread if lists are small
// . look at STAGE? in IndexReadInfo.cpp to see how we read in stages
// . it's always saying msg39 handler is hogging cpu...could this be it
//if ( m_msg2.getTotalRead() < 2000*8 ) goto skipThread;
// debug
//goto skipThread;
// . NOW! let's do this in a thread so we can continue to service
// incoming requests
// . don't launch more than 1 thread at a time for this
// . set callback when thread done
// breathe
QUICKPOLL ( m_r->m_niceness );
// . create the thread
// . only one of these type of threads should be launched at a time
if ( ! m_debug &&
g_threads.call ( INTERSECT_THREAD , // threadType
m_r->m_niceness ,
this , // top 4 bytes must be cback
controlLoopWrapper2,//threadDoneWrapper ,
addListsWrapper ) ) {
m_blocked = true;
return false;
}
// if it failed
//log(LOG_INFO,"query: Intersect thread creation failed. Doing "
// "blocking. Hurts performance.");
// check tree
if ( m_tt.m_nodes == NULL ) {
log(LOG_LOGIC,"query: msg39: Badness.");
char *xx = NULL; *xx = 0; }
// sometimes we skip the thread
//skipThread:
// . addLists() should never have a problem
// . g_errno should be set by prepareToAddLists() above if there is
// going to be a problem
//if ( m_r->m_useNewAlgo )
m_posdbTable.intersectLists10_r ( );
//else
// m_posdbTable.intersectLists9_r ( );
// time it
diff = gettimeofdayInMilliseconds() - start;
if ( diff > 10 ) log("query: Took %"INT64" ms for intersection",diff);
// returns false if blocked, true otherwise
//return addedLists ();
return true;
}
// . returns false if blocked, true otherwise
// . sets g_errno on error
// . called either from
// 1) doDocIdSplitLoop
// 2) or getDocIds2() if only 1 docidsplit
bool Msg39::getLists () {
if ( m_debug ) m_startTime = gettimeofdayInMilliseconds();
// . ask Indexdb for the IndexLists we need for these termIds
// . each rec in an IndexList is a termId/score/docId tuple
//
// restrict to docid range?
//
// . get the docid start and end
// . do docid paritioning so we can send to all hosts
// in the network, not just one stripe
int64_t docIdStart = 0;
int64_t docIdEnd = MAX_DOCID;
// . restrict to this docid?
// . will really make gbdocid:| searches much faster!
int64_t dr = m_tmpq.m_docIdRestriction;
if ( dr ) {
docIdStart = dr;
docIdEnd = dr + 1;
}
// . override
// . this is set from Msg39::doDocIdSplitLoop() to compute
// search results in stages, so that we do not load massive
// termlists into memory and got OOM (out of memory)
if ( m_r->m_minDocId != -1 ) docIdStart = m_r->m_minDocId;
if ( m_r->m_maxDocId != -1 ) docIdEnd = m_r->m_maxDocId+1;
// if we have twins, then make sure the twins read different
// pieces of the same docid range to make things 2x faster
//bool useTwins = false;
//if ( g_hostdb.getNumStripes() == 2 ) useTwins = true;
//if ( useTwins ) {
// int64_t delta2 = ( docIdEnd - docIdStart ) / 2;
// if ( m_r->m_stripe == 0 ) docIdEnd = docIdStart + delta2;
// else docIdStart = docIdStart + delta2;
//}
// new striping logic:
int32_t numStripes = g_hostdb.getNumStripes();
int64_t delta2 = ( docIdEnd - docIdStart ) / numStripes;
int32_t stripe = g_hostdb.getMyHost()->m_stripe;
docIdStart += delta2 * stripe; // is this right?
docIdEnd = docIdStart + delta2;
// add 1 to be safe so we don't lose a docid
docIdEnd++;
// TODO: add triplet support later for this to split the
// read 3 ways. 4 ways for quads, etc.
//if ( g_hostdb.getNumStripes() >= 3 ) { char *xx=NULL;*xx=0;}
// do not go over MAX_DOCID because it gets masked and
// ends up being 0!!! and we get empty lists
if ( docIdEnd > MAX_DOCID ) docIdEnd = MAX_DOCID;
// remember so Msg2.cpp can use them to restrict the termlists
// from "whiteList" as well
m_docIdStart = docIdStart;
m_docIdEnd = docIdEnd;
//
// set startkey/endkey for each term/termlist
//
for ( int32_t i = 0 ; i < m_tmpq.getNumTerms() ; i++ ) {
// breathe
QUICKPOLL ( m_r->m_niceness );
// int16_tcuts
QueryTerm *qterm = &m_tmpq.m_qterms[i];
char *sk = qterm->m_startKey;
char *ek = qterm->m_endKey;
// get the term id
int64_t tid = m_tmpq.getTermId(i);
// if only 1 stripe
//if ( g_hostdb.getNumStripes() == 1 ) {
// docIdStart = 0;
// docIdEnd = MAX_DOCID;
//}
// debug
if ( m_debug )
log("query: setting sk/ek for docids %"INT64""
" to %"INT64" for termid=%"INT64""
, docIdStart
, docIdEnd
, tid
);
// store now in qterm
g_posdb.makeStartKey ( sk , tid , docIdStart );
g_posdb.makeEndKey ( ek , tid , docIdEnd );
qterm->m_ks = sizeof(POSDBKEY);//key144_t);
}
// debug msg
if ( m_debug || g_conf.m_logDebugQuery ) {
for ( int32_t i = 0 ; i < m_tmpq.getNumTerms() ; i++ ) {
// get the term in utf8
//char bb[256];
QueryTerm *qt = &m_tmpq.m_qterms[i];
//utf16ToUtf8(bb, 256, qt->m_term, qt->m_termLen);
char *tpc = qt->m_term + qt->m_termLen;
char tmp = *tpc;
*tpc = '\0';
char sign = qt->m_termSign;
if ( sign == 0 ) sign = '0';
QueryWord *qw = qt->m_qword;
int32_t wikiPhrId = qw->m_wikiPhraseId;
if ( m_tmpq.isPhrase(i) ) wikiPhrId = 0;
char leftwikibigram = 0;
char rightwikibigram = 0;
if ( qt->m_leftPhraseTerm &&
qt->m_leftPhraseTerm->m_isWikiHalfStopBigram )
leftwikibigram = 1;
if ( qt->m_rightPhraseTerm &&
qt->m_rightPhraseTerm->m_isWikiHalfStopBigram )
rightwikibigram = 1;
/*
char c = m_tmpq.getTermSign(i);
char tt[512];
int32_t ttlen = m_tmpq.getTermLen(i);
if ( ttlen > 254 ) ttlen = 254;
if ( ttlen < 0 ) ttlen = 0;
// old:painful: convert each term from unicode to ascii
gbmemcpy ( tt , m_tmpq.getTerm(i) , ttlen );
*/
int32_t isSynonym = 0;
QueryTerm *st = qt->m_synonymOf;
if ( st ) isSynonym = true;
SafeBuf sb;
// now we can display it
//tt[ttlen]='\0';
//if ( c == '\0' ) c = ' ';
sb.safePrintf(
"query: msg39: [%"PTRFMT"] "
"query term #%"INT32" \"%s\" "
"phr=%"INT32" termId=%"UINT64" rawTermId=%"UINT64" "
//"estimatedTermFreq=%"INT64" (+/- ~16000) "
"tfweight=%.02f "
"sign=%c "
"numPlusses=%hhu "
"required=%"INT32" "
"fielcode=%"INT32" "
"ebit=0x%0"XINT64" "
"impBits=0x%0"XINT64" "
"wikiphrid=%"INT32" "
"leftwikibigram=%"INT32" "
"rightwikibigram=%"INT32" "
//"range.startTermNum=%hhi range.endTermNum=%hhi "
//"minRecSizes=%"INT32" "
"readSizeInBytes=%"INT32" "
//"ebit=0x%"XINT64" "
//"impBits=0x%"XINT64" "
"hc=%"INT32" "
"component=%"INT32" "
"otermLen=%"INT32" "
"isSynonym=%"INT32" "
"querylangid=%"INT32" " ,
(PTRTYPE)this ,
i ,
qt->m_term,//bb ,
(int32_t)m_tmpq.isPhrase (i) ,
m_tmpq.getTermId (i) ,
m_tmpq.getRawTermId (i) ,
((float *)m_r->ptr_termFreqWeights)[i] ,
sign , //c ,
0 ,
(int32_t)qt->m_isRequired,
(int32_t)qt->m_fieldCode,
(int64_t)qt->m_explicitBit ,
(int64_t)qt->m_implicitBits ,
wikiPhrId,
(int32_t)leftwikibigram,
(int32_t)rightwikibigram,
((int32_t *)m_r->ptr_readSizes)[i] ,
//(int64_t)m_tmpq.m_qterms[i].m_explicitBit ,
//(int64_t)m_tmpq.m_qterms[i].m_implicitBits ,
(int32_t)m_tmpq.m_qterms[i].m_hardCount ,
(int32_t)m_tmpq.m_componentCodes[i],
(int32_t)m_tmpq.getTermLen(i) ,
isSynonym,
(int32_t)m_tmpq.m_langId ); // ,tt
// put it back
*tpc = tmp;
if ( st ) {
int32_t stnum = st - m_tmpq.m_qterms;
sb.safePrintf("synofterm#=%"INT32"",stnum);
//sb.safeMemcpy(st->m_term,st->m_termLen);
sb.pushChar(' ');
sb.safePrintf("synwid0=%"INT64" ",qt->m_synWids0);
sb.safePrintf("synwid1=%"INT64" ",qt->m_synWids1);
sb.safePrintf("synalnumwords=%"INT32" ",
qt->m_numAlnumWordsInSynonym);
// like for synonym "nj" it's base,
// "new jersey" has 2 alnum words!
sb.safePrintf("synbasealnumwords=%"INT32" ",
qt->m_numAlnumWordsInBase);
}
logf(LOG_DEBUG,"%s",sb.getBufStart());
}
m_tmpq.printBooleanTree();
}
// timestamp log
if ( m_debug )
log(LOG_DEBUG,"query: msg39: [%"PTRFMT"] "
"Getting %"INT32" index lists ",
(PTRTYPE)this,m_tmpq.getNumTerms());
// . now get the index lists themselves
// . return if it blocked
// . not doing a merge (last parm) means that the lists we receive
// will be an appending of a bunch of lists so keys won't be in order
// . merging is uneccessary for us here because we hash the keys anyway
// . and merging takes up valuable cpu time
// . caution: the index lists returned from Msg2 are now compressed
// . now i'm merging because it's 10 times faster than hashing anyway
// and the reply buf should now always be <= minRecSizes so we can
// pre-allocate one better, and, 3) this should fix the yahoo.com
// reindex bug
char rdbId = RDB_POSDB;
// . TODO: MDW: fix
// . partap says there is a bug in this??? we can't cache UOR'ed lists?
bool checkCache = false;
// split is us????
//int32_t split = g_hostdb.m_myHost->m_group;
int32_t split = g_hostdb.m_myHost->m_shardNum;
// call msg2
if ( ! m_msg2.getLists ( rdbId ,
m_r->m_collnum,//m_r->ptr_coll ,
m_r->m_maxAge ,
m_r->m_addToCache ,
//m_tmpq.m_qterms ,
&m_tmpq,
m_r->ptr_whiteList,
// we need to restrict docid range for
// whitelist as well! this is from
// doDocIdSplitLoop()
m_docIdStart,
m_docIdEnd,
// how much of each termlist to read in bytes
(int32_t *)m_r->ptr_readSizes ,
//m_tmpq.getNumTerms() , // numLists
// 1-1 with query terms
m_lists ,
this ,
controlLoopWrapper,//gotListsWrapper ,
m_r ,
m_r->m_niceness ,
true , // do merge?
m_debug ,
NULL , // best hostids
m_r->m_restrictPosdbForQuery ,
split ,
checkCache )) {
m_blocked = true;
return false;
}
// error?
//if ( g_errno ) {
// log("msg39: Had error getting termlists2: %s.",
// mstrerror(g_errno));
// // don't bail out here because we are in docIdSplitLoop()
// //sendReply (m_slot,this,NULL,0,0,true);
// return true;
//}
//return gotLists ( true );
return true;
}
void Msg39::estimateHitsAndSendReply ( ) {
// no longer in use
m_inUse = false;
// now this for the query loop on the QueryLogEntries.
m_topDocId50 = 0LL;
m_topScore50 = 0.0;
// a little hack for the seo pipeline in xmldoc.cpp
m_topDocId = 0LL;
m_topScore = 0.0;
m_topDocId2 = 0LL;
m_topScore2 = 0.0;
int32_t ti = m_tt.getHighNode();
if ( ti >= 0 ) {
TopNode *t = &m_tt.m_nodes[ti];
m_topDocId = t->m_docId;
m_topScore = t->m_score;
}
// try the 2nd one too
int32_t ti2 = -1;
if ( ti >= 0 ) ti2 = m_tt.getNext ( ti );
if ( ti2 >= 0 ) {
TopNode *t2 = &m_tt.m_nodes[ti2];
m_topDocId2 = t2->m_docId;
m_topScore2 = t2->m_score;
}
// convenience ptrs. we will store the docids/scores into these arrays
int64_t *topDocIds;
double *topScores;
key_t *topRecs;
// numDocIds counts docs in all tiers when using toptree.
int32_t numDocIds = m_tt.m_numUsedNodes;
// the msg39 reply we send back
int32_t replySize;
char *reply;
//m_numTotalHits = m_posdbTable.m_docIdVoteBuf.length() / 6;
// make the reply?
Msg39Reply mr;
// this is what you want to look at if there is no seo.cpp module...
if ( ! m_callback ) {
// if we got clusterdb recs in here, use 'em
if ( m_gotClusterRecs ) numDocIds = m_numVisible;
// don't send more than the docs that are asked for
if ( numDocIds > m_r->m_docsToGet) numDocIds =m_r->m_docsToGet;
// # of QueryTerms in query
int32_t nqt = m_tmpq.m_numTerms;
// start setting the stuff
mr.m_numDocIds = numDocIds;
// copy # estiamted hits into 8 bytes of reply
//int64_t est = m_posdbTable.m_estimatedTotalHits;
// ensure it has at least as many results as we got
//if ( est < numDocIds ) est = numDocIds;
// or if too big...
//if ( numDocIds < m_r->m_docsToGet ) est = numDocIds;
// . total estimated hits
// . this is now an EXACT count!
mr.m_estimatedHits = m_numTotalHits;
// sanity check
mr.m_nqt = nqt;
// the m_errno if any
mr.m_errno = m_errno;
// int16_tcut
PosdbTable *pt = &m_posdbTable;
// the score info, in no particular order right now
mr.ptr_scoreInfo = pt->m_scoreInfoBuf.getBufStart();
mr.size_scoreInfo = pt->m_scoreInfoBuf.length();
// that has offset references into posdbtable::m_pairScoreBuf
// and m_singleScoreBuf, so we need those too now
mr.ptr_pairScoreBuf = pt->m_pairScoreBuf.getBufStart();
mr.size_pairScoreBuf = pt->m_pairScoreBuf.length();
mr.ptr_singleScoreBuf = pt->m_singleScoreBuf.getBufStart();
mr.size_singleScoreBuf = pt->m_singleScoreBuf.length();
// save some time since seo.cpp gets from posdbtable directly,
// so we can avoid serializing/copying this stuff at least
if ( ! m_r->m_makeReply ) {
mr.size_scoreInfo = 0;
mr.size_pairScoreBuf = 0;
mr.size_singleScoreBuf = 0;
}
//mr.m_sectionStats = pt->m_sectionStats;
// reserve space for these guys, we fill them in below
mr.ptr_docIds = NULL;
mr.ptr_scores = NULL;
mr.ptr_clusterRecs = NULL;
// this is how much space to reserve
mr.size_docIds = 8 * numDocIds; // int64_t
mr.size_scores = sizeof(double) * numDocIds; // float
// if not doing site clustering, we won't have these perhaps...
if ( m_gotClusterRecs )
mr.size_clusterRecs = sizeof(key_t) *numDocIds;
else
mr.size_clusterRecs = 0;
#define MAX_FACETS 20000
/////////////////
//
// FACETS
//
/////////////////
// We can have multiple gbfacet: terms in a query so
// serialize all the QueryTerm::m_facetHashTables into
// Msg39Reply::ptr_facetHashList.
//
// combine the facet hash lists of each query term into
// a list of lists. each lsit is preceeded by the query term
// id of the query term (like gbfacet:xpathsitehash12345)
// followed by a 4 byte length of the following 32-bit
// facet values
int32_t need = 0;
for ( int32_t i = 0 ; i < m_tmpq.m_numTerms; i++ ) {
QueryTerm *qt = &m_tmpq.m_qterms[i];
// skip if not facet
if ( qt->m_fieldCode != FIELD_GBFACETSTR &&
qt->m_fieldCode != FIELD_GBFACETINT &&
qt->m_fieldCode != FIELD_GBFACETFLOAT )
continue;
HashTableX *ft = &qt->m_facetHashTable;
if ( ft->m_numSlotsUsed == 0 ) continue;
int32_t used = ft->m_numSlotsUsed;
// limit for memory
if ( used > (int32_t)MAX_FACETS ) {
log("msg39: truncating facet list to 20000 "
"from %"INT32" for %s",used,qt->m_term);
used = (int32_t)MAX_FACETS;
}
// store query term id 64 bit
need += 8;
// then size
need += 4;
// then buckets. keys and counts
need += (4+sizeof(FacetEntry)) * used;
}
// allocate
SafeBuf tmp;
if ( ! tmp.reserve ( need ) ) {
log("query: Could not allocate memory "
"to hold reply facets");
sendReply(m_slot,this,NULL,0,0,true);
return;
}
// point to there
char *p = tmp.getBufStart();
for ( int32_t i = 0 ; i < m_tmpq.m_numTerms ; i++ ) {
QueryTerm *qt = &m_tmpq.m_qterms[i];
// skip if not facet
if ( qt->m_fieldCode != FIELD_GBFACETSTR &&
qt->m_fieldCode != FIELD_GBFACETINT &&
qt->m_fieldCode != FIELD_GBFACETFLOAT )
continue;
// get all the facet hashes and their counts
HashTableX *ft = &qt->m_facetHashTable;
// skip if none
if ( ft->m_numSlotsUsed == 0 ) continue;
// store query term id 64 bit
*(int64_t *)p = qt->m_termId;
p += 8;
int32_t used = ft->getNumSlotsUsed();
if ( used > (int32_t)MAX_FACETS )
used = (int32_t)MAX_FACETS;
// store count
*(int32_t *)p = used;
p += 4;
int32_t count = 0;
// for sanity check
char *pend = p + (used * (4+sizeof(FacetEntry)));
// serialize the key/val pairs
for ( int32_t k = 0 ; k < ft->m_numSlots ; k++ ) {
// skip empty buckets
if ( ! ft->m_flags[k] ) continue;
// store key. the hash of the facet value.
*(int32_t *)p = ft->getKey32FromSlot(k); p += 4;
// then store count
//*(int32_t *)p = ft->getVal32FromSlot(k); p += 4;
// now this has a docid on it so we can
// lookup the text of the facet in Msg40.cpp
FacetEntry *fe;
fe = (FacetEntry *)ft->getValFromSlot(k);
// sanity
// no, count can be zero if its a range facet
// that was never added to. we add those
// empty FaceEntries only for range facets
// in Posdb.cpp
//if(fe->m_count == 0 ) { char *xx=NULL;*xx=0;}
gbmemcpy ( p , fe , sizeof(FacetEntry) );
p += sizeof(FacetEntry);
// do not breach
if ( ++count >= (int32_t)MAX_FACETS ) break;
}
// sanity check
if ( p != pend ) { char *xx=NULL;*xx=0; }
// do the next query term
}
// now point to that so it can be serialized below
mr.ptr_facetHashList = tmp.getBufStart();
mr.size_facetHashList = p - tmp.getBufStart();//tmp.length();
/////////////
//
// END FACETS
//
/////////////
// . that is pretty much it,so serialize it into buffer,"reply"
// . mr.ptr_docIds, etc., will point into the buffer so we can
// re-serialize into it below from the tree
// . returns NULL and sets g_errno on error
// . "true" means we should make mr.ptr_* reference into the
// newly serialized buffer.
reply = serializeMsg ( sizeof(Msg39Reply), // baseSize
&mr.size_docIds, // firstSizeParm
&mr.size_clusterRecs,//lastSizePrm
&mr.ptr_docIds , // firstStrPtr
&mr , // thisPtr
&replySize ,
NULL ,
0 ,
true ) ;
if ( ! reply ) {
log("query: Could not allocated memory "
"to hold reply of docids to send back.");
sendReply(m_slot,this,NULL,0,0,true);
return;
}
topDocIds = (int64_t *) mr.ptr_docIds;
topScores = (double *) mr.ptr_scores;
topRecs = (key_t *) mr.ptr_clusterRecs;
}
int32_t docCount = 0;
// loop over all results in the TopTree
for ( int32_t ti = m_tt.getHighNode() ; ti >= 0 ;
ti = m_tt.getPrev(ti) ) {
// get the guy
TopNode *t = &m_tt.m_nodes[ti];
// skip if clusterLevel is bad!
if ( m_gotClusterRecs && t->m_clusterLevel != CR_OK )
continue;
// if not sending back a reply... we were called from seo.cpp
// State3f logic to evaluate a QueryLogEntry, etc.
if ( m_callback ) {
// skip results past #50
if ( docCount > 50 ) continue;
// set this
m_topScore50 = t->m_score;
m_topDocId50 = t->m_docId;
// that's it
continue;
}
// get the docid ptr
//char *diptr = t->m_docIdPtr;
//int64_t docId = getDocIdFromPtr(diptr);
// sanity check
if ( t->m_docId < 0 ) { char *xx=NULL; *xx=0; }
//add it to the reply
topDocIds [docCount] = t->m_docId;
topScores [docCount] = t->m_score;
if ( m_tt.m_useIntScores )
topScores[docCount] = (double)t->m_intScore;
// supply clusterdb rec? only for full splits
if ( m_gotClusterRecs )
topRecs [docCount] = t->m_clusterRec;
//topExplicits [docCount] =
// getNumBitsOn(t->m_explicits)
docCount++;
// 50th score? set this for seo.cpp. if less than 50 results
// we want the score of the last doc then.
if ( docCount <= 50 ) m_topScore50 = t->m_score;
if ( m_debug ) {
logf(LOG_DEBUG,"query: msg39: [%"PTRFMT"] "
"%03"INT32") docId=%012"UINT64" sum=%.02f",
(PTRTYPE)this, docCount,
t->m_docId,t->m_score);
}
//don't send more than the docs that are wanted
if ( docCount >= numDocIds ) break;
}
if ( docCount > 300 && m_debug )
log("query: Had %"INT32" nodes in top tree",docCount);
// this is sensitive info
if ( m_debug ) {
log(LOG_DEBUG,
"query: msg39: [%"PTRFMT"] "
"Intersected lists took %"INT64" (%"INT64") "
"ms "
"docIdsToGet=%"INT32" docIdsGot=%"INT32" "
"q=%s",
(PTRTYPE)this ,
m_posdbTable.m_addListsTime ,
gettimeofdayInMilliseconds() - m_startTime ,
m_r->m_docsToGet ,
numDocIds ,
m_tmpq.getQuery() );
}
// if we blocked because we used a thread then call callback if
// summoned from a msg3f handler and not a msg39 handler
if ( m_callback ) {
// if we blocked call user callback
if ( m_blocked ) m_callback ( m_state );
// if not sending back a udp reply, return now
return;
}
// now send back the reply
sendReply(m_slot,this,reply,replySize,replySize,false);
return;
}
// . THIS Msg0 class must be alloc'd, i.e. not on the stack, etc.
// . if list is stored locally this tries to get it locally
// . otherwise tries to get the list from the network
// . returns false if blocked, true otherwise
// . sets g_errno on error
// . NOTE: i was having problems with queries being cached too long, you
// see the cache here is a NETWORK cache, so when the machines that owns
// the list updates it on disk it can't flush our cache... so use a small
// maxCacheAge of like , 30 seconds or so...
bool Msg0::getList ( int64_t hostId , // host to ask (-1 if none)
int32_t ip , // info on hostId
int16_t port ,
int32_t maxCacheAge , // max cached age in seconds
bool addToCache , // add net recv'd list to cache?
char rdbId , // specifies the rdb
collnum_t collnum ,
RdbList *list ,
const char *startKey ,
const char *endKey ,
int32_t minRecSizes , // use -1 for no max
void *state ,
void (* callback)(void *state ),//, RdbList *list ) ,
int32_t niceness ,
bool doErrorCorrection ,
bool includeTree ,
bool doMerge ,
int32_t firstHostId ,
int32_t startFileNum ,
int32_t numFiles ,
int64_t timeout ,
int64_t syncPoint ,
int32_t preferLocalReads ,
Msg5 *msg5 ,
bool isRealMerge ,
bool allowPageCache ,
bool forceLocalIndexdb ,
bool noSplit ,
int32_t forceParitySplit ) {
logTrace( g_conf.m_logTraceMsg0, "BEGIN. hostId: %" PRId64", rdbId: %d", hostId, (int)rdbId );
// warning
if ( collnum < 0 ) log(LOG_LOGIC,"net: NULL collection. msg0.");
// reset the list they passed us
list->reset();
// get keySize of rdb
m_ks = getKeySizeFromRdbId ( rdbId );
// if( g_conf.m_logTraceMsg0 )
// {
// log("%s:%s:%d: rdbId. [%d]", __FILE__,__func__,__LINE__, (int)rdbId);
// log("%s:%s:%d: m_ks.. [%d]", __FILE__,__func__,__LINE__, (int)m_ks);
// log("%s:%s:%d: hostId [%" PRId64"]", __FILE__,__func__,__LINE__, hostId);
// }
// if startKey > endKey, don't read anything
//if ( startKey > endKey ) return true;
if ( KEYCMP(startKey,endKey,m_ks)>0 ) { char *xx=NULL;*xx=0; }//rettrue
// . reset hostid if it is dead
// . this is causing UOR queries to take forever when we have a dead
if ( hostId >= 0 && g_hostdb.isDead ( hostId ) ) hostId = -1;
// no longer accept negative minrecsize
if ( minRecSizes < 0 ) {
g_errno = EBADENGINEER;
logTrace( g_conf.m_logTraceMsg0, "END" );
log(LOG_LOGIC, "net: msg0: Negative minRecSizes no longer supported.");
char *xx=NULL;*xx=0;
}
// remember these
m_state = state;
m_callback = callback;
m_list = list;
m_hostId = hostId;
m_niceness = niceness;
m_addToCache = addToCache;
// . these define our request 100%
KEYSET(m_startKey,startKey,m_ks);
KEYSET(m_endKey,endKey,m_ks);
m_minRecSizes = minRecSizes;
m_rdbId = rdbId;
m_collnum = collnum;// = coll;
m_isRealMerge = isRealMerge;
m_allowPageCache = allowPageCache;
// . group to ask is based on the first key
// . we only do 1 group per call right now
// . groupMask must turn on higher bits first (count downwards kinda)
// . titledb and spiderdb use special masks to get groupId
// if diffbot.cpp is reading spiderdb from each shard we have to
// get groupid from hostid here lest we core in getGroupId() below.
// it does that for dumping spiderdb to the client browser. they
// can download the whole enchilada.
if ( hostId >= 0 && m_rdbId == RDB_SPIDERDB )
m_shardNum = 0;
// did they force it? core until i figure out what this is
else if ( forceParitySplit >= 0 )
//m_groupId = g_hostdb.getGroupId ( forceParitySplit );
m_shardNum = forceParitySplit;
else
//m_groupId = getGroupId ( m_rdbId , startKey , ! noSplit );
m_shardNum = getShardNum ( m_rdbId , startKey );
// if we are looking up a termlist in posdb that is split by termid and
// not the usual docid then we have to set this posdb key bit that tells
// us that ...
if ( noSplit && m_rdbId == RDB_POSDB )
m_shardNum = g_hostdb.getShardNumByTermId ( startKey );
// how is this used?
if ( forceLocalIndexdb ) m_shardNum = getMyShardNum();
// if( g_conf.m_logTraceMsg0 ) log("%s:%s:%d: shardNum [%" PRId32"]", __FILE__,__func__, __LINE__, m_shardNum);
// . store these parameters
// . get a handle to the rdb in case we can satisfy locally
// . returns NULL and sets g_errno on error
QUICKPOLL((m_niceness));
Rdb *rdb = getRdbFromId ( m_rdbId );
if ( ! rdb ) return true;
// we need the fixedDataSize
m_fixedDataSize = rdb->getFixedDataSize();
m_useHalfKeys = rdb->useHalfKeys();
// . debug msg
// . Msg2 does this when checking for a cached compound list.
// compound lists do not actually exist, they are merges of smaller
// UOR'd lists.
if ( maxCacheAge != 0 && ! addToCache && (numFiles > 0 || includeTree)) {
log( LOG_LOGIC, "net: msg0: Weird. check but don't add... rdbid=%" PRId32".", ( int32_t ) m_rdbId );
}
// set this here since we may not call msg5 if list not local
//m_list->setFixedDataSize ( m_fixedDataSize );
// . now that we do load balancing we don't want to do a disk lookup
// even if local if we are merging or dumping
// . UNLESS g_conf.m_preferLocalReads is true
if ( preferLocalReads == -1 )
preferLocalReads = g_conf.m_preferLocalReads;
// . always prefer local for full split clusterdb
// . and keep the tfndb/titledb lookups in the same stripe
// . so basically we can't do biased caches if fully split
//if ( g_conf.m_fullSplit ) preferLocalReads = true;
preferLocalReads = true;
// it it stored locally?
bool isLocal = ( m_hostId == -1 && //g_hostdb.m_groupId == m_groupId );
m_shardNum == getMyShardNum() );
// only do local lookups if this is true
if ( ! preferLocalReads ) isLocal = false;
/*
int64_t singleDocIdQuery = 0LL;
if ( rdbId == RDB_POSDB ) {
int64_t d1 = g_posdb.getDocId(m_startKey);
int64_t d2 = g_posdb.getDocId(m_endKey);
if ( d1+1 == d2 ) singleDocIdQuery = d1;
}
// . try the LOCAL termlist cache
// . so when msg2 is evaluating a gbdocid:| query and it has to
// use msg0 to go across the network to get the same damn termlist
// over and over again for the same docid, this will help alot.
// . ideally it'd be nice if the seo pipe in xmldoc.cpp can try to
// send the same gbdocid:xxxx docids to the same hosts. maybe hash
// based on docid into the list of hosts and if that host is busy
// just chain until we find someone not busy.
if ( singleDocIdQuery &&
getListFromTermListCache ( coll,
m_startKey,
m_endKey,
maxCacheAge,
list ) )
// found!
return true;
*/
// but always local if only one host
if ( g_hostdb.getNumHosts() == 1 ) isLocal = true;
// . if the group is local then do it locally
// . Msg5::getList() returns false if blocked, true otherwise
// . Msg5::getList() sets g_errno on error
// . don't do this if m_hostId was specified
if ( isLocal ) {
logTrace( g_conf.m_logTraceMsg0, "isLocal" );
if ( msg5 ) {
m_msg5 = msg5;
m_deleteMsg5 = false;
}
else {
try { m_msg5 = new ( Msg5 ); }
catch ( ... ) {
g_errno = ENOMEM;
log("net: Local alloc for disk read failed "
"while tring to read data for %s. "
"Trying remote request.",
getDbnameFromId(m_rdbId));
goto skip;
}
mnew ( m_msg5 , sizeof(Msg5) , "Msg0::Msg5" );
m_deleteMsg5 = true;
}
QUICKPOLL(m_niceness);
if ( ! m_msg5->getList ( rdbId,
m_collnum ,
m_list ,
m_startKey ,
m_endKey ,
m_minRecSizes ,
includeTree , // include Tree?
addToCache , // addToCache?
maxCacheAge ,
startFileNum ,
numFiles ,
this ,
gotListWrapper2 ,
niceness ,
doErrorCorrection ,
NULL , // cacheKeyPtr
0 , // retryNum
-1 , // maxRetries
true , // compensateForMerge
syncPoint ,
m_isRealMerge ,
m_allowPageCache ) ) {
logTrace( g_conf.m_logTraceMsg0, "END, return false" );
return false;
}
// nuke it
reset();
logTrace( g_conf.m_logTraceMsg0, "END, return true" );
return true;
}
skip:
// debug msg
if ( g_conf.m_logDebugQuery )
log(LOG_DEBUG,"net: msg0: Sending request for data to "
"shard=%" PRIu32" "
"listPtr=%" PTRFMT" minRecSizes=%" PRId32" termId=%" PRIu64" "
//"startKey.n1=%" PRIx32",n0=%" PRIx64" (niceness=%" PRId32")",
"startKey.n1=%" PRIx64",n0=%" PRIx64" (niceness=%" PRId32")",
//g_hostdb.makeHostId ( m_groupId ) ,
m_shardNum,
(PTRTYPE)m_list,
m_minRecSizes, g_posdb.getTermId(m_startKey) ,
//m_startKey.n1,m_startKey.n0 , (int32_t)m_niceness);
KEY1(m_startKey,m_ks),KEY0(m_startKey),
(int32_t)m_niceness);
char *replyBuf = NULL;
int32_t replyBufMaxSize = 0;
bool freeReply = true;
// . make a request with the info above (note: not in network order)
// . IMPORTANT!!!!! if you change this change
// Multicast.cpp::sleepWrapper1 too!!!!!!!!!!!!
// no, not anymore, we commented out that request peeking code
char *p = m_request;
*(int64_t *) p = syncPoint ; p += 8;
//*(key_t *) p = m_startKey ; p += sizeof(key_t);
//*(key_t *) p = m_endKey ; p += sizeof(key_t);
*(int32_t *) p = m_minRecSizes ; p += 4;
*(int32_t *) p = startFileNum ; p += 4;
*(int32_t *) p = numFiles ; p += 4;
*(int32_t *) p = maxCacheAge ; p += 4;
if ( p - m_request != RDBIDOFFSET ) { char *xx=NULL;*xx=0; }
*p = m_rdbId ; p++;
*p = addToCache ; p++;
*p = doErrorCorrection; p++;
*p = includeTree ; p++;
*p = (char)niceness ; p++;
*p = (char)m_allowPageCache; p++;
KEYSET(p,m_startKey,m_ks); ; p+=m_ks;
KEYSET(p,m_endKey,m_ks); ; p+=m_ks;
// NULL terminated collection name
//strcpy ( p , coll ); p += gbstrlen ( coll ); *p++ = '\0';
*(collnum_t *)p = m_collnum; p += sizeof(collnum_t);
m_requestSize = p - m_request;
// ask an individual host for this list if hostId is NOT -1
if ( m_hostId != -1 ) {
// get Host
Host *h = g_hostdb.getHost ( m_hostId );
if ( ! h ) {
g_errno = EBADHOSTID;
log(LOG_LOGIC,"net: msg0: Bad hostId of %" PRId64".", m_hostId);
logTrace( g_conf.m_logTraceMsg0, "END, return true. Bad hostId" );
return true;
}
// if niceness is 0, use the higher priority udpServer
UdpServer *us ;
uint16_t port;
QUICKPOLL(m_niceness);
us = &g_udpServer ; port = h->m_port ;
// . returns false on error and sets g_errno, true otherwise
// . calls callback when reply is received (or error)
// . we return true if it returns false
if ( ! us->sendRequest ( m_request ,
m_requestSize ,
0x00 , // msgType
h->m_ip ,
port ,
m_hostId ,
NULL , // the slotPtr
this ,
gotSingleReplyWrapper ,
timeout ,
-1 , // backoff
-1 , // maxwait
replyBuf ,
replyBufMaxSize ,
m_niceness ) ) { // cback niceness
logTrace( g_conf.m_logTraceMsg0, "END, return true. Request sent" );
return true;
}
// return false cuz it blocked
logTrace( g_conf.m_logTraceMsg0, "END, return false. sendRequest blocked" );
return false;
}
// timing debug
if ( g_conf.m_logTimingNet )
m_startTime = gettimeofdayInMilliseconds();
else
m_startTime = 0;
// . get the top int32_t of the key
// . i guess this will work for 128 bit keys... hmmmmm
int32_t keyTop = hash32 ( (char *)startKey , m_ks );
// . otherwise, multicast to a host in group "groupId"
// . returns false and sets g_errno on error
// . calls callback on completion
// . select first host to send to in group based on upper 32 bits
// of termId (m_startKey.n1)
// . need to send out to all the indexdb split hosts
m_numRequests = 0;
m_numReplies = 0;
//for ( int32_t i = 0; i < m_numSplit; i++ ) {
QUICKPOLL(m_niceness);
//int32_t gr;
char *buf;
buf = replyBuf;
// get the multicast
Multicast *m = &m_mcast;
if ( ! m->send ( m_request ,
m_requestSize,
0x00 , // msgType 0x00
false , // does multicast own request?
m_shardNum ,
false , // send to whole group?
//m_startKey.n1, // key is passed on startKey
keyTop , // key is passed on startKey
this , // state data
NULL , // state data
gotMulticastReplyWrapper0 ,
timeout*1000 , // timeout
niceness ,
firstHostId ,
buf ,
replyBufMaxSize ,
freeReply , // free reply buf?
true , // do disk load balancing?
maxCacheAge ,
//(key_t *)cacheKey ,
// multicast uses it for determining the best
// host to send the request to when doing
// disk load balancing. if the host has our
// data cached, then it will probably get to
// handle the request. for now let's just assume
// this is a 96-bit key. TODO: fix...
0 , // *(key_t *)cacheKey ,
rdbId ,
minRecSizes ) )
{
log(LOG_ERROR, "net: Failed to send request for data from %s in shard "
"#%" PRIu32" over network: %s.",
getDbnameFromId(m_rdbId),m_shardNum, mstrerror(g_errno));
// but speed it up
m_errno = g_errno;
m->reset();
if ( m_numRequests > 0 ) {
logTrace( g_conf.m_logTraceMsg0, "END - returning false" );
return false;
}
logTrace( g_conf.m_logTraceMsg0, "END - returning true" );
return true;
}
m_numRequests++;
// we blocked
logTrace( g_conf.m_logTraceMsg0, "END - returning false, blocked" );
return false;
}
// . slot should be auto-nuked upon transmission or error
// . TODO: ensure if this sendReply() fails does it really nuke the slot?
void gotListWrapper ( void *state , RdbList *listb , Msg5 *msg5xx ) {
logTrace( g_conf.m_logTraceMsg0, "BEGIN" );
// get the state
State00 *st0 = (State00 *)state;
// extract the udp slot and list and msg5
UdpSlot *slot = st0->m_slot;
RdbList *list = &st0->m_list;
Msg5 *msg5 = &st0->m_msg5;
UdpServer *us = st0->m_us;
// timing debug
if ( g_conf.m_logTimingNet || g_conf.m_logDebugNet ) {
//log("Msg0:hndled request %" PRIu64,gettimeofdayInMilliseconds());
int32_t size = -1;
if ( list ) size = list->getListSize();
log(LOG_TIMING|LOG_DEBUG,
"net: msg0: Handled request for data. "
"Now sending data termId=%" PRIu64" size=%" PRId32
" transId=%" PRId32" ip=%s port=%i took=%" PRId64" "
"(niceness=%" PRId32").",
g_posdb.getTermId(msg5->m_startKey),
size,slot->m_transId,
iptoa(slot->m_ip),slot->m_port,
gettimeofdayInMilliseconds() - st0->m_startTime ,
st0->m_niceness );
}
// on error nuke the list and it's data
if ( g_errno ) {
mdelete ( st0 , sizeof(State00) , "Msg0" );
delete (st0);
// TODO: free "slot" if this send fails
log(LOG_ERROR,"%s:%s:%d: call sendErrorReply.", __FILE__, __func__, __LINE__);
us->sendErrorReply ( slot , g_errno );
return;
}
QUICKPOLL(st0->m_niceness);
// point to the serialized list in "list"
char *data = list->getList();
int32_t dataSize = list->getListSize();
char *alloc = list->getAlloc();
int32_t allocSize = list->getAllocSize();
// tell list not to free the data since it is a reply so UdpServer
// will free it when it destroys the slot
list->setOwnData ( false );
// keep track of stats
Rdb *rdb = getRdbFromId ( st0->m_rdbId );
if ( rdb ) rdb->sentReplyGet ( dataSize );
// TODO: can we free any memory here???
// keep track of how long it takes to complete the send
st0->m_startTime = gettimeofdayInMilliseconds();
// debug point
int32_t oldSize = msg5->m_minRecSizes;
int32_t newSize = msg5->m_minRecSizes + 20;
// watch for wrap around
if ( newSize < oldSize ) newSize = 0x7fffffff;
if ( dataSize > newSize && list->getFixedDataSize() == 0 &&
// do not annoy me with these linkdb msgs
dataSize > newSize+100 )
log(LOG_LOGIC,"net: msg0: Sending more data than what was "
"requested. Ineffcient. Bad engineer. dataSize=%" PRId32" "
"minRecSizes=%" PRId32".",dataSize,oldSize);
//
// for linkdb lists, remove all the keys that have the same IP32
// and store a count of what we removed somewhere
//
if ( st0->m_rdbId == RDB_LINKDB ) {
// store compressed list on itself
char *dst = list->m_list;
// keep stats
int32_t totalOrigLinks = 0;
int32_t ipDups = 0;
int32_t lastIp32 = 0;
char *listEnd = list->getListEnd();
// compress the list
for ( ; ! list->isExhausted() ; list->skipCurrentRecord() ) {
// breathe
QUICKPOLL ( st0->m_niceness );
// count it
totalOrigLinks++;
// get rec
char *rec = list->getCurrentRec();
int32_t ip32 = g_linkdb.getLinkerIp_uk((key224_t *)rec );
// same as one before?
if ( ip32 == lastIp32 &&
// are we the last rec? include that for
// advancing the m_nextKey in Linkdb more
// efficiently.
rec + LDBKS < listEnd ) {
ipDups++;
continue;
}
// store it
gbmemcpy (dst , rec , LDBKS );
dst += LDBKS;
// update it
lastIp32 = ip32;
}
// . if we removed one key, store the stats
// . caller should recognize reply is not a multiple of
// the linkdb key size LDBKS and no its there!
if ( ipDups ) {
//*(int32_t *)dst = totalOrigLinks;
//dst += 4;
//*(int32_t *)dst = ipDups;
//dst += 4;
}
// update list parms
list->m_listSize = dst - list->m_list;
list->m_listEnd = list->m_list + list->m_listSize;
data = list->getList();
dataSize = list->getListSize();
}
//log("sending replySize=%" PRId32" min=%" PRId32,dataSize,msg5->m_minRecSizes);
// . TODO: dataSize may not equal list->getListMaxSize() so
// Mem class may show an imblanace
// . now g_udpServer is responsible for freeing data/dataSize
// . the "true" means to call doneSending_ass() from the signal handler
// if need be
st0->m_us->sendReply_ass( data, dataSize, alloc, allocSize, slot, st0, doneSending_ass, -1, -1, true );
logTrace( g_conf.m_logTraceMsg0, "END" );
}
// . but now that we may get a list remotely to fix data corruption,
// this may indeed block
bool Msg3::doneScanning ( ) {
QUICKPOLL(m_niceness);
// . did we have any error on any scan?
// . if so, repeat ALL of the scans
g_errno = m_errno;
// 2 retry is the default
int32_t max = 2;
// see if explicitly provided by the caller
if ( m_maxRetries >= 0 ) max = m_maxRetries;
// now use -1 (no max) as the default no matter what
max = -1;
// ENOMEM is particulary contagious, so watch out with it...
if ( g_errno == ENOMEM && m_maxRetries == -1 ) max = 0;
// msg0 sets maxRetries to 2, don't let max stay set to -1
if ( g_errno == ENOMEM && m_maxRetries != -1 ) max = m_maxRetries;
// when thread cannot alloc enough read buf it keeps the read buf
// set to NULL and BigFile.cpp sets g_errno to EBUFTOOSMALL
if ( g_errno == EBUFTOOSMALL && m_maxRetries == -1 ) max = 0;
// msg0 sets maxRetries to 2, don't let max stay set to -1
if ( g_errno == EBUFTOOSMALL && m_maxRetries != -1 ) max = m_maxRetries;
// . if no thread slots available, that hogs up serious memory.
// the size of Msg3 is 82k, so having just 5000 of them is 430MB.
// . i just made Msg3 alloc mem when it needs more than about 2k
// so this problem is greatly reduced, therefore let's keep
// retrying... forever if no thread slots in thread queue since
// we become the thread queue in a way.
if ( g_errno == ENOTHREADSLOTS ) max = -1;
// this is set above if the map has the same consecutive key repeated
// and the read is enormous
if ( g_errno == ECORRUPTDATA ) max = 0;
// usually bad disk failures, don't retry those forever
//if ( g_errno == EIO ) max = 3;
// no, now our hitachis return these even when they're good so
// we have to keep retrying forever
if ( g_errno == EIO ) max = -1;
// count these so we do not take drives offline just because
// kernel ring buffer complains...
if ( g_errno == EIO ) g_numIOErrors++;
// bail early on high priority reads for these errors
if ( g_errno == EDISKSTUCK && m_niceness == 0 ) max = 0;
if ( g_errno == EIO && m_niceness == 0 ) max = 0;
// how does this happen? we should never bail out on a low priority
// disk read... we just wait for it to complete...
if ( g_errno == EDISKSTUCK && m_niceness != 0 ) { char *xx=NULL;*xx=0;}
// on I/O, give up at call it corrupt after a while. some hitachis
// have I/O errros on little spots, like gk88, maybe we can fix him
if ( g_errno == EIO && m_retryNum >= 5 ) {
m_errno = ECORRUPTDATA;
m_hadCorruption = true;
// do not do any retries any more
max = 0;
}
// convert m_errno to ECORRUPTDATA if it is EBUFTOOSMALL and the
// max of the bytesToRead are over 500MB.
// if bytesToRead was ludicrous, then assume that the data file
// was corrupted, the map was regenerated and it patched
// over the corrupted bits which were 500MB or more in size.
// we cannot practically allocate that much, so let's just
// give back an empty buffer. treat it like corruption...
// the way it patches is to store the same key over all the corrupted
// pages, which can get pretty big. so if you read a range with that
// key you will be hurting!!
// this may be the same scenario as when the rdbmap has consecutive
// same keys. see above where we set m_errno to ECORRUPTDATA...
if ( g_errno == EBUFTOOSMALL ) {
int32_t biggest = 0;
for ( int32_t i = 0 ; i < m_numFileNums ; i++ ) {
if ( m_scans[i].m_bytesToRead < biggest ) continue;
biggest = m_scans[i].m_bytesToRead;
}
if ( biggest > 500000000 ) {
log("db: Max read size was %" PRId32" > 500000000. Assuming "
"corrupt data in data file.",biggest);
m_errno = ECORRUPTDATA;
m_hadCorruption = true;
// do not do any retries on this, the read was > 500MB
max = 0;
}
}
// if shutting down gb then limit to 20 so we can shutdown because
// it can't shutdown until all threads are out of the queue i think
if ( g_process.m_mode == EXIT_MODE && max < 0 ) {
//log("msg3: forcing retries to 0 because shutting down");
max = 0;
}
// get base, returns NULL and sets g_errno to ENOCOLLREC on error
RdbBase *base = getRdbBase( m_rdbId, m_collnum );
if ( ! base ) {
return true;
}
// this really slows things down because it blocks the cpu so
// leave it out for now
#ifdef GBSANITYCHECK
// check for corruption here, do not do it again in Msg5 if we pass
if ( ! g_errno ) { // && g_conf.m_doErrorCorrection ) {
int32_t i;
for ( i = 0 ; i < m_numFileNums ; i++ )
if ( ! m_lists[i].checkList_r ( false, false ) ) break;
if ( i < m_numFileNums ) {
g_errno = ECORRUPTDATA;
m_errno = ECORRUPTDATA;
max = g_conf.m_corruptRetries; // try 100 times
log("db: Encountered corrupt list in file %s.",
base->getFile(m_fileNums[i])->getFilename());
}
else
m_listsChecked = true;
}
#endif
// try to fix this error i've seen
if ( g_errno == EBADENGINEER && max == -1 )
max = 100;
// . if we had a ETRYAGAIN error, then try again now
// . it usually means the whole file or a part of it was deleted
// before we could finish reading it, so we should re-read all now
// . RdbMerge deletes BigFiles after it merges them and also chops
// off file heads
// . now that we have threads i'd imagine we'd get EBADFD or something
// . i've also seen "illegal seek" as well
if ( m_errno && (m_retryNum < max || max < 0) &&
// this will complete in due time, we can't call a sleep wrapper
// on it because the read is really still pending...
m_errno != EDISKSTUCK ) {
// print the error
static time_t s_time = 0;
time_t now = getTime();
if ( now - s_time > 5 || g_errno != ENOTHREADSLOTS ) {
log("net: Had error reading %s: %s. Retrying. "
"(retry #%" PRId32")",
base->m_dbname,mstrerror(m_errno) , m_retryNum );
s_time = now;
}
// send email alert if in an infinite loop, but don't send
// more than once every 2 hours
static int32_t s_lastSendTime = 0;
if ( m_retryNum == 100 && getTime() - s_lastSendTime > 3600*2){
// remove this for now it is going off all the time
//g_pingServer.sendEmail(NULL,//g_hostdb.getMyHost(),
// "100 read retries",true);
s_lastSendTime = getTime();
}
// clear g_errno cuz we should for call to readList()
g_errno = 0;
// free the list buffer since if we have 1000 Msg3s retrying
// it will totally use all of our memory
for ( int32_t i = 0 ; i < m_numChunks ; i++ )
m_lists[i].destructor();
// count retries
m_retryNum++;
// backoff scheme, wait 100ms more each time
int32_t wait ;
if ( m_retryNum == 1 ) wait = 10;
else wait = 200 * m_retryNum;
// . don't wait more than 10 secs between tries
// . i've seen gf0 and gf16 get mega saturated
if ( wait > 10000 ) wait = 10000;
// wait 500 ms
if ( g_loop.registerSleepCallback ( wait , // ms
this ,
doneSleepingWrapper3,
m_niceness))
return false;
// otherwise, registration failed
log(
"net: Failed to register sleep callback for retry. "
"Abandoning read. This is bad.");
// return, g_errno should be set
g_errno = EBUFTOOSMALL;
m_errno = EBUFTOOSMALL;
return true;
}
// if we got an error and should not retry any more then give up
if ( g_errno ) {
log(
"net: Had error reading %s: %s. Giving up after %" PRId32" "
"retries.",
base->m_dbname,mstrerror(g_errno) , m_retryNum );
return true;
}
// note it if the retry finally worked
if ( m_retryNum > 0 )
log(LOG_INFO,"disk: Read succeeded after retrying %" PRId32" times.",
(int32_t)m_retryNum);
// count total bytes for logging
int32_t count = 0;
// . constrain all lists to make merging easier
// . if we have only one list, then that's nice cuz the constrain
// will allow us to send it right away w/ zero copying
// . if we have only 1 list, it won't be merged into a final list,
// that is, we'll just set m_list = &m_lists[i]
for ( int32_t i = 0 ; i < m_numFileNums ; i++ ) {
QUICKPOLL(m_niceness);
// count total bytes for logging
count += m_lists[i].getListSize();
// . hint offset is relative to the offset of first key we read
// . if that key was only 6 bytes RdbScan shift the list buf
// down 6 bytes to make the first key 12 bytes... a
// requirement for all RdbLists
// . don't inc it, though, if it was 0, pointing to the start
// of the list because our shift won't affect that
if ( m_scans[i].m_shifted == 6 && m_hintOffsets[i] > 0 )
m_hintOffsets[i] += 6;
// posdb double compression
if ( m_scans[i].m_shifted == 12 && m_hintOffsets[i] > 0 )
m_hintOffsets[i] += 12;
// . don't constrain on minRecSizes here because it may
// make our endKey smaller, which will cause problems
// when Msg5 merges these lists.
// . If all lists have different endKeys RdbList's merge
// chooses the min and will merge in recs beyond that
// causing a bad list BECAUSE we don't check to make
// sure that recs we are adding are below the endKey
// . if we only read from one file then constrain based
// on minRecSizes so we can send the list back w/o merging
// OR if just merging with RdbTree's list
int32_t mrs ;
// . constrain to m_minRecSizesOrig, not m_minRecSizes cuz
// that could be adjusted by compensateForNegativeRecs()
// . but, really, they should be the same if we only read from
// the root file
if ( m_numFileNums == 1 ) mrs = m_minRecSizesOrig;
else mrs = -1;
// . this returns false and sets g_errno on error
// . like if data is corrupt
BigFile *ff = base->getFile(m_fileNums[i]);
// if we did a merge really quick and delete one of the
// files we were reading, i've seen 'ff' be NULL
char *filename = "lostfilename";
if ( ff ) filename = ff->getFilename();
// compute cache info
RdbCache *rpc = getDiskPageCache ( m_rdbId );
if ( ! m_allowPageCache ) rpc = NULL;
int64_t vfd ;
if ( ff ) vfd = ff->getVfd();
key192_t ck ;
if ( ff )
ck = makeCacheKey ( vfd ,
m_scans[i].m_offset ,
m_scans[i].m_bytesToRead );
if ( m_validateCache && ff && rpc && vfd != -1 ) {
bool inCache;
char *rec; int32_t recSize;
inCache = rpc->getRecord ( (collnum_t)0 , // collnum
(char *)&ck ,
&rec ,
&recSize ,
true , // copy?
-1 , // maxAge, none
true ); // inccounts?
if ( inCache &&
// 1st byte is RdbScan::m_shifted
( m_lists[i].m_listSize != recSize-1 ||
memcmp ( m_lists[i].m_list , rec+1,recSize-1) ||
*rec != m_scans[i].m_shifted ) ) {
log("msg3: cache did not validate");
char *xx=NULL;*xx=0;
}
mfree ( rec , recSize , "vca" );
}
///////
//
// STORE IN PAGE CACHE
//
///////
// store what we read in the cache. don't bother storing
// if it was a retry, just in case something strange happened.
// store pre-constrain call is more efficient.
if ( m_retryNum<=0 && ff && rpc && vfd != -1 &&
! m_scans[i].m_inPageCache )
rpc->addRecord ( (collnum_t)0 , // collnum
(char *)&ck ,
// rec1 is this little thingy
&m_scans[i].m_shifted,
1,
// rec2
m_lists[i].getList() ,
m_lists[i].getListSize() ,
0 ); // timestamp. 0 = now
QUICKPOLL(m_niceness);
// if from our 'page' cache, no need to constrain
if ( ! m_lists[i].constrain ( m_startKey ,
m_constrainKey , // m_endKey
mrs , // m_minRecSizes
m_hintOffsets[i] ,
//m_hintKeys [i] ,
&m_hintKeys [i*m_ks] ,
filename,//ff->getFilename() ,
m_niceness ) ) {
log("net: Had error while constraining list read from "
"%s: %s/%s. vfd=%" PRId32" parts=%" PRId32". "
"This is likely caused by corrupted "
"data on disk.",
mstrerror(g_errno), ff->getDir(),
ff->getFilename(), ff->m_vfd ,
(int32_t)ff->m_numParts );
continue;
}
}
// print the time
if ( g_conf.m_logTimingDb ) {
int64_t now = gettimeofdayInMilliseconds();
int64_t took = now - m_startTime;
log(LOG_TIMING,
"net: Took %" PRId64" ms to read %" PRId32" lists of %" PRId32" bytes total"
" from %s (niceness=%" PRId32").",
took,m_numFileNums,count,base->m_dbname,m_niceness);
}
return true;
}
// . return false if blocked, true otherwise
// . set g_errno on error
// . read list of keys in [startKey,endKey] range
// . read at least "minRecSizes" bytes of keys in that range
// . the "m_endKey" of resulting, merged list may have a smaller endKey
// than the argument, "endKey" due to limitation by "minRecSizes"
// . resulting list will contain ALL keys between ITS [m_startKey,m_endKey]
// . final merged list "should" try to have a size of at least "minRecSizes"
// but due to negative/postive rec elimination may be less
// . the endKey of the lists we read may be <= "endKey" provided
// . we try to shrink the endKey if minRecSizes is >= 0 in order to
// avoid excessive reading
// . by shrinking the endKey we cannot take into account the size of deleted
// records, so therefore we may fall short of "minRecSizes" in actuality,
// in fact, the returned list may even be empty with a shrunken endKey
// . we merge all lists read from disk into the provided "list"
// . caller should call Msg3.getList(int32_t i) and Msg3:getNumLists() to retrieve
// . this makes the query engine faster since we don't need to merge the docIds
// and can just send them across the network separately and they will be
// hashed into IndexTable's table w/o having to do time-wasting merging.
// . caller can specify array of filenums to read from so incremental syncing
// in Sync class can just read from titledb*.dat files that were formed
// since the last sync point.
bool Msg3::readList ( char rdbId ,
collnum_t collnum ,
const char *startKeyArg ,
const char *endKeyArg ,
int32_t minRecSizes , // max size of scan
int32_t startFileNum , // first file to scan
int32_t numFiles , // rel. to startFileNum
void *state , // for callback
void (* callback ) ( void *state ) ,
int32_t niceness ,
int32_t retryNum ,
int32_t maxRetries ,
bool compensateForMerge ,
bool justGetEndKey ,
bool allowPageCache ,
bool hitDisk ) {
// set this to true to validate
m_validateCache = false;//true;
// clear, this MUST be done so if we return true g_errno is correct
g_errno = 0;
// assume lists are not checked for corruption
m_listsChecked = false;
// warn
if ( minRecSizes < -1 ) {
log(LOG_LOGIC,"db: Msg3 got minRecSizes of %" PRId32", changing "
"to -1.",minRecSizes);
minRecSizes = -1;
}
// reset m_alloc and data in all lists in case we are a re-call
reset();
// warning
if ( collnum < 0 ) log(LOG_LOGIC,"net: NULL collection. msg3.");
// remember the callback
m_rdbId = rdbId;
m_collnum = collnum;
m_callback = callback;
m_state = state;
m_niceness = niceness;
m_numScansCompleted = 0;
m_retryNum = retryNum;
m_maxRetries = maxRetries;
m_compensateForMerge = compensateForMerge;
m_allowPageCache = allowPageCache;
m_hitDisk = hitDisk;
m_hadCorruption = false;
// get keySize of rdb
m_ks = getKeySizeFromRdbId ( m_rdbId );
// reset the group error
m_errno = 0;
// . reset all our lists
// . these are reset in call the RdbScan::setRead() below
//for ( int32_t i = 0 ; i < MAX_RDB_FILES ; i++ ) m_lists[i].reset();
// . ensure startKey last bit clear, endKey last bit set
// . no! this warning is now only in Msg5
// . if RdbMerge is merging some files, not involving the root
// file, then we can expect to get a lot of unmatched negative recs.
// . as a consequence, our endKeys may often be negative. This means
// it may not annihilate with the positive key, but we should only
// miss like this at the boundaries of the lists we fetch.
// . so in that case RdbList::merge will stop merging once the
// minRecSizes limit is reached even if it means ending on a negative
// rec key
//if ( (startKey.n0 & 0x01) == 0x01 )
if ( !KEYNEG(startKeyArg) )
log(LOG_REMIND,"net: msg3: StartKey lastbit set.");
if ( KEYNEG(endKeyArg) )
log(LOG_REMIND,"net: msg3: EndKey lastbit clear.");
// declare vars here becaues of 'goto skip' below
int32_t mergeFileNum = -1 ;
int32_t max ;
// get base, returns NULL and sets g_errno to ENOCOLLREC on error
RdbBase *base = getRdbBase( m_rdbId, m_collnum );
if ( ! base ) {
return true;
}
// store the file numbers in the array, these are the files we read
m_numFileNums = 0;
// save startFileNum here, just for recall
m_startFileNum = startFileNum;
m_numFiles = numFiles;
// . if we have a merge going on, we may have to change startFileNum
// . if some files get unlinked because merge completes then our
// reads will detect the error and loop back here
// . we launch are reads right after this without giving up the cpu
// and we use file descriptors, so any changes to Rdb::m_files[]
// should not hurt us
// . WARNING: just make sure you don't lose control of cpu until after
// you call RdbScan::set()
// . we use hasMergeFile() instead of isMerging() because he may not
// be merging cuz he got suspended or he restarted and
// hasn't called attemptMerge() yet, but he may still contain it
if ( g_conf.m_logDebugQuery )
log(LOG_DEBUG,
"net: msg3: "
"c=%" PRId32" hmf=%" PRId32" sfn=%" PRId32" msfn=%" PRId32" nf=%" PRId32" db=%s.",
(int32_t)compensateForMerge,(int32_t)base->hasMergeFile(),
(int32_t)startFileNum,(int32_t)base->m_mergeStartFileNum-1,
(int32_t)numFiles,base->m_dbname);
int32_t pre = -10;
if ( compensateForMerge && base->hasMergeFile() &&
startFileNum >= base->m_mergeStartFileNum - 1 &&
(startFileNum > 0 || numFiles != -1) ) {
// now also include the file being merged into, but only
// if we are reading from a file being merged...
if ( startFileNum < base->m_mergeStartFileNum +
base->m_numFilesToMerge - 1 )
//m_fileNums [ m_numFileNums++ ] =
// base->m_mergeStartFileNum - 1;
pre = base->m_mergeStartFileNum - 1;
// debug msg
if ( g_conf.m_logDebugQuery )
log(LOG_DEBUG,
"net: msg3: startFileNum from %" PRId32" to %" PRId32" (mfn=%" PRId32")",
startFileNum,startFileNum+1,mergeFileNum);
// if merge file was inserted before us, inc our file number
startFileNum++;
}
// adjust num files if we need to, as well
if ( compensateForMerge && base->hasMergeFile() &&
startFileNum < base->m_mergeStartFileNum - 1 &&
numFiles != -1 &&
startFileNum + numFiles - 1 >= base->m_mergeStartFileNum - 1 ) {
// debug msg
if ( g_conf.m_logDebugQuery )
log(LOG_DEBUG,"net: msg3: numFiles up one.");
// if merge file was inserted before us, inc our file number
numFiles++;
}
// . how many rdb files does this base have?
// . IMPORTANT: this can change since files are unstable because they
// might have all got merged into one!
// . so do this check to make sure we're safe... especially if
// there was an error before and we called readList() on ourselves
max = base->getNumFiles();
// -1 means we should scan ALL the files in the base
if ( numFiles == -1 ) numFiles = max;
// limit it by startFileNum, however
if ( numFiles > max - startFileNum ) numFiles = max - startFileNum;
// set g_errno and return true if it is < 0
if ( numFiles < 0 ) {
log(LOG_LOGIC,
"net: msg3: readList: numFiles = %" PRId32" < 0 (max=%" PRId32")(sf=%" PRId32")",
numFiles , max , startFileNum );
g_errno = EBADENGINEER;
// force core dump
char *xx=NULL;*xx=0;
return true;
}
// . allocate buffer space
// . m_scans, m_startpg, m_endpg, m_hintKeys, m_hintOffsets,
// m_fileNums, m_lists
int32_t chunk = sizeof(RdbScan) + // m_scans
4 + // m_startpg
4 + // m_endpg
//sizeof(key_t) + // m_hintKeys
m_ks + // m_hintKeys
4 + // m_hintOffsets
4 + // m_fileNums
sizeof(RdbList) ; // m_lists
int32_t nn = numFiles;
if ( pre != -10 ) nn++;
m_numChunks = nn;
int32_t need = nn * (chunk);
m_alloc = m_buf;
if ( need > (int32_t)MSG3_BUF_SIZE ) {
m_allocSize = need;
m_alloc = (char *)mcalloc ( need , "Msg3" );
if ( ! m_alloc ) {
log("disk: Could not allocate %" PRId32" bytes read "
"structures to read %s.",need,base->m_dbname);
return true;
}
}
char *p = m_alloc;
m_scans = (RdbScan *)p; p += nn * sizeof(RdbScan);
m_startpg = (int32_t *)p; p += nn * 4;
m_endpg = (int32_t *)p; p += nn * 4;
//m_hintKeys = (key_t *)p; p += nn * sizeof(key_t);
m_hintKeys = (char *)p; p += nn * m_ks;
m_hintOffsets = (int32_t *)p; p += nn * 4;
m_fileNums = (int32_t *)p; p += nn * 4;
m_lists = (RdbList *)p; p += nn * sizeof(RdbList);
// sanity check
if ( p - m_alloc != need ) {
log(LOG_LOGIC,"disk: Bad malloc in Msg3.cpp.");
char *xx = NULL; *xx = 0;
}
// call constructors
for ( int32_t i = 0 ; i < nn ; i++ ) m_lists[i].constructor();
// make fix from up top
if ( pre != -10 ) m_fileNums [ m_numFileNums++ ] = pre;
// store them all
for ( int32_t i = startFileNum ; i < startFileNum + numFiles ; i++ )
m_fileNums [ m_numFileNums++ ] = i;
// . remove file nums that are being unlinked after a merge now
// . keep it here (below skip: label) so sync point reads can use it
int32_t n = 0;
for ( int32_t i = 0 ; i < m_numFileNums ; i++ ) {
// skip those that are being unlinked after the merge
if ( base->m_isUnlinking &&
m_fileNums[i] >= base->m_mergeStartFileNum &&
m_fileNums[i] < base->m_mergeStartFileNum +
base->m_numFilesToMerge )
continue;
// otherwise, keep it
m_fileNums[n++] = m_fileNums[i];
}
m_numFileNums = n;
// . if root file is being merged, he's file #0, & root file is file #1
// . this is a hack so caller gets what he wants
//if ( startFileNum == 0 && base->getFileId(0) == 0 && numFiles == 1 )
// numFiles = 2;
// remember the file range we should scan
m_numScansStarted = 0;
m_numScansCompleted = 0;
//m_startKey = startKey;
//m_endKey = endKey;
//m_constrainKey = endKey; // set in case justGetEndKey is true
KEYSET(m_startKey,startKeyArg,m_ks);
KEYSET(m_endKey,endKeyArg,m_ks);
KEYSET(m_constrainKey,endKeyArg,m_ks);//set incase justGetEndKey istrue
m_minRecSizes = minRecSizes;
m_compensateForMerge = compensateForMerge;
// bail if 0 files to scan -- no! need to set startKey/endKey
if ( numFiles == 0 ) return true;
// don't read anything if endKey < startKey
//if ( m_startKey > m_endKey ) return true;
if ( KEYCMP(m_startKey,m_endKey,m_ks)>0 ) return true;
// keep the original in tact in case g_errno == ETRYAGAIN
//m_endKeyOrig = endKey;
KEYSET(m_endKeyOrig,endKeyArg,m_ks);
m_minRecSizesOrig = minRecSizes;
// start reading at this key
m_fileStartKey = startKeyArg;
// start the timer, keep it fast for clusterdb though
if ( g_conf.m_logTimingDb ) m_startTime = gettimeofdayInMilliseconds();
// translate base to an id, for the sake of m_msg0
//char baseId = m_msg0->getRdbId ( base );
// map ptrs
RdbMap **maps = base->getMaps();
// . we now boost m_minRecSizes to account for negative recs
// . but not if only reading one list, cuz it won't get merged and
// it will be too big to send back
if ( m_numFileNums > 1 ) compensateForNegativeRecs ( base );
// . often endKey is too big for an efficient read of minRecSizes bytes
// because we end up reading too much from all the files
// . this will set m_startpg[i], m_endpg[i] for each RdbScan/RdbFile
// to ensure we read "minRecSizes" worth of records, not much more
// . returns the new endKey for all ranges
// . now this just overwrites m_endKey
//m_endKey = setPageRanges ( base ,
setPageRanges ( base ,
m_fileNums ,
m_numFileNums ,
m_fileStartKey , // start reading @ key
m_endKey , // stop reading @ key
m_minRecSizes );
// . NEVER let m_endKey be a negative key, because it will
// always be unmatched, since delbit is cleared
// . adjusting it here ensures our generated hints are valid
// . we will use this key to call constrain() with
//m_constrainKey = m_endKey;
//if ( ( m_constrainKey.n0 & 0x01) == 0x00 )
// m_constrainKey -= (uint32_t)1;
KEYSET(m_constrainKey,m_endKey,m_ks);
if ( KEYNEG(m_constrainKey) )
KEYSUB(m_constrainKey,m_ks);
// Msg5 likes to get the endkey for getting the list from the tree
if ( justGetEndKey ) return true;
// sanity check
if ( m_numFileNums > nn ) {
log(LOG_LOGIC,"disk: Failed sanity check in Msg3.");
char *xx = NULL; *xx = 0;
}
// debug msg
//log("msg3 getting list (msg5=%" PRIu32")",m_state);
// . MDW removed this -- go ahead an end on a delete key
// . RdbMerge might not pick it up this round, but oh well
// . so we can have both positive and negative co-existing in same file
// make sure the last bit is set so we don't end on a delete key
//m_endKey.n0 |= 0x01LL;
// . now start reading/scanning the files
// . our m_scans array starts at 0
for ( int32_t i = 0 ; i < m_numFileNums ; i++ ) {
// get the page range
//int32_t p1 = m_startpg [ i ];
//int32_t p2 = m_endpg [ i ];
//#ifdef GBSANITYCHECK
int32_t fn = m_fileNums[i];
// this can happen somehow!
if ( fn < 0 ) {
log(LOG_LOGIC,"net: msg3: fn=%" PRId32". Bad engineer.",fn);
continue;
}
// sanity check
if ( i > 0 && m_fileNums[i-1] >= fn ) {
log(LOG_LOGIC,
"net: msg3: files must be read in order "
"from oldest to newest so RdbList::indexMerge_r "
"works properly. Otherwise, corruption will "
"result. ");
char *xx = NULL; *xx = 0;
return true;
}
// . sanity check?
// . no, we must get again since we turn on endKey's last bit
int32_t p1 , p2;
maps[fn]->getPageRange ( m_fileStartKey ,
m_endKey ,
&p1 ,
&p2 ,
NULL );
//if ( p1 != p1c || p2 != p2c ) {
// fprintf(stderr,"Msg3::bad page range\n");
// sleep(50000);
//}
// sanity check, each endpg's key should be > endKey
//if ( p2 < maps[fn]->getNumPages() &&
// maps[fn]->getKey ( p2 ) <= m_endKey ) {
// fprintf(stderr,"Msg3::bad page range 2\n");
// sleep(50000);
//}
//#endif
//int32_t p1 , p2;
//maps[fn]->getPageRange (startKey,endKey,minRecSizes,&p1,&p2);
// now get some read info
int64_t offset = maps[fn]->getAbsoluteOffset ( p1 );
int32_t bytesToRead = maps[fn]->getRecSizes ( p1, p2, false);
// max out the endkey for this list
// debug msg
//#ifdef _DEBUG_
//if ( minRecSizes == 2000000 )
//log("Msg3:: reading %" PRId32" bytes from file #%" PRId32,bytesToRead,i);
//#endif
// inc our m_numScans
m_numScansStarted++;
// . keep stats on our disk accesses
// . count disk seeks (assuming no fragmentation)
// . count disk bytes read
if ( bytesToRead > 0 ) {
base->m_rdb->didSeek ( );
base->m_rdb->didRead ( bytesToRead );
}
// . the startKey may be different for each RdbScan class
// . RdbLists must have all keys within their [startKey,endKey]
// . therefore set startKey individually from first page in map
// . this endKey must be >= m_endKey
// . this startKey must be < m_startKey
//key_t startKey = maps[fn]->getKey ( p1 );
//key_t endKey = maps[fn]->getKey ( p2 );
char startKey2 [ MAX_KEY_BYTES ];
char endKey2 [ MAX_KEY_BYTES ];
maps[fn]->getKey ( p1 , startKey2 );
maps[fn]->getKey ( p2 , endKey2 );
//char *startKey = maps[fn]->getKeyPtr ( p1 );
//char *endKey = maps[fn]->getKeyPtr ( p2 );
// store in here
m_startpg [ i ] = p1;
m_endpg [ i ] = p2;
// . we read UP TO that endKey, so reduce by 1
// . but iff p2 is NOT the last page in the map/file
// . maps[fn]->getKey(lastPage) will return the LAST KEY
// and maps[fn]->getOffset(lastPage) the length of the file
//if ( maps[fn]->getNumPages()!=p2) endKey -=(uint32_t)1;
if ( maps[fn]->getNumPages() != p2 ) KEYSUB(endKey2,m_ks);
// otherwise, if we're reading all pages, then force the
// endKey to virtual inifinite
//else endKey.setMax();
else KEYMAX(endKey2,m_ks);
// . set up the hints
// . these are only used if we are only reading from 1 file
// . these are used to call constrain() so we can constrain
// the end of the list w/o looping through all the recs
// in the list
int32_t h2 = p2 ;
// decrease by one page if we're on the last page
if ( h2 > p1 && maps[fn]->getNumPages() == h2 ) h2--;
// . decrease hint page until key is <= endKey on that page
// AND offset is NOT -1 because the old way would give
// us hints passed the endkey
// . also decrease so we can constrain on minRecSizes in
// case we're the only list being read
// . use >= m_minRecSizes instead of >, otherwise we may
// never be able to set "size" in RdbList::constrain()
// because "p" could equal "maxPtr" right away
while ( h2 > p1 &&
//( maps[fn]->getKey (h2) > m_constrainKey ||
(KEYCMP(maps[fn]->getKeyPtr(h2),m_constrainKey,m_ks)>0||
maps[fn]->getOffset(h2) == -1 ||
maps[fn]->getAbsoluteOffset(h2) - offset >=
m_minRecSizes ) )
h2--;
// now set the hint
m_hintOffsets [ i ] = maps[fn]->getAbsoluteOffset ( h2 ) -
maps[fn]->getAbsoluteOffset ( p1 ) ;
//m_hintKeys [ i ] = maps[fn]->getKey ( h2 );
KEYSET(&m_hintKeys[i*m_ks],maps[fn]->getKeyPtr(h2),m_ks);
// reset g_errno before calling setRead()
g_errno = 0;
// . this fix is now in RdbList::checklist_r()
// . we can now have dup keys, so, we may read in
// a rec with key "lastMinKey" even though we don't read
// in the first key on the end page, so don't subtract 1...
//if ( endKey != m_endKeyOrig )
// endKey += (uint32_t) 1;
// timing debug
if ( g_conf.m_logTimingDb )
log(LOG_TIMING,
"net: msg: reading %" PRId32" bytes from %s file #%" PRId32" "
"(niceness=%" PRId32")",
bytesToRead,base->m_dbname,i,m_niceness);
// log huge reads, those hurt us
if ( bytesToRead > 150000000 ) {
logf(LOG_INFO,"disk: Reading %" PRId32" bytes at offset %" PRId64" "
"from %s.",
bytesToRead,offset,base->m_dbname);
}
// if any keys in the map are the same report corruption
char tmpKey [16];
char lastTmpKey[16];
int32_t ccount = 0;
if ( bytesToRead > 10000000 &&
bytesToRead / 2 > m_minRecSizes &&
base->m_fixedDataSize >= 0 ) {
for ( int32_t pn = p1 ; pn <= p2 ; pn++ ) {
maps[fn]->getKey ( pn , tmpKey );
if ( KEYCMP(tmpKey,lastTmpKey,m_ks) == 0 )
ccount++;
gbmemcpy(lastTmpKey,tmpKey,m_ks);
}
}
if ( ccount > 10 ) {
logf(LOG_INFO,"disk: Reading %" PRId32" bytes from %s file #"
"%" PRId32" when min "
"required is %" PRId32". Map is corrupt and has %" PRId32" "
"identical consecutive page keys because the "
"map was \"repaired\" because out of order keys "
"in the index.",
(int32_t)bytesToRead,
base->m_dbname,fn,
(int32_t)m_minRecSizes,
(int32_t)ccount);
m_numScansCompleted++;
m_errno = ECORRUPTDATA;
m_hadCorruption = true;
//m_maxRetries = 0;
break;
}
////////
//
// try to get from PAGE CACHE
//
////////
BigFile *ff = base->getFile(m_fileNums[i]);
RdbCache *rpc = getDiskPageCache ( m_rdbId );
if ( ! m_allowPageCache ) rpc = NULL;
// . vfd is unique 64 bit file id
// . if file is opened vfd is -1, only set in call to open()
int64_t vfd = ff->getVfd();
key192_t ck = makeCacheKey ( vfd , offset, bytesToRead);
char *rec; int32_t recSize;
bool inCache = false;
if ( rpc && vfd != -1 && ! m_validateCache )
inCache = rpc->getRecord ( (collnum_t)0 , // collnum
(char *)&ck ,
&rec ,
&recSize ,
true , // copy?
-1 , // maxAge, none
true ); // inccounts?
m_scans[i].m_inPageCache = false;
if ( inCache ) {
m_scans[i].m_inPageCache = true;
m_numScansCompleted++;
// now we have to store this value, 6 or 12 so
// we can modify the hint appropriately
m_scans[i].m_shifted = *rec;
m_lists[i].set ( rec +1,
recSize-1 ,
rec , // alloc
recSize , // allocSize
startKey2 ,
endKey2 ,
base->m_fixedDataSize ,
true , // owndata
base->useHalfKeys() ,
getKeySizeFromRdbId ( m_rdbId ) );
continue;
}
// . do the scan/read of file #i
// . this returns false if blocked, true otherwise
// . this will set g_errno on error
bool done = m_scans[i].setRead (base->getFile(m_fileNums[i]),
base->m_fixedDataSize ,
offset ,
bytesToRead ,
startKey2 ,
endKey2 ,
m_ks ,
&m_lists[i] ,
this ,
doneScanningWrapper ,
base->useHalfKeys() ,
m_rdbId,
m_niceness ,
m_allowPageCache ,
m_hitDisk ) ;
// . damn, usually the above will indirectly launch a thread
// to do the reading, but it sets g_errno to EINTR,
// "interrupted system call"!
// . i guess the thread does the read w/o blocking and then
// queues the signal on g_loop's queue before it exits
// . try ignoring, and keep going
if ( g_errno == EINTR ) {
log("net: Interrupted system call while reading file. "
"Ignoring.");
g_errno = 0;
}
// debug msg
//fprintf(stderr,"Msg3:: reading %" PRId32" bytes from file #%" PRId32","
// "done=%" PRId32",offset=%" PRId64",g_errno=%s,"
// "startKey=n1=%" PRIu32",n0=%" PRIu64", "
// "endKey=n1=%" PRIu32",n0=%" PRIu64"\n",
// bytesToRead,i,(int32_t)done,offset,mstrerror(g_errno),
// m_startKey,m_endKey);
//if ( bytesToRead == 0 )
// fprintf(stderr,"shit\n");
// if it did not block then it completed, so count it
if ( done ) m_numScansCompleted++;
// break on an error, and remember g_errno in case we block
if ( g_errno && g_errno != ENOTHREADSLOTS ) {
int32_t tt = LOG_WARN;
if ( g_errno == EFILECLOSED ) tt = LOG_INFO;
log(tt,"disk: Reading %s had error: %s.",
base->m_dbname, mstrerror(g_errno));
m_errno = g_errno;
break;
}
}
// debug test
//if ( rand() % 100 <= 10 ) m_errno = EIO;
// if we blocked, return false
if ( m_numScansCompleted < m_numScansStarted ) return false;
// . if all scans completed without blocking then wrap it up & ret true
// . doneScanning may now block if it finds data corruption and must
// get the list remotely
return doneScanning();
}
void gotDocWrapper ( void *state , TcpSocket *s ) {
// no longer launched
s_launched--;
char* url = (char*)state;
// bail if got cut off
if ( s->m_readOffset == 0 ) {
log("lost %s",(char *) state);
if(s_server) mfree(url, gbstrlen(url)+1, "saved url");
return;
}
// got one more result page
s_total++;
// allow printing
s_printIt = true;
// get time now
int64_t now = gettimeofdayInMilliseconds();
// get hash
char *reply = s->m_readBuf ;
int32_t size = s->m_readOffset;
HttpMime mime;
mime.set ( reply , size , NULL );
char *content = reply + mime.getMimeLen();
int32_t contentLen = size - mime.getMimeLen();
int32_t status = mime.getHttpStatus();
uint32_t h = hash32 ( content , contentLen );
char *p = mime.getMime();
char *pend = p + mime.getMimeLen();
char message[256];
int32_t mlen = 0;
// parse status message out of response
// HTTP/1.0
while ( p < pend && !isspace(*p) ) p++;
// skip space
while ( p < pend && isspace(*p) ) p++;
// copy to end of line
while (p < pend && mlen < 255 && *p != '\r' && *p != '\n'){
message[mlen++] = *p;
}
message[mlen] = '\0';
// log msg
if ( g_errno )
logf(LOG_INFO,"blaster: got doc (status=%"INT32") (%"INT32") (%"INT32"ms) %s : "
"%s", status,
s->m_readOffset ,
(int32_t)(now - s->m_startTime) ,
(char *)state ,
mstrerror(g_errno) );
else
logf(LOG_INFO,"blaster: got doc (status=%"INT32") (%"INT32") (%"INT32"ms) "
"(hash=%"XINT32") %s", status,
s->m_readOffset ,
(int32_t)(now - s->m_startTime) ,
h ,
(char *)state );
if(s_server) mfree(url, gbstrlen(url)+1, "saved url");
// try to launch another
startSpidering();
}
// . set table size to "n" slots
// . rehashes the termId/score pairs into new table
// . returns false and sets errno on error
bool HashTableX::setTableSize ( long oldn , char *buf , long bufSize ) {
// don't change size if we do not need to
if ( oldn == m_numSlots ) return true;
long long n = (long long)oldn;
// make it a power of 2 for speed if small
n = getHighestLitBitValueLL((unsigned long long)oldn * 2LL -1);
// sanity check, must be less than 1B
if ( n > 1000000000 ) { char *xx=NULL;*xx=0; }
// limit...
//if ( n > m_maxSlots ) n = m_maxSlots;
// do not go negative on me
if ( oldn == 0 ) n = 0;
// sanity check
if ( n < oldn ) { char *xx = NULL; *xx = 0; }
// do we have a buf?
long need = (m_ks+m_ds+1) * n;
// sanity check, buf should also meet what we need
if ( buf && bufSize < need ) { char *xx = NULL; *xx = 0; }
// we grow kinda slow, it slows things down, so note it
long long startTime =0LL;
long old = -1;
if ( m_numSlots > 2000 ) {
startTime = gettimeofdayInMilliseconds();
old = m_numSlots;
}
// if we should not free note that
bool savedDoFree = m_doFree ;
char *savedBuf = m_buf;
long savedBufSize = m_bufSize;
// use what they gave us if we can
m_buf = buf;
m_doFree = false;
// alloc if we should
if ( ! m_buf ) {
m_buf = (char *)mmalloc ( need , m_allocName);
m_bufSize = need;
m_doFree = true;
if ( ! m_buf ) return false;
}
// save the old junk
char *oldFlags = m_flags;
char *oldKeys = m_keys;
char *oldVals = m_vals;
// now point to the new bigger and empty table
m_keys = m_buf;
m_vals = m_buf + m_ks * n;
m_flags = m_buf + m_ks * n + m_ds * n;
// clear flags only
//bzero ( m_flags , n );
memset ( m_flags , 0 , n );
// rehash the slots if we had some
long ns = m_numSlots; if ( ! m_keys ) ns = 0;
// update these for the new empty table
m_numSlots = n;
m_mask = n - 1;
long oldUsed = m_numSlotsUsed;
// reset this before re-adding all of them
m_numSlotsUsed = 0;
// loop over results in old table, if any
for ( long i = 0 ; i < ns ; i++ ) {
// breathe
QUICKPOLL ( m_niceness );
// skip the empty slots
if ( oldFlags [ i ] == 0 ) continue;
// add old key/val into the empty table
if ( m_ks == sizeof(key144_t) )
// use this special adder that hashes it up better!
addTerm144 ( (key144_t *)(oldKeys + m_ks * i) ,
*(long *)(oldVals + m_ds * i) );
else
addKey ( oldKeys + m_ks * i , oldVals + m_ds * i );
}
if ( startTime ) {
char *name ="";
if ( m_allocName ) name = m_allocName;
//if ( name && strcmp(name,"HashTableX")==0 )
// log("hey");
long long now = gettimeofdayInMilliseconds();
logf(LOG_DEBUG,"table: grewtable %s from %li to %li slots "
"in %lli ms (this=0x%lx) (used=%li)",
name,old,m_numSlots ,now - startTime,(long)this,oldUsed);
}
// free the old guys
if ( ! savedDoFree ) return true;
if ( ! savedBuf ) return true;
// let the old table go
mfree ( savedBuf , savedBufSize , m_allocName );
return true;
}
// . reply to a request for an RdbList
// . MUST call g_udpServer::sendReply or sendErrorReply() so slot can
// be destroyed
void handleRequest0 ( UdpSlot *slot , int32_t netnice ) {
logTrace( g_conf.m_logTraceMsg0, "BEGIN. Got request for an RdbList" );
// if niceness is 0, use the higher priority udpServer
UdpServer *us = &g_udpServer;
//if ( netnice == 0 ) us = &g_udpServer2;
// get the request
char *request = slot->m_readBuf;
int32_t requestSize = slot->m_readBufSize;
// collection is now stored in the request, so i commented this out
//if ( requestSize != MSG0_REQ_SIZE ) {
// log("net: Received bad data request size of %" PRId32" bytes. "
// "Should be %" PRId32".", requestSize ,(int32_t)MSG0_REQ_SIZE);
// us->sendErrorReply ( slot , EBADREQUESTSIZE );
// return;
//}
// parse the request
char *p = request;
int64_t syncPoint = *(int64_t *)p ; p += 8;
//key_t startKey = *(key_t *)p ; p += sizeof(key_t);
//key_t endKey = *(key_t *)p ; p += sizeof(key_t);
int32_t minRecSizes = *(int32_t *)p ; p += 4;
int32_t startFileNum = *(int32_t *)p ; p += 4;
int32_t numFiles = *(int32_t *)p ; p += 4;
int32_t maxCacheAge = *(int32_t *)p ; p += 4;
char rdbId = *p++;
char addToCache = *p++;
char doErrorCorrection = *p++;
char includeTree = *p++;
// this was messing up our niceness conversion logic
int32_t niceness = slot->m_niceness;//(int32_t)(*p++);
// still need to skip it though!
p++;
bool allowPageCache = (bool)(*p++);
char ks = getKeySizeFromRdbId ( rdbId );
char *startKey = p; p+=ks;
char *endKey = p; p+=ks;
collnum_t collnum = *(collnum_t *)p; p += sizeof(collnum_t);
CollectionRec *xcr = g_collectiondb.getRec ( collnum );
if ( ! xcr ) g_errno = ENOCOLLREC;
if( g_conf.m_logTraceMsg0 ) {
logTrace( g_conf.m_logTraceMsg0, "rdbId....... %d", (int)rdbId );
logTrace( g_conf.m_logTraceMsg0, "key size.... %d", (int)ks );
logTrace( g_conf.m_logTraceMsg0, "startFileNum %" PRId32, startFileNum );
logTrace( g_conf.m_logTraceMsg0, "numFiles.... %" PRId32, numFiles );
}
// error set from XmlDoc::cacheTermLists()?
if ( g_errno ) {
logTrace( g_conf.m_logTraceMsg0, "END. Invalid collection" );
log(LOG_ERROR,"%s:%s:%d: call sendErrorReply.", __FILE__, __func__, __LINE__);
us->sendErrorReply ( slot , EBADRDBID );
return;
}
// . get the rdb we need to get the RdbList from
// . returns NULL and sets g_errno on error
//Msg0 msg0;
//Rdb *rdb = msg0.getRdb ( rdbId );
Rdb *rdb = getRdbFromId ( rdbId );
if ( ! rdb ) {
logTrace( g_conf.m_logTraceMsg0, "END. Invalid rdbId" );
log(LOG_ERROR,"%s:%s:%d: call sendErrorReply.", __FILE__, __func__, __LINE__);
us->sendErrorReply ( slot , EBADRDBID );
return;
}
// keep track of stats
rdb->readRequestGet ( requestSize );
// . do a local get
// . create a msg5 to get the list
State00 *st0 ;
try { st0 = new (State00); }
catch ( ... ) {
g_errno = ENOMEM;
log("Msg0: new(%" PRId32"): %s",
(int32_t)sizeof(State00),mstrerror(g_errno));
log(LOG_ERROR,"%s:%s:%d: call sendErrorReply.", __FILE__, __func__, __LINE__);
us->sendErrorReply ( slot , g_errno );
return;
}
mnew ( st0 , sizeof(State00) , "State00" );
// timing debug
if ( g_conf.m_logTimingNet )
st0->m_startTime = gettimeofdayInMilliseconds();
// save slot in state
st0->m_slot = slot;
// save udp server to send back reply on
st0->m_us = us;
// init this one
st0->m_niceness = niceness;
st0->m_rdbId = rdbId;
QUICKPOLL(niceness);
// debug msg
if ( maxCacheAge != 0 && ! addToCache ) {
log( LOG_LOGIC, "net: msg0: check but don't add... rdbid=%" PRId32".", ( int32_t ) rdbId );
}
// . if this request came over on the high priority udp server
// make sure the priority gets passed along
// . return if this blocks
// . we'll call sendReply later
if ( ! st0->m_msg5.getList ( rdbId ,
collnum ,
&st0->m_list ,
startKey ,
endKey ,
minRecSizes ,
includeTree , // include tree?
addToCache , // addToCache?
maxCacheAge ,
startFileNum ,
numFiles ,
st0 ,
gotListWrapper ,
niceness ,
doErrorCorrection ,
NULL , // cacheKeyPtr
0 , // retryNum
2 , // maxRetries
true , // compensateForMerge
syncPoint ,
false,
allowPageCache ) ) {
logTrace( g_conf.m_logTraceMsg0, "END. m_msg5.getList returned false" );
return;
}
// call wrapper ouselves
logTrace( g_conf.m_logTraceMsg0, "Calling gotListWrapper" );
gotListWrapper ( st0 , NULL , NULL );
logTrace( g_conf.m_logTraceMsg0, "END" );
}
