// . slot should be auto-nuked upon transmission or error // . TODO: ensure if this sendReply() fails does it really nuke the slot? void gotListWrapper ( void *state , RdbList *listb , Msg5 *msg5xx ) { logTrace( g_conf.m_logTraceMsg0, "BEGIN" ); // get the state State00 *st0 = (State00 *)state; // extract the udp slot and list and msg5 UdpSlot *slot = st0->m_slot; RdbList *list = &st0->m_list; Msg5 *msg5 = &st0->m_msg5; UdpServer *us = st0->m_us; // timing debug if ( g_conf.m_logTimingNet || g_conf.m_logDebugNet ) { //log("Msg0:hndled request %" PRIu64,gettimeofdayInMilliseconds()); int32_t size = -1; if ( list ) size = list->getListSize(); log(LOG_TIMING|LOG_DEBUG, "net: msg0: Handled request for data. " "Now sending data termId=%" PRIu64" size=%" PRId32 " transId=%" PRId32" ip=%s port=%i took=%" PRId64" " "(niceness=%" PRId32").", g_posdb.getTermId(msg5->m_startKey), size,slot->m_transId, iptoa(slot->m_ip),slot->m_port, gettimeofdayInMilliseconds() - st0->m_startTime , st0->m_niceness ); } // on error nuke the list and it's data if ( g_errno ) { mdelete ( st0 , sizeof(State00) , "Msg0" ); delete (st0); // TODO: free "slot" if this send fails log(LOG_ERROR,"%s:%s:%d: call sendErrorReply.", __FILE__, __func__, __LINE__); us->sendErrorReply ( slot , g_errno ); return; } QUICKPOLL(st0->m_niceness); // point to the serialized list in "list" char *data = list->getList(); int32_t dataSize = list->getListSize(); char *alloc = list->getAlloc(); int32_t allocSize = list->getAllocSize(); // tell list not to free the data since it is a reply so UdpServer // will free it when it destroys the slot list->setOwnData ( false ); // keep track of stats Rdb *rdb = getRdbFromId ( st0->m_rdbId ); if ( rdb ) rdb->sentReplyGet ( dataSize ); // TODO: can we free any memory here??? // keep track of how long it takes to complete the send st0->m_startTime = gettimeofdayInMilliseconds(); // debug point int32_t oldSize = msg5->m_minRecSizes; int32_t newSize = msg5->m_minRecSizes + 20; // watch for wrap around if ( newSize < oldSize ) newSize = 0x7fffffff; if ( dataSize > newSize && list->getFixedDataSize() == 0 && // do not annoy me with these linkdb msgs dataSize > newSize+100 ) log(LOG_LOGIC,"net: msg0: Sending more data than what was " "requested. Ineffcient. Bad engineer. dataSize=%" PRId32" " "minRecSizes=%" PRId32".",dataSize,oldSize); // // for linkdb lists, remove all the keys that have the same IP32 // and store a count of what we removed somewhere // if ( st0->m_rdbId == RDB_LINKDB ) { // store compressed list on itself char *dst = list->m_list; // keep stats int32_t totalOrigLinks = 0; int32_t ipDups = 0; int32_t lastIp32 = 0; char *listEnd = list->getListEnd(); // compress the list for ( ; ! list->isExhausted() ; list->skipCurrentRecord() ) { // breathe QUICKPOLL ( st0->m_niceness ); // count it totalOrigLinks++; // get rec char *rec = list->getCurrentRec(); int32_t ip32 = g_linkdb.getLinkerIp_uk((key224_t *)rec ); // same as one before? if ( ip32 == lastIp32 && // are we the last rec? include that for // advancing the m_nextKey in Linkdb more // efficiently. rec + LDBKS < listEnd ) { ipDups++; continue; } // store it gbmemcpy (dst , rec , LDBKS ); dst += LDBKS; // update it lastIp32 = ip32; } // . if we removed one key, store the stats // . caller should recognize reply is not a multiple of // the linkdb key size LDBKS and no its there! if ( ipDups ) { //*(int32_t *)dst = totalOrigLinks; //dst += 4; //*(int32_t *)dst = ipDups; //dst += 4; } // update list parms list->m_listSize = dst - list->m_list; list->m_listEnd = list->m_list + list->m_listSize; data = list->getList(); dataSize = list->getListSize(); } //log("sending replySize=%" PRId32" min=%" PRId32,dataSize,msg5->m_minRecSizes); // . TODO: dataSize may not equal list->getListMaxSize() so // Mem class may show an imblanace // . now g_udpServer is responsible for freeing data/dataSize // . the "true" means to call doneSending_ass() from the signal handler // if need be st0->m_us->sendReply_ass( data, dataSize, alloc, allocSize, slot, st0, doneSending_ass, -1, -1, true ); logTrace( g_conf.m_logTraceMsg0, "END" ); }
main ( int argc , char *argv[] ) { // seed with same value so we get same rand sequence for all srand ( 1945687 ); // # of keys to in each list long nk = 200000; // # keys wanted long numKeysWanted = 200000; // get # lists to merge long numToMerge = atoi ( argv[1] ); // print start time fprintf (stderr,"smt:: randomizing begin. %li lists of %li keys.\n", numToMerge, nk); // make a list of compressed (6 byte) docIds key_t *keys0 = (key_t *) malloc ( sizeof(key_t) * nk ); key_t *keys1 = (key_t *) malloc ( sizeof(key_t) * nk ); key_t *keys2 = (key_t *) malloc ( sizeof(key_t) * nk ); key_t *keys3 = (key_t *) malloc ( sizeof(key_t) * nk ); // store radnom docIds in this list unsigned long *p = (unsigned long *) keys0; // random docIds for ( long i = 0 ; i < nk ; i++ ) { *p++ = rand() ; *p++ = rand() ; *p++ = rand() ; } p = (unsigned long *) keys1; for ( long i = 0 ; i < nk ; i++ ) { *p++ = rand() ; *p++ = rand() ; *p++ = rand() ; } p = (unsigned long *) keys2; for ( long i = 0 ; i < nk ; i++ ) { *p++ = rand() ; *p++ = rand() ; *p++ = rand() ; } p = (unsigned long *) keys3; for ( long i = 0 ; i < nk ; i++ ) { *p++ = rand() ; *p++ = rand() ; *p++ = rand() ; } // sort em up gbsort ( keys0 , nk , sizeof(key_t) , cmp ); gbsort ( keys1 , nk , sizeof(key_t) , cmp ); gbsort ( keys2 , nk , sizeof(key_t) , cmp ); gbsort ( keys3 , nk , sizeof(key_t) , cmp ); // set lists RdbList list0; RdbList list1; RdbList list2; RdbList list3; key_t minKey; minKey.n0 = 0LL; minKey.n1 = 0LL; key_t maxKey; maxKey.setMax(); list0.set ( (char *)keys0 , nk * sizeof(key_t), nk * sizeof(key_t), minKey , maxKey , 0 , false ); list1.set ( (char *)keys1 , nk * sizeof(key_t), nk * sizeof(key_t), minKey , maxKey , 0 , false ); list2.set ( (char *)keys2 , nk * sizeof(key_t), nk * sizeof(key_t), minKey , maxKey , 0 , false ); list3.set ( (char *)keys3 , nk * sizeof(key_t), nk * sizeof(key_t), minKey , maxKey , 0 , false ); // mergee RdbList list; RdbList *lists[2]; lists[0] = &list0; lists[1] = &list1; lists[2] = &list2; lists[3] = &list3; //list.prepareForMerge ( lists , 3 , numKeysWanted * sizeof(key_t)); list.prepareForMerge (lists,numToMerge,numKeysWanted * sizeof(key_t)); // start time fprintf(stderr,"starting merge\n"); long long t = gettimeofdayInMilliseconds(); // do it if ( numToMerge == 2 ) list.superMerge2 ( &list0 , &list1 , minKey , maxKey , false ); if ( numToMerge == 3 ) list.superMerge3 ( &list0 , &list1 , &list2 , minKey , maxKey ); // completed long long now = gettimeofdayInMilliseconds(); fprintf(stderr,"smt:: %li list NEW MERGE took %llu ms\n", numToMerge,now-t); // time per key long size = list.getListSize() / sizeof(key_t); double tt = ((double)(now - t))*1000000.0 / ((double)size); fprintf (stderr,"smt:: %f nanoseconds per key\n", tt); // stats //double d = (1000.0*(double)nk*2.0) / ((double)(now - t)); double d = (1000.0*(double)(size)) / ((double)(now - t)); fprintf (stderr,"smt:: %f cycles per final key\n" , 400000000.0 / d ); fprintf (stderr,"smt:: we can do %li adds per second\n" ,(long)d); fprintf (stderr,"smt:: final list size = %li\n",list.getListSize()); // now get list from the old merge routine RdbList listOld; listOld.prepareForMerge (lists,numToMerge,numKeysWanted*sizeof(key_t)); t = gettimeofdayInMilliseconds(); listOld.merge_r ( lists , numToMerge , true , minKey , maxKey , false , numKeysWanted * sizeof(key_t)); now = gettimeofdayInMilliseconds(); fprintf(stderr,"smt:: %li list OLD MERGE took %llu ms\n", numToMerge,now-t); // then compare // exit gracefully exit ( 0 ); }
// . slot should be auto-nuked upon transmission or error // . TODO: ensure if this sendReply() fails does it really nuke the slot? void gotListWrapper ( void *state , RdbList *listb , Msg5 *msg5xx ) { // get the state State00 *st0 = (State00 *)state; // extract the udp slot and list and msg5 UdpSlot *slot = st0->m_slot; RdbList *list = &st0->m_list; Msg5 *msg5 = &st0->m_msg5; UdpServer *us = st0->m_us; // sanity check -- ensure they match //if ( niceness != st0->m_niceness ) // log("Msg0: niceness mismatch"); // debug msg //if ( niceness != 0 ) // log("HEY! niceness is not 0"); // timing debug if ( g_conf.m_logTimingNet || g_conf.m_logDebugNet ) { //log("Msg0:hndled request %"UINT64"",gettimeofdayInMilliseconds()); int32_t size = -1; if ( list ) size = list->getListSize(); log(LOG_TIMING|LOG_DEBUG, "net: msg0: Handled request for data. " "Now sending data termId=%"UINT64" size=%"INT32"" " transId=%"INT32" ip=%s port=%i took=%"INT64" " "(niceness=%"INT32").", g_posdb.getTermId(msg5->m_startKey), size,slot->m_transId, iptoa(slot->m_ip),slot->m_port, gettimeofdayInMilliseconds() - st0->m_startTime , st0->m_niceness ); } // debug //if ( ! msg5->m_includeTree ) // log("hotit\n"); // on error nuke the list and it's data if ( g_errno ) { mdelete ( st0 , sizeof(State00) , "Msg0" ); delete (st0); // TODO: free "slot" if this send fails us->sendErrorReply ( slot , g_errno ); return; } QUICKPOLL(st0->m_niceness); // point to the serialized list in "list" char *data = list->getList(); int32_t dataSize = list->getListSize(); char *alloc = list->getAlloc(); int32_t allocSize = list->getAllocSize(); // tell list not to free the data since it is a reply so UdpServer // will free it when it destroys the slot list->setOwnData ( false ); // keep track of stats Rdb *rdb = getRdbFromId ( st0->m_rdbId ); if ( rdb ) rdb->sentReplyGet ( dataSize ); // TODO: can we free any memory here??? // keep track of how long it takes to complete the send st0->m_startTime = gettimeofdayInMilliseconds(); // debug point int32_t oldSize = msg5->m_minRecSizes; int32_t newSize = msg5->m_minRecSizes + 20; // watch for wrap around if ( newSize < oldSize ) newSize = 0x7fffffff; if ( dataSize > newSize && list->getFixedDataSize() == 0 && // do not annoy me with these linkdb msgs dataSize > newSize+100 ) log(LOG_LOGIC,"net: msg0: Sending more data than what was " "requested. Ineffcient. Bad engineer. dataSize=%"INT32" " "minRecSizes=%"INT32".",dataSize,oldSize); /* // always compress these lists if ( st0->m_rdbId == RDB_SECTIONDB ) { // && 1 == 3) { // get sh48, the sitehash key128_t *startKey = (key128_t *)msg5->m_startKey ; int64_t sh48 = g_datedb.getTermId(startKey); // debug //log("msg0: got sectiondblist from disk listsize=%"INT32"", // list->getListSize()); if ( dataSize > 50000 ) log("msg0: sending back list rdb=%"INT32" " "listsize=%"INT32" sh48=0x%"XINT64"", (int32_t)st0->m_rdbId, dataSize, sh48); // save it int32_t origDataSize = dataSize; // store compressed list on itself char *dst = list->m_list; // warn if niceness is 0! if ( st0->m_niceness == 0 ) log("msg0: compressing sectiondb list at niceness 0!"); // compress the list uint32_t lastVoteHash32 = 0LL; SectionVote *lastVote = NULL; for ( ; ! list->isExhausted() ; list->skipCurrentRecord() ) { // breathe QUICKPOLL ( st0->m_niceness ); // get rec char *rec = list->getCurrentRec(); // for ehre key128_t *key = (key128_t *)rec; // the score is the bit which is was set in // Section::m_flags for that docid int32_t secType = g_indexdb.getScore ( (char *)key ); // 0 means it probably used to count # of voters // from this site, so i don't think xmldoc uses // that any more if ( secType == SV_SITE_VOTER ) continue; // treat key like a datedb key and get the taghash uint32_t h32 = g_datedb.getDate ( key ); // get data/vote from the current record in the // sectiondb list SectionVote *sv=(SectionVote *)list->getCurrentData (); // get the average score for this doc float avg = sv->m_score ; if ( sv->m_numSampled > 0.0 ) avg /= sv->m_numSampled; // if same as last guy, add to it if ( lastVoteHash32 == h32 && lastVote ) { // turn possible multi-vote into single docid // into a single vote, with the score averaged. lastVote->m_score += avg; lastVote->m_numSampled++; continue; } // otherwise, add in a new guy! *(key128_t *)dst = *key; dst += sizeof(key128_t); // the new vote SectionVote *dsv = (SectionVote *)dst; dsv->m_score = avg; dsv->m_numSampled = 1; // set this lastVote = dsv; lastVoteHash32 = h32; // skip over dst += sizeof(SectionVote); } // update the list size now for sending back dataSize = dst - data; // if the list was over the requested minrecsizes we need // to set a flag so that the caller will do a re-call. // so making the entire odd, will be the flag. if ( origDataSize > msg5->m_minRecSizes && dataSize < origDataSize ) { *dst++ = '\0'; dataSize++; } // debug //log("msg0: compressed sectiondblist from disk " // "newlistsize=%"INT32"", dataSize); // use this timestamp int32_t now = getTimeLocal();//Global(); // finally, cache this sucker s_sectiondbCache.addRecord ( msg5->m_coll, (char *)startKey,//(char *)&sh48 data, dataSize , now ); // ignore errors g_errno = 0; } */ // // for linkdb lists, remove all the keys that have the same IP32 // and store a count of what we removed somewhere // if ( st0->m_rdbId == RDB_LINKDB ) { // store compressed list on itself char *dst = list->m_list; // keep stats int32_t totalOrigLinks = 0; int32_t ipDups = 0; int32_t lastIp32 = 0; char *listEnd = list->getListEnd(); // compress the list for ( ; ! list->isExhausted() ; list->skipCurrentRecord() ) { // breathe QUICKPOLL ( st0->m_niceness ); // count it totalOrigLinks++; // get rec char *rec = list->getCurrentRec(); int32_t ip32 = g_linkdb.getLinkerIp_uk((key224_t *)rec ); // same as one before? if ( ip32 == lastIp32 && // are we the last rec? include that for // advancing the m_nextKey in Linkdb more // efficiently. rec + LDBKS < listEnd ) { ipDups++; continue; } // store it gbmemcpy (dst , rec , LDBKS ); dst += LDBKS; // update it lastIp32 = ip32; } // . if we removed one key, store the stats // . caller should recognize reply is not a multiple of // the linkdb key size LDBKS and no its there! if ( ipDups ) { //*(int32_t *)dst = totalOrigLinks; //dst += 4; //*(int32_t *)dst = ipDups; //dst += 4; } // update list parms list->m_listSize = dst - list->m_list; list->m_listEnd = list->m_list + list->m_listSize; data = list->getList(); dataSize = list->getListSize(); } //log("sending replySize=%"INT32" min=%"INT32"",dataSize,msg5->m_minRecSizes); // . TODO: dataSize may not equal list->getListMaxSize() so // Mem class may show an imblanace // . now g_udpServer is responsible for freeing data/dataSize // . the "true" means to call doneSending_ass() from the signal handler // if need be st0->m_us->sendReply_ass ( data , dataSize , alloc , // alloc allocSize , // alloc size slot , 60 , st0 , doneSending_ass , -1 , -1 , true ); }
// . sets m_errno to g_errno if not already set void Msg51::gotClusterRec(Slot *slot) { // count it m_numReplies++; // free up slot->m_inUse = false; RdbList *list = slot->m_msg0.m_list; // update m_errno if we had an error if ( ! m_errno ) m_errno = g_errno; if ( g_errno ) // print error log(LOG_DEBUG, "query: Had error getting cluster info got docId=d: " "%s.",mstrerror(g_errno)); // this doubles as a ptr to a cluster rec int32_t ci = slot->m_ci; // get docid int64_t docId = m_docIds[ci]; // assume error! m_clusterLevels[ci] = CR_ERROR_CLUSTERDB; // bail on error if ( g_errno || list->getListSize() < 12 ) { //log(LOG_DEBUG, // "build: clusterdb rec for d=%" PRId64" dptr=%" PRIu32" " // "not found. where is it?", docId, (int32_t)ci); g_errno = 0; return; } // . steal rec from this multicast // . point to cluster rec, a int32_t key96_t *rec = &m_clusterRecs[ci]; // store the cluster rec itself *rec = *(key96_t *)(list->getList()); // debug note log(LOG_DEBUG, "build: had clusterdb SUCCESS for d=%" PRId64" dptr=%" PRIu32" " "rec.n1=%" PRIx32",%016" PRIx64" sitehash26=0x%" PRIx32".", (int64_t)docId, (int32_t)ci, rec->n1,rec->n0, g_clusterdb.getSiteHash26((char *)rec)); // check for docid mismatch int64_t docId2 = g_clusterdb.getDocId ( rec ); if ( docId != docId2 ) { logf(LOG_DEBUG,"query: docid mismatch in clusterdb."); return; } // it is legit, set to CR_OK m_clusterLevels[ci] = CR_OK; // . init the quick cache // . use 100k if ( ! s_cacheInit && s_clusterdbQuickCache.init(200*1024 , // maxMem sizeof(key96_t) , // fixedDataSize (clusterdb rec) false , // support lists 10000 , // max recs false , // use half keys? "clusterdbQuickCache" , false , // load from disk? sizeof(key96_t) , // cache key size sizeof(key96_t) )) // cache key size // only init once if successful s_cacheInit = true; // debug msg //logf(LOG_DEBUG,"query: msg51 addRec k.n0=%" PRIu64" rec.n0=%" PRIu64,docId, // rec->n0); // . add the record to our quick cache as a int64_t // . ignore any error if ( s_cacheInit ) s_clusterdbQuickCache.addRecord(m_collnum, (key96_t)docId, // docid is key (char *)rec, sizeof(key96_t), // recSize 0); // clear it in case the cache set it, we don't care g_errno = 0; }