static std::vector<keypair_t> parseMetaList(const char *metaList, int32_t metaListSize) { std::vector<keypair_t> result; const char *p = metaList; const char *pend = metaList + metaListSize; for (; p < pend;) { // get rdbId rdbid_t rdbId = (rdbid_t)(*p & 0x7f); p++; // key size int32_t ks = getKeySizeFromRdbId(rdbId); // get key const char *key = p; p += ks; // . if key is negative, no data is present // . the doledb key is negative for us here int32_t ds = KEYNEG(key) ? 0 : getDataSizeFromRdbId(rdbId); // if datasize variable, read it in if (ds == -1) { // get data size ds = *(int32_t *)p; // skip data size int32_t p += 4; } // skip data if not zero p += ds; result.push_back(std::make_pair(rdbId,key)); } return result; }
// . returns false if blocked, true otherwise // . sets g_errno on error // . if the list is sorted by keys this will be the most efficient bool Msg1::sendSomeOfList ( ) { // sanity check if ( m_list->m_ks != 8 && m_list->m_ks != 12 && m_list->m_ks != 16 && m_list->m_ks != 24 ) { g_process.shutdownAbort(true); } // debug msg //log("sendSomeOfList: mcast=%" PRIu32" exhausted=%" PRId32, // (int32_t)&m_mcast,(int32_t)m_list->isExhausted()); loop: // return true if list exhausted and nothing left to add if ( m_list->isExhausted() ) return true; // get key of the first record in the list //key_t firstKey = m_list->getCurrentKey(); char firstKey[MAX_KEY_BYTES]; m_list->getCurrentKey(firstKey); QUICKPOLL(m_niceness); // get groupId from this key //uint32_t groupId ; // . use the new Hostdb.h inlined function uint32_t shardNum = getShardNum ( m_rdbId , firstKey ); // point to start of data we're going to send char *dataStart = m_list->getListPtr(); // how many records belong to the same group as "firstKey" //key_t key; char key[MAX_KEY_BYTES]; while ( ! m_list->isExhausted() ) { //key = m_list->getCurrentKey(); m_list->getCurrentKey(key); #ifdef GBSANITYCHECK // no half bits in here! // debug point if ( m_list->useHalfKeys() && m_list->isHalfBitOn ( m_list->getCurrentRec() ) ) log(LOG_LOGIC,"net: msg1: Got half bit. Bad " "engineer."); #endif // . if key belongs to same group as firstKey then continue // . titledb now uses last bits of docId to determine groupId // . but uses the top 32 bits of key still // . spiderdb uses last 64 bits to determine groupId // . tfndb now is like titledb(top 32 bits are top 32 of docId) //if ( getGroupId(m_rdbId,key) != groupId ) goto done; if ( getShardNum(m_rdbId,key) != shardNum ) goto done; // . break so we don't send more than MAX_DGRAMS defined in // UdpServer.cpp. // . let's boost it from 16k to 64k for speed if ( m_list->getListPtr() - dataStart > 64*1024 ) goto done; // . point to next record // . will point passed records if no more left! QUICKPOLL(m_niceness); //int32_t crec = m_list->getCurrentRecSize(); m_list->skipCurrentRecord(); // sanity check if ( m_list->m_listPtr > m_list->m_listEnd ) { g_process.shutdownAbort(true); } } done: // now point to the end of the data char *dataEnd = m_list->getListPtr(); // . if force local is true we force the data to be added locally // . this fixes the bug we had from spiderdb since a key got corrupted // just enough to put it into a different groupId (but not out // of order) so we couldn't delete it cuz our delete keys would go // elsewhere if ( m_forceLocal && shardNum != getMyShardNum() && ! g_conf.m_interfaceMachine ) { // make the groupId local, our group //groupId = g_hostdb.m_groupId; // bitch about this to log it log("net: Data does not belong in shard %" PRIu32", but adding " "to %s anyway. Probable data corruption.", (uint32_t)shardNum,getDbnameFromId(m_rdbId)); } QUICKPOLL(m_niceness); // sanity test for new rdbs if ( m_list->m_fixedDataSize != getDataSizeFromRdbId(m_rdbId) ) { g_process.shutdownAbort(true); } // . now send this list to the host // . this returns false if blocked, true otherwise // . it also sets g_errno on error // . if it blocked return false if ( ! sendData ( shardNum , dataStart , dataEnd - dataStart ) ) return false; // if there was an error return true if ( g_errno ) return true; // otherwise, keep adding goto loop; }
// . returns false if blocked, true otherwise // . sets g_errno on error // . if the list is sorted by keys this will be the most efficient bool Msg1::sendSomeOfList ( ) { // sanity check if ( m_list->m_ks != 8 && m_list->m_ks != 12 && m_list->m_ks != 16 && m_list->m_ks != 24 ) { char *xx=NULL;*xx=0; } // debug msg //log("sendSomeOfList: mcast=%lu exhausted=%li", // (long)&m_mcast,(long)m_list->isExhausted()); loop: // return true if list exhausted and nothing left to add if ( m_list->isExhausted() ) return true; // get key of the first record in the list //key_t firstKey = m_list->getCurrentKey(); char firstKey[MAX_KEY_BYTES]; m_list->getCurrentKey(firstKey); QUICKPOLL(m_niceness); // get groupId from this key //unsigned long groupId ; // . use the new Hostdb.h inlined function uint32_t shardNum = getShardNum ( m_rdbId , firstKey ); // . default is to use top bits of the key // . but if we're adding to titledb use last bits in the top of key // . but if we're adding to spiderdb we use the last long in the key // . tfndb urlRec key same as titleRec key /* if ( m_rdbId == RDB_INDEXDB ) groupId = g_indexdb.getGroupIdFromKey((key_t *)firstKey); else if ( m_rdbId == RDB_DATEDB ) groupId = g_datedb.getGroupIdFromKey((key128_t *)firstKey); else if ( m_rdbId == RDB_TITLEDB) groupId = g_titledb.getGroupIdFromKey((key_t *)firstKey); else if ( m_rdbId == RDB_CHECKSUMDB) groupId = g_checksumdb.getGroupId ( firstKey ); else if ( m_rdbId == RDB_SPIDERDB ) groupId = g_spiderdb.getGroupId ( (key_t *)firstKey ); else if ( m_rdbId == RDB_TFNDB ) groupId = g_tfndb.getGroupId ( (key_t *)firstKey ); else if ( m_rdbId == RDB_CLUSTERDB ) groupId = g_clusterdb.getGroupIdFromKey((key_t *)firstKey); else if ( m_rdbId == RDB2_INDEXDB2 ) groupId = g_indexdb.getGroupIdFromKey((key_t *)firstKey); else if ( m_rdbId == RDB2_DATEDB2 ) groupId = g_datedb.getGroupIdFromKey((key128_t *)firstKey); else if ( m_rdbId == RDB2_TITLEDB2) groupId = g_titledb.getGroupIdFromKey((key_t *)firstKey); else if ( m_rdbId == RDB2_CHECKSUMDB2) groupId = g_checksumdb.getGroupId ( firstKey ); else if ( m_rdbId == RDB2_SPIDERDB2 ) groupId = g_spiderdb.getGroupId ( (key_t *)firstKey ); else if ( m_rdbId == RDB2_TFNDB2 ) groupId = g_tfndb.getGroupId ( (key_t *)firstKey ); else if ( m_rdbId == RDB2_CLUSTERDB2 ) groupId = g_clusterdb.getGroupIdFromKey((key_t *)firstKey); //else groupId=firstKey.n1 & g_hostdb.m_groupMask; else groupId = (((key_t *)firstKey)->n1) & g_hostdb.m_groupMask; */ // point to start of data we're going to send char *dataStart = m_list->getListPtr(); // how many records belong to the same group as "firstKey" //key_t key; char key[MAX_KEY_BYTES]; while ( ! m_list->isExhausted() ) { //key = m_list->getCurrentKey(); m_list->getCurrentKey(key); #ifdef _SANITYCHECK_ // no half bits in here! // debug point if ( m_list->useHalfKeys() && m_list->isHalfBitOn ( m_list->getCurrentRec() ) ) log(LOG_LOGIC,"net: msg1: Got half bit. Bad " "engineer."); #endif // . if key belongs to same group as firstKey then continue // . titledb now uses last bits of docId to determine groupId // . but uses the top 32 bits of key still // . spiderdb uses last 64 bits to determine groupId // . tfndb now is like titledb(top 32 bits are top 32 of docId) //if ( getGroupId(m_rdbId,key) != groupId ) goto done; if ( getShardNum(m_rdbId,key) != shardNum ) goto done; /* switch ( m_rdbId ) { case RDB_TITLEDB: if(g_titledb.getGroupIdFromKey((key_t *)key)!=groupId) goto done; break; case RDB_CHECKSUMDB: if(g_checksumdb.getGroupId ( key)!=groupId) goto done; break; case RDB_SPIDERDB: if ( g_spiderdb.getGroupId ((key_t *)key) != groupId) goto done; break; case RDB_TFNDB: if ( g_tfndb.getGroupId ((key_t *)key) != groupId) goto done; break; case RDB_CLUSTERDB: if(g_clusterdb.getGroupIdFromKey((key_t *)key)!=groupId) goto done; break; case RDB_DATEDB: if(g_datedb.getGroupIdFromKey((key128_t *)key)!=groupId) goto done; break; case RDB_INDEXDB: if(g_indexdb.getGroupIdFromKey((key_t *)key)!=groupId) goto done; break; //default:if ((key.n1&g_hostdb.m_groupMask) != groupId) default: if ( ((((key_t *)key)->n1) & g_hostdb.m_groupMask) != groupId) goto done; } */ // . break so we don't send more than MAX_DGRAMS defined in // UdpServer.cpp. // . let's boost it from 16k to 64k for speed if ( m_list->getListPtr() - dataStart > 64*1024 ) goto done; // . point to next record // . will point passed records if no more left! QUICKPOLL(m_niceness); //long crec = m_list->getCurrentRecSize(); m_list->skipCurrentRecord(); // sanity check if ( m_list->m_listPtr > m_list->m_listEnd ) { char *xx=NULL;*xx=0; } } done: // now point to the end of the data char *dataEnd = m_list->getListPtr(); // . if force local is true we force the data to be added locally // . this fixes the bug we had from spiderdb since a key got corrupted // just enough to put it into a different groupId (but not out // of order) so we couldn't delete it cuz our delete keys would go // elsewhere if ( m_forceLocal && shardNum != getMyShardNum() && ! g_conf.m_interfaceMachine ) { // make the groupId local, our group //groupId = g_hostdb.m_groupId; // bitch about this to log it log("net: Data does not belong in shard %lu, but adding " "to %s anyway. Probable data corruption.", (unsigned long)shardNum,getDbnameFromId(m_rdbId)); } QUICKPOLL(m_niceness); // sanity test for new rdbs if ( m_list->m_fixedDataSize != getDataSizeFromRdbId(m_rdbId) ) { char *xx=NULL;*xx=0; } // little debug thing for genCatdb from msg9b's huge list add //if ( m_list->m_listSize > 10000000 ) // log("msg1: adding chunk @ %li of %li bytes", // (long)(dataStart - m_list->m_list) , // (long)m_list->m_listSize ); // . now send this list to the host // . this returns false if blocked, true otherwise // . it also sets g_errno on error // . if it blocked return false if ( ! sendData ( shardNum , dataStart , dataEnd - dataStart ) ) return false; // if there was an error return true if ( g_errno ) return true; // otherwise, keep adding goto loop; }