static std::vector<keypair_t> parseMetaList(const char *metaList, int32_t metaListSize) {
std::vector<keypair_t> result;
const char *p = metaList;
const char *pend = metaList + metaListSize;
for (; p < pend;) {
// get rdbId
rdbid_t rdbId = (rdbid_t)(*p & 0x7f);
p++;
// key size
int32_t ks = getKeySizeFromRdbId(rdbId);
// get key
const char *key = p;
p += ks;
// . if key is negative, no data is present
// . the doledb key is negative for us here
int32_t ds = KEYNEG(key) ? 0 : getDataSizeFromRdbId(rdbId);
// if datasize variable, read it in
if (ds == -1) {
// get data size
ds = *(int32_t *)p;
// skip data size int32_t
p += 4;
}
// skip data if not zero
p += ds;
result.push_back(std::make_pair(rdbId,key));
}
return result;
}
// . returns false if blocked, true otherwise
// . sets g_errno on error
// . if the list is sorted by keys this will be the most efficient
bool Msg1::sendSomeOfList ( ) {
// sanity check
if ( m_list->m_ks != 8 &&
m_list->m_ks != 12 &&
m_list->m_ks != 16 &&
m_list->m_ks != 24 ) {
g_process.shutdownAbort(true); }
// debug msg
//log("sendSomeOfList: mcast=%" PRIu32" exhausted=%" PRId32,
// (int32_t)&m_mcast,(int32_t)m_list->isExhausted());
loop:
// return true if list exhausted and nothing left to add
if ( m_list->isExhausted() ) return true;
// get key of the first record in the list
//key_t firstKey = m_list->getCurrentKey();
char firstKey[MAX_KEY_BYTES];
m_list->getCurrentKey(firstKey);
QUICKPOLL(m_niceness);
// get groupId from this key
//uint32_t groupId ;
// . use the new Hostdb.h inlined function
uint32_t shardNum = getShardNum ( m_rdbId , firstKey );
// point to start of data we're going to send
char *dataStart = m_list->getListPtr();
// how many records belong to the same group as "firstKey"
//key_t key;
char key[MAX_KEY_BYTES];
while ( ! m_list->isExhausted() ) {
//key = m_list->getCurrentKey();
m_list->getCurrentKey(key);
#ifdef GBSANITYCHECK
// no half bits in here!
// debug point
if ( m_list->useHalfKeys() &&
m_list->isHalfBitOn ( m_list->getCurrentRec() ) )
log(LOG_LOGIC,"net: msg1: Got half bit. Bad "
"engineer.");
#endif
// . if key belongs to same group as firstKey then continue
// . titledb now uses last bits of docId to determine groupId
// . but uses the top 32 bits of key still
// . spiderdb uses last 64 bits to determine groupId
// . tfndb now is like titledb(top 32 bits are top 32 of docId)
//if ( getGroupId(m_rdbId,key) != groupId ) goto done;
if ( getShardNum(m_rdbId,key) != shardNum ) goto done;
// . break so we don't send more than MAX_DGRAMS defined in
// UdpServer.cpp.
// . let's boost it from 16k to 64k for speed
if ( m_list->getListPtr() - dataStart > 64*1024 ) goto done;
// . point to next record
// . will point passed records if no more left!
QUICKPOLL(m_niceness);
//int32_t crec = m_list->getCurrentRecSize();
m_list->skipCurrentRecord();
// sanity check
if ( m_list->m_listPtr > m_list->m_listEnd ) {
g_process.shutdownAbort(true); }
}
done:
// now point to the end of the data
char *dataEnd = m_list->getListPtr();
// . if force local is true we force the data to be added locally
// . this fixes the bug we had from spiderdb since a key got corrupted
// just enough to put it into a different groupId (but not out
// of order) so we couldn't delete it cuz our delete keys would go
// elsewhere
if ( m_forceLocal && shardNum != getMyShardNum() &&
! g_conf.m_interfaceMachine ) {
// make the groupId local, our group
//groupId = g_hostdb.m_groupId;
// bitch about this to log it
log("net: Data does not belong in shard %" PRIu32", but adding "
"to %s anyway. Probable data corruption.",
(uint32_t)shardNum,getDbnameFromId(m_rdbId));
}
QUICKPOLL(m_niceness);
// sanity test for new rdbs
if ( m_list->m_fixedDataSize != getDataSizeFromRdbId(m_rdbId) ) {
g_process.shutdownAbort(true); }
// . now send this list to the host
// . this returns false if blocked, true otherwise
// . it also sets g_errno on error
// . if it blocked return false
if ( ! sendData ( shardNum , dataStart , dataEnd - dataStart ) )
return false;
// if there was an error return true
if ( g_errno ) return true;
// otherwise, keep adding
goto loop;
}
// . returns false if blocked, true otherwise
// . sets g_errno on error
// . if the list is sorted by keys this will be the most efficient
bool Msg1::sendSomeOfList ( ) {
// sanity check
if ( m_list->m_ks != 8 &&
m_list->m_ks != 12 &&
m_list->m_ks != 16 &&
m_list->m_ks != 24 ) {
char *xx=NULL;*xx=0; }
// debug msg
//log("sendSomeOfList: mcast=%lu exhausted=%li",
// (long)&m_mcast,(long)m_list->isExhausted());
loop:
// return true if list exhausted and nothing left to add
if ( m_list->isExhausted() ) return true;
// get key of the first record in the list
//key_t firstKey = m_list->getCurrentKey();
char firstKey[MAX_KEY_BYTES];
m_list->getCurrentKey(firstKey);
QUICKPOLL(m_niceness);
// get groupId from this key
//unsigned long groupId ;
// . use the new Hostdb.h inlined function
uint32_t shardNum = getShardNum ( m_rdbId , firstKey );
// . default is to use top bits of the key
// . but if we're adding to titledb use last bits in the top of key
// . but if we're adding to spiderdb we use the last long in the key
// . tfndb urlRec key same as titleRec key
/*
if ( m_rdbId == RDB_INDEXDB )
groupId = g_indexdb.getGroupIdFromKey((key_t *)firstKey);
else if ( m_rdbId == RDB_DATEDB )
groupId = g_datedb.getGroupIdFromKey((key128_t *)firstKey);
else if ( m_rdbId == RDB_TITLEDB)
groupId = g_titledb.getGroupIdFromKey((key_t *)firstKey);
else if ( m_rdbId == RDB_CHECKSUMDB)
groupId = g_checksumdb.getGroupId ( firstKey );
else if ( m_rdbId == RDB_SPIDERDB )
groupId = g_spiderdb.getGroupId ( (key_t *)firstKey );
else if ( m_rdbId == RDB_TFNDB )
groupId = g_tfndb.getGroupId ( (key_t *)firstKey );
else if ( m_rdbId == RDB_CLUSTERDB )
groupId = g_clusterdb.getGroupIdFromKey((key_t *)firstKey);
else if ( m_rdbId == RDB2_INDEXDB2 )
groupId = g_indexdb.getGroupIdFromKey((key_t *)firstKey);
else if ( m_rdbId == RDB2_DATEDB2 )
groupId = g_datedb.getGroupIdFromKey((key128_t *)firstKey);
else if ( m_rdbId == RDB2_TITLEDB2)
groupId = g_titledb.getGroupIdFromKey((key_t *)firstKey);
else if ( m_rdbId == RDB2_CHECKSUMDB2)
groupId = g_checksumdb.getGroupId ( firstKey );
else if ( m_rdbId == RDB2_SPIDERDB2 )
groupId = g_spiderdb.getGroupId ( (key_t *)firstKey );
else if ( m_rdbId == RDB2_TFNDB2 )
groupId = g_tfndb.getGroupId ( (key_t *)firstKey );
else if ( m_rdbId == RDB2_CLUSTERDB2 )
groupId = g_clusterdb.getGroupIdFromKey((key_t *)firstKey);
//else groupId=firstKey.n1 & g_hostdb.m_groupMask;
else groupId = (((key_t *)firstKey)->n1) & g_hostdb.m_groupMask;
*/
// point to start of data we're going to send
char *dataStart = m_list->getListPtr();
// how many records belong to the same group as "firstKey"
//key_t key;
char key[MAX_KEY_BYTES];
while ( ! m_list->isExhausted() ) {
//key = m_list->getCurrentKey();
m_list->getCurrentKey(key);
#ifdef _SANITYCHECK_
// no half bits in here!
// debug point
if ( m_list->useHalfKeys() &&
m_list->isHalfBitOn ( m_list->getCurrentRec() ) )
log(LOG_LOGIC,"net: msg1: Got half bit. Bad "
"engineer.");
#endif
// . if key belongs to same group as firstKey then continue
// . titledb now uses last bits of docId to determine groupId
// . but uses the top 32 bits of key still
// . spiderdb uses last 64 bits to determine groupId
// . tfndb now is like titledb(top 32 bits are top 32 of docId)
//if ( getGroupId(m_rdbId,key) != groupId ) goto done;
if ( getShardNum(m_rdbId,key) != shardNum ) goto done;
/*
switch ( m_rdbId ) {
case RDB_TITLEDB:
if(g_titledb.getGroupIdFromKey((key_t *)key)!=groupId)
goto done;
break;
case RDB_CHECKSUMDB:
if(g_checksumdb.getGroupId ( key)!=groupId)
goto done;
break;
case RDB_SPIDERDB:
if ( g_spiderdb.getGroupId ((key_t *)key) != groupId)
goto done;
break;
case RDB_TFNDB:
if ( g_tfndb.getGroupId ((key_t *)key) != groupId)
goto done;
break;
case RDB_CLUSTERDB:
if(g_clusterdb.getGroupIdFromKey((key_t *)key)!=groupId)
goto done;
break;
case RDB_DATEDB:
if(g_datedb.getGroupIdFromKey((key128_t *)key)!=groupId)
goto done;
break;
case RDB_INDEXDB:
if(g_indexdb.getGroupIdFromKey((key_t *)key)!=groupId)
goto done;
break;
//default:if ((key.n1&g_hostdb.m_groupMask) != groupId)
default: if ( ((((key_t *)key)->n1) & g_hostdb.m_groupMask) !=
groupId)
goto done;
}
*/
// . break so we don't send more than MAX_DGRAMS defined in
// UdpServer.cpp.
// . let's boost it from 16k to 64k for speed
if ( m_list->getListPtr() - dataStart > 64*1024 ) goto done;
// . point to next record
// . will point passed records if no more left!
QUICKPOLL(m_niceness);
//long crec = m_list->getCurrentRecSize();
m_list->skipCurrentRecord();
// sanity check
if ( m_list->m_listPtr > m_list->m_listEnd ) {
char *xx=NULL;*xx=0; }
}
done:
// now point to the end of the data
char *dataEnd = m_list->getListPtr();
// . if force local is true we force the data to be added locally
// . this fixes the bug we had from spiderdb since a key got corrupted
// just enough to put it into a different groupId (but not out
// of order) so we couldn't delete it cuz our delete keys would go
// elsewhere
if ( m_forceLocal && shardNum != getMyShardNum() &&
! g_conf.m_interfaceMachine ) {
// make the groupId local, our group
//groupId = g_hostdb.m_groupId;
// bitch about this to log it
log("net: Data does not belong in shard %lu, but adding "
"to %s anyway. Probable data corruption.",
(unsigned long)shardNum,getDbnameFromId(m_rdbId));
}
QUICKPOLL(m_niceness);
// sanity test for new rdbs
if ( m_list->m_fixedDataSize != getDataSizeFromRdbId(m_rdbId) ) {
char *xx=NULL;*xx=0; }
// little debug thing for genCatdb from msg9b's huge list add
//if ( m_list->m_listSize > 10000000 )
// log("msg1: adding chunk @ %li of %li bytes",
// (long)(dataStart - m_list->m_list) ,
// (long)m_list->m_listSize );
// . now send this list to the host
// . this returns false if blocked, true otherwise
// . it also sets g_errno on error
// . if it blocked return false
if ( ! sendData ( shardNum , dataStart , dataEnd - dataStart ) )
return false;
// if there was an error return true
if ( g_errno ) return true;
// otherwise, keep adding
goto loop;
}
