void Indexdb::deepVerify ( char *coll ) {
log ( LOG_INFO, "db: Deep Verifying Indexdb for coll %s...", coll );
g_threads.disableThreads();
Msg5 msg5;
Msg5 msg5b;
RdbList list;
key_t startKey;
key_t endKey;
startKey.setMin();
endKey.setMax();
//long minRecSizes = 64000;
collnum_t collnum = g_collectiondb.getCollnum(coll);
RdbBase *rdbBase = g_indexdb.m_rdb.getBase(collnum);
long numFiles = rdbBase->getNumFiles();
long currentFile = 0;
deepLoop:
// done after scanning all files
if ( currentFile >= numFiles ) {
g_threads.enableThreads();
log ( LOG_INFO, "db: Finished deep verify for %li files.",
numFiles );
return;
}
// scan this file
if ( ! msg5.getList ( RDB_INDEXDB ,
coll ,
&list ,
startKey ,
endKey ,
64000 , // minRecSizes ,
true , // includeTree ,
false , // add to cache?
0 , // max cache age
currentFile , // startFileNum ,
1 , // numFiles ,
NULL , // state
NULL , // callback
0 , // niceness
false , // err correction?
NULL ,
0 ,
-1 ,
true ,
-1LL ,
&msg5b ,
false )) {
g_threads.enableThreads();
log("db: HEY! it did not block");
return;
}
long count = 0;
long got = 0;
for ( list.resetListPtr() ; ! list.isExhausted() ;
list.skipCurrentRecord() ) {
key_t k = list.getCurrentKey();
count++;
//unsigned long groupId = k.n1 & g_hostdb.m_groupMask;
unsigned long groupId = getGroupId ( RDB_INDEXDB , &k );
if ( groupId == g_hostdb.m_groupId ) got++;
}
if ( got != count ) {
BigFile *f = rdbBase->getFile(currentFile);
log ("db: File %s: Out of first %li records in indexdb, "
"only %li belong to our group.",
f->getFilename(),count,got );
}
//else
// log ( LOG_INFO, "db: File %li: Indexdb passed verification "
// "successfully for %li recs.",currentFile,count );
// next file
currentFile++;
goto deepLoop;
}
/*
bool Monitordb::addColl ( char *coll, bool doVerify ) {
if ( ! m_rdb.addColl ( coll ) ) return false;
if ( ! doVerify ) return true;
// verify
if ( verify(coll) ) return true;
// if not allowing scale, return false
if ( ! g_conf.m_allowScale ) return false;
// otherwise let it go
log ( "db: Verify failed, but scaling is allowed, passing." );
return true;
}
*/
bool Monitordb::verify ( char *coll ) {
log ( LOG_INFO, "db: Verifying Monitordb for coll %s...", coll );
g_threads.disableThreads();
Msg5 msg5;
Msg5 msg5b;
RdbList list;
key224_t startKey;
key224_t endKey;
startKey.setMin();
endKey.setMax();
long minRecSizes = 64000;
CollectionRec *cr = g_collectiondb.getRec(coll);
if ( ! msg5.getList ( RDB_MONITORDB ,
cr->m_collnum,
&list ,
(char*)&startKey ,
(char*)&endKey ,
minRecSizes ,
true , // includeTree ,
false , // add to cache?
0 , // max cache age
0 , // startFileNum ,
-1 , // numFiles ,
NULL , // state
NULL , // callback
0 , // niceness
false , // err correction?
NULL ,
0 ,
-1 ,
true ,
-1LL ,
&msg5b ,
true )) {
g_threads.enableThreads();
return log("db: HEY! it did not block");
}
long count = 0;
long got = 0;
for ( list.resetListPtr() ; ! list.isExhausted() ;
list.skipCurrentRecord() ) {
key224_t k;
list.getCurrentKey((char*)&k);
count++;
uint32_t shardNum = getShardNum ( RDB_MONITORDB , &k );
if ( shardNum == getMyShardNum() ) got++;
}
if ( got != count ) {
log ("db: Out of first %li records in Monitordb , "
"only %li belong to our group.",count,got);
/*
// repeat with log
for ( list.resetListPtr() ; ! list.isExhausted() ;
list.skipCurrentRecord() ) {
key224_t k;
list.getCurrentKey((char*)&k);
uint32_t shardNum = getShardNum ( RDB_MONITORDB , &k );
long groupNum = g_hostdb.getGroupNum(groupId);
unsigned long sh32 ;
sh32 = g_monitordb.getLinkeeSiteHash32_uk(&k);
uint16_t sh16 = sh32 >> 19;
log("db: sh16=0x%lx group=%li",
(long)sh16,groupNum);
}
*/
// exit if NONE, we probably got the wrong data
if ( got == 0 ) log("db: Are you sure you have the "
"right "
"data in the right directory? "
"Exiting.");
log ( "db: Exiting due to inconsistency.");
g_threads.enableThreads();
return g_conf.m_bypassValidation;
}
log ( LOG_INFO, "db: Monitordb passed verification successfully for "
"%li recs.", count );
// DONE
g_threads.enableThreads();
return true;
}
/*
bool Titledb::addColl ( char *coll, bool doVerify ) {
if ( ! m_rdb.addColl ( coll ) ) return false;
if ( ! doVerify ) return true;
// verify
if ( verify(coll) ) return true;
// if not allowing scale, return false
if ( ! g_conf.m_allowScale ) return false;
// otherwise let it go
log ( "db: Verify failed, but scaling is allowed, passing." );
return true;
}
*/
bool Titledb::verify(const char *coll) {
log ( LOG_DEBUG, "db: Verifying Titledb for coll %s...", coll );
Msg5 msg5;
RdbList list;
key96_t startKey;
key96_t endKey;
startKey.setMin();
endKey.setMax();
//int32_t minRecSizes = 64000;
const CollectionRec *cr = g_collectiondb.getRec(coll);
if ( ! msg5.getList ( RDB_TITLEDB ,
cr->m_collnum ,
&list ,
startKey ,
endKey ,
1024*1024 , // minRecSizes ,
true , // includeTree ,
0 , // max cache age
0 , // startFileNum ,
-1 , // numFiles ,
NULL , // state
NULL , // callback
0 , // niceness
false , // err correction?
NULL , // cache key ptr
0 , // retry num
-1 , // maxRetries
-1LL , // sync point
false , // isRealMerge
true)) // allowPageCache
{
log(LOG_DEBUG, "db: HEY! it did not block");
return false;
}
int32_t count = 0;
int32_t got = 0;
for ( list.resetListPtr() ; ! list.isExhausted() ;
list.skipCurrentRecord() ) {
key96_t k = list.getCurrentKey();
// skip negative keys
if ( (k.n0 & 0x01) == 0x00 ) continue;
count++;
//uint32_t groupId = getGroupId ( RDB_TITLEDB , &k );
//if ( groupId == g_hostdb.m_groupId ) got++;
uint32_t shardNum = getShardNum ( RDB_TITLEDB, &k );
if ( shardNum == getMyShardNum() ) got++;
}
if ( got != count ) {
// tally it up
g_rebalance.m_numForeignRecs += count - got;
log ("db: Out of first %" PRId32" records in titledb, "
"only %" PRId32" belong to our shard. c=%s",count,got,coll);
// exit if NONE, we probably got the wrong data
if ( count > 10 && got == 0 )
log("db: Are you sure you have the right "
"data in the right directory? "
"coll=%s "
"Exiting.",
coll);
// repeat with log
for ( list.resetListPtr() ; ! list.isExhausted() ;
list.skipCurrentRecord() ) {
key96_t k = list.getCurrentKey();
//uint32_t groupId = getGroupId ( RDB_TITLEDB,&k);
//int32_t groupNum = g_hostdb.getGroupNum(groupId);
int32_t shardNum = getShardNum ( RDB_TITLEDB, &k );
log("db: docid=%" PRId64" shard=%" PRId32,
getDocId(&k),shardNum);
}
//if ( g_conf.m_bypassValidation ) return true;
//if ( g_conf.m_allowScale ) return true;
// don't exit any more, allow it, but do not delete
// recs that belong to different shards when we merge now!
log ( "db: db shards unbalanced. "
"Click autoscale in master controls.");
//return false;
return true;
}
log ( LOG_DEBUG, "db: Titledb passed verification successfully for %" PRId32
" recs.", count );
// DONE
return true;
}
bool Indexdb::verify ( char *coll ) {
return true;
log ( LOG_INFO, "db: Verifying Indexdb for coll %s...", coll );
g_threads.disableThreads();
Msg5 msg5;
Msg5 msg5b;
RdbList list;
key_t startKey;
key_t endKey;
startKey.setMin();
endKey.setMax();
//long minRecSizes = 64000;
if ( ! msg5.getList ( RDB_INDEXDB ,
coll ,
&list ,
startKey ,
endKey ,
64000 , // minRecSizes ,
true , // includeTree ,
false , // add to cache?
0 , // max cache age
0 , // startFileNum ,
-1 , // numFiles ,
NULL , // state
NULL , // callback
0 , // niceness
false , // err correction?
NULL ,
0 ,
-1 ,
true ,
-1LL ,
&msg5b ,
true )) {
g_threads.enableThreads();
return log("db: HEY! it did not block");
}
long count = 0;
long got = 0;
bool printedKey = false;
bool printedZeroKey = false;
for ( list.resetListPtr() ; ! list.isExhausted() ;
list.skipCurrentRecord() ) {
key_t k = list.getCurrentKey();
count++;
//unsigned long groupId = k.n1 & g_hostdb.m_groupMask;
unsigned long groupId = getGroupId ( RDB_INDEXDB , &k );
if ( groupId == g_hostdb.m_groupId ) got++;
else if ( !printedKey ) {
log ( "db: Found bad key in list (only printing once): "
"%lx %llx", k.n1, k.n0 );
printedKey = true;
}
if ( k.n1 == 0 && k.n0 == 0 ) {
if ( !printedZeroKey ) {
log ( "db: Found Zero key in list, passing. "
"(only printing once)." );
printedZeroKey = true;
}
if ( groupId != g_hostdb.m_groupId )
got++;
}
}
if ( got != count ) {
log ("db: Out of first %li records in indexdb, only %li belong "
"to our group.",count,got);
// exit if NONE, we probably got the wrong data
if ( got == 0 ) log("db: Are you sure you have the "
"right "
"data in the right directory? "
"Exiting.");
log ( "db: Exiting due to Indexdb inconsistency." );
g_threads.enableThreads();
return g_conf.m_bypassValidation;
}
log ( LOG_INFO, "db: Indexdb passed verification successfully for %li "
"recs.", count );
// DONE
g_threads.enableThreads();
return true;
}
void gotDatedbList ( State60 *st ) {
// must only be run on host #0 since we need just one lock table
if ( g_hostdb.m_myHost->m_hostId != 0 ) { char *xx=NULL;*xx=0; }
// load turk lock table if we need to
bool s_init = false;
if ( ! s_init ) {
s_init = true;
if ( ! g_turkLocks.set(8,sizeof(TurkLock),256) )
log("turk: failed to init turk lock table");
if ( ! g_turkLocks.load(g_conf.m_dir,"turkdir/docidlocks.dat"))
log("turk: failed to load turk lock table");
}
time_t now = getTimeGlobal();
// int16_tcut
RdbList *list = &st->m_list;
// the best docid
int64_t best = 0LL;
// scan the list to get urls/docids to turk out
for ( ; ! list->isExhausted() ; ) {
// get rec
char *k = list->getCurrentKey();
// skip that
list->skipCurrentRecord();
// skip if negative
if ( (k[0] & 0x01) == 0x00 ) continue;
// get the docid
int64_t docid = g_datedb.getDocId ( k );
// skip if locked
TurkLock *tt = (TurkLock *)g_turkLock.getValue(&docid);
// if there check time
if ( tt && now - tt->m_lockTime > 3600 ) {
// remove it
g_turkLock.removeKey(&docId);
// nuke tt
tt = NULL;
}
// if still there, skip it and try next one
if ( tt ) continue;
// ok, we got a good docid to dish out
best = docId;
break;
}
SafeBuf sb;
// print description so they can clikc a button to start the turk
sb.safePrintf("<html>\n"
"<title>Event Editor</title>\n"
"<body>\n"
"<table width=\"100%%\" border=\"0\">\n"
"<tr><td style=\"background-color:#0079ba;\">\n"
"<center><font color=#00000>"
"<h2>Event Editor</h2>\n"
"</font></center></td>"
"</tr></table>");
// if we had no docid, give user an empty msg
if ( ! best ) {
sb.safePrintf("<center>Nothing currently available to edit. "
"Please try again later.</center>"
"</body></html>\n");
sendReply ( &sb );
return;
}
// lock it!
TurkLock tt;
strcpy ( tt.m_user , st->m_user );
tt.m_lockTime = now;
if ( ! g_lockTable.addLock ( &tt ) ) {
sendErrorReply ( st , g_errno );
return;
}
// . fetch the TitleRec
// . a max cache age of 0 means not to read from the cache
XmlDoc *xd = &st->m_xd;
// . when getTitleRec() is called it will load the old one
// since XmlDoc::m_setFromTitleRec will be true
// . niceness is 0
xd->set3 ( best , st->m_coll , 0 );
// if it blocks while it loads title rec, it will re-call this routine
xd->setCallback ( st , processLoopWrapper );
// good to go!
return processLoop ( st );
}
bool Clusterdb::verify ( char *coll ) {
log ( LOG_DEBUG, "db: Verifying Clusterdb for coll %s...", coll );
g_jobScheduler.disallow_new_jobs();
Msg5 msg5;
RdbList list;
key_t startKey;
key_t endKey;
startKey.setMin();
endKey.setMax();
//int32_t minRecSizes = 64000;
CollectionRec *cr = g_collectiondb.getRec(coll);
if ( ! msg5.getList ( RDB_CLUSTERDB ,
cr->m_collnum ,
&list ,
startKey ,
endKey ,
64000 , // minRecSizes ,
true , // includeTree ,
false , // add to cache?
0 , // max cache age
0 , // startFileNum ,
-1 , // numFiles ,
NULL , // state
NULL , // callback
0 , // niceness
false , // err correction?
NULL ,
0 ,
-1 ,
true ,
-1LL ,
true )) {
g_jobScheduler.allow_new_jobs();
log("db: HEY! it did not block");
return false;
}
int32_t count = 0;
int32_t got = 0;
for ( list.resetListPtr() ; ! list.isExhausted() ;
list.skipCurrentRecord() ) {
key_t k = list.getCurrentKey();
// skip negative keys
if ( (k.n0 & 0x01) == 0x00 ) continue;
count++;
//uint32_t groupId = getGroupId ( RDB_CLUSTERDB , &k );
//if ( groupId == g_hostdb.m_groupId ) got++;
uint32_t shardNum = getShardNum( RDB_CLUSTERDB , &k );
if ( shardNum == getMyShardNum() ) got++;
}
if ( got != count ) {
// tally it up
g_rebalance.m_numForeignRecs += count - got;
log ("db: Out of first %" PRId32" records in clusterdb, "
"only %" PRId32" belong to our group.",count,got);
// exit if NONE, we probably got the wrong data
if ( got == 0 ) log("db: Are you sure you have the "
"right "
"data in the right directory? "
"Exiting.");
log ( "db: Exiting due to Clusterdb inconsistency." );
g_jobScheduler.allow_new_jobs();
return g_conf.m_bypassValidation;
}
log ( LOG_DEBUG, "db: Clusterdb passed verification successfully for "
"%" PRId32" recs.", count );
// DONE
g_jobScheduler.allow_new_jobs();
return true;
}
bool Syncdb::verify ( char *coll ) {
log ( LOG_INFO, "db: Verifying Syncdb for coll %s...", coll );
g_threads.disableThreads();
Msg5 msg5;
Msg5 msg5b;
RdbList list;
key_t startKey;
key_t endKey;
startKey.setMin();
endKey.setMax();
CollectionRec *cr = g_collectiondb.getRec(coll);
if ( ! msg5.getList ( RDB_SYNCDB ,
cr->m_collnum ,
&list ,
startKey ,
endKey ,
64000 , // minRecSizes ,
true , // includeTree ,
false , // add to cache?
0 , // max cache age
0 , // startFileNum ,
-1 , // numFiles ,
NULL , // state
NULL , // callback
0 , // niceness
false , // err correction?
NULL ,
0 ,
-1 ,
true ,
-1LL ,
&msg5b ,
true )) {
g_threads.enableThreads();
return log("db: HEY! it did not block");
}
long count = 0;
long got = 0;
for ( list.resetListPtr() ; ! list.isExhausted() ;
list.skipCurrentRecord() ) {
key_t k = list.getCurrentKey();
count++;
//unsigned long groupId = getGroupId ( RDB_SYNCDB , &k );
//if ( groupId == g_hostdb.m_groupId ) got++;
uint32_t shardNum = getShardNum ( RDB_SYNCDB , (char *)&k );
if ( shardNum == getMyShardNum() ) got++;
}
if ( got != count ) {
log ("db: Out of first %li records in syncdb, "
"only %li belong to our group.",count,got);
// exit if NONE, we probably got the wrong data
if ( got == 0 ) log("db: Are you sure you have the "
"right "
"data in the right directory? "
"Exiting.");
log ( "db: Exiting due to Syncdb inconsistency." );
g_threads.enableThreads();
return g_conf.m_bypassValidation;
}
log ( LOG_INFO, "db: Syncdb passed verification successfully for "
"%li recs.", count );
// DONE
g_threads.enableThreads();
return true;
}
/*
bool Revdb::addColl ( char *coll, bool doVerify ) {
if ( ! m_rdb.addColl ( coll ) ) return false;
if ( ! doVerify ) return true;
// verify
if ( verify(coll) ) return true;
// if not allowing scale, return false
if ( ! g_conf.m_allowScale ) return false;
// otherwise let it go
log ( "db: Verify failed, but scaling is allowed, passing." );
return true;
}
*/
bool Revdb::verify ( char *coll ) {
log ( LOG_INFO, "db: Verifying Revdb for coll %s...", coll );
g_threads.disableThreads();
Msg5 msg5;
Msg5 msg5b;
RdbList list;
key_t startKey;
key_t endKey;
startKey.setMin();
endKey.setMax();
//int32_t minRecSizes = 64000;
CollectionRec *cr = g_collectiondb.getRec(coll);
if ( ! msg5.getList ( RDB_REVDB ,
cr->m_collnum ,
&list ,
startKey ,
endKey ,
1024*1024 , // minRecSizes ,
true , // includeTree ,
false , // add to cache?
0 , // max cache age
0 , // startFileNum ,
-1 , // numFiles ,
NULL , // state
NULL , // callback
0 , // niceness
false , // err correction?
NULL , // cache key ptr
0 , // retry num
-1 , // maxRetries
true , // compensate for merge
-1LL , // sync point
&msg5b ,
false )) {
g_threads.enableThreads();
return log("db: HEY! it did not block");
}
int32_t count = 0;
int32_t got = 0;
for ( list.resetListPtr() ; ! list.isExhausted() ;
list.skipCurrentRecord() ) {
key_t k = list.getCurrentKey();
count++;
//uint32_t groupId = getGroupId ( RDB_REVDB , &k );
//if ( groupId == g_hostdb.m_groupId ) got++;
uint32_t shardNum = getShardNum( RDB_REVDB , &k );
if ( shardNum == getMyShardNum() ) got++;
}
if ( got != count ) {
log ("db: Out of first %"INT32" records in revdb, "
"only %"INT32" belong to our group.",count,got);
// exit if NONE, we probably got the wrong data
if ( count > 10 && got == 0 )
log("db: Are you sure you have the right "
"data in the right directory? "
"Exiting.");
log ( "db: Exiting due to Revdb inconsistency." );
g_threads.enableThreads();
return g_conf.m_bypassValidation;
}
log ( LOG_INFO, "db: Revdb passed verification successfully for %"INT32""
" recs.", count );
// DONE
g_threads.enableThreads();
return true;
}
/*
bool Placedb::addColl ( char *coll, bool doVerify ) {
if ( ! m_rdb.addColl ( coll ) ) return false;
if ( ! doVerify ) return true;
// verify
if ( verify(coll) ) return true;
// if not allowing scale, return false
if ( ! g_conf.m_allowScale ) return false;
// otherwise let it go
log ( "db: Verify failed, but scaling is allowed, passing." );
return true;
}
*/
bool Placedb::verify ( char *coll ) {
log ( LOG_INFO, "db: Verifying Placedb for coll %s...", coll );
g_threads.disableThreads();
Msg5 msg5;
Msg5 msg5b;
RdbList list;
key_t startKey;
startKey.setMin();
key_t endKey;
endKey.setMax();
if ( ! msg5.getList ( RDB_PLACEDB ,
coll ,
&list ,
startKey ,
endKey ,
64000 , // minRecSizes ,
true , // includeTree ,
false , // add to cache?
0 , // max cache age
0 , // startFileNum ,
-1 , // numFiles ,
NULL , // state
NULL , // callback
0 , // niceness
false , // err correction?
NULL ,
0 ,
-1 ,
true ,
-1LL ,
&msg5b ,
true ,
false )) { // allow page cache?
g_threads.enableThreads();
return log("db: HEY! it did not block");
}
long count = 0;
long got = 0;
bool printedKey = false;
bool printedZeroKey = false;
for ( list.resetListPtr() ; ! list.isExhausted() ;
list.skipCurrentRecord() ) {
key_t k = list.getCurrentKey();
count++;
// verify the group
uint32_t shardNum = getShardNum ( RDB_PLACEDB , (char *)&k );
if ( shardNum == getMyShardNum() )
got++;
else if ( !printedKey ) {
log ("db: Found bad key in list (only printing once): "
"%lx %llx", k.n1, k.n0 );
printedKey = true;
}
if ( k.n1 == 0 && k.n0 == 0 ) {
if ( !printedZeroKey ) {
log ( "db: Found Zero key in list, passing. "
"(only printing once)." );
printedZeroKey = true;
}
// pass if we didn't match above
if ( shardNum != getMyShardNum() )
got++;
}
}
if ( got != count ) {
log("db: Out of first %li records in placedb, only %li passed "
"verification.",count,got);
// exit if NONE, we probably got the wrong data
if ( got == 0 ) log("db: Are you sure you have the "
"right "
"data in the right directory? "
"Exiting.");
g_threads.enableThreads();
// if only one let it slide, i saw this happen on gb1 cluster
if ( got - count >= -1 && got - count <= 1 )
return true;
log ( "db: Exiting due to Placedb inconsistency." );
return g_conf.m_bypassValidation;
}
log ( LOG_INFO, "db: Placedb passed verification successfully for %li "
"recs.", count );
// DONE
g_threads.enableThreads();
return true;
}
