void gotDatedbList ( State60 *st ) { // must only be run on host #0 since we need just one lock table if ( g_hostdb.m_myHost->m_hostId != 0 ) { char *xx=NULL;*xx=0; } // load turk lock table if we need to bool s_init = false; if ( ! s_init ) { s_init = true; if ( ! g_turkLocks.set(8,sizeof(TurkLock),256) ) log("turk: failed to init turk lock table"); if ( ! g_turkLocks.load(g_conf.m_dir,"turkdir/docidlocks.dat")) log("turk: failed to load turk lock table"); } time_t now = getTimeGlobal(); // int16_tcut RdbList *list = &st->m_list; // the best docid int64_t best = 0LL; // scan the list to get urls/docids to turk out for ( ; ! list->isExhausted() ; ) { // get rec char *k = list->getCurrentKey(); // skip that list->skipCurrentRecord(); // skip if negative if ( (k[0] & 0x01) == 0x00 ) continue; // get the docid int64_t docid = g_datedb.getDocId ( k ); // skip if locked TurkLock *tt = (TurkLock *)g_turkLock.getValue(&docid); // if there check time if ( tt && now - tt->m_lockTime > 3600 ) { // remove it g_turkLock.removeKey(&docId); // nuke tt tt = NULL; } // if still there, skip it and try next one if ( tt ) continue; // ok, we got a good docid to dish out best = docId; break; } SafeBuf sb; // print description so they can clikc a button to start the turk sb.safePrintf("<html>\n" "<title>Event Editor</title>\n" "<body>\n" "<table width=\"100%%\" border=\"0\">\n" "<tr><td style=\"background-color:#0079ba;\">\n" "<center><font color=#00000>" "<h2>Event Editor</h2>\n" "</font></center></td>" "</tr></table>"); // if we had no docid, give user an empty msg if ( ! best ) { sb.safePrintf("<center>Nothing currently available to edit. " "Please try again later.</center>" "</body></html>\n"); sendReply ( &sb ); return; } // lock it! TurkLock tt; strcpy ( tt.m_user , st->m_user ); tt.m_lockTime = now; if ( ! g_lockTable.addLock ( &tt ) ) { sendErrorReply ( st , g_errno ); return; } // . fetch the TitleRec // . a max cache age of 0 means not to read from the cache XmlDoc *xd = &st->m_xd; // . when getTitleRec() is called it will load the old one // since XmlDoc::m_setFromTitleRec will be true // . niceness is 0 xd->set3 ( best , st->m_coll , 0 ); // if it blocks while it loads title rec, it will re-call this routine xd->setCallback ( st , processLoopWrapper ); // good to go! return processLoop ( st ); }
void processReply ( char *reply , long replyLen ) { // store our current reply SafeBuf fb2; fb2.safeMemcpy(reply,replyLen ); fb2.nullTerm(); // log that we got the reply log("qa: got reply(len=%li)(errno=%s)=%s", replyLen,mstrerror(g_errno),reply); char *content = NULL; long contentLen = 0; // get mime if ( reply ) { HttpMime mime; mime.set ( reply, replyLen , NULL ); // only hash content since mime has a timestamp in it content = mime.getContent(); contentLen = mime.getContentLen(); if ( content && contentLen>0 && content[contentLen] ) { char *xx=NULL;*xx=0; } } if ( ! content ) { content = ""; contentLen = 0; } s_content = content; // take out <responseTimeMS> markOut ( content , "<currentTimeUTC>"); markOut ( content , "<responseTimeMS>"); // until i figure this one out, take it out markOut ( content , "<docsInCollection>"); // until i figure this one out, take it out markOut ( content , "<hits>"); // for those links in the html pages markOut ( content, "rand64="); // for json markOut ( content , "\"currentTimeUTC\":" ); markOut ( content , "\"responseTimeMS\":"); markOut ( content , "\"docsInCollection\":"); // for xml markOut ( content , "<currentTimeUTC>" ); markOut ( content , "<responseTimeMS>"); markOut ( content , "<docsInCollection>"); // indexed 1 day ago markOut ( content,"indexed:"); // modified 1 day ago markOut ( content,"modified:"); // s_gigabitCount... it is perpetually incrementing static counter // in PageResults.cpp markOut(content,"ccc("); markOut(content,"id=fd"); markOut(content,"id=sd"); // for some reason the term freq seems to change a little in // the scoring table markOut(content,"id=tf"); // make checksum. we ignore back to back spaces so this // hash works for <docsInCollection>10 vs <docsInCollection>9 long contentCRC = 0; if ( content ) contentCRC = qa_hash32 ( content ); // note it log("qa: got contentCRC of %lu",contentCRC); // if what we expected, save to disk if not there yet, then // call s_callback() to resume the qa pipeline /* if ( contentCRC == s_expectedCRC ) { // save content if good char fn3[1024]; sprintf(fn3,"%sqa/content.%lu",g_hostdb.m_dir,contentCRC); File ff; ff.set ( fn3 ); if ( ! ff.doesExist() ) { // if not there yet then save it fb2.save(fn3); } // . continue on with the qa process // . which qa function that may be //s_callback(); return; } */ // // if crc of content does not match what was expected then do a diff // so we can see why not // // this means caller does not care about the response if ( ! s_checkCRC ) { //s_callback(); return; } //const char *emsg = "qa: bad contentCRC of %li should be %li " // "\n";//"phase=%li\n"; //fprintf(stderr,emsg,contentCRC,s_expectedCRC);//,s_phase-1); // hash url long urlHash32 = hash32n ( s_url.getUrl() ); // combine test function too since two tests may use the same url long nameHash = hash32n ( s_qt->m_testName ); // combine together urlHash32 = hash32h ( nameHash , urlHash32 ); static bool s_init = false; if ( ! s_init ) { s_init = true; s_ht.set(4,4,1024,NULL,0,false,0,"qaht"); // make symlink //char cmd[512]; //snprintf(cmd,"cd %s/html ;ln -s ../qa ./qa", g_hostdb.m_dir); //system(cmd); char dir[1024]; snprintf(dir,1000,"%sqa",g_hostdb.m_dir); long status = ::mkdir ( dir , S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | S_IWGRP | S_IXGRP | S_IROTH | S_IXOTH ); if ( status == -1 && errno != EEXIST && errno ) log("qa: Failed to make directory %s: %s.", dir,mstrerror(errno)); // try to load from disk SafeBuf fn; fn.safePrintf("%s/qa/",g_hostdb.m_dir); log("qa: loading crctable.dat"); s_ht.load ( fn.getBufStart() , "crctable.dat" ); } // break up into lines char fn2[1024]; sprintf(fn2,"%sqa/content.%lu",g_hostdb.m_dir,contentCRC); fb2.save ( fn2 ); // look up in hashtable to see what reply crc should be long *val = (long *)s_ht.getValue ( &urlHash32 ); // just return if the same if ( val && contentCRC == *val ) { g_qaOutput.safePrintf("<b style=color:green;>" "passed test</b><br>%s : " "<a href=%s>%s</a> (urlhash=%lu " "crc=<a href=/qa/content.%lu>" "%lu</a>)<br>" "<hr>", s_qt->m_testName, s_url.getUrl(), s_url.getUrl(), urlHash32, contentCRC, contentCRC); return; } if ( ! val ) { // add it so we know s_ht.addKey ( &urlHash32 , &contentCRC ); g_qaOutput.safePrintf("<b style=color:blue;>" "first time testing</b><br>%s : " "<a href=%s>%s</a> " "(urlhash=%lu " "crc=<a href=/qa/content.%lu>%lu" "</a>)<br>" "<hr>", s_qt->m_testName, s_url.getUrl(), s_url.getUrl(), urlHash32, contentCRC, contentCRC); return; } log("qa: crc changed for url %s from %li to %li", s_url.getUrl(),*val,contentCRC); // get response on file SafeBuf fb1; char fn1[1024]; sprintf(fn1,"%sqa/content.%lu",g_hostdb.m_dir, *val); fb1.load(fn1); fb1.nullTerm(); // do the diff between the two replies so we can see what changed char cmd[1024]; sprintf(cmd,"diff %s %s > /tmp/diffout",fn1,fn2); log("qa: %s\n",cmd); system(cmd); g_numErrors++; g_qaOutput.safePrintf("<b style=color:red;>FAILED TEST</b><br>%s : " "<a href=%s>%s</a> (urlhash=%lu)<br>" "<input type=checkbox name=urlhash%lu value=1 " // use ajax to update test crc. if you undo your // check then it should put the old val back. // when you first click the checkbox it should // gray out the diff i guess. "onclick=submitchanges(%lu,%lu);> " "Accept changes" "<br>" "original on left, new on right. " "oldcrc = <a href=/qa/content.%lu>%lu</a>" " != <a href=/qa/content.%lu>%lu</a> = newcrc" "<br>diff output follows:<br>" "<pre id=%lu style=background-color:0xffffff;>", s_qt->m_testName, s_url.getUrl(), s_url.getUrl(), urlHash32, // input checkbox name field urlHash32, // submitchanges() parms urlHash32, contentCRC, // original/old content.%lu *val, *val, // new content.%lu contentCRC, contentCRC, // for the pre tag id: urlHash32); // store in output SafeBuf sb; sb.load("/tmp/diffout"); g_qaOutput.htmlEncode ( sb.getBufStart() ); g_qaOutput.safePrintf("</pre><br><hr>"); // if this is zero allow it to slide by. it is learning mode i guess. // so we can learn what crc we need to use. // otherwise, stop right there for debugging //if ( s_expectedCRC != 0 ) exit(1); // keep on going //s_callback(); }