// . but now that we may get a list remotely to fix data corruption, // this may indeed block bool Msg3::doneScanning ( ) { QUICKPOLL(m_niceness); // . did we have any error on any scan? // . if so, repeat ALL of the scans g_errno = m_errno; // 2 retry is the default int32_t max = 2; // see if explicitly provided by the caller if ( m_maxRetries >= 0 ) max = m_maxRetries; // now use -1 (no max) as the default no matter what max = -1; // ENOMEM is particulary contagious, so watch out with it... if ( g_errno == ENOMEM && m_maxRetries == -1 ) max = 0; // msg0 sets maxRetries to 2, don't let max stay set to -1 if ( g_errno == ENOMEM && m_maxRetries != -1 ) max = m_maxRetries; // when thread cannot alloc enough read buf it keeps the read buf // set to NULL and BigFile.cpp sets g_errno to EBUFTOOSMALL if ( g_errno == EBUFTOOSMALL && m_maxRetries == -1 ) max = 0; // msg0 sets maxRetries to 2, don't let max stay set to -1 if ( g_errno == EBUFTOOSMALL && m_maxRetries != -1 ) max = m_maxRetries; // . if no thread slots available, that hogs up serious memory. // the size of Msg3 is 82k, so having just 5000 of them is 430MB. // . i just made Msg3 alloc mem when it needs more than about 2k // so this problem is greatly reduced, therefore let's keep // retrying... forever if no thread slots in thread queue since // we become the thread queue in a way. if ( g_errno == ENOTHREADSLOTS ) max = -1; // this is set above if the map has the same consecutive key repeated // and the read is enormous if ( g_errno == ECORRUPTDATA ) max = 0; // usually bad disk failures, don't retry those forever //if ( g_errno == EIO ) max = 3; // no, now our hitachis return these even when they're good so // we have to keep retrying forever if ( g_errno == EIO ) max = -1; // count these so we do not take drives offline just because // kernel ring buffer complains... if ( g_errno == EIO ) g_numIOErrors++; // bail early on high priority reads for these errors if ( g_errno == EDISKSTUCK && m_niceness == 0 ) max = 0; if ( g_errno == EIO && m_niceness == 0 ) max = 0; // how does this happen? we should never bail out on a low priority // disk read... we just wait for it to complete... if ( g_errno == EDISKSTUCK && m_niceness != 0 ) { char *xx=NULL;*xx=0;} // on I/O, give up at call it corrupt after a while. some hitachis // have I/O errros on little spots, like gk88, maybe we can fix him if ( g_errno == EIO && m_retryNum >= 5 ) { m_errno = ECORRUPTDATA; m_hadCorruption = true; // do not do any retries any more max = 0; } // convert m_errno to ECORRUPTDATA if it is EBUFTOOSMALL and the // max of the bytesToRead are over 500MB. // if bytesToRead was ludicrous, then assume that the data file // was corrupted, the map was regenerated and it patched // over the corrupted bits which were 500MB or more in size. // we cannot practically allocate that much, so let's just // give back an empty buffer. treat it like corruption... // the way it patches is to store the same key over all the corrupted // pages, which can get pretty big. so if you read a range with that // key you will be hurting!! // this may be the same scenario as when the rdbmap has consecutive // same keys. see above where we set m_errno to ECORRUPTDATA... if ( g_errno == EBUFTOOSMALL ) { int32_t biggest = 0; for ( int32_t i = 0 ; i < m_numFileNums ; i++ ) { if ( m_scans[i].m_bytesToRead < biggest ) continue; biggest = m_scans[i].m_bytesToRead; } if ( biggest > 500000000 ) { log("db: Max read size was %" PRId32" > 500000000. Assuming " "corrupt data in data file.",biggest); m_errno = ECORRUPTDATA; m_hadCorruption = true; // do not do any retries on this, the read was > 500MB max = 0; } } // if shutting down gb then limit to 20 so we can shutdown because // it can't shutdown until all threads are out of the queue i think if ( g_process.m_mode == EXIT_MODE && max < 0 ) { //log("msg3: forcing retries to 0 because shutting down"); max = 0; } // get base, returns NULL and sets g_errno to ENOCOLLREC on error RdbBase *base = getRdbBase( m_rdbId, m_collnum ); if ( ! base ) { return true; } // this really slows things down because it blocks the cpu so // leave it out for now #ifdef GBSANITYCHECK // check for corruption here, do not do it again in Msg5 if we pass if ( ! g_errno ) { // && g_conf.m_doErrorCorrection ) { int32_t i; for ( i = 0 ; i < m_numFileNums ; i++ ) if ( ! m_lists[i].checkList_r ( false, false ) ) break; if ( i < m_numFileNums ) { g_errno = ECORRUPTDATA; m_errno = ECORRUPTDATA; max = g_conf.m_corruptRetries; // try 100 times log("db: Encountered corrupt list in file %s.", base->getFile(m_fileNums[i])->getFilename()); } else m_listsChecked = true; } #endif // try to fix this error i've seen if ( g_errno == EBADENGINEER && max == -1 ) max = 100; // . if we had a ETRYAGAIN error, then try again now // . it usually means the whole file or a part of it was deleted // before we could finish reading it, so we should re-read all now // . RdbMerge deletes BigFiles after it merges them and also chops // off file heads // . now that we have threads i'd imagine we'd get EBADFD or something // . i've also seen "illegal seek" as well if ( m_errno && (m_retryNum < max || max < 0) && // this will complete in due time, we can't call a sleep wrapper // on it because the read is really still pending... m_errno != EDISKSTUCK ) { // print the error static time_t s_time = 0; time_t now = getTime(); if ( now - s_time > 5 || g_errno != ENOTHREADSLOTS ) { log("net: Had error reading %s: %s. Retrying. " "(retry #%" PRId32")", base->m_dbname,mstrerror(m_errno) , m_retryNum ); s_time = now; } // send email alert if in an infinite loop, but don't send // more than once every 2 hours static int32_t s_lastSendTime = 0; if ( m_retryNum == 100 && getTime() - s_lastSendTime > 3600*2){ // remove this for now it is going off all the time //g_pingServer.sendEmail(NULL,//g_hostdb.getMyHost(), // "100 read retries",true); s_lastSendTime = getTime(); } // clear g_errno cuz we should for call to readList() g_errno = 0; // free the list buffer since if we have 1000 Msg3s retrying // it will totally use all of our memory for ( int32_t i = 0 ; i < m_numChunks ; i++ ) m_lists[i].destructor(); // count retries m_retryNum++; // backoff scheme, wait 100ms more each time int32_t wait ; if ( m_retryNum == 1 ) wait = 10; else wait = 200 * m_retryNum; // . don't wait more than 10 secs between tries // . i've seen gf0 and gf16 get mega saturated if ( wait > 10000 ) wait = 10000; // wait 500 ms if ( g_loop.registerSleepCallback ( wait , // ms this , doneSleepingWrapper3, m_niceness)) return false; // otherwise, registration failed log( "net: Failed to register sleep callback for retry. " "Abandoning read. This is bad."); // return, g_errno should be set g_errno = EBUFTOOSMALL; m_errno = EBUFTOOSMALL; return true; } // if we got an error and should not retry any more then give up if ( g_errno ) { log( "net: Had error reading %s: %s. Giving up after %" PRId32" " "retries.", base->m_dbname,mstrerror(g_errno) , m_retryNum ); return true; } // note it if the retry finally worked if ( m_retryNum > 0 ) log(LOG_INFO,"disk: Read succeeded after retrying %" PRId32" times.", (int32_t)m_retryNum); // count total bytes for logging int32_t count = 0; // . constrain all lists to make merging easier // . if we have only one list, then that's nice cuz the constrain // will allow us to send it right away w/ zero copying // . if we have only 1 list, it won't be merged into a final list, // that is, we'll just set m_list = &m_lists[i] for ( int32_t i = 0 ; i < m_numFileNums ; i++ ) { QUICKPOLL(m_niceness); // count total bytes for logging count += m_lists[i].getListSize(); // . hint offset is relative to the offset of first key we read // . if that key was only 6 bytes RdbScan shift the list buf // down 6 bytes to make the first key 12 bytes... a // requirement for all RdbLists // . don't inc it, though, if it was 0, pointing to the start // of the list because our shift won't affect that if ( m_scans[i].m_shifted == 6 && m_hintOffsets[i] > 0 ) m_hintOffsets[i] += 6; // posdb double compression if ( m_scans[i].m_shifted == 12 && m_hintOffsets[i] > 0 ) m_hintOffsets[i] += 12; // . don't constrain on minRecSizes here because it may // make our endKey smaller, which will cause problems // when Msg5 merges these lists. // . If all lists have different endKeys RdbList's merge // chooses the min and will merge in recs beyond that // causing a bad list BECAUSE we don't check to make // sure that recs we are adding are below the endKey // . if we only read from one file then constrain based // on minRecSizes so we can send the list back w/o merging // OR if just merging with RdbTree's list int32_t mrs ; // . constrain to m_minRecSizesOrig, not m_minRecSizes cuz // that could be adjusted by compensateForNegativeRecs() // . but, really, they should be the same if we only read from // the root file if ( m_numFileNums == 1 ) mrs = m_minRecSizesOrig; else mrs = -1; // . this returns false and sets g_errno on error // . like if data is corrupt BigFile *ff = base->getFile(m_fileNums[i]); // if we did a merge really quick and delete one of the // files we were reading, i've seen 'ff' be NULL char *filename = "lostfilename"; if ( ff ) filename = ff->getFilename(); // compute cache info RdbCache *rpc = getDiskPageCache ( m_rdbId ); if ( ! m_allowPageCache ) rpc = NULL; int64_t vfd ; if ( ff ) vfd = ff->getVfd(); key192_t ck ; if ( ff ) ck = makeCacheKey ( vfd , m_scans[i].m_offset , m_scans[i].m_bytesToRead ); if ( m_validateCache && ff && rpc && vfd != -1 ) { bool inCache; char *rec; int32_t recSize; inCache = rpc->getRecord ( (collnum_t)0 , // collnum (char *)&ck , &rec , &recSize , true , // copy? -1 , // maxAge, none true ); // inccounts? if ( inCache && // 1st byte is RdbScan::m_shifted ( m_lists[i].m_listSize != recSize-1 || memcmp ( m_lists[i].m_list , rec+1,recSize-1) || *rec != m_scans[i].m_shifted ) ) { log("msg3: cache did not validate"); char *xx=NULL;*xx=0; } mfree ( rec , recSize , "vca" ); } /////// // // STORE IN PAGE CACHE // /////// // store what we read in the cache. don't bother storing // if it was a retry, just in case something strange happened. // store pre-constrain call is more efficient. if ( m_retryNum<=0 && ff && rpc && vfd != -1 && ! m_scans[i].m_inPageCache ) rpc->addRecord ( (collnum_t)0 , // collnum (char *)&ck , // rec1 is this little thingy &m_scans[i].m_shifted, 1, // rec2 m_lists[i].getList() , m_lists[i].getListSize() , 0 ); // timestamp. 0 = now QUICKPOLL(m_niceness); // if from our 'page' cache, no need to constrain if ( ! m_lists[i].constrain ( m_startKey , m_constrainKey , // m_endKey mrs , // m_minRecSizes m_hintOffsets[i] , //m_hintKeys [i] , &m_hintKeys [i*m_ks] , filename,//ff->getFilename() , m_niceness ) ) { log("net: Had error while constraining list read from " "%s: %s/%s. vfd=%" PRId32" parts=%" PRId32". " "This is likely caused by corrupted " "data on disk.", mstrerror(g_errno), ff->getDir(), ff->getFilename(), ff->m_vfd , (int32_t)ff->m_numParts ); continue; } } // print the time if ( g_conf.m_logTimingDb ) { int64_t now = gettimeofdayInMilliseconds(); int64_t took = now - m_startTime; log(LOG_TIMING, "net: Took %" PRId64" ms to read %" PRId32" lists of %" PRId32" bytes total" " from %s (niceness=%" PRId32").", took,m_numFileNums,count,base->m_dbname,m_niceness); } return true; }
// . sets m_errno to g_errno if not already set void Msg51::gotClusterRec(Slot *slot) { // count it m_numReplies++; // free up slot->m_inUse = false; RdbList *list = slot->m_msg0.m_list; // update m_errno if we had an error if ( ! m_errno ) m_errno = g_errno; if ( g_errno ) // print error log(LOG_DEBUG, "query: Had error getting cluster info got docId=d: " "%s.",mstrerror(g_errno)); // this doubles as a ptr to a cluster rec int32_t ci = slot->m_ci; // get docid int64_t docId = m_docIds[ci]; // assume error! m_clusterLevels[ci] = CR_ERROR_CLUSTERDB; // bail on error if ( g_errno || list->getListSize() < 12 ) { //log(LOG_DEBUG, // "build: clusterdb rec for d=%" PRId64" dptr=%" PRIu32" " // "not found. where is it?", docId, (int32_t)ci); g_errno = 0; return; } // . steal rec from this multicast // . point to cluster rec, a int32_t key96_t *rec = &m_clusterRecs[ci]; // store the cluster rec itself *rec = *(key96_t *)(list->getList()); // debug note log(LOG_DEBUG, "build: had clusterdb SUCCESS for d=%" PRId64" dptr=%" PRIu32" " "rec.n1=%" PRIx32",%016" PRIx64" sitehash26=0x%" PRIx32".", (int64_t)docId, (int32_t)ci, rec->n1,rec->n0, g_clusterdb.getSiteHash26((char *)rec)); // check for docid mismatch int64_t docId2 = g_clusterdb.getDocId ( rec ); if ( docId != docId2 ) { logf(LOG_DEBUG,"query: docid mismatch in clusterdb."); return; } // it is legit, set to CR_OK m_clusterLevels[ci] = CR_OK; // . init the quick cache // . use 100k if ( ! s_cacheInit && s_clusterdbQuickCache.init(200*1024 , // maxMem sizeof(key96_t) , // fixedDataSize (clusterdb rec) false , // support lists 10000 , // max recs false , // use half keys? "clusterdbQuickCache" , false , // load from disk? sizeof(key96_t) , // cache key size sizeof(key96_t) )) // cache key size // only init once if successful s_cacheInit = true; // debug msg //logf(LOG_DEBUG,"query: msg51 addRec k.n0=%" PRIu64" rec.n0=%" PRIu64,docId, // rec->n0); // . add the record to our quick cache as a int64_t // . ignore any error if ( s_cacheInit ) s_clusterdbQuickCache.addRecord(m_collnum, (key96_t)docId, // docid is key (char *)rec, sizeof(key96_t), // recSize 0); // clear it in case the cache set it, we don't care g_errno = 0; }