IndexReaderPtr IndexReader::open(const IndexCommitPtr& commit, const IndexDeletionPolicyPtr& deletionPolicy, bool readOnly) {
return open(commit->getDirectory(), deletionPolicy, commit, readOnly, DEFAULT_TERMS_INDEX_DIVISOR);
}
IndexReaderPtr IndexReader::open(const IndexCommitPtr& commit, const IndexDeletionPolicyPtr& deletionPolicy, bool readOnly, int32_t termInfosIndexDivisor) {
return open(commit->getDirectory(), deletionPolicy, commit, readOnly, termInfosIndexDivisor);
}
void FindSegmentsFile::doRun(IndexCommitPtr commit)
{
if (commit)
{
if (directory != commit->getDirectory())
boost::throw_exception(IOException(L"The specified commit does not match the specified Directory"));
runBody(commit->getSegmentsFileName());
return;
}
String segmentFileName;
int64_t lastGen = -1;
int64_t gen = 0;
int32_t genLookaheadCount = 0;
bool retry = false;
LuceneException exc;
SegmentInfosPtr segmentInfos(_segmentInfos);
int32_t method = 0;
// Loop until we succeed in calling runBody() without hitting an IOException. An IOException most likely
// means a commit was in process and has finished, in the time it took us to load the now-old infos files
// (and segments files). It's also possible it's a true error (corrupt index). To distinguish these,
// on each retry we must see "forward progress" on which generation we are trying to load. If we don't,
// then the original error is real and we throw it.
// We have three methods for determining the current generation. We try the first two in parallel, and
// fall back to the third when necessary.
while (true)
{
if (method == 0)
{
// Method 1: list the directory and use the highest segments_N file. This method works well as long
// as there is no stale caching on the directory contents (NOTE: NFS clients often have such stale caching)
HashSet<String> files(directory->listAll());
int64_t genA = segmentInfos->getCurrentSegmentGeneration(files);
segmentInfos->message(L"directory listing genA=" + StringUtils::toString(genA));
// Method 2: open segments.gen and read its contents. Then we take the larger of the two gens. This way,
// if either approach is hitting a stale cache (NFS) we have a better chance of getting the right generation.
int64_t genB = -1;
for (int32_t i = 0; i < SegmentInfos::defaultGenFileRetryCount; ++i)
{
IndexInputPtr genInput;
try
{
genInput = directory->openInput(IndexFileNames::SEGMENTS_GEN());
}
catch (FileNotFoundException& e)
{
segmentInfos->message(L"Segments.gen open: FileNotFoundException " + e.getError());
break;
}
catch (IOException& e)
{
segmentInfos->message(L"Segments.gen open: IOException " + e.getError());
}
if (genInput)
{
LuceneException finally;
bool fileConsistent = false;
try
{
int32_t version = genInput->readInt();
if (version == SegmentInfos::FORMAT_LOCKLESS)
{
int64_t gen0 = genInput->readLong();
int64_t gen1 = genInput->readLong();
segmentInfos->message(L"fallback check: " + StringUtils::toString(gen0) + L"; " + StringUtils::toString(gen1));
if (gen0 == gen1)
{
// the file is consistent
genB = gen0;
fileConsistent = true;
}
}
}
catch (IOException&)
{
// will retry
}
catch (LuceneException& e)
{
finally = e;
}
genInput->close();
finally.throwException();
if (fileConsistent)
break;
}
LuceneThread::threadSleep(SegmentInfos::defaultGenFileRetryPauseMsec);
}
segmentInfos->message(String(IndexFileNames::SEGMENTS_GEN()) + L" check: genB=" + StringUtils::toString(genB));
// pick the larger of the two gen's
gen = std::max(genA, genB);
// neither approach found a generation
if (gen == -1)
boost::throw_exception(FileNotFoundException(L"No segments* file found in directory"));
}
// Third method (fallback if first & second methods are not reliable): since both directory cache and
// file contents cache seem to be stale, just advance the generation.
if (method == 1 || (method == 0 && lastGen == gen && retry))
{
method = 1;
if (genLookaheadCount < SegmentInfos::defaultGenLookaheadCount)
{
++gen;
++genLookaheadCount;
segmentInfos->message(L"look ahead increment gen to " + StringUtils::toString(gen));
}
}
if (lastGen == gen)
{
// This means we're about to try the same segments_N last tried. This is allowed, exactly once, because
// writer could have been in the process of writing segments_N last time.
if (retry)
{
// OK, we've tried the same segments_N file twice in a row, so this must be a real error.
exc.throwException();
}
else
retry = true;
}
else if (method == 0)
{
// Segment file has advanced since our last loop, so reset retry
retry = false;
}
lastGen = gen;
segmentFileName = IndexFileNames::fileNameFromGeneration(IndexFileNames::SEGMENTS(), L"", gen);
try
{
runBody(segmentFileName);
segmentInfos->message(L"success on " + segmentFileName);
return;
}
catch (LuceneException& err)
{
// Save the original root cause
if (exc.isNull())
exc = err;
segmentInfos->message(L"primary Exception on '" + segmentFileName + L"': " + err.getError() + L"'; will retry: retry=" + StringUtils::toString(retry) + L"; gen = " + StringUtils::toString(gen));
if (!retry && gen > 1)
{
// This is our first time trying this segments file (because retry is false), and, there is possibly a
// segments_(N-1) (because gen > 1). So, check if the segments_(N-1) exists and try it if so.
String prevSegmentFileName(IndexFileNames::fileNameFromGeneration(IndexFileNames::SEGMENTS(), L"", gen - 1));
if (directory->fileExists(prevSegmentFileName))
{
segmentInfos->message(L"fallback to prior segment file '" + prevSegmentFileName + L"'");
try
{
runBody(prevSegmentFileName);
if (!exc.isNull())
segmentInfos->message(L"success on fallback " + prevSegmentFileName);
return;
}
catch (LuceneException& err2)
{
segmentInfos->message(L"secondary Exception on '" + prevSegmentFileName + L"': " + err2.getError() + L"'; will retry");
}
}
}
}
}
}
