bool DirectoryFileFetcher::fetchNext(RawDocumentPtr& pRawDoc)
{
while (m_pDirIterator->hasNext())
{
const File& file = m_pDirIterator->next();
if (!file.isFile())
{
continue;
}
if (m_pFileFilter)
{
const Path& filePath = m_pDirIterator->getPath();
if (m_pFileFilter->isFiltered(filePath))
{
FX_DEBUG("File : [%s] filtered", filePath.toString().c_str());
continue;
}
}
FX_DEBUG("Fetching file : [%s].", file.getPath().c_str());
pRawDoc->setPath(file.getPath());
return true;
}
return false;
}
void IndexWriter::addDocument(const DocumentPtr& pDocument)
{
FX_DEBUG("Add one document");
pDocument->setAction(Document::AT_ADD);
m_pDocConsumer->consume(pDocument);
++m_nOpCountSinceLastCommit;
if (autoCommit())
{
// Perform commit automatically
commit();
}
}
void KeywordIndexMerger::endMerge()
{
m_pDictWriter->finalize();
FX_DEBUG("End merge: total terms: %llu, distinct terms: %llu",
m_nTotalTerms, m_pDictWriter->size());
m_pDictOutStream->close();
m_pDocOutStream->close();
m_pDictOutStream.reset();
m_pDocOutStream.reset();
}
/* static */
void EvHttpSyncClient::handleResponse(struct evhttp_request* req, void* arg)
{
FX_DEBUG("handleResponse");
EvHttpSyncClient* pThis = (EvHttpSyncClient*)arg;
try
{
pThis->done(req);
}
catch(std::exception& e)
{
FX_LOG(ERROR, "Response exception thrown: [%s]: ", e.what());
}
catch(const FirteXException& e)
{
FX_LOG(ERROR, "Response exception thrown: [%s]: ", e.what().c_str());
}
}
void EvHttpSyncClient::close()
{
FX_DEBUG("Close client");
if (m_pConn)
{
evhttp_connection_free(m_pConn);
m_pConn = NULL;
}
if (m_evbase && m_bOwnEventBase)
{
event_base_loopbreak(m_evbase);
event_base_free(m_evbase);
}
m_evbase = NULL;
m_lastState = ST_DISCONNECTED;
m_pLastRequest = NULL;
}
void IndexWriter::commit()
{
FX_DEBUG("Commit in-memory data");
m_pDocConsumer->commit();
m_nOpCountSinceLastCommit = 0;
}
offset_t DocPostingMerger::endMerge()
{
FX_DEBUG("end merge doc posting");
OutputStreamPtr nullStream;
return commit(nullStream);
}
void IndexBarrelKeeper::sealBarrel(const IndexBarrelWriterPtr& pInMemBarrel)
{
FX_DEBUG("Seal barrel");
IndexBarrelWriterPtr pBarrelWriter = pInMemBarrel;
InMemIndexMergerPtr pInMemBarrelMerger;
pBarrelWriter->sealBarrel();
{
ScopedRWLock lock(m_lock, true);
IndexBarrelPtr pLastBarrel;
if (!m_commitMap.empty())
{
pLastBarrel = m_commitMap.rbegin()->second;
}
BarrelsInfoPtr pNewBarrelsInfo;
if (pLastBarrel)
{
pNewBarrelsInfo.reset(pLastBarrel->getBarrelsInfo()->clone());
}
else
{
pNewBarrelsInfo.reset(new BarrelsInfo);
}
/// Commit barrel to barrels info
if (!pBarrelWriter->commitBarrelInfo(pNewBarrelsInfo))
{
FX_DEBUG("No barrel to commit");
return ;
}
FX_DEBUG("Committed barrel info: commit: [%d], baseDocId: [%d]",
pNewBarrelsInfo->getLastBarrel().getCommitId(),
pNewBarrelsInfo->getLastBarrel().getBaseDocId());
//TODO: implement real-time search
IndexBarrelReaderPtr pReader;
if (pLastBarrel)
{
pReader = pLastBarrel->getReader();
}
PrimaryKeyIndexPtr pPrimKey;
if (m_pOnDiskPrimKeyIndex)
{
pPrimKey.reset(m_pOnDiskPrimKeyIndex->clone());
if (pPrimKey)
{
const PrimaryKeyIndexPtr& pPkIndex = pBarrelWriter->getPrimaryKey();
if (pPkIndex)
{
pPrimKey->reopen(pNewBarrelsInfo, pPkIndex->getHashMap());
}
}
}
else
{
pPrimKey = createPrimaryKeyIndex();
if (pPrimKey)
{
const PrimaryKeyIndexPtr& pPkIndex = pBarrelWriter->getPrimaryKey();
FIRTEX_ASSERT2(pPkIndex);
pPrimKey->open(m_pFileSys, pNewBarrelsInfo, pPkIndex->getHashMap());
}
}
IndexBarrelPtr pBarrel(new IndexBarrel(pNewBarrelsInfo, pReader, m_pOnDiskDocFilter));
pBarrel->setEncoding(m_sEncoding);
bool bInserted = doInsertCommit(pNewBarrelsInfo->getCommitId(), pBarrel);
(void)bInserted;
FIRTEX_ASSERT2(bInserted);
m_pOnDiskDocFilter.reset(m_pOnDiskDocFilter->clone());
m_pOnDiskPrimKeyIndex = pPrimKey;
if (!m_pInMemBarrelMerger)
{
m_pInMemBarrelMerger.reset(new InMemIndexMerger(this));
}
m_pInMemBarrelMerger->addToMerge(pNewBarrelsInfo->getLastBarrel(),
pBarrelWriter);
if (m_pInMemBarrelMerger->getInMemBarrelCount() >= (size_t)GLOBAL_CONF().Build.buildThreadCount)
{
pInMemBarrelMerger = m_pInMemBarrelMerger;
m_pInMemBarrelMerger.reset();
}
}
if (pInMemBarrelMerger)
{
CommittablePtr pCommitObj = std::dynamic_pointer_cast<Committable>(pInMemBarrelMerger);
m_pCommitScheduler->commit(pCommitObj);
}
}
