void IndexWriter::close()
{
if (m_pDocConsumer.isNotNull())
{
FX_TRACE("Close index writer");
m_pDocConsumer->stop();
m_pDocConsumer.reset();
m_pKeeper->waitCommit();
///perform last merge
if (m_pIndexMerger.isNotNull())
{
CommittablePtr pCommitObj = m_pIndexMerger.cast<Committable>();
m_pCommitScheduler->commit(pCommitObj);
m_pCommitScheduler->waitCommit();
m_pIndexMerger.reset();
}
m_pCommitScheduler->close();
m_pCommitScheduler.reset();
}
if (m_pAutoCommitTask.isNotNull())
{
m_pAutoCommitTask->stop();
m_autoCommitThread.join();
m_pAutoCommitTask.reset();
}
}
void EvHttpSyncClient::get(const std::string& sURI)
{
struct evhttp_request* pReq = newReqest();
if (!pReq)
{
setError(ST_ERROR, "new request FAILED");
return ;
}
int rv = evhttp_make_request(m_pConn, pReq, EVHTTP_REQ_GET, sURI.c_str());
if (rv != 0)
{
FX_LOG(ERROR, "evhttp_make_request FAILED");
setError(ST_ERROR, "evhttp_make_request FAILED");
return;
}
rv = event_base_dispatch(m_evbase);
if (rv != 0)
{
setError(ST_ERROR, "event loop failed");
FX_LOG(ERROR, "event loop failed: [%d]", rv);
}
FX_TRACE("made request: [%s]", sURI.c_str());
return ;
}
void DocPostingMerger::doMerge(const MergingTerm* pTerm)
{
docid_t docBuffer[RECORD_SIZE];
docid_t tfBuffer[RECORD_SIZE];
PostingDecoderPtr pDecoder = pTerm->getPostingIterator()->getPostingDecoder();
const DocIdRecycling* pDocIdRecycle = pTerm->getDocIdRecycling();
if (pDocIdRecycle->hasDeletions())
{
docid_t baseDocId = pTerm->getNewBaseDocId();
uint32_t nDecoded = 0;
docid_t lastDocId = 0;
while ((nDecoded = pDecoder->decodeDocRecord(docBuffer, lastDocId)) > 0)
{
if (pDecoder->decodeTfRecord(tfBuffer) != nDecoded)
{
FIRTEX_THROW(IndexCollapseException,
"Doc and Tf record is inconsistant.");
}
lastDocId = docBuffer[nDecoded - 1] + 1;
size_t i, j;
for (i = 0, j = 0; j < nDecoded; ++j)
{
docid_t docId = docBuffer[j];
docid_t newDocId = pDocIdRecycle->remap(docId);
if (newDocId != INVALID_DOCID)
{
docBuffer[i] = newDocId;
tfBuffer[i] = tfBuffer[j];
++i;
}
}
if (i > 0)
{
commitDocuments(baseDocId, docBuffer, tfBuffer, (uint32_t)i);
}
}
}
else
{
docid_t baseDocId = pTerm->getNewBaseDocId();
uint32_t nDecoded = 0;
docid_t lastDocId = 0;
FX_TRACE("Merge doc list: base doc id: %d", baseDocId);
while ((nDecoded = pDecoder->decodeDocRecord(docBuffer, lastDocId)) > 0)
{
if (pDecoder->decodeTfRecord(tfBuffer) != nDecoded)
{
FIRTEX_THROW(IndexCollapseException,
"Doc and Tf record is inconsistant.");
}
commitDocuments(baseDocId, docBuffer, tfBuffer, nDecoded);
lastDocId = docBuffer[nDecoded - 1] + 1;
}
}
}
bool IndexBarrelKeeper::deleteOnDiskDocument(const char* szPrimKey)
{
ScopedRWLock lock(m_lock, false);
if (!m_pOnDiskPrimKeyIndex)
{
return false;
}
docid_t docId = m_pOnDiskPrimKeyIndex->lookup(szPrimKey);
if (docId != INVALID_DOCID)
{
FX_TRACE("Delete on-disk document: [%s]->[%d]", szPrimKey, docId);
return m_pOnDiskDocFilter->deleteDocument(docId);
}
return false;
}
void DocPostingMerger::commitDocuments(docid_t baseDocId, docid_t* docBuffer,
tf_t* tfBuffer, uint32_t nSize)
{
FX_TRACE("commit documents: last committed doc: %d, base doc id: %d, "
"first doc: %d, length: %d",
m_lastDocId, baseDocId, docBuffer[0], (int32_t)nSize);
if (m_nUpTo == RECORD_SIZE)
{
flushBuffer(false);
}
m_pDocBuffer[m_nUpTo] = docBuffer[0] + baseDocId - m_lastDocId;
m_lastDocId = docBuffer[0] + baseDocId;
ctf_t totalTf = tfBuffer[0];
m_pTfBuffer[m_nUpTo] = (tf_t)totalTf;
++m_nUpTo;
tf_t tf;
for (uint32_t i = 1; i < nSize; ++i)
{
if (m_nUpTo == RECORD_SIZE)
{
getTermMeta().getCTF() += totalTf;
flushBuffer(false);
totalTf = 0;
}
m_pDocBuffer[m_nUpTo] = docBuffer[i] - docBuffer[i - 1];
tf = tfBuffer[i];
m_pTfBuffer[m_nUpTo] = tf;
totalTf += tf;
m_lastDocId = docBuffer[i] + baseDocId;
++m_nUpTo;
}
TermMeta& termMeta = getTermMeta();
termMeta.getDocFreq() += nSize;
termMeta.getCTF() += totalTf;
m_curTermFreq = 0;
}
void QueryExecutor::execute(HitCollectorPtr& pCollector,
ScorerPtr& pScorer,
const QueryTracerPtr& pTracer)
{
//TODO: allocate from mempool
MatchedDocSet machedDocs(m_pPool, DEFAULT_MATCHED_DOC_BUFFER_SIZE, size());
machedDocs.setTracer(pTracer);
QueryFeature queryFeature;
queryFeature.reserve(size());
extractFeature(queryFeature);
pScorer->beginQuery(m_pFeatureProvider, queryFeature);
while (advance(machedDocs) > 0)
{
FX_TRACE("Matched doc count: [%u]", (uint32_t)machedDocs.size());
pScorer->score(machedDocs);
pCollector->collect(machedDocs);
machedDocs.reset();
}
pScorer->endQuery();
}
