AsyncDocumentConsumer::AsyncDocumentConsumer(
const IndexBarrelKeeperPtr& pBarrelKeeper)
: m_pKeeper(pBarrelKeeper)
, m_threadPool(GLOBAL_CONF().Build.buildThreadCount, GLOBAL_CONF().Build.buildThreadCount)
, m_docQueue(GLOBAL_CONF().Build.documentQueueSize)
, m_docUpdateQueue(GLOBAL_CONF().Build.documentQueueSize)
{
}
void AsyncDocumentConsumer::start()
{
size_t nSizeInMB = (size_t)GLOBAL_CONF().Build.memory;
FX_LOG(INFO, "Allocate memory for building, total size: [%u] MB, segment count: [%u]",
(uint32_t)nSizeInMB, (uint32_t)m_threadPool.capacity());
m_pAllocator.reset(new SyncSegregatedAllocator((size_t)(nSizeInMB * 1024 * 1024),
m_threadPool.capacity() * MAX_CHUNK_COUNT_PER_SEGMENT));
m_pUpdateTask.reset(new OnDiskUpdateTask(m_pKeeper, m_docQueue, m_docUpdateQueue));
m_updateThread.start(*m_pUpdateTask);
taskid_t taskId = 0, nextTaskId = INVALID_TASKID;
for (size_t i = 0; i < m_threadPool.capacity(); i++)
{
IndexBarrelWriterPtr pIndexBarrelWriter = m_pKeeper->createBarrelWriter();
if (!pIndexBarrelWriter->init(m_pAllocator))
{
FIRTEX_THROW_AND_LOG(RuntimeException, "Init index writer FAILED.");
break;
}
if (i + 1 != m_threadPool.capacity())
{
nextTaskId = taskId + 1;
}
TaskPtr pTask(new Task(taskId, nextTaskId, pIndexBarrelWriter,
m_docQueue, *this));
m_threadPool.start(*pTask);
m_tasks.push_back(pTask);
}
}
void Index::open(const std::string& sIndexPath, AccessMode am,
const DocumentSchema* pDocSchema)
{
std::string sFs = GLOBAL_CONF().Storage.filesystem;
FileSystemPtr pFileSys = FileSystemFactory::instance()->createFileSystem(sFs);
if (pFileSys.isNull())
{
FIRTEX_THROW(InvalidConfigException, "Create file system: [%s] FAILED", sFs.c_str());
}
FileSystem::OpenMode om = FileSystem::READ;
switch (am)
{
case READ:
om = FileSystem::READ;
break;
case WRITE:
om = FileSystem::CREATE;
break;
case APPEND:
case RDWR:
om = FileSystem::APPEND;
break;
}
pFileSys->open(sIndexPath, om);
if ((am == APPEND) && (!pFileSys->fileExists(SCHEMA_FILENAME)))
{
am = WRITE;
}
open(pFileSys, am, pDocSchema);
}
void Index::remove(const std::string& sIndexPath)
{
std::string sFs =GLOBAL_CONF().Storage.filesystem;
FileSystemPtr pFileSys = FileSystemFactory::instance()->createFileSystem(sFs);
if (pFileSys.isNull())
{
FIRTEX_THROW(InvalidConfigException, "Create file system: [%s] FAILED", sFs.c_str());
}
pFileSys->open(sIndexPath, FileSystem::CREATE);
remove(pFileSys);
}
void StandardAnalyzer::init()
{
tstring dict = getCoreDictPath(GLOBAL_CONF().General.dictionaryPath);
File f(dict);
if (!f.exists())
{
FX_LOG(ERROR, _T("Core dictionary: [%s] not found"), dict.c_str());
FIRTEX_THROW(FileIOException, _T("Load dictionary FAILED."));
return;
}
init(dict);
}
void IndexBarrelKeeper::setupIndexCleaner()
{
string sCleaner = GLOBAL_CONF().Build.IndexCleaner.strategy;
string sParam = GLOBAL_CONF().Build.IndexCleaner.param;
if (sCleaner.empty())
{
sCleaner = "keep_by_commit";
}
if (sParam.empty())
{
sParam = "keep_count=";
NumberFormatter::append(sParam, DEFAULT_COMMIT_KEEP_COUNT);
}
m_pIndexCleaner.reset(IndexCleanerFactory::instance()->createIndexCleaner(sCleaner));
if (!m_pIndexCleaner)
{
FX_LOG(ERROR, "Create index cleaner: [%s] FAILED.", sCleaner.c_str());
return;
}
m_pIndexCleaner->init(m_pFileSys, sParam);
}
void IndexWriter::createMerger()
{
if (m_pIndexMerger.isNull())
{
m_pIndexMerger.assign(new IndexMerger(m_pKeeper.get()));
std::string sIdent = GLOBAL_CONF().Merge.strategy;
MergePolicyPtr pMergePolicy =
MergePolicyFactory::instance()->createMergePolicy(sIdent);
if (pMergePolicy.isNull())
{
FX_LOG(WARN, "Invalid merge policy identifier: [%s]",
sIdent.c_str());
}
else
{
m_pIndexMerger->setMergePolicy(pMergePolicy);
}
}
}
void IndexBarrelKeeper::sealBarrel(const IndexBarrelWriterPtr& pInMemBarrel)
{
FX_DEBUG("Seal barrel");
IndexBarrelWriterPtr pBarrelWriter = pInMemBarrel;
InMemIndexMergerPtr pInMemBarrelMerger;
pBarrelWriter->sealBarrel();
{
ScopedRWLock lock(m_lock, true);
IndexBarrelPtr pLastBarrel;
if (!m_commitMap.empty())
{
pLastBarrel = m_commitMap.rbegin()->second;
}
BarrelsInfoPtr pNewBarrelsInfo;
if (pLastBarrel)
{
pNewBarrelsInfo.reset(pLastBarrel->getBarrelsInfo()->clone());
}
else
{
pNewBarrelsInfo.reset(new BarrelsInfo);
}
/// Commit barrel to barrels info
if (!pBarrelWriter->commitBarrelInfo(pNewBarrelsInfo))
{
FX_DEBUG("No barrel to commit");
return ;
}
FX_DEBUG("Committed barrel info: commit: [%d], baseDocId: [%d]",
pNewBarrelsInfo->getLastBarrel().getCommitId(),
pNewBarrelsInfo->getLastBarrel().getBaseDocId());
//TODO: implement real-time search
IndexBarrelReaderPtr pReader;
if (pLastBarrel)
{
pReader = pLastBarrel->getReader();
}
PrimaryKeyIndexPtr pPrimKey;
if (m_pOnDiskPrimKeyIndex)
{
pPrimKey.reset(m_pOnDiskPrimKeyIndex->clone());
if (pPrimKey)
{
const PrimaryKeyIndexPtr& pPkIndex = pBarrelWriter->getPrimaryKey();
if (pPkIndex)
{
pPrimKey->reopen(pNewBarrelsInfo, pPkIndex->getHashMap());
}
}
}
else
{
pPrimKey = createPrimaryKeyIndex();
if (pPrimKey)
{
const PrimaryKeyIndexPtr& pPkIndex = pBarrelWriter->getPrimaryKey();
FIRTEX_ASSERT2(pPkIndex);
pPrimKey->open(m_pFileSys, pNewBarrelsInfo, pPkIndex->getHashMap());
}
}
IndexBarrelPtr pBarrel(new IndexBarrel(pNewBarrelsInfo, pReader, m_pOnDiskDocFilter));
pBarrel->setEncoding(m_sEncoding);
bool bInserted = doInsertCommit(pNewBarrelsInfo->getCommitId(), pBarrel);
(void)bInserted;
FIRTEX_ASSERT2(bInserted);
m_pOnDiskDocFilter.reset(m_pOnDiskDocFilter->clone());
m_pOnDiskPrimKeyIndex = pPrimKey;
if (!m_pInMemBarrelMerger)
{
m_pInMemBarrelMerger.reset(new InMemIndexMerger(this));
}
m_pInMemBarrelMerger->addToMerge(pNewBarrelsInfo->getLastBarrel(),
pBarrelWriter);
if (m_pInMemBarrelMerger->getInMemBarrelCount() >= (size_t)GLOBAL_CONF().Build.buildThreadCount)
{
pInMemBarrelMerger = m_pInMemBarrelMerger;
m_pInMemBarrelMerger.reset();
}
}
if (pInMemBarrelMerger)
{
CommittablePtr pCommitObj = std::dynamic_pointer_cast<Committable>(pInMemBarrelMerger);
m_pCommitScheduler->commit(pCommitObj);
}
}
