示例#1
0
struct evhttp_request* EvHttpSyncClient::newReqest()
{
    struct evhttp_request* pReq = evhttp_request_new(handleResponse, this);
    if (pReq == NULL) 
    {
        FX_LOG(ERROR, "evhttp_request_new FAILED");
        setError(ST_ERROR, "evhttp_request_new FAILED");
        return NULL;
    }

    int rv = evhttp_add_header(evhttp_request_get_output_headers(pReq), 
                               "Host", m_sHost.c_str());
    if (rv != 0) 
    {
        FX_LOG(ERROR, "evhttp_add_header FAILED");
        setError(ST_ERROR, "evhttp_add_header FAILED");
        return NULL;
    }

    rv = evhttp_add_header(evhttp_request_get_output_headers(pReq),
                           "Connection", "keep-alive");
    if (rv != 0) 
    {
        FX_LOG(ERROR, "evhttp_add_header FAILED");
        setError(ST_ERROR, "evhttp_add_header FAILED");
        return NULL;
    }
    return pReq;
}
示例#2
0
void EvHttpSyncClient::get(const std::string& sURI)
{
    struct evhttp_request* pReq = newReqest();
    if (!pReq)
    {
        setError(ST_ERROR, "new request FAILED");
        return ;
    }

    int rv = evhttp_make_request(m_pConn, pReq, EVHTTP_REQ_GET, sURI.c_str());
    if (rv != 0) 
    {
        FX_LOG(ERROR, "evhttp_make_request FAILED");
        setError(ST_ERROR, "evhttp_make_request FAILED");
        return;
    }

    rv = event_base_dispatch(m_evbase);
    if (rv != 0)
    {
        setError(ST_ERROR, "event loop failed");
        FX_LOG(ERROR, "event loop failed: [%d]", rv);
    }

    FX_TRACE("made request: [%s]", sURI.c_str());
    return ;
}
示例#3
0
void StandardStopFilter::setParam(const tstring& sParam)
{
    KeyValueParser parser;
    if (parser.parse(sParam, TokenFilter::PARAM_SEPERATOR,
                     TokenFilter::EQUAL_MARK))
    {
        size_t nWords = 0;
        string sValue;
        if (parser.getValue("words", sValue))
        {
            nWords = loadWords(sValue);
        }
        else if (parser.getValue("file", sValue))
        {
            nWords = loadFile(sValue);
        }
        else 
        {
            return ;
        }

        if (nWords == 0)
        {
            FX_LOG(ERROR, "Load stop words: [%s] FAILED.",
                   sValue.c_str());
        }
        else
        {
            FX_LOG(INFO, "Loaded [%u] stop words.", (uint32_t)nWords);
        }

    }
}
示例#4
0
void IndexBarrelKeeper::housekeep()
{
    ScopedRWLock lock(m_lock, true);

    CommitMap toReserve;
    CommitMap::reverse_iterator it = m_commitMap.rbegin();
    // Always reserve the last element 
    toReserve.insert(*it++);

    for (; it != m_commitMap.rend(); it++)
    {
        if (it->second.use_count() > 1)
        {
            FX_LOG(INFO, "Staled commit is in use: [%d]", it->first);
            toReserve.insert(*it);
        }
        else
        {
            FX_LOG(INFO, "Clean staled commit: [%d]", it->first);
            it->second.reset();
        }
    }

    toReserve.swap(m_commitMap);
}
示例#5
0
void FormulaScorer::setParameter(const std::string& sParam)
{
    KeyValueParser kvParser;
    if (!kvParser.parse(sParam, ",", "="))
    {
        FX_LOG(ERROR, "Parse formula parameter : [%s] FAILED", sParam.c_str());
        return;
    }
    if (!kvParser.getValue("formula", m_sFormula))
    {
        FX_LOG(ERROR, "Extract formula from [%s] FAILED", sParam.c_str());
        return;
    }
}
示例#6
0
void StandardAnalyzer::setParam(const tstring& sParam)
{
    KeyValueParser kvParser;
    bool ret = kvParser.parse(sParam, ";", "=");
    if (!ret)
    {
        FX_LOG(ERROR, "Invalid analyzer parameter: [%s]",
               sParam.c_str());
    }
    else
    {
        tstring sValue;
        if (kvParser.getValue(PARAM_ALGORITHM, sValue))
        {
            if (!strCompareNoCase(sValue.c_str(), "max_forward"))
            {
                m_eSegAlgorithm = SA_MAX_FORWARD;
            }
            else if (!strCompareNoCase(sValue.c_str(), "unigram"))
            {
                m_eSegAlgorithm = SA_UNIGRAM;
            }
            else
            {
                FX_LOG(ERROR, "Invalid parameter: [algorithm=%s]",
                       sValue.c_str());
                m_eSegAlgorithm = SA_MAX_FORWARD;
            }
        }

        if (kvParser.getValue(PARAM_ENCODE, sValue))
        {
            if ((!strCompareNoCase(sValue.c_str(), "utf-8"))
                    || (!strCompareNoCase(sValue.c_str(), "gbk"))
                    || (!strCompareNoCase(sValue.c_str(), "gb2312")))
            {
                m_sEncodeName = toLower(sValue);
            }
            else
            {
                FX_LOG(ERROR, "Invalid parameter: [encode=%s], "
                       "using default [encode=utf-8]",
                       sValue.c_str());
                m_sEncodeName = toLower(sValue);
            }
        }
    }
}
示例#7
0
void AsyncDocumentConsumer::start()
{
    size_t nSizeInMB = (size_t)GLOBAL_CONF().Build.memory;
    FX_LOG(INFO, "Allocate memory for building, total size: [%u] MB, segment count: [%u]",
           (uint32_t)nSizeInMB, (uint32_t)m_threadPool.capacity());
    m_pAllocator.reset(new SyncSegregatedAllocator((size_t)(nSizeInMB * 1024 * 1024),
                    m_threadPool.capacity() * MAX_CHUNK_COUNT_PER_SEGMENT));

    m_pUpdateTask.reset(new OnDiskUpdateTask(m_pKeeper, m_docQueue, m_docUpdateQueue));
    m_updateThread.start(*m_pUpdateTask);
    
    taskid_t taskId = 0, nextTaskId = INVALID_TASKID;
    for (size_t i = 0; i < m_threadPool.capacity(); i++)
    {
        IndexBarrelWriterPtr pIndexBarrelWriter = m_pKeeper->createBarrelWriter(); 
        if (!pIndexBarrelWriter->init(m_pAllocator))
        {
            FIRTEX_THROW_AND_LOG(RuntimeException, "Init index writer FAILED.");
            break;
        }
        
        if (i + 1 != m_threadPool.capacity())
        {
            nextTaskId = taskId + 1;
        }

        TaskPtr pTask(new Task(taskId, nextTaskId, pIndexBarrelWriter,
                               m_docQueue, *this));
        
        m_threadPool.start(*pTask);
        m_tasks.push_back(pTask);
    }
}
示例#8
0
void StandardStopFilter::init(const tstring& stopfile)
{
    if (loadFile(stopfile) == 0)
    {
        FX_LOG(ERROR, "Load stop word file: [%s] FAILED.", stopfile.c_str());
    }
}
示例#9
0
void EvHttpSyncClient::post(const std::string& sURI, const std::string& sData)
{
    struct evhttp_request* pReq = newReqest();
    if (!pReq)
    {
        return;
    }
    
    int rv = evbuffer_add(evhttp_request_get_output_buffer(pReq),
                                 sData.c_str(), sData.length());
    if (rv != 0) 
    {
        setError(ST_ERROR, "evhttp_add_buffer FAILED");
        return;
    }

    rv = evhttp_make_request(m_pConn, pReq, EVHTTP_REQ_POST, sURI.c_str());
    if (rv != 0) 
    {
        setError(ST_ERROR, "evhttp_make_request FAILED");
        return;
    }

    rv = event_base_dispatch(m_evbase);
    if (rv != 0)
    {
        FX_LOG(WARN, "event loop failed: [%d]", rv);
    }
}
示例#10
0
void StandardAnalyzer::init(const tstring& sDictPath)
{
    TokenizerPtr pTok;
    if (m_sEncodeName == "gbk" || m_sEncodeName == "gb2312")
    {
        pTok = new GBKStandardTokenizer();
    }
    else
    {
        pTok = new UTF8StandardTokenizer();
    }

    if (m_eSegAlgorithm == SA_UNIGRAM)
    {
        m_pSegmenter = new UnigramSegmenter(pTok);
    }
    else
    {
        m_pSegmenter = new MaxForwardSegmenter(pTok);
    }

    try
    {
        m_pSegmenter->init(sDictPath);
    }
    catch(const FirteXException& e)
    {
        FX_LOG(ERROR, _T("Load dictionary [%s] FAILED: [%s]"),
               sDictPath.c_str(), e.what().c_str());
        throw;
    }
}
示例#11
0
/* static */ 
void EvHttpSyncClient::handleResponse(struct evhttp_request* req, void* arg) 
{
    FX_DEBUG("handleResponse");
    EvHttpSyncClient* pThis = (EvHttpSyncClient*)arg;
    try
    {
        pThis->done(req);
    }
    catch(std::exception& e) 
    {
        FX_LOG(ERROR, "Response exception thrown: [%s]: ", e.what());
    }
    catch(const FirteXException& e) 
    {
        FX_LOG(ERROR, "Response exception thrown: [%s]: ", e.what().c_str());
    }
}
示例#12
0
void IndexBarrelKeeper::waitCommit()
{
    FX_LOG(INFO, "Waiting last commit...");
    forceCommit();

    //Wait until all documents committed to file system
    m_pCommitScheduler->waitCommit();

    m_pInMemBarrelMerger.reset();
}
示例#13
0
void HTMLParser::parse(const tstring& sHtmlFile)
{
    BinaryFile bf;
    try
    {
        bf.open(sHtmlFile, BinaryFile::READ);
    
        m_nFileSize = (size_t)bf.getLength();
        if(m_nFileSize > MAX_FILESIZE - 1)
        {
            m_nFileSize = MAX_FILESIZE - 1;
        }

        if(!m_pReadBuffer)
        {
            m_nReadBufferSize = DEFAULT_READBUFFER_SIZE;
            if(m_nReadBufferSize < m_nFileSize + 1)
                m_nReadBufferSize = m_nFileSize + 1;
            m_pReadBuffer = new char[m_nReadBufferSize];
        }
        else if(m_nFileSize + 1 > m_nReadBufferSize)
        {
            m_nReadBufferSize = m_nFileSize + 1;
            delete[] m_pReadBuffer;
            m_pReadBuffer = new char[m_nReadBufferSize];
        }
        size_t nRet = bf.read(m_pReadBuffer, m_nFileSize);
        if(nRet != m_nFileSize)
        {
            FX_LOG(WARN, "Read file [%s] error", sHtmlFile.c_str());
            bf.close();
            return;
        }
        bf.close();
        parse(m_pReadBuffer, m_nFileSize);
    }
    catch(const FirteXException& e)
    {
        FX_LOG(ERROR, "Parse file: [%s] FAILED. Error message: [%s]",
               sHtmlFile.c_str(), e.what().c_str());
    }
}
示例#14
0
IndexWriterPtr Index::acquireWriter()
{
    ScopedRWLock lock(m_lock, false);
    if (m_accessMode == READ)
    {
        FX_LOG(INFO, "The index is in READ mode, "
               "can't aquire writer of the index");
        return IndexWriterPtr();
    }
    return m_pWriter;
}
示例#15
0
void StandardAnalyzer::init()
{
    tstring dict = getCoreDictPath(GLOBAL_CONF().General.dictionaryPath);
    File f(dict);
    if (!f.exists())
    {
        FX_LOG(ERROR, _T("Core dictionary: [%s] not found"), dict.c_str());
        FIRTEX_THROW(FileIOException, _T("Load dictionary FAILED."));
        return;
    }

    init(dict);
}
示例#16
0
//static
void Index::remove(FileSystemPtr& pFileSys)
{
    BarrelsInfoPtr barrelsInfoPtr(new BarrelsInfo());
    try
    {
        barrelsInfoPtr->read(pFileSys); //read barrels Info
        barrelsInfoPtr->remove(pFileSys);
    }
    catch(const IndexCollapseException& e)
    {
        FX_LOG(ERROR, "Remove index FAILED: [%s]", e.what().c_str());
        FIRTEX_RETHROW(e);
    }
}
示例#17
0
void DocumentTemplate::makeSureMeta(Meta& meta, const DocumentSchema* pSchema)
{
    const FieldSchema* pFieldSchema = 
        pSchema->getSchema(meta.getFieldName().c_str());
    if (!pFieldSchema)
    {
        FX_LOG(ERROR, "No field: [%s] in schema match the meta: [%s]", 
               meta.getFieldName().c_str(), meta.getMetaName().c_str());
        FIRTEX_THROW(InvalidConfigException, "No field: [%s] in schema match "
                     "the meta: [%s]", meta.getFieldName().c_str(),
                     meta.getMetaName().c_str());
    }
    
    meta.setId(pFieldSchema->getId());
}
示例#18
0
IndexReaderPtr Index::acquireReader(bool bRefresh)
{
    if ((m_accessMode != READ) && (m_accessMode != RDWR))
    {
        FX_LOG(INFO, _T("The access mode is WRITE or APPEND, "
                        "can't aquire reader of the index"));
        return IndexReaderPtr();
    }

    if (bRefresh)
    {
        CommitList commits;
        commits.load(m_pFileSys);
        commitid_t lastCommit = commits.getLastCommit();
        commitid_t curCommit = m_pReader->getBarrelsInfo()->getCommitId();
        if (lastCommit > curCommit)
        {
            m_pIndexBarrelKeeper->refresh();

            IndexReaderPtr pNewReader(m_pReader->clone());
            pNewReader->reopen();
            
            ScopedRWLock lock(m_lock, true);
            m_pReader = pNewReader;
            pNewReader.reset();

            if (m_pIndexBarrelKeeper->getHeldCommitCount() >= 2)
            {
                m_pIndexBarrelKeeper->housekeep();
            }

            return m_pReader;
        }
        if (lastCommit < curCommit)
        {
            FIRTEX_THROW(IndexCollapseException, "Invalid commit id: "
                            "current commit: [%d], last comit: [%d]", 
                         curCommit, lastCommit);
        }
    }

    ScopedRWLock lock(m_lock, false);
    return m_pReader;
}
示例#19
0
void IndexWriter::createMerger()
{
    if (m_pIndexMerger.isNull())
    {
        m_pIndexMerger.assign(new IndexMerger(m_pKeeper.get()));
        std::string sIdent = GLOBAL_CONF().Merge.strategy;
        MergePolicyPtr pMergePolicy =
            MergePolicyFactory::instance()->createMergePolicy(sIdent);
        if (pMergePolicy.isNull())
        {
            FX_LOG(WARN, "Invalid merge policy identifier: [%s]",
                   sIdent.c_str());
        }
        else
        {
            m_pIndexMerger->setMergePolicy(pMergePolicy);
        }
    }
}
示例#20
0
void IndexBarrelKeeper::forceCommit()
{
    FX_LOG(INFO, "Waiting last commit...");
	InMemIndexMergerPtr pInMemBarrelMerger;

	{
		ScopedRWLock lock(m_lock, true);
		if (m_pInMemBarrelMerger)
		{
			pInMemBarrelMerger = m_pInMemBarrelMerger;
			m_pInMemBarrelMerger.reset(); 
		}
	}

	if (pInMemBarrelMerger)
	{
            CommittablePtr pCommitObj = std::dynamic_pointer_cast<Committable>(pInMemBarrelMerger);
            m_pCommitScheduler->commit(pCommitObj);
	}
}
示例#21
0
void IndexBarrelKeeper::init(const BarrelsInfoPtr& pBarrelsInfo, Mode mode)
{
    FX_LOG(INFO, "Initialize index barrel keeper.");
    m_mode = mode;
    m_sEncoding = pBarrelsInfo->getEncoding();

    switch(mode)
    {
    case READ:
        loadOnDiskBarrel(pBarrelsInfo);
        break;
    case RDWR:
        setupIndexCleaner();
        loadOnDiskBarrel(pBarrelsInfo);
        break;
    case WRITE:
        loadOnDiskDataForUpdate(pBarrelsInfo);
        setupIndexCleaner();
        break;
    }
}
示例#22
0
void IndexBarrelKeeper::setupIndexCleaner()
{
    string sCleaner = GLOBAL_CONF().Build.IndexCleaner.strategy;
    string sParam = GLOBAL_CONF().Build.IndexCleaner.param;
    if (sCleaner.empty())
    {
        sCleaner = "keep_by_commit";
    }
    if (sParam.empty())
    {
        sParam = "keep_count=";
        NumberFormatter::append(sParam, DEFAULT_COMMIT_KEEP_COUNT);
    }
    m_pIndexCleaner.reset(IndexCleanerFactory::instance()->createIndexCleaner(sCleaner));
    if (!m_pIndexCleaner)
    {
        FX_LOG(ERROR, "Create index cleaner: [%s] FAILED.", sCleaner.c_str());
        return;
    }

    m_pIndexCleaner->init(m_pFileSys, sParam);
}
示例#23
0
bool IndexBarrelKeeper::doInsertCommit(
        commitid_t commId, const IndexBarrelPtr& pIndexBarrel)
{
    commitid_t lastCommit = INVALID_COMMIT;
    if (!m_commitMap.empty())
    {
        IndexBarrelPtr pLastBarrel = m_commitMap.rbegin()->second;
        lastCommit = pLastBarrel->getBarrelsInfo()->getCommitId();
    }

    if (lastCommit >= commId)
    {
        return false;
    }

    FX_LOG(INFO, "Insert new commit: [%d]", commId);

    m_commitMap.insert(make_pair(commId, pIndexBarrel));
    m_latestOnDiskCommit = commId;

    return true;
}
示例#24
0
InputStreamPool::~InputStreamPool() 
{
    for (StreamMap::iterator it = m_streamMap.begin();
         it != m_streamMap.end(); ++it)
    {
        StreamVector& streams = it->second;
        for (size_t i = 0; i < streams.size(); ++i)
        {
            try
            {
                streams[i]->close();
            }
            catch(const FileIOException& e)
            {
                FX_LOG(ERROR, "Close file FAILED: [%s]", e.what().c_str());
            }
        }
        streams.clear();
    }
    m_streamMap.clear();

    m_nOpenedStreamCount = 0;
}
示例#25
0
commitid_t IndexBarrelKeeper::refreshDataForRead()
{
    BarrelsInfoPtr pBarrelsInfo(new BarrelsInfo());
    pBarrelsInfo->read(m_pFileSys);
    commitid_t latestCommit = pBarrelsInfo->getCommitId();

    commitid_t prevCommit = INVALID_COMMIT;
    IndexBarrelPtr pLastBarrel;
    {
        ScopedRWLock lock(m_lock, false);
        if (!m_commitMap.empty())
        {
            pLastBarrel = m_commitMap.rbegin()->second;
            prevCommit = pLastBarrel->getBarrelsInfo()->getCommitId();
        }
    }
    if (latestCommit == prevCommit)
    {
        return prevCommit;
    }

    if (prevCommit == INVALID_COMMIT)
    {
        return createOnDiskBarrelReader(pBarrelsInfo);
    }

    FX_LOG(INFO, "Begin reopen index database, commitId: [%d]", latestCommit);

    IndexBarrelReaderPtr pReader;

    DeletedDocumentFilterPtr pDocFilter(pLastBarrel->getDeletedDocFilter()->clone());
    pDocFilter->reopen(pBarrelsInfo);

    PrimaryKeyIndexPtr pPrimKey;
    if (m_pOnDiskPrimKeyIndex)
    {
        pPrimKey.reset(m_pOnDiskPrimKeyIndex->clone());
        pPrimKey->reopen(pBarrelsInfo);
    }

    BarrelsInfoPtr pLastBarrelsInfo = pLastBarrel->getBarrelsInfo();
    IndexBarrelReaderPtr pLastReader = pLastBarrel->getReader();

    size_t bc = pBarrelsInfo->getBarrelCount();
    if (bc == 1)
    {
        if (pLastBarrelsInfo->getBarrelCount() > 1 
            || pBarrelsInfo->getCommitId() - pLastBarrelsInfo->getCommitId() > 1)
        {
            const BarrelInfo& lastBarrelInfo = pBarrelsInfo->getLastBarrel();
            const BitVector* pBitVector =
                pDocFilter->getDocFilter(lastBarrelInfo.getBaseDocId());
            SingleIndexBarrelReader* pSingleReader = new SingleIndexBarrelReader(
                    m_pFileSys, m_pDocSchema, m_pComponentBuilder);
            pReader.reset(pSingleReader);
            
            pSingleReader->open(&lastBarrelInfo, pBitVector);
        }
        else
        {
            const BarrelInfo& lastBarrelInfo = pBarrelsInfo->getLastBarrel();
            const BitVector* pBitVector =
                pDocFilter->getDocFilter(lastBarrelInfo.getBaseDocId());
                
            pReader.reset(pLastReader->clone());
            SingleIndexBarrelReaderPtr pSingleReader =
                std::dynamic_pointer_cast<SingleIndexBarrelReader>(pReader);

            FIRTEX_ASSERT2(pSingleReader);
            pSingleReader->reopen(&lastBarrelInfo, pBitVector);
        }
    }
    else if (bc > 1)
    {
        if (pLastBarrelsInfo->getBarrelCount() > 1)
        {
            pReader.reset(pLastReader->clone());
            MultiIndexBarrelReaderPtr pMultiReader =
                std::dynamic_pointer_cast<MultiIndexBarrelReader>(pReader);
            FIRTEX_ASSERT2(pMultiReader);
            pMultiReader->reopen(pBarrelsInfo, pDocFilter);
        }
        else
        {
            //TODO: optimize?
            MultiIndexBarrelReader* pMultiReader = new MultiIndexBarrelReader(
                    m_pFileSys, m_pDocSchema, m_pComponentBuilder);
            pReader.reset(pMultiReader);

            pMultiReader->open(pBarrelsInfo, pDocFilter);
        }
    }
    FX_LOG(INFO, "End reopen index database.");

    IndexBarrelPtr pBarrel(new IndexBarrel(pBarrelsInfo, pReader, pDocFilter));
    pBarrel->setEncoding(m_sEncoding);
    {
        ScopedRWLock lock(m_lock, true);
        bool bInserted = doInsertCommit(latestCommit, pBarrel);
        if (bInserted)
        {
            m_pOnDiskDocFilter = pDocFilter;
            m_pOnDiskPrimKeyIndex = pPrimKey;
            return latestCommit;
        }
        return prevCommit;
    }
}
示例#26
0
IndexBarrelWriterPtr IndexBarrelKeeper::createBarrelWriter()
{
    FastMutex::Guard g(m_counterLock);
    FX_LOG(INFO, "Create new in-memory barrel: [%d]", m_barrelCounter);
    return IndexBarrelWriterPtr(new IndexBarrelWriter(this, m_barrelCounter++));
}