void StandardAnalyzer::setParam(const tstring& sParam) { KeyValueParser kvParser; bool ret = kvParser.parse(sParam, ";", "="); if (!ret) { FX_LOG(ERROR, "Invalid analyzer parameter: [%s]", sParam.c_str()); } else { tstring sValue; if (kvParser.getValue(PARAM_ALGORITHM, sValue)) { if (!strCompareNoCase(sValue.c_str(), "max_forward")) { m_eSegAlgorithm = SA_MAX_FORWARD; } else if (!strCompareNoCase(sValue.c_str(), "unigram")) { m_eSegAlgorithm = SA_UNIGRAM; } else { FX_LOG(ERROR, "Invalid parameter: [algorithm=%s]", sValue.c_str()); m_eSegAlgorithm = SA_MAX_FORWARD; } } if (kvParser.getValue(PARAM_ENCODE, sValue)) { if ((!strCompareNoCase(sValue.c_str(), "utf-8")) || (!strCompareNoCase(sValue.c_str(), "gbk")) || (!strCompareNoCase(sValue.c_str(), "gb2312"))) { m_sEncodeName = toLower(sValue); } else { FX_LOG(ERROR, "Invalid parameter: [encode=%s], " "using default [encode=utf-8]", sValue.c_str()); m_sEncodeName = toLower(sValue); } } } }
TermReaderPtr SingleIndexBarrelReader::termReader(const std::string& sField) const { if (m_pTermReader.isNull()) return TermReaderPtr(); if (m_bMultiIndexFields) { MultiFieldTermReaderPtr pMultiFieldTermReader = m_pTermReader.cast<MultiFieldTermReader>(); FIRTEX_ASSERT2(!pMultiFieldTermReader.isNull()); return pMultiFieldTermReader->termReader(sField); } else { if (!strCompareNoCase(m_pTermReader->getFieldSchema()->getName().c_str(), sField.c_str())) { return TermReaderPtr(m_pTermReader->clone()); } } return TermReaderPtr(); }
void BM25DocScorer::init(const FeatureProvider* pProvider, const QueryFeature& queryFeature, const std::string& sField) { const IndexFeature& indexFeature = pProvider->getIndexFeature(); for (size_t i = 0; i < queryFeature.size(); ++i) { const TermQueryFeature& tqFeature = queryFeature[i]; FIRTEX_ASSERT2(tqFeature.getNumSuccessors() < queryFeature.size() - i); const Term* pTerm = tqFeature.getTerm(); if (!tqFeature.isProhibited() && (sField.empty() || !strCompareNoCase(pTerm->getField().c_str(), sField.c_str()))) { BM25Weight w; w.init(indexFeature, &tqFeature); BM25TermScorer* pDocScore = new BM25TermScorer(indexFeature, w, &tqFeature, (uint32_t)i); m_termScorers.push_back(pDocScore); } } }