void checkNorms(IndexReaderPtr reader) { // test omit norms for (int32_t i = 0; i < DocHelper::fields.size(); ++i) { FieldPtr f = DocHelper::fields[i]; if (f->isIndexed()) { BOOST_CHECK_EQUAL(reader->hasNorms(f->name()), !f->getOmitNorms()); BOOST_CHECK_EQUAL(reader->hasNorms(f->name()), !DocHelper::noNorms.contains(f->name())); if (!reader->hasNorms(f->name())) { // test for fake norms of 1.0 or null depending on the flag ByteArray norms = reader->norms(f->name()); uint8_t norm1 = DefaultSimilarity::encodeNorm(1.0); BOOST_CHECK(!norms); norms.resize(reader->maxDoc()); reader->norms(f->name(), norms, 0); for (int32_t j = 0; j < reader->maxDoc(); ++j) BOOST_CHECK_EQUAL(norms[j], norm1); } } } }
void checkNorms(IndexReaderPtr reader) { for (Collection<FieldPtr>::iterator field = DocHelper::fields.begin(); field != DocHelper::fields.end(); ++field) { if ((*field)->isIndexed()) { BOOST_CHECK_EQUAL(reader->hasNorms((*field)->name()), !(*field)->getOmitNorms()); BOOST_CHECK_EQUAL(reader->hasNorms((*field)->name()), !DocHelper::noNorms.contains((*field)->name())); if (!reader->hasNorms((*field)->name())) { // test for fake norms of 1.0 or null depending on the flag ByteArray norms = reader->norms((*field)->name()); uint8_t norm1 = DefaultSimilarity::encodeNorm(1.0); BOOST_CHECK(!norms); norms = ByteArray::newInstance(reader->maxDoc()); reader->norms((*field)->name(), norms, 0); for (int32_t j = 0; j < reader->maxDoc(); ++j) BOOST_CHECK_EQUAL(norms[j], norm1); } } } }
void verifyIndex(const IndexReaderPtr& ir) { for (int32_t i = 0; i < NUM_FIELDS; ++i) { String field = L"f" + StringUtils::toString(i); ByteArray b = ir->norms(field); EXPECT_EQ(numDocNorms, b.size()); Collection<double> storedNorms = (i == 1 ? modifiedNorms : norms); for (int32_t j = 0; j < b.size(); ++j) { double norm = Similarity::decodeNorm(b[j]); double norm1 = storedNorms[j]; EXPECT_EQ(norm, norm1); // 0.000001 ?? } } }
ExplanationPtr PhraseWeight::explain(IndexReaderPtr reader, int32_t doc) { ExplanationPtr result(newLucene<Explanation>()); result->setDescription(L"weight(" + query->toString() + L" in " + StringUtils::toString(doc) + L"), product of:"); StringStream docFreqsBuffer; StringStream queryBuffer; queryBuffer << L"\""; docFreqsBuffer << idfExp->explain(); for (Collection<TermPtr>::iterator term = query->terms.begin(); term != query->terms.end(); ++term) { if (term != query->terms.begin()) queryBuffer << L" "; queryBuffer << (*term)->text(); } queryBuffer << L"\""; ExplanationPtr idfExpl(newLucene<Explanation>(idf, L"idf(" + query->field + L":" + docFreqsBuffer.str() + L")")); // explain query weight ExplanationPtr queryExpl(newLucene<Explanation>()); queryExpl->setDescription(L"queryWeight(" + query->toString() + L"), product of:"); ExplanationPtr boostExpl(newLucene<Explanation>(query->getBoost(), L"boost")); if (query->getBoost() != 1.0) queryExpl->addDetail(boostExpl); queryExpl->addDetail(idfExpl); ExplanationPtr queryNormExpl(newLucene<Explanation>(queryNorm, L"queryNorm")); queryExpl->addDetail(queryNormExpl); queryExpl->setValue(boostExpl->getValue() * idfExpl->getValue() * queryNormExpl->getValue()); result->addDetail(queryExpl); // explain field weight ExplanationPtr fieldExpl(newLucene<Explanation>()); fieldExpl->setDescription(L"fieldWeight(" + query->field + L":" + query->toString() + L" in " + StringUtils::toString(doc) + L"), product of:"); PhraseScorerPtr phraseScorer(boost::dynamic_pointer_cast<PhraseScorer>(scorer(reader, true, false))); if (!phraseScorer) return newLucene<Explanation>(0.0, L"no matching docs"); ExplanationPtr tfExplanation(newLucene<Explanation>()); int32_t d = phraseScorer->advance(doc); double phraseFreq = d == doc ? phraseScorer->currentFreq() : 0.0; tfExplanation->setValue(similarity->tf(phraseFreq)); tfExplanation->setDescription(L"tf(phraseFreq=" + StringUtils::toString(phraseFreq) + L")"); fieldExpl->addDetail(tfExplanation); fieldExpl->addDetail(idfExpl); ExplanationPtr fieldNormExpl(newLucene<Explanation>()); ByteArray fieldNorms(reader->norms(query->field)); double fieldNorm = fieldNorms ? Similarity::decodeNorm(fieldNorms[doc]) : 1.0; fieldNormExpl->setValue(fieldNorm); fieldNormExpl->setDescription(L"fieldNorm(field=" + query->field + L", doc=" + StringUtils::toString(doc) + L")"); fieldExpl->addDetail(fieldNormExpl); fieldExpl->setValue(tfExplanation->getValue() * idfExpl->getValue() * fieldNormExpl->getValue()); result->addDetail(fieldExpl); // combine them result->setValue(queryExpl->getValue() * fieldExpl->getValue()); if (queryExpl->getValue() == 1.0) return fieldExpl; return result; }
ScorerPtr PhraseWeight::scorer(IndexReaderPtr reader, bool scoreDocsInOrder, bool topScorer) { if (query->terms.empty()) // optimize zero-term case return ScorerPtr(); Collection<TermPositionsPtr> tps(Collection<TermPositionsPtr>::newInstance(query->terms.size())); for (int32_t i = 0; i < tps.size(); ++i) { TermPositionsPtr p(reader->termPositions(query->terms[i])); if (!p) return ScorerPtr(); tps[i] = p; } if (query->slop == 0) // optimize exact case return newLucene<ExactPhraseScorer>(shared_from_this(), tps, query->getPositions(), similarity, reader->norms(query->field)); else return newLucene<SloppyPhraseScorer>(shared_from_this(), tps, query->getPositions(), similarity, query->slop, reader->norms(query->field)); }
ScorerPtr PayloadTermWeight::scorer(const IndexReaderPtr& reader, bool scoreDocsInOrder, bool topScorer) { return newLucene<PayloadTermSpanScorer>(boost::dynamic_pointer_cast<TermSpans>(query->getSpans(reader)), shared_from_this(), similarity, reader->norms(query->getField())); }