void check_query(index::ir_eval& eval,
const std::vector<std::pair<doc_id, double>>& ranking,
query_id qid, double e_f1, double e_p, double e_r,
double e_avg_p, double e_ndcg,
uint64_t num_docs = std::numeric_limits<uint64_t>::max())
{
auto f1 = eval.f1(ranking, qid, num_docs);
auto p = eval.precision(ranking, qid, num_docs);
auto r = eval.recall(ranking, qid, num_docs);
auto avg_p = eval.avg_p(ranking, qid, num_docs);
auto ndcg = eval.ndcg(ranking, qid, num_docs);
ASSERT_APPROX_EQUAL(f1, e_f1);
ASSERT_APPROX_EQUAL(p, e_p);
ASSERT_APPROX_EQUAL(r, e_r);
ASSERT_APPROX_EQUAL(avg_p, e_avg_p);
ASSERT_APPROX_EQUAL(ndcg, e_ndcg);
}
int test_correctness(std::vector<double>& v)
{
// this makes sure every single element is touched exactly once
return testing::run_test("parallel-correctness", [&]()
{
std::fill(v.begin(), v.end(), 1.0);
std::mutex mtx;
parallel::parallel_for(v.begin(), v.end(), easy_func<double>);
ASSERT_APPROX_EQUAL(std::accumulate(v.begin(), v.end(), 0.0), 0.0);
});
}
void check_ceeaus_doc_id(Index& idx)
{
doc_id d_id{47};
term_id first;
double second;
std::ifstream in{"../data/ceeaus-doc-count.txt"};
auto pdata = idx.search_primary(d_id);
for (auto& count : pdata->counts())
{
in >> first;
in >> second;
ASSERT_EQUAL(first, count.first);
ASSERT_APPROX_EQUAL(second, count.second);
}
}
void check_bcancer_doc_id(Index& idx)
{
doc_id d_id{47};
term_id first;
double second;
std::ifstream in{"../data/bcancer-doc-count.txt"};
auto pdata = idx.search_primary(d_id);
for (auto& count : pdata->counts())
{
in >> first;
in >> second;
ASSERT_EQUAL(first - 1, count.first); // - 1 because libsvm format
ASSERT_APPROX_EQUAL(second, count.second);
}
}
void test_rank(Ranker& r, Index& idx, const std::string& encoding)
{
for (size_t i = 0; i < idx.num_docs(); ++i)
{
auto d_id = idx.docs()[i];
corpus::document query{idx.doc_path(d_id), doc_id{i}};
query.encoding(encoding);
auto ranking = r.score(idx, query);
ASSERT_EQUAL(ranking.size(), 10); // default is 10 docs
// since we're searching for a document already in the index, the same
// document should be ranked first, but there are a few duplicate
// documents......
if (ranking[0].first != i)
{
ASSERT_EQUAL(ranking[1].first, i);
ASSERT_APPROX_EQUAL(ranking[0].second, ranking[1].second);
}
}
}
int ir_eval_results()
{
return testing::run_test(
"ir-eval-results", [&]()
{
create_config("file");
index::ir_eval eval{"test-config.toml"};
ASSERT_APPROX_EQUAL(eval.map(), 0.0);
ASSERT_APPROX_EQUAL(eval.gmap(), 0.0);
// make some fake results based on the loaded qrels file
std::vector<std::pair<doc_id, double>> results;
query_id qid{0};
auto idcg_5 = 1.0 + 1.0 / std::log2(3.0) + 1.0 / std::log2(4.0)
+ 1.0 / std::log2(5.0) + 1.0 / std::log2(6.0);
auto idcg = idcg_5 + 1.0 / std::log2(7.0) + 1.0 / std::log2(8.0)
+ 1.0 / std::log2(9.0) + 1.0 / std::log2(10.0)
+ 1.0 / std::log2(11.0);
results.emplace_back(doc_id{0}, 1.0); // relevant
check_query(eval, results, qid, 0.2 / 1.1, 1, 0.1, 0.1, 1.0 / idcg);
check_query(eval, results, qid, 0.2 / 1.1, 1, 0.1, 0.2,
1.0 / idcg_5, 5);
results.emplace_back(doc_id{2}, 0.9); // not relevant
check_query(eval, results, qid, 0.1 / 0.6, 0.5, 0.1, 0.1,
1.0 / idcg);
check_query(eval, results, qid, 0.1 / 0.6, 0.5, 0.1, 0.2,
1.0 / idcg_5, 5);
results.emplace_back(doc_id{1}, 0.8); // relevant
check_query(eval, results, qid,
(2.0 * (2.0 / 3.0) * 0.2) / (2.0 / 3.0 + 0.2),
2.0 / 3.0, 0.2, 1.0 / 6.0, 1.5 / idcg);
check_query(eval, results, qid,
(2.0 * (2.0 / 3.0) * 0.2) / (2.0 / 3.0 + 0.2),
2.0 / 3.0, 0.2, 1.0 / 3.0, 1.5 / idcg_5, 5);
results.emplace_back(doc_id{30}, 0.8); // relevant
results.emplace_back(doc_id{6}, 0.7); // relevant
results.emplace_back(doc_id{43}, 0.6); // relevant
results.emplace_back(doc_id{24}, 0.5); // relevant
results.emplace_back(doc_id{34}, 0.4); // relevant
results.emplace_back(doc_id{35}, 0.3); // relevant
results.emplace_back(doc_id{38}, 0.2); // relevant
results.emplace_back(doc_id{754}, 0.1); // relevant
auto avg_p_5 = (1.0 + 2.0 / 3.0 + 3.0 / 4.0 + 4.0 / 5.0 + 5.0 / 6.0)
/ 5.0;
auto avg_p = (1.0 + 2.0 / 3.0 + 3.0 / 4.0 + 4.0 / 5.0 + 5.0 / 6.0
+ 6.0 / 7.0 + 7.0 / 8.0 + 8.0 / 9.0 + 9.0 / 10.0
+ 10.0 / 11.0) / 10.0;
auto dcg_5 = 1.0 + 1.0 / std::log2(4.0) + 1.0 / std::log2(5.0)
+ 1.0 / std::log2(6.0); // 4 terms, 1 zero term
auto dcg = dcg_5 + 1.0 / std::log2(7.0) + 1.0 / std::log2(8.0)
+ 1.0 / std::log2(9.0) + 1.0 / std::log2(10.0)
+ 1.0 / std::log2(11.0) + 1.0 / std::log2(12.0);
check_query(eval, results, qid,
(2.0 * (10.0 / 11.0)) / ((10.0 / 11.0) + 1.0),
10.0 / 11.0, 1.0, avg_p, dcg / idcg);
check_query(eval, results, qid,
(2.0 * (4.0 / 5.0) * 0.4) / ((4.0 / 5.0) + 0.4),
4.0 / 5.0, 0.4, avg_p_5, dcg_5 / idcg_5, 5);
results.erase(results.begin() + 1); // remove non-relevant result
check_query(eval, results, qid, 1.0, 1.0, 1.0, 1.0, 1.0);
// recall is still not perfect @5
check_query(eval, results, qid, 1.0 / 1.5, 1.0, 0.5, 1.0, 1.0, 5);
system("rm test-config.toml");
});
}
