void forward_index::impl::uninvert(const inverted_index& inv_idx) { io::compressed_file_reader inv_reader{inv_idx.index_name() + idx_->impl_->files[POSTINGS], io::default_compression_reader_func}; term_id t_id{0}; chunk_handler<forward_index> handler{idx_->index_name()}; { auto producer = handler.make_producer(); while (inv_reader.has_next()) { inverted_pdata_type pdata{t_id}; pdata.read_compressed(inv_reader); producer(pdata.primary_key(), pdata.counts()); ++t_id; } } handler.merge_chunks(); compressed_postings_to_libsvm(inv_idx.num_docs()); }
std::vector<std::pair<doc_id, double>> ranker::score(inverted_index& idx, corpus::document& query, uint64_t num_results /* = 10 */, const std::function<bool(doc_id d_id)>& filter /* return true */) { if (query.counts().empty()) idx.tokenize(query); score_data sd{idx, idx.avg_doc_length(), idx.num_docs(), idx.total_corpus_terms(), query}; // zeros out elements and (if necessary) resizes the vector; this eliminates // constructing a new vector each query for the same index results_.assign(sd.num_docs, std::numeric_limits<double>::lowest()); for (auto& tpair : query.counts()) { term_id t_id{idx.get_term_id(tpair.first)}; auto pdata = idx.search_primary(t_id); sd.doc_count = pdata->counts().size(); sd.t_id = t_id; sd.query_term_count = tpair.second; sd.corpus_term_count = idx.total_num_occurences(sd.t_id); for (auto& dpair : pdata->counts()) { sd.d_id = dpair.first; sd.doc_term_count = dpair.second; sd.doc_size = idx.doc_size(dpair.first); sd.doc_unique_terms = idx.unique_terms(dpair.first); // if this is the first time we've seen this document, compute // its initial score if (results_[dpair.first] == std::numeric_limits<double>::lowest()) results_[dpair.first] = initial_score(sd); results_[dpair.first] += score_one(sd); } } using doc_pair = std::pair<doc_id, double>; auto doc_pair_comp = [](const doc_pair& a, const doc_pair& b) { return a.second > b.second; }; std::priority_queue<doc_pair, std::vector<doc_pair>, decltype(doc_pair_comp)> pq{doc_pair_comp}; for (uint64_t id = 0; id < results_.size(); ++id) { if (!filter(doc_id{id})) continue; pq.emplace(doc_id{id}, results_[id]); if (pq.size() > num_results) pq.pop(); } std::vector<doc_pair> sorted; while (!pq.empty()) { sorted.emplace_back(pq.top()); pq.pop(); } std::reverse(sorted.begin(), sorted.end()); return sorted; }