index_sada(collection& col)
{
file_name = col.path +"index/"+name+"-"+sdsl::util::class_to_hash(*this)+".idx";
if (utils::file_exists(file_name)) { // load
LOG(INFO) << "LOAD from file '" << file_name << "'";
std::ifstream ifs(file_name);
load(ifs);
m_doc_rmin_marked = sdsl::bit_vector(m_doc_cnt, 0);
m_doc_rmax_marked = sdsl::bit_vector(m_doc_cnt, 0);
} else { // construct
LOG(INFO) << "CONSTRUCT sada index";
LOG(INFO) << "CONSTRUCT DOC ISA";
{
m_doc_isa.resize(m_doc_cnt);
std::vector<uint64_t> doc_buffer;
sdsl::int_vector_mapper<> text(col.file_map[KEY_TEXTPERM]);
size_type doc_id = 0;
for (size_type i = 0; i < text.size(); ++i) {
if (1ULL == text[i]) {
if (doc_buffer.size() > 0) {
doc_buffer.push_back(0);
{
sdsl::int_vector<> sa(doc_buffer.size(), 0, sdsl::bits::hi(doc_buffer.size())+1);
sdsl::qsufsort::construct_sa(sa, doc_buffer);
sdsl::util::bit_compress(sa);
m_doc_isa[doc_id] = sa;
for (size_type j = 0; j < doc_buffer.size(); ++j) {
m_doc_isa[doc_id][sa[j]] = j;
}
}
}
++doc_id;
doc_buffer.clear();
} else {
doc_buffer.push_back(text[i]);
}
}
}
{
const sdsl::int_vector_mapper<0,std::ios_base::in> D(col.file_map[KEY_D]);
{
LOG(INFO) << "CONSTRUCT CPrev";
sdsl::int_vector<> Cprev(D.size(), 0, sdsl::bits::hi(D.size())+1);
{
sdsl::int_vector<> last_occ(m_doc_cnt+1, 0, sdsl::bits::hi(D.size())+1);
for (size_type i = 0; i < D.size(); ++i) {
size_type doc = D[i];
Cprev[i] = last_occ[doc];
last_occ[doc] = i;
}
}
m_rminq = range_min_type(&Cprev);
}
{
LOG(INFO) << "CONSTRUCT CNext";
sdsl::int_vector<> Cnext(D.size(), 0, sdsl::bits::hi(D.size())+1);
{
sdsl::int_vector<> last_occ(m_doc_cnt+1, D.size(), sdsl::bits::hi(D.size())+1);
for (size_type i = 0, j = D.size()-1; i < D.size(); ++i, --j) {
size_type doc = D[j];
Cnext[j] = last_occ[doc];
last_occ[doc] = j;
}
}
m_rmaxq = range_max_type(&Cnext);
}
}
m_doc_rmin_marked = sdsl::bit_vector(m_doc_cnt, 0);
m_doc_rmax_marked = sdsl::bit_vector(m_doc_cnt, 0);
LOG(INFO) << "CONSTRUCT CSA";
sdsl::cache_config cfg;
cfg.delete_files = false;
cfg.dir = col.path + "/tmp/";
cfg.id = "TMP";
cfg.file_map[sdsl::conf::KEY_SA] = col.file_map[KEY_SA];
cfg.file_map[sdsl::conf::KEY_TEXT_INT] = col.file_map[KEY_TEXTPERM];
construct(m_csa_full,col.file_map[KEY_TEXTPERM],cfg,0);
LOG(INFO) << "CONSTRUCT DOC BORDER RANK";
sdsl::bit_vector dbv;
sdsl::load_from_file(dbv,col.file_map[KEY_DBV]);
m_doc_border = doc_border_type(dbv);
m_doc_border_rank = doc_border_rank_type(&m_doc_border);
m_doc_border_select = doc_border_select_type(&m_doc_border);
m_doc_cnt = m_doc_border_rank(m_doc_border.size());
LOG(INFO) << "STORE to file '" << file_name << "'";
std::ofstream ofs(file_name);
auto bytes = serialize(ofs);
std::ofstream vofs(file_name+".html");
sdsl::write_structure<sdsl::HTML_FORMAT>(vofs,*this);
LOG(INFO) << "sada index size : " << bytes / (1024*1024) << " MB";
}
}
static void prune(collection& col, bool rebuild, uint64_t target_dict_size_bytes, uint64_t num_threads)
{
auto start_total = hrclock::now();
auto start_dict_size_bytes = 0ULL;
{
const sdsl::int_vector_mapper<8, std::ios_base::in> dict(col.file_map[KEY_DICT]);
start_dict_size_bytes = dict.size();
}
if (start_dict_size_bytes == target_dict_size_bytes) {
LOG(INFO) << "\t"
<< "No dictionary pruning necessary.";
return;
}
auto dict_hash = col.param_map[PARAM_DICT_HASH];
auto new_dict_file = file_name(col, target_dict_size_bytes, dict_hash);
if (!rebuild && utils::file_exists(new_dict_file)) {
LOG(INFO) << "\t"
<< "Pruned dictionary exists at '" << new_dict_file << "'";
col.file_map[KEY_DICT] = new_dict_file;
col.compute_dict_hash();
return;
}
/* (1) create or load statistics */
LOG(INFO) << "\t"
<< "Create or load statistics.";
auto dict_file = col.file_map[KEY_DICT];
auto dict_stats_file = dict_file + "-" + KEY_DICT_STATISTICS + "-" + t_factorization_strategy::type() + ".sdsl";
factorization_statistics fstats;
if (rebuild || !utils::file_exists(dict_stats_file)) {
fstats = t_factorization_strategy::template parallel_factorize<factor_tracker>(col, rebuild, num_threads);
sdsl::store_to_file(fstats, dict_stats_file);
}
else {
sdsl::load_from_file(fstats, dict_stats_file);
}
/* (2) find segments */
LOG(INFO) << "\t"
<< "Find potential segments to remove.";
size_t total_len = 0;
auto bytes_to_remove = start_dict_size_bytes - target_dict_size_bytes;
auto freq_threshold = t_freq_threshold;
std::vector<segment_info> segments;
while (total_len < bytes_to_remove) {
segments.clear();
total_len = 0;
size_t run_len = 0;
size_t total_byte_usage = 0;
for (size_t i = 0; i < fstats.dict_usage.size(); i++) {
if (fstats.dict_usage[i] <= freq_threshold) {
run_len++;
total_byte_usage += fstats.dict_usage[i];
}
else {
if (run_len >= t_length_threshold) {
auto seg_start = i - run_len;
segments.emplace_back(seg_start, run_len, total_byte_usage, 0);
total_len += run_len;
}
run_len = 0;
total_byte_usage = 0;
}
}
LOG(INFO) << "Freq threshold = " << freq_threshold << " Found bytes = " << total_len << " Req = " << bytes_to_remove;
freq_threshold *= 2;
}
LOG(INFO) << "\t"
<< "Freq threshold = " << freq_threshold / 2 << " Length threshold = " << t_length_threshold << " Found bytes = " << total_len;
LOG(INFO) << "\t"
<< "Found " << segments.size() << " segments of total length " << total_len << " (" << total_len / (1024 * 1024) << " MiB)";
/* (3) compute the metric for those segments */
{
LOG(INFO) << "Create/Load dictionary index";
t_dict_idx idx(col, rebuild);
const sdsl::int_vector_mapper<8, std::ios_base::in> dict(col.file_map[KEY_DICT]);
for (size_t i = 0; i < segments.size(); i++) {
compute_nfac(idx, dict, segments[i]);
}
}
/* (3) sort by method */
LOG(INFO) << "Sort segments by weight";
if (t_method == FF) {
/* FF */
std::sort(segments.begin(), segments.end(), [](const segment_info& a, const segment_info& b) {
double score_a = (double)a.num_factors_req * ((double)a.total_byte_usage / (double)a.length);
double score_b = (double)b.num_factors_req * ((double)b.total_byte_usage / (double)b.length);
return score_a < score_b;
});
}
else {
/* FFT */
std::sort(segments.begin(), segments.end(), [](const segment_info& a, const segment_info& b) {
double score_a = (double)a.num_factors_req * ((double)a.total_byte_usage / (double)a.length);
score_a = score_a / (double)a.length;
double score_b = (double)b.num_factors_req * ((double)b.total_byte_usage / (double)b.length);
score_b = score_b / (double)b.length;
return score_a < score_b;
});
}
size_t segments_to_remove = 0;
size_t segment_cum_len = 0;
for (size_t i = 0; i < segments.size(); i++) {
segments_to_remove++;
segment_cum_len += segments[i].length;
if (segment_cum_len >= bytes_to_remove) {
// update the length so it fits into the size we want
auto new_len = segment_cum_len - bytes_to_remove;
segments[i].length -= (new_len + 1);
break;
}
}
segments.resize(segments_to_remove);
LOG(INFO) << "Selected " << segments_to_remove << " for removal";
LOG(INFO) << "Creating pruned dictionary";
LOG(INFO) << "Sorting segments into offset order";
std::sort(segments.begin(), segments.end(), [](const segment_info& a, const segment_info& b) {
return a.offset < b.offset;
});
{
const sdsl::int_vector_mapper<8, std::ios_base::in> dict(col.file_map[KEY_DICT]);
auto wdict = sdsl::write_out_buffer<8>::create(new_dict_file);
size_t cur_segment = 0;
for (size_t i = 0; i < dict.size() - 2;) {
if (segments[cur_segment].offset == i) {
/* skip the segment */
i += segments[cur_segment].length;
cur_segment++;
}
else {
wdict.push_back(dict[i]);
i++;
}
}
wdict.push_back(0);
LOG(INFO) << "\t"
<< "Pruned dictionary size = " << wdict.size() / (1024 * 1024) << " MiB";
}
col.file_map[KEY_DICT] = new_dict_file;
auto end_total = hrclock::now();
LOG(INFO) << "\n"
<< "\t" << type() + " Total time = " << duration_cast<milliseconds>(end_total - start_total).count() / 1000.0f << " sec";
col.compute_dict_hash();
}