csa_wt<t_wt, t_dens, t_inv_dens, t_sa_sample_strat, t_isa, t_alphabet_strat>::csa_wt(cache_config& config)
{
if (!cache_file_exists(key_trait<alphabet_type::int_width>::KEY_BWT, config)) {
return;
}
{
auto event = memory_monitor::event("construct csa-alpbabet");
int_vector_buffer<alphabet_type::int_width> bwt_buf(cache_file_name(key_trait<alphabet_type::int_width>::KEY_BWT,config));
size_type n = bwt_buf.size();
alphabet_type tmp_alphabet(bwt_buf, n);
m_alphabet.swap(tmp_alphabet);
}
{
auto event = memory_monitor::event("construct wavelet tree");
int_vector_buffer<alphabet_type::int_width> bwt_buf(cache_file_name(key_trait<alphabet_type::int_width>::KEY_BWT,config));
size_type n = bwt_buf.size();
wavelet_tree_type tmp_wt(bwt_buf, n);
m_wavelet_tree.swap(tmp_wt);
}
{
auto event = memory_monitor::event("sample SA");
sa_sample_type tmp_sa_sample(config);
m_sa_sample.swap(tmp_sa_sample);
}
{
auto event = memory_monitor::event("sample ISA");
int_vector_buffer<> sa_buf(cache_file_name(conf::KEY_SA, config));
set_isa_samples<csa_wt>(sa_buf, m_isa_sample);
}
}
void construct_lcp_PHI(cache_config& config)
{
static_assert(t_width == 0 or t_width == 8 , "construct_lcp_PHI: width must be `0` for integer alphabet and `8` for byte alphabet");
typedef int_vector<>::size_type size_type;
typedef int_vector<t_width> text_type;
const char* KEY_TEXT = key_text_trait<t_width>::KEY_TEXT;
int_vector_buffer<> sa_buf(config.file_map[conf::KEY_SA]);
size_type n = sa_buf.size();
assert(n > 0);
if (1 == n) { // Handle special case: Input only the sentinel character.
int_vector<> lcp(1, 0);
store_to_cache(lcp, conf::KEY_LCP, config);
return;
}
// (1) Calculate PHI (stored in array plcp)
int_vector<> plcp(n, 0, sa_buf.width());
for (size_type i=0, sai_1 = 0; i < n; ++i) {
size_type sai = sa_buf[i];
plcp[ sai ] = sai_1;
sai_1 = sai;
}
// (2) Load text from disk
text_type text;
load_from_cache(text, KEY_TEXT, config);
// (3) Calculate permuted LCP array (text order), called PLCP
size_type max_l = 0;
for (size_type i=0, l=0; i < n-1; ++i) {
size_type phii = plcp[i];
while (text[i+l] == text[phii+l]) {
++l;
}
plcp[i] = l;
if (l) {
max_l = std::max(max_l, l);
--l;
}
}
util::clear(text);
uint8_t lcp_width = bits::hi(max_l)+1;
// (4) Transform PLCP into LCP
std::string lcp_file = cache_file_name(conf::KEY_LCP, config);
size_type buffer_size = 1000000; // buffer_size is a multiple of 8!
int_vector_buffer<> lcp_buf(lcp_file, std::ios::out, buffer_size, lcp_width); // open buffer for lcp
lcp_buf[0] = 0;
sa_buf.buffersize(buffer_size);
for (size_type i=1; i < n; ++i) {
size_type sai = sa_buf[i];
lcp_buf[i] = plcp[sai];
}
lcp_buf.close();
register_cache_file(conf::KEY_LCP, config);
}
csa_sada<t_enc_vec, t_dens, t_inv_dens, t_sa_sample_strat, t_isa, t_alphabet_strat>::csa_sada(cache_config& config)
{
create_buffer();
if (!cache_file_exists(key_trait<alphabet_type::int_width>::KEY_BWT, config)) {
return;
}
int_vector_buffer<alphabet_type::int_width> bwt_buf(cache_file_name(key_trait<alphabet_type::int_width>::KEY_BWT,config));
size_type n = bwt_buf.size();
{
auto event = memory_monitor::event("construct csa-alpbabet");
alphabet_type tmp_alphabet(bwt_buf, n);
m_alphabet.swap(tmp_alphabet);
}
int_vector<> cnt_chr(sigma, 0, bits::hi(n)+1);
for (typename alphabet_type::sigma_type i=0; i < sigma; ++i) {
cnt_chr[i] = C[i];
}
// calculate psi
{
auto event = memory_monitor::event("construct PSI");
// TODO: move PSI construct into construct_PSI.hpp
int_vector<> psi(n, 0, bits::hi(n)+1);
for (size_type i=0; i < n; ++i) {
psi[ cnt_chr[ char2comp[bwt_buf[i]] ]++ ] = i;
}
std::string psi_file = cache_file_name(conf::KEY_PSI, config);
if (!store_to_cache(psi, conf::KEY_PSI, config)) {
return;
}
}
{
auto event = memory_monitor::event("encode PSI");
int_vector_buffer<> psi_buf(cache_file_name(conf::KEY_PSI, config));
t_enc_vec tmp_psi(psi_buf);
m_psi.swap(tmp_psi);
}
{
auto event = memory_monitor::event("sample SA");
sa_sample_type tmp_sa_sample(config);
m_sa_sample.swap(tmp_sa_sample);
}
{
auto event = memory_monitor::event("sample ISA");
int_vector_buffer<> sa_buf(cache_file_name(conf::KEY_SA, config));
set_isa_samples<csa_sada>(sa_buf, m_isa_sample);
}
}
void construct_int_bwt(int_vector<> &bwt, const cache_config& config){
int_vector<> text;
tMSS::const_iterator text_entry = config.file_map.find(constants::KEY_TEXT_INT);
if ( config.file_map.end() == text_entry ){ return; }
util::load_from_file(text, text_entry->second.c_str());
tMSS::const_iterator sa_entry = config.file_map.find(constants::KEY_SA);
if ( config.file_map.end() == sa_entry ){ return; }
int_vector_file_buffer<> sa_buf(sa_entry->second.c_str());
util::assign(bwt, int_vector<>(text.size(), 0, text.get_int_width()));
int_vector_size_type to_add[2] = {-1,text.size()-1};
for (int_vector_size_type i=0, r_sum=0, r = 0; r_sum < text.size();) {
for (; i<r_sum+r; ++i) {
bwt[i] = text[ sa_buf[i-r_sum]+to_add[sa_buf[i-r_sum]==0] ];
}
r_sum += r; r = sa_buf.load_next_block();
}
}
//bool construct_bwt(tMSS& file_map, const std::string& dir, const std::string& id) {
void construct_bwt(int_vector<8>& bwt, const cache_config& config){
typedef int_vector<>::size_type size_type;
tMSS::const_iterator key;
key = config.file_map.find(constants::KEY_BWT);
if ( config.file_map.end() == key) { // if bwt is not already registered in file_map
std::string bwt_tmp_file_name = config.dir+"/tmp_bwt_"+util::to_string(util::get_pid())+"_"+util::to_string(util::get_id());
write_R_output("csa", "construct BWT", "begin", 1, 0);
const size_type buffer_size = 1000000;
int_vector<8> text;
util::load_from_file(text, config.file_map.find(constants::KEY_TEXT)->second.c_str() );
size_type n = text.size();
int_vector_file_buffer<> sa_buf(config.file_map.find(constants::KEY_SA)->second.c_str(), buffer_size);
std::ofstream bwt_out_buf(bwt_tmp_file_name.c_str(), std::ios::binary | std::ios::trunc | std::ios::out); // open out file stream
int_vector<8> bwt_buf(buffer_size);
size_type bit_size = n*8;
bwt_out_buf.write((char*) &(bit_size), sizeof(int_vector_size_type)); // write size
size_type wb = 0; // written bytes
size_type to_add[2] = {-1,n-1};
for (size_type i=0, r_sum=0, r = 0; r_sum < n;) {
for (; i<r_sum+r; ++i) {
bwt_buf[i-r_sum] = text[ sa_buf[i-r_sum]+to_add[sa_buf[i-r_sum]==0] ];
}
if (r > 0) {
bwt_out_buf.write((const char*)bwt_buf.data(), r);
wb += r;
}
r_sum += r; r = sa_buf.load_next_block();
}
if (wb%8) {
bwt_out_buf.write("\0\0\0\0\0\0\0\0", 8-wb%8);
}
bwt_out_buf.close();
util::clear(text);
util::load_from_file(bwt, bwt_tmp_file_name.c_str()); // load bwt in memory
std::remove(bwt_tmp_file_name.c_str()); // remove temporary file
write_R_output("csa", "construct BWT", "end", 1, 0);
}else{
util::load_from_file(bwt, key->second.c_str()); // load bwt in memory
}
}
//! Constructor
csa_bitcompressed(cache_config& config) {
std::string text_file = cache_file_name(key_trait<alphabet_type::int_width>::KEY_TEXT,config);
int_vector_buffer<alphabet_type::int_width> text_buf(text_file);
int_vector_buffer<> sa_buf(cache_file_name(conf::KEY_SA,config));
size_type n = text_buf.size();
{
alphabet_type tmp_alphabet(text_buf, n);
m_alphabet.swap(tmp_alphabet);
}
{
sa_sample_type tmp_sample(config);
m_sa.swap(tmp_sample);
}
set_isa_samples<csa_bitcompressed>(sa_buf, m_isa);
if (!store_to_file(m_isa, cache_file_name(conf::KEY_ISA,config), true)) {
throw std::ios_base::failure("#csa_bitcompressed: Cannot store ISA to file system!");
} else {
register_cache_file(conf::KEY_ISA, config);
}
}
doc_list_index_greedy(std::string file_name, sdsl::cache_config& cconfig, uint8_t num_bytes) {
construct(m_csa_full, file_name, cconfig, num_bytes);
const char* KEY_TEXT = key_text_trait<WIDTH>::KEY_TEXT;
std::string text_file = cache_file_name(KEY_TEXT, cconfig);
bit_vector doc_border;
construct_doc_border(text_file,doc_border);
bit_vector::rank_1_type doc_border_rank(&doc_border);
m_doc_cnt = doc_border_rank(doc_border.size());
int_vector_buffer<0> sa_buf(cache_file_name(conf::KEY_SA, cconfig));
{
int_vector<> D;
construct_D_array(sa_buf, doc_border_rank, m_doc_cnt, D);
std::string d_file = cache_file_name("DARRAY", cconfig);
store_to_file(D, d_file);
util::clear(D);
construct(m_wtd, d_file);
sdsl::remove(d_file);
}
}
void construct_lcp_PHI(cache_config& config)
{
typedef int_vector<>::size_type size_type;
typedef int_vector<t_width> text_type;
const char* KEY_TEXT = key_text_trait<t_width>::KEY_TEXT;
int_vector_file_buffer<> sa_buf(config.file_map[constants::KEY_SA]);
size_type n = sa_buf.int_vector_size;
assert(n > 0);
if (1 == n) { // Handle special case: Input only the sentinel character.
int_vector<> lcp(1, 0);
store_to_cache(lcp, constants::KEY_LCP, config);
return;
}
// (1) Calculate PHI (stored in array plcp)
int_vector<> plcp(n, 0, sa_buf.width);
for (size_type i=0, r_sum=0, r=sa_buf.load_next_block(), sai_1 = 0; r_sum < n;) {
for (; i < r_sum+r; ++i) {
size_type sai = sa_buf[i-r_sum];
plcp[ sai ] = sai_1;
sai_1 = sai;
}
r_sum += r; r = sa_buf.load_next_block();
}
// (2) Load text from disk
text_type text;
load_from_cache(text, KEY_TEXT, config);
// (3) Calculate permuted LCP array (text order), called PLCP
size_type max_l = 0;
for (size_type i=0, l=0; i < n-1; ++i) {
size_type phii = plcp[i];
while (text[i+l] == text[phii+l]) {
++l;
}
plcp[i] = l;
if (l) {
max_l = std::max(max_l, l);
--l;
}
}
util::clear(text);
uint8_t lcp_width = bits::hi(max_l)+1;
// (4) Transform PLCP into LCP
std::string lcp_file = cache_file_name(constants::KEY_LCP, config);
osfstream lcp_out_buf(lcp_file, std::ios::binary | std::ios::app | std::ios::out); // open buffer for lcp
size_type bit_size = n*lcp_width;
lcp_out_buf.write((char*) &(bit_size), sizeof(bit_size)); // write size of vector
lcp_out_buf.write((char*) &(lcp_width),sizeof(lcp_width)); // write int_width of vector
size_type wb = 0; // bytes written into lcp int_vector
size_type buffer_size = 1000000; // buffer_size is a multiple of 8!
int_vector<> lcp_buf(buffer_size, 0, lcp_width);
lcp_buf[0] = 0;
sa_buf.reset(buffer_size);
size_type r = 0;// sa_buf.load_next_block();
for (size_type i=1, r_sum=0; r_sum < n;) {
for (; i < r_sum+r; ++i) {
size_type sai = sa_buf[i-r_sum];
lcp_buf[ i-r_sum ] = plcp[sai];
}
if (r > 0) {
size_type cur_wb = (r*lcp_buf.width()+7)/8;
lcp_out_buf.write((const char*)lcp_buf.data(), cur_wb);
wb += cur_wb;
}
r_sum += r; r = sa_buf.load_next_block();
}
if (wb%8) {
lcp_out_buf.write("\0\0\0\0\0\0\0\0", 8-wb%8);
}
lcp_out_buf.close();
register_cache_file(constants::KEY_LCP, config);
}
