void construct_lcp_PHI(cache_config& config)
{
static_assert(t_width == 0 or t_width == 8 , "construct_lcp_PHI: width must be `0` for integer alphabet and `8` for byte alphabet");
typedef int_vector<>::size_type size_type;
typedef int_vector<t_width> text_type;
const char* KEY_TEXT = key_text_trait<t_width>::KEY_TEXT;
int_vector_buffer<> sa_buf(config.file_map[conf::KEY_SA]);
size_type n = sa_buf.size();
assert(n > 0);
if (1 == n) { // Handle special case: Input only the sentinel character.
int_vector<> lcp(1, 0);
store_to_cache(lcp, conf::KEY_LCP, config);
return;
}
// (1) Calculate PHI (stored in array plcp)
int_vector<> plcp(n, 0, sa_buf.width());
for (size_type i=0, sai_1 = 0; i < n; ++i) {
size_type sai = sa_buf[i];
plcp[ sai ] = sai_1;
sai_1 = sai;
}
// (2) Load text from disk
text_type text;
load_from_cache(text, KEY_TEXT, config);
// (3) Calculate permuted LCP array (text order), called PLCP
size_type max_l = 0;
for (size_type i=0, l=0; i < n-1; ++i) {
size_type phii = plcp[i];
while (text[i+l] == text[phii+l]) {
++l;
}
plcp[i] = l;
if (l) {
max_l = std::max(max_l, l);
--l;
}
}
util::clear(text);
uint8_t lcp_width = bits::hi(max_l)+1;
// (4) Transform PLCP into LCP
std::string lcp_file = cache_file_name(conf::KEY_LCP, config);
size_type buffer_size = 1000000; // buffer_size is a multiple of 8!
int_vector_buffer<> lcp_buf(lcp_file, std::ios::out, buffer_size, lcp_width); // open buffer for lcp
lcp_buf[0] = 0;
sa_buf.buffersize(buffer_size);
for (size_type i=1; i < n; ++i) {
size_type sai = sa_buf[i];
lcp_buf[i] = plcp[sai];
}
lcp_buf.close();
register_cache_file(conf::KEY_LCP, config);
}
void construct_sa(cache_config& config)
{
static_assert(t_width == 0 or t_width == 8 , "construct_sa: width must be `0` for integer alphabet and `8` for byte alphabet");
const char* KEY_TEXT = key_text_trait<t_width>::KEY_TEXT;
if (t_width == 8) {
typedef int_vector<t_width> text_type;
text_type text;
load_from_cache(text, KEY_TEXT, config);
// call divsufsort
int_vector<> sa(text.size(), 0, bits::hi(text.size())+1);
algorithm::calculate_sa((const unsigned char*)text.data(), text.size(), sa);
store_to_cache(sa, conf::KEY_SA, config);
} else if (t_width == 0) {
// call qsufsort
int_vector<> sa;
sdsl::qsufsort::construct_sa(sa, config.file_map[KEY_TEXT].c_str(), 0);
store_to_cache(sa, conf::KEY_SA, config);
} else {
std::cerr << "Unknown alphabet type" << std::endl;
}
}
void construct(t_index& idx, const std::string& file, cache_config& config, uint8_t num_bytes, lcp_tag)
{
auto event = memory_monitor::event("construct compressed LCP");
const char* KEY_TEXT = key_text_trait<t_width>::KEY_TEXT;
typedef int_vector<t_width> text_type;
{
// (2) check, if the longest common prefix array is cached
auto event = memory_monitor::event("LCP");
if (!cache_file_exists(conf::KEY_LCP, config)) {
{
auto event = memory_monitor::event("parse input text");
// (1) check, if the text is cached
if (!cache_file_exists(KEY_TEXT, config)) {
text_type text;
load_vector_from_file(text, file, num_bytes);
if (contains_no_zero_symbol(text, file)) {
append_zero_symbol(text);
store_to_cache(text,KEY_TEXT, config);
}
}
register_cache_file(KEY_TEXT, config);
}
{
// (2) check, if the suffix array is cached
auto event = memory_monitor::event("SA");
if (!cache_file_exists(conf::KEY_SA, config)) {
construct_sa<t_width>(config);
}
register_cache_file(conf::KEY_SA, config);
}
if (t_width==8) {
construct_lcp_semi_extern_PHI(config);
} else {
construct_lcp_PHI<t_width>(config);
}
}
register_cache_file(conf::KEY_LCP, config);
}
{
auto event = memory_monitor::event("compressed LCP");
t_index tmp(config);
tmp.swap(idx);
}
if (config.delete_files) {
auto event = memory_monitor::event("delete temporary files");
util::delete_all_files(config.file_map);
}
}
csa_sada<t_enc_vec, t_dens, t_inv_dens, t_sa_sample_strat, t_isa, t_alphabet_strat>::csa_sada(cache_config& config)
{
create_buffer();
if (!cache_file_exists(key_trait<alphabet_type::int_width>::KEY_BWT, config)) {
return;
}
int_vector_buffer<alphabet_type::int_width> bwt_buf(cache_file_name(key_trait<alphabet_type::int_width>::KEY_BWT,config));
size_type n = bwt_buf.size();
{
auto event = memory_monitor::event("construct csa-alpbabet");
alphabet_type tmp_alphabet(bwt_buf, n);
m_alphabet.swap(tmp_alphabet);
}
int_vector<> cnt_chr(sigma, 0, bits::hi(n)+1);
for (typename alphabet_type::sigma_type i=0; i < sigma; ++i) {
cnt_chr[i] = C[i];
}
// calculate psi
{
auto event = memory_monitor::event("construct PSI");
// TODO: move PSI construct into construct_PSI.hpp
int_vector<> psi(n, 0, bits::hi(n)+1);
for (size_type i=0; i < n; ++i) {
psi[ cnt_chr[ char2comp[bwt_buf[i]] ]++ ] = i;
}
std::string psi_file = cache_file_name(conf::KEY_PSI, config);
if (!store_to_cache(psi, conf::KEY_PSI, config)) {
return;
}
}
{
auto event = memory_monitor::event("encode PSI");
int_vector_buffer<> psi_buf(cache_file_name(conf::KEY_PSI, config));
t_enc_vec tmp_psi(psi_buf);
m_psi.swap(tmp_psi);
}
{
auto event = memory_monitor::event("sample SA");
sa_sample_type tmp_sa_sample(config);
m_sa_sample.swap(tmp_sa_sample);
}
{
auto event = memory_monitor::event("sample ISA");
isa_sample_type isa_s(config, &m_sa_sample);
util::swap_support(m_isa_sample, isa_s, &m_sa_sample, (const sa_sample_type*)nullptr);
}
}
void construct(t_index& idx, const std::string& file, cache_config& config, uint8_t num_bytes, csa_tag)
{
auto event = memory_monitor::event("construct CSA");
const char* KEY_TEXT = key_text_trait<t_index::alphabet_category::WIDTH>::KEY_TEXT;
const char* KEY_BWT = key_bwt_trait<t_index::alphabet_category::WIDTH>::KEY_BWT;
typedef int_vector<t_index::alphabet_category::WIDTH> text_type;
{
auto event = memory_monitor::event("parse input text");
// (1) check, if the text is cached
if (!cache_file_exists(KEY_TEXT, config)) {
text_type text;
load_vector_from_file(text, file, num_bytes);
if (contains_no_zero_symbol(text, file)) {
append_zero_symbol(text);
store_to_cache(text,KEY_TEXT, config);
}
}
register_cache_file(KEY_TEXT, config);
}
{
// (2) check, if the suffix array is cached
auto event = memory_monitor::event("SA");
if (!cache_file_exists(conf::KEY_SA, config)) {
construct_sa<t_index::alphabet_category::WIDTH>(config);
}
register_cache_file(conf::KEY_SA, config);
}
{
// (3) construct BWT
auto event = memory_monitor::event("BWT");
if (!cache_file_exists(KEY_BWT, config)) {
construct_bwt<t_index::alphabet_category::WIDTH>(config);
}
register_cache_file(KEY_BWT, config);
}
{
// (4) use BWT to construct the CSA
auto event = memory_monitor::event("construct CSA");
t_index tmp(config);
idx.swap(tmp);
}
if (config.delete_files) {
auto event = memory_monitor::event("delete temporary files");
util::delete_all_files(config.file_map);
}
}
void construct_lcp_kasai(cache_config& config)
{
int_vector<> lcp;
typedef int_vector<>::size_type size_type;
construct_isa(config);
{
int_vector<t_width> text;
if (!load_from_cache(text, key_text_trait<t_width>::KEY_TEXT, config)) {
return;
}
int_vector_file_buffer<> isa_buf(config.file_map[constants::KEY_ISA], 1000000); // init isa file_buffer
int_vector<> sa;
if (!load_from_cache(sa, constants::KEY_SA, config)) {
return;
}
// use Kasai algorithm to compute the lcp values
for (size_type i=0,j=0,sa_1=0,l=0, r_sum=0, r=isa_buf.load_next_block(), n=isa_buf.int_vector_size; r_sum < n;) {
for (; i < r_sum+r; ++i) {
sa_1 = isa_buf[i-r_sum]; // = isa[i]
if (sa_1) {
j = sa[sa_1-1];
if (l) --l;
assert(i!=j);
while (text[i+l]==text[j+l]) { // i+l < n and j+l < n are not necessary, since text[n]=0 and text[i]!=0 (i<n) and i!=j
++l;
}
sa[ sa_1-1 ] = l; //overwrite sa array with lcp values
} else {
l = 0;
sa[ n-1 ] = 0;
}
}
r_sum += r;
r = isa_buf.load_next_block();
}
for (size_type i=sa.size(); i>1; --i) {
sa[i-1] = sa[i-2];
}
sa[0] = 0;
lcp.swap(sa);
}
store_to_cache(lcp, constants::KEY_LCP, config);
}
void construct(t_index& idx, const std::string& file, cache_config& config, uint8_t num_bytes, cst_tag)
{
auto event = memory_monitor::event("construct CST");
const char* KEY_TEXT = key_text_trait<t_index::alphabet_category::WIDTH>::KEY_TEXT;
const char* KEY_BWT = key_bwt_trait<t_index::alphabet_category::WIDTH>::KEY_BWT;
csa_tag csa_t;
{
// (1) check, if the compressed suffix array is cached
typename t_index::csa_type csa;
if (!cache_file_exists(std::string(conf::KEY_CSA)+"_"+util::class_to_hash(csa), config)) {
cache_config csa_config(false, config.dir, config.id, config.file_map);
construct(csa, file, csa_config, num_bytes, csa_t);
auto event = memory_monitor::event("store CSA");
config.file_map = csa_config.file_map;
store_to_cache(csa,std::string(conf::KEY_CSA)+"_"+util::class_to_hash(csa), config);
}
register_cache_file(std::string(conf::KEY_CSA)+"_"+util::class_to_hash(csa), config);
}
{
// (2) check, if the longest common prefix array is cached
auto event = memory_monitor::event("LCP");
register_cache_file(KEY_TEXT, config);
register_cache_file(KEY_BWT, config);
register_cache_file(conf::KEY_SA, config);
if (!cache_file_exists(conf::KEY_LCP, config)) {
if (t_index::alphabet_category::WIDTH==8) {
construct_lcp_semi_extern_PHI(config);
} else {
construct_lcp_PHI<t_index::alphabet_category::WIDTH>(config);
}
}
register_cache_file(conf::KEY_LCP, config);
}
{
auto event = memory_monitor::event("CST");
t_index tmp(config);
tmp.swap(idx);
}
if (config.delete_files) {
auto event = memory_monitor::event("delete temporary files");
util::delete_all_files(config.file_map);
}
}
void construct_lcp_kasai(cache_config& config)
{
static_assert(t_width == 0 or t_width == 8 , "construct_lcp_kasai: width must be `0` for integer alphabet and `8` for byte alphabet");
int_vector<> lcp;
typedef int_vector<>::size_type size_type;
construct_isa(config);
{
int_vector<t_width> text;
if (!load_from_cache(text, key_text_trait<t_width>::KEY_TEXT, config)) {
return;
}
int_vector_buffer<> isa_buf(config.file_map[conf::KEY_ISA], std::ios::in, 1000000); // init isa file_buffer
int_vector<> sa;
if (!load_from_cache(sa, conf::KEY_SA, config)) {
return;
}
// use Kasai algorithm to compute the lcp values
for (size_type i=0,j=0,sa_1=0,l=0, n=isa_buf.size(); i < n; ++i) {
sa_1 = isa_buf[i]; // = isa[i]
if (sa_1) {
j = sa[sa_1-1];
if (l) --l;
assert(i!=j);
while (text[i+l]==text[j+l]) { // i+l < n and j+l < n are not necessary, since text[n]=0 and text[i]!=0 (i<n) and i!=j
++l;
}
sa[ sa_1-1 ] = l; //overwrite sa array with lcp values
} else {
l = 0;
sa[ n-1 ] = 0;
}
}
for (size_type i=sa.size(); i>1; --i) {
sa[i-1] = sa[i-2];
}
sa[0] = 0;
lcp.swap(sa);
}
store_to_cache(lcp, conf::KEY_LCP, config);
}
void construct_lcp_PHI(cache_config& config)
{
typedef int_vector<>::size_type size_type;
typedef int_vector<t_width> text_type;
const char* KEY_TEXT = key_text_trait<t_width>::KEY_TEXT;
int_vector_file_buffer<> sa_buf(config.file_map[constants::KEY_SA]);
size_type n = sa_buf.int_vector_size;
assert(n > 0);
if (1 == n) { // Handle special case: Input only the sentinel character.
int_vector<> lcp(1, 0);
store_to_cache(lcp, constants::KEY_LCP, config);
return;
}
// (1) Calculate PHI (stored in array plcp)
int_vector<> plcp(n, 0, sa_buf.width);
for (size_type i=0, r_sum=0, r=sa_buf.load_next_block(), sai_1 = 0; r_sum < n;) {
for (; i < r_sum+r; ++i) {
size_type sai = sa_buf[i-r_sum];
plcp[ sai ] = sai_1;
sai_1 = sai;
}
r_sum += r; r = sa_buf.load_next_block();
}
// (2) Load text from disk
text_type text;
load_from_cache(text, KEY_TEXT, config);
// (3) Calculate permuted LCP array (text order), called PLCP
size_type max_l = 0;
for (size_type i=0, l=0; i < n-1; ++i) {
size_type phii = plcp[i];
while (text[i+l] == text[phii+l]) {
++l;
}
plcp[i] = l;
if (l) {
max_l = std::max(max_l, l);
--l;
}
}
util::clear(text);
uint8_t lcp_width = bits::hi(max_l)+1;
// (4) Transform PLCP into LCP
std::string lcp_file = cache_file_name(constants::KEY_LCP, config);
osfstream lcp_out_buf(lcp_file, std::ios::binary | std::ios::app | std::ios::out); // open buffer for lcp
size_type bit_size = n*lcp_width;
lcp_out_buf.write((char*) &(bit_size), sizeof(bit_size)); // write size of vector
lcp_out_buf.write((char*) &(lcp_width),sizeof(lcp_width)); // write int_width of vector
size_type wb = 0; // bytes written into lcp int_vector
size_type buffer_size = 1000000; // buffer_size is a multiple of 8!
int_vector<> lcp_buf(buffer_size, 0, lcp_width);
lcp_buf[0] = 0;
sa_buf.reset(buffer_size);
size_type r = 0;// sa_buf.load_next_block();
for (size_type i=1, r_sum=0; r_sum < n;) {
for (; i < r_sum+r; ++i) {
size_type sai = sa_buf[i-r_sum];
lcp_buf[ i-r_sum ] = plcp[sai];
}
if (r > 0) {
size_type cur_wb = (r*lcp_buf.width()+7)/8;
lcp_out_buf.write((const char*)lcp_buf.data(), cur_wb);
wb += cur_wb;
}
r_sum += r; r = sa_buf.load_next_block();
}
if (wb%8) {
lcp_out_buf.write("\0\0\0\0\0\0\0\0", 8-wb%8);
}
lcp_out_buf.close();
register_cache_file(constants::KEY_LCP, config);
}
errno_t name2ip( in_addr_t *out, const char *name, int flags )
{
int ia, ib, ic, id;
if( 4 == sscanf( name, "%d.%d.%d.%d", &ia, &ib, &ic, &id ) )
{
// No resolver required, ip4 addr given
ipv4_addr iaddr = IPV4_DOTADDR_TO_ADDR( ia, ib, ic, id);
*out = htonl( iaddr );
SHOW_FLOW( 2, "parsed %s to %s", name, inet_ntoa(* (struct in_addr*)out) );
return 0;
}
if(!inited)
return ENXIO;
int tries = 20;
if(flags & RESOLVER_FLAG_NORETRY)
tries = 1;
ipv4_addr result;
//ipv4_addr next_servers[MAX_DNS_SERVERS];
SHOW_FLOW( 1, "request '%s'", name );
ipv4_addr * sptr = servers;
int sleft = MAX_DNS_SERVERS;
if( !(flags & RESOLVER_FLAG_NORCACHE) )
if( lookup_cache( out, name ) == 0 )
{
SHOW_FLOW0( 1, "got from cache");
return 0;
}
// On OS stop don't produce network traffic
if( (flags & RESOLVER_FLAG_NOWAIT) || phantom_stop_level )
return ESRCH;
while(tries--)
{
ipv4_addr server = *sptr++;
if(sleft-- <= 0 || server == 0)
{
SHOW_ERROR0( 1, "No more places to look in, give up\n");
return ENOENT;
}
SHOW_FLOW( 2, "look in %s", inet_ntoa(* (struct in_addr*)&server) );
errno_t res = dns_request( (const unsigned char *)name, server, &result );
if( res == 0 )//|| result != 0 )
{
SHOW_FLOW( 1, "answer is %s", inet_ntoa(* (struct in_addr*)&result) );
*out = result;
if( !(flags & RESOLVER_FLAG_NOWCACHE) )
store_to_cache( result, name );
return 0;
}
}
return ENOENT;
}
