int main(int argc, char** argv)
{
::testing::InitGoogleTest(&argc, argv);
if (argc < 3) {
// LCOV_EXCL_START
cout << "Usage: " << argv[0] << " test_file temp_file [in-memory]" << endl;
cout << " (1) Generates a WT out of test_file; stores it in temp_file." << endl;
cout << " If `in-memory` is specified, the in-memory construction is tested." << endl;
cout << " (2) Performs tests." << endl;
cout << " (3) Deletes temp_file." << endl;
return 1;
// LCOV_EXCL_STOP
}
test_file = argv[1];
temp_file = argv[2];
in_memory = argc > 3;
if (in_memory) {
int_vector<> data;
load_from_file(data, test_file);
test_file = ram_file_name(test_file);
store_to_file(data, test_file);
temp_file = ram_file_name(temp_file);
}
return RUN_ALL_TESTS();
}
void construct_im(t_index& idx, t_data data, uint8_t num_bytes=0)
{
std::string tmp_file = ram_file_name(util::to_string(util::pid())+"_"+util::to_string(util::id()));
store_to_file(data, tmp_file);
construct(idx, tmp_file, num_bytes);
ram_fs::remove(tmp_file);
}
int main (int argc, char *argv[]) {
char * in_name, * in_ext, * in_filename, * out_filename;
if (argc != 2) {
printf("Usage %s <grammar_filename>\n", argv[0]);
return 1;
}
in_filename = argv[1];
infile = fopen(in_filename, "r");
if (infile == NULL) {
fprintf(stderr, "ERROR: Could not open file \"%s\"\n", in_filename);
return 1;
}
in_ext = get_filename_extension(in_filename);
in_name = get_filename(in_filename);
out_filename = store_to_file(in_name, in_ext);
printf("###\tRecursive Generator\t###\n");
outfile = fopen(out_filename, "w");
if (outfile == NULL) {
fprintf(stderr, "ERROR:Could not open file \"%s\"\n", outfile);
fclose(infile);
return 1;
}
printf("Parsing file %s...\n", in_filename);
parse_file(infile);
//print_grammar();
printf("Output filename: %s\n", out_filename);
printf("Generating sentence... \t");
generate_sentence(START_ELEMENT);
printf("Done!\n");
fprintf(outfile, "\n");
fclose(infile);
fclose(outfile);
printf("Quiting...\n");
destroy_grammar();
return 0;
}
void build_or_load_csa(t_csa& csa,std::string csa_file,std::string text_file,int type,std::string test_case)
{
if(file_exists(csa_file)) {
sdsl::load_from_file(csa,csa_file);
} else {
std::cout << "BUILD INDEX ("<<IDX_NAME<<") " << test_case << " input type = " << type << std::endl;
std::string tmp_dir = std::string(TMP_DIR);
std::string id = test_case;
sdsl::cache_config cfg;
cfg.delete_files = false;
cfg.dir = tmp_dir;
cfg.id = id;
construct(csa, text_file, cfg, type);
store_to_file(csa, csa_file);
}
}
void construct(t_index& idx, const std::string& file, cache_config& config, uint8_t num_bytes, wt_tag)
{
auto event = memory_monitor::event("construct wavelet tree");
int_vector<t_index::alphabet_category::WIDTH> text;
load_vector_from_file(text, file, num_bytes);
std::string tmp_key = util::to_string(util::pid())+"_"+util::to_string(util::id());
std::string tmp_file_name = cache_file_name(tmp_key, config);
store_to_file(text, tmp_file_name);
util::clear(text);
{
int_vector_buffer<t_index::alphabet_category::WIDTH> text_buf(tmp_file_name);
t_index tmp(text_buf, text_buf.size());
idx.swap(tmp);
}
sdsl::remove(tmp_file_name);
}
//! Constructor
csa_bitcompressed(cache_config& config) {
std::string text_file = cache_file_name(key_trait<alphabet_type::int_width>::KEY_TEXT,config);
int_vector_buffer<alphabet_type::int_width> text_buf(text_file);
int_vector_buffer<> sa_buf(cache_file_name(conf::KEY_SA,config));
size_type n = text_buf.size();
{
alphabet_type tmp_alphabet(text_buf, n);
m_alphabet.swap(tmp_alphabet);
}
{
sa_sample_type tmp_sample(config);
m_sa.swap(tmp_sample);
}
set_isa_samples<csa_bitcompressed>(sa_buf, m_isa);
if (!store_to_file(m_isa, cache_file_name(conf::KEY_ISA,config), true)) {
throw std::ios_base::failure("#csa_bitcompressed: Cannot store ISA to file system!");
} else {
register_cache_file(conf::KEY_ISA, config);
}
}
doc_list_index_greedy(std::string file_name, sdsl::cache_config& cconfig, uint8_t num_bytes) {
construct(m_csa_full, file_name, cconfig, num_bytes);
const char* KEY_TEXT = key_text_trait<WIDTH>::KEY_TEXT;
std::string text_file = cache_file_name(KEY_TEXT, cconfig);
bit_vector doc_border;
construct_doc_border(text_file,doc_border);
bit_vector::rank_1_type doc_border_rank(&doc_border);
m_doc_cnt = doc_border_rank(doc_border.size());
int_vector_buffer<0> sa_buf(cache_file_name(conf::KEY_SA, cconfig));
{
int_vector<> D;
construct_D_array(sa_buf, doc_border_rank, m_doc_cnt, D);
std::string d_file = cache_file_name("DARRAY", cconfig);
store_to_file(D, d_file);
util::clear(D);
construct(m_wtd, d_file);
sdsl::remove(d_file);
}
}
int main(int argc, char** argv)
{
::testing::InitGoogleTest(&argc, argv);
if (argc < 4) {
// LCOV_EXCL_START
cout << "Usage: " << argv[0] << " test_file num_bytes temp_file tmp_dir" << endl;
cout << " (1) Generates a CSA out of test_file; stores it in temp_file." << endl;
cout << " Temporary files (SA/BWT/TEXT) are stored in tmp_dir." << endl;
cout << " num_bytes specifies who many bytes make a symbol in the"<< endl;
cout << " input sequence" << endl;
cout << " If `in-memory` is specified, the in-memory construction is tested." << endl;
cout << " (2) Performs tests." << endl;
cout << " (3) Deletes temp_file." << endl;
return 1;
// LCOV_EXCL_STOP
}
test_file = argv[1];
num_bytes = atoi(argv[2]);
temp_file = argv[3];
temp_dir = argv[4];
in_memory = argc > 5;
if (in_memory) {
temp_dir = "@";
int_vector<> data;
load_vector_from_file(data, test_file, num_bytes);
test_file = ram_file_name(test_file);
switch (num_bytes) {
case 0: store_to_file(data, test_file); break;
case 1: store_to_plain_array<uint8_t>(data, test_file); break;
case 2: store_to_plain_array<uint16_t>(data, test_file); break;
case 3: store_to_plain_array<uint32_t>(data, test_file); break;
case 4: store_to_plain_array<uint64_t>(data, test_file); break;
}
temp_file = ram_file_name(temp_file);
}
return RUN_ALL_TESTS();
}
//! Constructor
lcp_wt(cache_config& config, std::string other_key="") {
std::string temp_file = tmp_file(config, "_lcp_sml");
std::string lcp_key = conf::KEY_LCP;
if ("" != other_key) {
lcp_key = other_key;
}
int_vector_buffer<> lcp_buf(cache_file_name(lcp_key, config));
size_type l=0, max_l=0, big_sum=0, n = lcp_buf.size();
{
int_vector<8> small_lcp = int_vector<8>(n);
for (size_type i=0; i < n; ++i) {
if ((l=lcp_buf[i]) < 255) {
small_lcp[i] = l;
} else {
small_lcp[i] = 255;
if (l > max_l) max_l = l;
++big_sum;
}
}
store_to_file(small_lcp, temp_file);
}
{
int_vector_buffer<8> lcp_sml_buf(temp_file);
small_lcp_type tmp(lcp_sml_buf, lcp_sml_buf.size());
m_small_lcp.swap(tmp);
}
sdsl::remove(temp_file);
m_big_lcp = int_vector<>(big_sum, 0, bits::hi(max_l)+1);
{
for (size_type i=0, ii=0; i < n; ++i) {
if (lcp_buf[i] >= 255) {
m_big_lcp[ ii++ ] = lcp_buf[i];
}
}
}
}
/*! \param text_buf A int_vector_buffer to the original text.
* \param size The length of the prefix of the text, for which the wavelet tree should be build.
*/
wt_int_rlmn(int_vector_buffer<>& text_buf, size_type size):m_size(size), sigma(m_wt.sigma) {
if (0 == text_buf.size() or 0 == size)
return;
int_vector<> condensed_bwt;
{
// scope for bl and bf
bit_vector bl = bit_vector(size, 0);
std::map<uint64_t, uint64_t> C;
uint64_t last_c = 0;
size_type runs = 0;
for (size_type i=0; i < size; ++i) {
uint64_t c = text_buf[i];
if (last_c != c or i==0) {
bl[i] = 1;
++runs;
}
++C[c];
last_c = c;
}
uint64_t max_symbol = (--C.end())->first;
m_C = int_vector<>(max_symbol+1, 0, bits::hi(size)+1);
for (size_type i=0, prefix_sum=0; i<=max_symbol; ++i) {
m_C[i] = prefix_sum;
prefix_sum += C[i];
}
int_vector<> lf_map = m_C;
bit_vector bf = bit_vector(size+1, 0);
bf[size] = 1; // initialize last element
condensed_bwt = int_vector<>(runs, 0, bits::hi(max_symbol)+1);
runs = 0;
for (size_type i=0; i < size; ++i) {
uint64_t c = text_buf[i];
if (bl[i]) {
bf[lf_map[c]] = 1;
condensed_bwt[runs++] = c;
}
++lf_map[c];
}
{
// TODO: remove absolute file name
std::string temp_file = "tmp_wt_int_rlmn_" + util::to_string(util::pid()) + "_" + util::to_string(util::id());
store_to_file(condensed_bwt, temp_file);
util::clear(condensed_bwt);
int_vector_buffer<> temp_bwt_buf(temp_file);
m_wt = std::move(wt_type(temp_bwt_buf, temp_bwt_buf.size()));
temp_bwt_buf.close(true);
}
m_bl = std::move(bit_vector_type(bl));
m_bf = std::move(bit_vector_type(bf));
}
util::init_support(m_bl_rank, &m_bl);
util::init_support(m_bf_rank, &m_bf);
util::init_support(m_bf_select, &m_bf);
util::init_support(m_bl_select, &m_bl);
m_C_bf_rank = int_vector<>(m_C.size(), 0, bits::hi(size)+1);
for (size_type i=0; i<m_C.size(); ++i) {
m_C_bf_rank[i] = m_bf_rank(m_C[i]);
}
}
void *file_read_thread_run(void *param)
{
FILE *file = fopen(input_file, "r");
int offset = 0;
int i = 0;
printf("DEBUG:\tFile read thread has started\n");
if (file == NULL) {
printf("ERROR:\tCould not open file %s: %s\n", input_file,
strerror(errno));
return NULL;
}
while (!stop_file_read_run) {
sleep(1);
if (!feof(file)) {
fread(fr_in, sizeof(float), MAX_FRAMESIZE, file);
} else {
printf("DEBUG:\tReached end of file\n");
break;
}
if (ferror(file)) {
printf("ERROR:\tAn error occurred during file "
"read\n");
break;
}
set_time_stamp();
frerr = applyPowerQuality(
frpPQInst,
fr_in,
&frpqResult,
NULL,
timestamp_buffer + offset,
FRAMES_PER_BLOCK);
offset = i * 32;
if (i == 4)
i = 0;
else
i++;
/* exit processing on error */
if(frerr != PQ_NO_ERROR) {
printf("TODO:\t\tError applying PQ-Lib\n\t\t\t");
print_PQ_Error(frerr);
stop_powqutyd_file_read();
break;
}
/* EN50160 event detected */
if (frpqResult.nmbPqEvents > 0)
handle_event(frpqResult, config);
if(frpqResult.HarmonicsExist) {
store_to_file(frpqResult, config);
#ifdef MQTT
publish_measurements(frpqResult);
#endif
}
}
fclose(file);
printf("DEBUG:\tFile read thread has ended\n");
stop_powqutyd_file_read();
return NULL;
}
int main(int argc, char *argv[]) {
ParamProgram *par = new ParamProgram();
char fileName[300];
if(argc != 2){
cout << "ERRORR !! " << endl;
cout << "buildDL_LZ's usage requires the Properties File as parameter !! " << endl;
cout << "Example for the file 'config.txt' in the same directory: ./buildDL_LZ config.txt" << endl;
exit(1);
}
par->configFile = string(argv[1]);
cout << "Congif file: " << par->configFile << endl;
ConfigFile cf(par->configFile);
TRACE = cf.Value("GLOBALS","TRACE");
TEST = cf.Value("GLOBALS","TEST");
N_REP = cf.Value("GLOBALS","N_REP");
strcpy(par->inputFile, ((string)(cf.Value("DL","inputFile"))).c_str());
par->filesInList = cf.Value("DL","filesInList");
par->boundSymbol = cf.Value("DL","boundSymbol");
par->cutDoc = cf.Value("DL","cutDoc");
par->lowercase = cf.Value("DL","lowercase");
par->levRMQ_range = cf.Value("DL","levRMQ");
strcpy(par->dirStore, ((string)(cf.Value("DL","dirStore"))).c_str());
cout << "buildDL_LZ config parameters..." << endl;
cout << "Input File : " << par->inputFile << endl;
cout << "File in list : " << par->filesInList << endl;
cout << "Boundary Symbol(code) : " << (int)par->boundSymbol << endl;
cout << "Cut Doc. Symbol(code) : " << (int)par->cutDoc << endl;
cout << "Lowercase : " << par->lowercase << endl;
cout << "Range levels with RMQ : " << par->levRMQ_range << endl;
cout << "Store Folder : " << par->dirStore << endl;
LZDocList64::TRACE = TRACE;
LZDocList64::TEST = TEST;
par->index = new LZDocList64(par->levRMQ_range, par->inputFile, par->filesInList, par->cutDoc, par->lowercase, par->dirStore, par->boundSymbol, false);
par->n = par->index->n;
par->EndDocs = par->index->EndDocs;
cout << "____________________________________________________" << endl;
cout << "*** Index size " << par->index->sizeDS << " bytes = " << (float)par->index->sizeDS*8.0/(float)par->n << " bpc" << endl;
cout << "*** SizeUpRange = " << par->index->sizeUpRange << " bytes = " << (float)par->index->sizeUpRange*8.0/(float)par->n << " bpc" << endl;
cout << "====================================================" << endl;
par->index->saveDS(true);
{
// create the FMI to execute test...
cout << "____________________________________________________" << endl;
cout << " Make the FMI ..." << endl;
strcpy(fileName, "");
strcpy(fileName, par->inputFile);
cout << " Reading... " << fileName << endl;
strcat(fileName, "_copy.txt");
construct(par->index->fmi, fileName, 1); // generate index
cout << " ** FMI size " << size_in_bytes(par->index->fmi) << " bytes = " << (float)size_in_bytes(par->index->fmi)/(float)par->n << "|T|" << endl;
cout << " ** FMI length " << par->index->fmi.size() << endl;
if (par->index->fmi.size() != par->n){
cout << "ERROR. FMI length != n = " << par->n << endl;
exit(1);
}
strcpy(fileName, "");
strcpy(fileName, par->dirStore);
strcat(fileName, "fmi.test");
store_to_file(par->index->fmi, fileName);
}
if (TEST){
strcpy(fileName, "");
strcpy(fileName, par->dirStore);
strcat(fileName, "EndDocs.test");
ofstream os2 (fileName, ios::binary);
os2.write((const char*)par->EndDocs, par->index->nDocs*sizeof(ulong)); // save LbRev[]
if(TRACE) cout << " .- EndDocs[] " << par->index->nDocs*sizeof(ulong) << " Bytes" << endl;
os2.close();
par->index->sep_rrr = rrr_vector<127>(par->index->sepPhrase_b);
par->index->sep_rank = rrr_vector<127>::rank_1_type(&par->index->sep_rrr);
std::cout << " sep_rrr uses " << size_in_bytes(par->index->sep_rrr) << " Bytes" << endl;
strcpy(fileName, "");
strcpy(fileName, par->dirStore);
strcat(fileName, "sep_rrr.test");
store_to_file(par->index->sep_rrr, fileName);
strcpy(fileName, "");
strcpy(fileName, par->dirStore);
strcat(fileName, "sep_rank.test");
store_to_file(par->index->sep_rank, fileName);
strcpy(fileName, "");
strcpy(fileName, par->dirStore);
strcat(fileName, "EndDocs.test");
ofstream os3 (fileName, ios::binary);
os3.write((const char*)par->EndDocs, par->index->nDocs*sizeof(ulong)); // save LbRev[]
if(TRACE) cout << " .- EndDocs[] " << par->index->nDocs*sizeof(ulong) << " Bytes" << endl;
os3.close();
// load Sequence...
strcpy(fileName, "");
strcpy(fileName, par->dirStore);
strcat(fileName, "sequence.test");
ifstream is(fileName, ios::binary);
par->seq = new uchar[par->n];
is.read((char*)par->seq, par->n*sizeof(uchar));
is.close();
cout << "Running test searchPattern.." << endl;
testSearchPatt_DL_CSA_LZ(par);
cout << "Test searchPattern OK !!" << endl;
}
par->index->~LZDocList64();
cout << "$$$$$$$$$$$$$$$$$$$$$" << endl;
return 0;
}
