void init(uint64_t offset = 0, uint64_t * psymoffset = 0) { if ( ((idda.kvec.size()!=0) && (idda.kvec[idda.kvec.size()-1] != 0)) ) { if ( offset >= idda.kvec[idda.kvec.size()-1] ) { fileptr = idda.data.size(); blockptr = 0; } else { ::libmaus::huffman::FileBlockOffset const FBO = idda.findKBlock(offset); fileptr = FBO.fileptr; blockptr = FBO.blockptr; offset = FBO.offset; /* open file and seek to block */ openNewFile(); /* decode block in question */ bool const blockok = decodeBlock(); assert ( blockok ); assert ( static_cast<int64_t>(offset) < (pe-pc) ); /* symbol offset of block (sum over elements of previous blocks) */ uint64_t symoffset = idda.data[FBO.fileptr].getValueCnt(FBO.blockptr); /* decode symbols up to offset in block */ for ( uint64_t i = 0; i < offset; ++i ) symoffset += decode(); /* store prefix sum if pointer is given */ if ( psymoffset ) *psymoffset = symoffset; } } }
void SplitJobOutputStream::writeRecord(Record const & rec) { JobOutputStream::writeRecord(rec); numberOfFiles++; totalSize += atof(rec[SIZE].c_str()); if(numberOfFiles >= Constants::MAX_ENTRIES_IN_JOB_FILE || totalSize >= Constants::MAX_JOB_FILE_SIZE) { openNewFile(); } }
/* decode next block */ bool decodeBlock() { /* open new file if necessary */ bool changedfile = false; while ( fileptr < idda.data.size() && blockptr == idda.data[fileptr].numentries ) { fileptr++; blockptr = 0; changedfile = true; } if ( fileptr == idda.data.size() ) return false; if ( changedfile ) openNewFile(); /* align to word boundary */ GD->flush(); /* read block size */ uint64_t const blocksize = GD->decodeWord(32); /* increase size of memory buffer if necessary */ if ( blocksize > decodebuf.size() ) decodebuf.resize(blocksize); /* set buffer pointers */ pa = decodebuf.begin(); pc = pa; pe = pa + blocksize; /* decode block */ for ( uint64_t i = 0; i < blocksize; ++i ) decodebuf[i] = GD->decode(); /* increment block pointer */ blockptr++; return true; }
void jwd::writeData(){ FILE *file; char line[128]; time_t time_in_sec; struct tm *date; int day_min, i; std::string data_set, template_filename; if (fdata <= 0) return; // Analyse time stamp, if new day a new file needs to generated if (filename != getDataFilename()) { openNewFile(); } // Read the template header if (datafileTemplate.length() == 0) throw std::invalid_argument("No template file given"); template_filename = configDir + datafileTemplate; printf("Reading data from template file %s\n", template_filename.c_str()); // open template file file = fopen(template_filename.c_str(), "r"); // throw away 2 header lines for dd* data file if (sensorGroup == "dd") { for (i = 0; i < 2; i++) { fgets(line, 128, file); } } // rd* data files don't have a header // get minute of day time(&time_in_sec); date = gmtime((const time_t *) &time_in_sec); day_min = date->tm_hour * 60 + date->tm_min; // throw away last (day_min - 1) data sets for (i = 0; i < day_min; i++) { fgets(line, 128, file); } // read actual data set fgets(line, 128, file); fclose(file); // replace date and year_day data_set = line; // print date string of file name to buffer sprintf(line, "%02d%02d%02d", date->tm_year - 100, date->tm_mon + 1, date->tm_mday); if (sensorGroup == "dd") { data_set.replace(1, 6, line); } else if (sensorGroup == "rd") { data_set.replace(0, 6, line); } sprintf(line, "%3d", date->tm_yday); if (sensorGroup == "dd") { data_set.replace(10, 3, line); } else if (sensorGroup == "rd") { data_set.replace(9, 3, line); } fprintf(fdata, "%s", data_set.c_str()); fflush(fdata); if (debug > 1) printf("%s", data_set.c_str()); }
void initKV(uint64_t kvtarget, ::libmaus::huffman::KvInitResult & result) { result = ::libmaus::huffman::KvInitResult(); if ( ( (idda.kvec.size()!=0) && (idda.kvec[idda.kvec.size()-1] != 0) ) ) { if ( kvtarget >= idda.kvec[idda.kvec.size()-1] + idda.vvec[idda.vvec.size()-1] ) { fileptr = idda.data.size(); blockptr = 0; result.koffset = idda.kvec[idda.kvec.size()-1]; result.voffset = idda.vvec[idda.vvec.size()-1]; result.kvoffset = result.koffset + result.voffset; result.kvtarget = 0; } else { ::libmaus::huffman::FileBlockOffset const FBO = idda.findKVBlock(kvtarget); fileptr = FBO.fileptr; blockptr = FBO.blockptr; /* open file and seek to block */ openNewFile(); /* decode block in question */ bool const blockok = decodeBlock(); assert ( blockok ); /* key/symbol offset of block (sum over elements of previous blocks) */ uint64_t kvoffset = idda.data[FBO.fileptr].getKeyValueCnt(FBO.blockptr); uint64_t voffset = idda.data[FBO.fileptr].getValueCnt(FBO.blockptr); uint64_t koffset = idda.data[FBO.fileptr].getKeyCnt(FBO.blockptr); assert ( kvtarget >= kvoffset ); kvtarget -= kvoffset; // std::cerr << "fileptr=" << fileptr << " blockptr=" << blockptr << " kvtarget=" << kvtarget << std::endl; while ( kvtarget >= peek() + 1 ) { uint64_t const gi = decode(); kvoffset += (gi+1); kvtarget -= (gi+1); voffset += gi; koffset += 1; } if ( koffset + 1 == getN() && kvtarget >= peek() ) { uint64_t const gi = decode(); kvoffset += gi; kvtarget -= gi; voffset += gi; koffset += 0; } else { assert ( pc != pe ); assert ( kvtarget <= peek() ); assert ( kvtarget <= *pc ); *pc -= kvtarget; } result.koffset = koffset; result.voffset = voffset; result.kvoffset = kvoffset; result.kvtarget = kvtarget; } } }