void init(uint64_t offset = 0, uint64_t * psymoffset = 0)
{
if ( ((idda.kvec.size()!=0) && (idda.kvec[idda.kvec.size()-1] != 0)) )
{
if ( offset >= idda.kvec[idda.kvec.size()-1] )
{
fileptr = idda.data.size();
blockptr = 0;
}
else
{
::libmaus::huffman::FileBlockOffset const FBO = idda.findKBlock(offset);
fileptr = FBO.fileptr;
blockptr = FBO.blockptr;
offset = FBO.offset;
/* open file and seek to block */
openNewFile();
/* decode block in question */
bool const blockok = decodeBlock();
assert ( blockok );
assert ( static_cast<int64_t>(offset) < (pe-pc) );
/* symbol offset of block (sum over elements of previous blocks) */
uint64_t symoffset = idda.data[FBO.fileptr].getValueCnt(FBO.blockptr);
/* decode symbols up to offset in block */
for ( uint64_t i = 0; i < offset; ++i )
symoffset += decode();
/* store prefix sum if pointer is given */
if ( psymoffset )
*psymoffset = symoffset;
}
}
}
void SplitJobOutputStream::writeRecord(Record const & rec) {
JobOutputStream::writeRecord(rec);
numberOfFiles++;
totalSize += atof(rec[SIZE].c_str());
if(numberOfFiles >= Constants::MAX_ENTRIES_IN_JOB_FILE
|| totalSize >= Constants::MAX_JOB_FILE_SIZE)
{
openNewFile();
}
}
/* decode next block */
bool decodeBlock()
{
/* open new file if necessary */
bool changedfile = false;
while ( fileptr < idda.data.size() && blockptr == idda.data[fileptr].numentries )
{
fileptr++;
blockptr = 0;
changedfile = true;
}
if ( fileptr == idda.data.size() )
return false;
if ( changedfile )
openNewFile();
/* align to word boundary */
GD->flush();
/* read block size */
uint64_t const blocksize = GD->decodeWord(32);
/* increase size of memory buffer if necessary */
if ( blocksize > decodebuf.size() )
decodebuf.resize(blocksize);
/* set buffer pointers */
pa = decodebuf.begin();
pc = pa;
pe = pa + blocksize;
/* decode block */
for ( uint64_t i = 0; i < blocksize; ++i )
decodebuf[i] = GD->decode();
/* increment block pointer */
blockptr++;
return true;
}
void jwd::writeData(){
FILE *file;
char line[128];
time_t time_in_sec;
struct tm *date;
int day_min, i;
std::string data_set, template_filename;
if (fdata <= 0) return;
// Analyse time stamp, if new day a new file needs to generated
if (filename != getDataFilename()) {
openNewFile();
}
// Read the template header
if (datafileTemplate.length() == 0) throw std::invalid_argument("No template file given");
template_filename = configDir + datafileTemplate;
printf("Reading data from template file %s\n", template_filename.c_str());
// open template file
file = fopen(template_filename.c_str(), "r");
// throw away 2 header lines for dd* data file
if (sensorGroup == "dd") {
for (i = 0; i < 2; i++) {
fgets(line, 128, file);
}
}
// rd* data files don't have a header
// get minute of day
time(&time_in_sec);
date = gmtime((const time_t *) &time_in_sec);
day_min = date->tm_hour * 60 + date->tm_min;
// throw away last (day_min - 1) data sets
for (i = 0; i < day_min; i++) {
fgets(line, 128, file);
}
// read actual data set
fgets(line, 128, file);
fclose(file);
// replace date and year_day
data_set = line;
// print date string of file name to buffer
sprintf(line, "%02d%02d%02d", date->tm_year - 100, date->tm_mon + 1, date->tm_mday);
if (sensorGroup == "dd") {
data_set.replace(1, 6, line);
} else if (sensorGroup == "rd") {
data_set.replace(0, 6, line);
}
sprintf(line, "%3d", date->tm_yday);
if (sensorGroup == "dd") {
data_set.replace(10, 3, line);
} else if (sensorGroup == "rd") {
data_set.replace(9, 3, line);
}
fprintf(fdata, "%s", data_set.c_str());
fflush(fdata);
if (debug > 1)
printf("%s", data_set.c_str());
}
void initKV(uint64_t kvtarget, ::libmaus::huffman::KvInitResult & result)
{
result = ::libmaus::huffman::KvInitResult();
if (
(
(idda.kvec.size()!=0)
&&
(idda.kvec[idda.kvec.size()-1] != 0)
)
)
{
if (
kvtarget >=
idda.kvec[idda.kvec.size()-1] + idda.vvec[idda.vvec.size()-1]
)
{
fileptr = idda.data.size();
blockptr = 0;
result.koffset = idda.kvec[idda.kvec.size()-1];
result.voffset = idda.vvec[idda.vvec.size()-1];
result.kvoffset = result.koffset + result.voffset;
result.kvtarget = 0;
}
else
{
::libmaus::huffman::FileBlockOffset const FBO = idda.findKVBlock(kvtarget);
fileptr = FBO.fileptr;
blockptr = FBO.blockptr;
/* open file and seek to block */
openNewFile();
/* decode block in question */
bool const blockok = decodeBlock();
assert ( blockok );
/* key/symbol offset of block (sum over elements of previous blocks) */
uint64_t kvoffset = idda.data[FBO.fileptr].getKeyValueCnt(FBO.blockptr);
uint64_t voffset = idda.data[FBO.fileptr].getValueCnt(FBO.blockptr);
uint64_t koffset = idda.data[FBO.fileptr].getKeyCnt(FBO.blockptr);
assert ( kvtarget >= kvoffset );
kvtarget -= kvoffset;
// std::cerr << "fileptr=" << fileptr << " blockptr=" << blockptr << " kvtarget=" << kvtarget << std::endl;
while ( kvtarget >= peek() + 1 )
{
uint64_t const gi = decode();
kvoffset += (gi+1);
kvtarget -= (gi+1);
voffset += gi;
koffset += 1;
}
if ( koffset + 1 == getN() && kvtarget >= peek() )
{
uint64_t const gi = decode();
kvoffset += gi;
kvtarget -= gi;
voffset += gi;
koffset += 0;
}
else
{
assert ( pc != pe );
assert ( kvtarget <= peek() );
assert ( kvtarget <= *pc );
*pc -= kvtarget;
}
result.koffset = koffset;
result.voffset = voffset;
result.kvoffset = kvoffset;
result.kvtarget = kvtarget;
}
}
}
