static void concatenate(std::vector<std::string> const & infilenames, std::string const & outfilename, bool const removeinput = false)
{
uint64_t const n = ::libmaus2::gamma::GammaRLDecoder::getLength(infilenames);
unsigned int const albits = infilenames.size() ? ::libmaus2::gamma::GammaRLDecoder::getAlBits(infilenames[0]) : 0;
::libmaus2::aio::OutputStreamInstance COS(outfilename);
::libmaus2::aio::SynchronousGenericOutput<uint64_t> SGO(COS,64);
SGO.put(n);
SGO.put(albits);
SGO.flush();
uint64_t const headerlen = 2*sizeof(uint64_t);
std::vector < ::libmaus2::huffman::IndexEntry > index;
uint64_t ioff = headerlen;
for ( uint64_t i = 0; i < infilenames.size(); ++i )
{
uint64_t const indexpos = ::libmaus2::huffman::IndexLoaderBase::getIndexPos(infilenames[i]);
uint64_t const datalen = indexpos-headerlen;
// copy data
::libmaus2::aio::InputStreamInstance CIS(infilenames[i]);
CIS.seekg(headerlen);
::libmaus2::util::GetFileSize::copy(CIS,COS,datalen);
// add entries to index
::libmaus2::huffman::IndexLoaderSequential indexdata(infilenames[i]);
::libmaus2::huffman::IndexEntry ij = indexdata.getNext();
// ::libmaus2::huffman::IndexDecoderData indexdata(infilenames[i]);
for ( uint64_t j = 0; j < indexdata.numentries; ++j )
{
::libmaus2::huffman::IndexEntry ij1 = indexdata.getNext();
/*
::libmaus2::huffman::IndexEntry const ij = indexdata.readEntry(j);
::libmaus2::huffman::IndexEntry const ij1 = indexdata.readEntry(j+1);
*/
index.push_back(::libmaus2::huffman::IndexEntry((ij.pos - headerlen) + ioff, ij1.kcnt - ij.kcnt, ij1.vcnt - ij.vcnt));
ij = ij1;
}
// update position pointer
ioff += datalen;
if ( removeinput )
libmaus2::aio::FileRemoval::removeFile(infilenames[i]);
}
// write index
::libmaus2::aio::SynchronousGenericOutput<uint8_t> SGO8(COS,64*1024);
::libmaus2::aio::SynchronousGenericOutput<uint8_t>::iterator_type it(SGO8);
::libmaus2::bitio::FastWriteBitWriterStream8Std FWBWS(it);
writeIndex(index,FWBWS,ioff);
FWBWS.flush();
SGO8.flush();
COS.flush();
}
GammaRLEncoder(std::string const & filename, unsigned int const ralbits, uint64_t const n, uint64_t const rblocksize, uint64_t const rbufsize = 64*1024)
:
blocksize(rblocksize),
COS(filename), SGO(COS,rbufsize), GE(SGO),
A(blocksize), pa(A.begin()), pc(pa), pe(A.end()),
cursym(0), curcnt(0), indexwritten(false), albits(ralbits)
{
SGO.put(n);
SGO.put(albits);
}
void flush()
{
if ( curcnt )
{
*(pc++) = ptype(cursym,curcnt);
implicitFlush();
curcnt = 0;
}
SGO.flush();
uint64_t const indexpos = SGO.getWrittenBytes();
writeIndex(indexpos);
}
void writeIndex(uint64_t const indexpos)
{
if ( ! indexwritten )
{
::libmaus::aio::SynchronousGenericOutput<uint8_t> SGO(COS,64*1024);
::libmaus::aio::SynchronousGenericOutput<uint8_t>::iterator_type it(SGO);
::libmaus::bitio::FastWriteBitWriterStream8Std FWBWS(it);
writeIndex(index,FWBWS,indexpos);
FWBWS.flush();
SGO.flush();
COS.flush();
indexwritten = true;
}
}
void implicitFlush()
{
uint64_t const bs = pc-pa;
if ( bs )
{
uint64_t acc = 0;
uint64_t const pos = SGO.getWrittenBytes();
GE.encode(bs);
for ( uint64_t i = 0; i < bs; ++i )
{
acc += pa[i].second;
GE.encodeWord(pa[i].first,albits);
GE.encode(pa[i].second);
}
GE.flush();
::libmaus::huffman::IndexEntry const entry(pos,bs,acc);
index.push_back(entry);
pc = pa;
}
}