void testgammagap()
{
unsigned int n = 512*1024+199481101;
std::vector<uint64_t> V (n);
std::vector<uint64_t> V2(n);
for ( uint64_t i = 0; i < V.size(); ++i )
{
V[i] = i & 0xFFull;
V2[i] = rand() % 0xFFull;
}
std::string const fn("tmpfile");
std::string const fn2("tmpfile2");
std::string const fnm("tmpfile.merged");
::libmaus::util::TempFileRemovalContainer::setup();
::libmaus::util::TempFileRemovalContainer::addTempFile(fn);
::libmaus::util::TempFileRemovalContainer::addTempFile(fn2);
::libmaus::util::TempFileRemovalContainer::addTempFile(fnm);
::libmaus::gamma::GammaGapEncoder GGE(fn);
GGE.encode(V.begin(),V.end());
::libmaus::gamma::GammaGapEncoder GGE2(fn2);
GGE2.encode(V2.begin(),V2.end());
::libmaus::huffman::IndexDecoderData IDD(fn);
::libmaus::gamma::GammaGapDecoder GGD(std::vector<std::string>(1,fn));
bool ok = true;
for ( uint64_t i = 0; i < n; ++i )
{
uint64_t const v = GGD.decode();
ok = ok && (v == V[i]);
}
std::cout << "decoding " << (ok ? "ok" : "fail") << std::endl;
std::vector < std::vector<std::string> > merin;
merin.push_back(std::vector<std::string>(1,fn));
merin.push_back(std::vector<std::string>(1,fn2));
::libmaus::gamma::GammaGapEncoder::merge(merin,fnm);
}
/*
* load index for one file
*/
static libmaus::autoarray::AutoArray< IndexEntry > loadIndex(std::string const & filename)
{
uint64_t const indexpos = getIndexPos(filename);
std::ifstream indexistr(filename.c_str(),std::ios::binary);
if ( ! indexistr.is_open() )
{
::libmaus::exception::LibMausException se;
se.getStream() << "RLDecoder::loadIndex(): Failed to open file " << filename << std::endl;
se.finish();
throw se;
}
// seek to index position
indexistr.seekg(indexpos,std::ios::beg);
if ( static_cast<int64_t>(indexistr.tellg()) != static_cast<int64_t>(indexpos) )
{
::libmaus::exception::LibMausException se;
se.getStream() << "Failed to seek to index position " << indexpos << " in file " << filename << " of size "
<< ::libmaus::util::GetFileSize::getFileSize(filename) << std::endl;
se.finish();
throw se;
}
::libmaus::bitio::StreamBitInputStream SBIS(indexistr);
// read size of index
uint64_t const numentries = ::libmaus::bitio::readElias2(SBIS);
// pos bits
unsigned int const posbits = ::libmaus::bitio::readElias2(SBIS);
// k bits
unsigned int const kbits = ::libmaus::bitio::readElias2(SBIS);
// k acc
/* uint64_t const symacc = */ ::libmaus::bitio::readElias2(SBIS);
// v bits
unsigned int const vbits = ::libmaus::bitio::readElias2(SBIS);
// v acc
/* uint64_t const symacc = */ ::libmaus::bitio::readElias2(SBIS);
// align
SBIS.flush();
SBIS.getBitsRead();
// std::cerr << "numentries " << numentries << std::endl;
// read index
libmaus::autoarray::AutoArray< IndexEntry > index(numentries,false);
// #define INDEXLOADERDEBUG
#if defined(INDEXLOADERDEBUG)
libmaus::autoarray::AutoArray< IndexEntry > tindex(numentries+1);
#endif
for ( uint64_t i = 0; i < numentries; ++i )
{
uint64_t const pos = SBIS.read(posbits);
uint64_t const kcnt = SBIS.read(kbits);
uint64_t const vcnt = SBIS.read(vbits);
index[i] = IndexEntry(pos,kcnt,vcnt);
#if defined(INDEXLOADERDEBUG)
tindex[i] = index[i];
#endif
}
if ( numentries )
{
assert ( index[0].kcnt == 0 );
assert ( index[0].vcnt == 0 );
for ( uint64_t i = 1; i < numentries; ++i )
{
index[i-1].kcnt = index[i].kcnt;
index[i-1].vcnt = index[i].vcnt;
}
/* uint64_t const pos = */ SBIS.read(posbits);
index[numentries-1].kcnt = SBIS.read(kbits);
index[numentries-1].vcnt = SBIS.read(vbits);
#if defined(INDEXLOADERDEBUG)
tindex[numentries].kcnt = index[numentries-1].kcnt;
tindex[numentries].vcnt = index[numentries-1].vcnt;
#endif
}
for ( uint64_t i = numentries-1; i >= 1; --i )
{
index[i].kcnt -= index[i-1].kcnt;
index[i].vcnt -= index[i-1].vcnt;
}
#if defined(INDEXLOADERDEBUG)
for ( uint64_t i = 0; i < numentries; ++i )
{
assert ( index[i].kcnt == tindex[i+1].kcnt-tindex[i].kcnt );
assert ( index[i].vcnt == tindex[i+1].vcnt-tindex[i].vcnt );
}
#endif
#if defined(INDEXLOADERDEBUG)
IndexDecoderData IDD(filename);
assert ( IDD.numentries == numentries );
assert ( IDD.posbits == posbits );
assert ( IDD.kbits == kbits );
assert ( IDD.vbits == vbits );
// assert ( IDD.symacc == symacc );
std::cerr << "((**++CHECKING " << numentries << "...";
uint64_t tkacc = 0;
uint64_t tvacc = 0;
for ( uint64_t i = 0; i < numentries; ++i )
{
IndexEntry const P = IDD.readEntry(i);
assert ( P.pos == index[i].pos );
assert ( P.kcnt == tkacc );
assert ( P.vcnt == tvacc );
tkacc += index[i].kcnt;
tvacc += index[i].vcnt;
}
assert ( tkacc == IDD.readEntry(numentries).kcnt );
assert ( tvacc == IDD.readEntry(numentries).vcnt );
std::cerr << "**++))" << std::endl;
#endif
// std::cerr << "loaded index of size " << numentries << std::endl;
return index;
}
void testgammarl()
{
srand(time(0));
unsigned int n = 128*1024*1024;
unsigned int n2 = 64*1024*1024;
std::vector<uint64_t> V (n);
std::vector<uint64_t> V2 (n2);
std::vector<uint64_t> Vcat;
unsigned int const albits = 3;
uint64_t const almask = (1ull << albits)-1;
for ( uint64_t i = 0; i < V.size(); ++i )
{
V[i] = rand() & almask;
Vcat.push_back(V[i]);
}
for ( uint64_t i = 0; i < V2.size(); ++i )
{
V2[i] = rand() & almask;
Vcat.push_back(V2[i]);
}
std::string const fn("tmpfile");
std::string const fn2("tmpfile2");
std::string const fn3("tmpfile3");
::libmaus::util::TempFileRemovalContainer::setup();
::libmaus::util::TempFileRemovalContainer::addTempFile(fn);
::libmaus::util::TempFileRemovalContainer::addTempFile(fn2);
::libmaus::util::TempFileRemovalContainer::addTempFile(fn3);
::libmaus::gamma::GammaRLEncoder GE(fn,albits,V.size(),256*1024);
for ( uint64_t i = 0; i < V.size(); ++i )
GE.encode(V[i]);
GE.flush();
::libmaus::gamma::GammaRLEncoder GE2(fn2,albits,V2.size(),256*1024);
for ( uint64_t i = 0; i < V2.size(); ++i )
GE2.encode(V2[i]);
GE2.flush();
#if 0
::libmaus::huffman::IndexDecoderData IDD(fn);
for ( uint64_t i = 0; i < IDD.numentries+1; ++i )
std::cerr << IDD.readEntry(i) << std::endl;
#endif
::libmaus::gamma::GammaRLDecoder GD(std::vector<std::string>(1,fn));
assert ( GD.getN() == n );
for ( uint64_t i = 0; i < n; ++i )
assert ( GD.decode() == static_cast<int64_t>(V[i]) );
uint64_t const off = n / 2 + 1031;
::libmaus::gamma::GammaRLDecoder GD2(std::vector<std::string>(1,fn),off);
for ( uint64_t i = off; i < n; ++i )
assert ( GD2.decode() == static_cast<int64_t>(V[i]) );
std::vector<std::string> fnv;
fnv.push_back(fn);
fnv.push_back(fn2);
::libmaus::gamma::GammaRLEncoder::concatenate(fnv,fn3);
for ( uint64_t off = 0; off < Vcat.size(); off += 18521 )
{
::libmaus::gamma::GammaRLDecoder GD3(std::vector<std::string>(1,fn3),off);
assert ( GD3.getN() == Vcat.size() );
for ( uint64_t i = 0; i < std::min(static_cast<uint64_t>(1024ull),Vcat.size()-off); ++i )
assert ( GD3.decode() == static_cast<int64_t>(Vcat[off+i]) );
}
remove ( fn.c_str() );
remove ( fn2.c_str() );
remove ( fn3.c_str() );
}
static uint64_t getKVAcc(std::string const & filename)
{
IndexDecoderData IDD(filename);
return IDD.kacc + IDD.vacc;
}
int Dialog::DoModal(HINSTANCE hInst, HWND parent)
{
return DialogBoxParam(hInst, MAKEINTRESOURCE(IDD()), parent, Proc,
reinterpret_cast<LPARAM>(this));
}
