uint64_t bamrecompress(libmaus2::util::ArgInfo const & arginfo)
{
int const level = libmaus2::bambam::BamBlockWriterBaseFactory::checkCompressionLevel(arginfo.getValue<int>("level",getDefaultLevel()));
int const verbose = arginfo.getValue<int>("verbose",getDefaultVerbose());
int const numthreads = std::max(1,arginfo.getValue<int>("numthreads",getDefaultNumThreads()));
/*
* start index/md5 callbacks
*/
std::string const tmpfilenamebase = arginfo.getValue<std::string>("tmpfile",arginfo.getDefaultTmpFileName());
std::string const tmpfileindex = tmpfilenamebase + "_index";
::libmaus2::util::TempFileRemovalContainer::addTempFile(tmpfileindex);
std::string md5filename;
std::string indexfilename;
std::vector< ::libmaus2::lz::BgzfDeflateOutputCallback * > cbs;
::libmaus2::lz::BgzfDeflateOutputCallbackMD5::unique_ptr_type Pmd5cb;
if ( arginfo.getValue<unsigned int>("md5",getDefaultMD5()) )
{
if ( arginfo.hasArg("md5filename") && arginfo.getUnparsedValue("md5filename","") != "" )
md5filename = arginfo.getUnparsedValue("md5filename","");
else
std::cerr << "[V] no filename for md5 given, not creating hash" << std::endl;
if ( md5filename.size() )
{
::libmaus2::lz::BgzfDeflateOutputCallbackMD5::unique_ptr_type Tmd5cb(new ::libmaus2::lz::BgzfDeflateOutputCallbackMD5);
Pmd5cb = UNIQUE_PTR_MOVE(Tmd5cb);
cbs.push_back(Pmd5cb.get());
}
}
libmaus2::bambam::BgzfDeflateOutputCallbackBamIndex::unique_ptr_type Pindex;
if ( arginfo.getValue<unsigned int>("index",getDefaultIndex()) )
{
if ( arginfo.hasArg("indexfilename") && arginfo.getUnparsedValue("indexfilename","") != "" )
indexfilename = arginfo.getUnparsedValue("indexfilename","");
else
std::cerr << "[V] no filename for index given, not creating index" << std::endl;
if ( indexfilename.size() )
{
libmaus2::bambam::BgzfDeflateOutputCallbackBamIndex::unique_ptr_type Tindex(new libmaus2::bambam::BgzfDeflateOutputCallbackBamIndex(tmpfileindex));
Pindex = UNIQUE_PTR_MOVE(Tindex);
cbs.push_back(Pindex.get());
}
}
/*
* end md5/index callbacks
*/
libmaus2::lz::BgzfInflateDeflateParallel::unique_ptr_type BIDP(new libmaus2::lz::BgzfInflateDeflateParallel(std::cin,std::cout,level,numthreads,4*numthreads));
for ( uint64_t i = 0; i < cbs.size(); ++i )
BIDP->registerBlockOutputCallback(cbs[i]);
libmaus2::autoarray::AutoArray<char> B(64*1024,false);
int r;
uint64_t t = 0;
uint64_t last = std::numeric_limits<uint64_t>::max();
uint64_t lcnt = 0;
uint64_t const mod = 64*1024*1024;
libmaus2::timing::RealTimeClock rtc; rtc.start();
libmaus2::timing::RealTimeClock lrtc; lrtc.start();
while ( (r = BIDP->read(B.begin(),B.size())) )
{
BIDP->write(B.begin(),r);
lcnt += r;
t += r;
if ( t/mod != last/mod )
{
if ( verbose )
{
if ( isatty(STDERR_FILENO) )
std::cerr
<< "\r" << std::string(60,' ') << "\r";
std::cerr
<< rtc.formatTime(rtc.getElapsedSeconds()) << " " << t/(1024*1024) << "MB, " << (lcnt/lrtc.getElapsedSeconds())/(1024.0*1024.0) << "MB/s";
if ( isatty(STDERR_FILENO) )
std::cerr << std::flush;
else
std::cerr << std::endl;
}
lrtc.start();
last = t;
lcnt = 0;
}
}
if ( verbose )
{
if ( isatty(STDERR_FILENO) )
std::cerr
<< "\r" << std::string(60,' ') << "\r";
std::cerr
<< rtc.formatTime(rtc.getElapsedSeconds()) << " " << t/(1024*1024) << "MB, " << (t/rtc.getElapsedSeconds())/(1024.0*1024.0) << "MB/s";
std::cerr << std::endl;
}
BIDP.reset();
if ( Pmd5cb )
{
Pmd5cb->saveDigestAsFile(md5filename);
}
if ( Pindex )
{
Pindex->flush(std::string(indexfilename));
}
return 0;
}
int main(int argc, char *argv[])
{
{
libmaus2::lz::LineSplittingGzipOutputStream LSG("gzsplit",4,17);
for ( uint64_t i = 0; i < 17; ++i )
LSG << "line_" << i << "\n";
}
{
libmaus2::lz::LineSplittingGzipOutputStream LSG("nogzsplit",4,17);
}
testGzip();
testlz4();
#if 0
maskBamDuplicateFlag(std::cin,std::cout);
return 0;
#endif
#if 0
{
libmaus2::lz::BgzfInflateDeflateParallel BIDP(std::cin,std::cout,Z_DEFAULT_COMPRESSION,32,128);
libmaus2::autoarray::AutoArray<char> B(64*1024,false);
int r;
uint64_t t = 0;
uint64_t last = std::numeric_limits<uint64_t>::max();
uint64_t lcnt = 0;
uint64_t const mod = 64*1024*1024;
libmaus2::timing::RealTimeClock rtc; rtc.start();
libmaus2::timing::RealTimeClock lrtc; lrtc.start();
while ( (r = BIDP.read(B.begin(),B.size())) )
{
BIDP.write(B.begin(),r);
lcnt += r;
t += r;
if ( t/mod != last/mod )
{
if ( isatty(STDERR_FILENO) )
std::cerr
<< "\r" << std::string(60,' ') << "\r";
std::cerr
<< rtc.formatTime(rtc.getElapsedSeconds()) << " " << t/(1024*1024) << "MB, " << (lcnt/lrtc.getElapsedSeconds())/(1024.0*1024.0) << "MB/s";
if ( isatty(STDERR_FILENO) )
std::cerr << std::flush;
else
std::cerr << std::endl;
lrtc.start();
last = t;
lcnt = 0;
}
}
if ( isatty(STDERR_FILENO) )
std::cerr
<< "\r" << std::string(60,' ') << "\r";
std::cerr
<< rtc.formatTime(rtc.getElapsedSeconds()) << " " << t/(1024*1024) << "MB, " << (t/rtc.getElapsedSeconds())/(1024.0*1024.0) << "MB/s";
std::cerr << std::endl;
return 0;
}
#endif
#if 0
{
::libmaus2::lz::BgzfDeflateParallel BDP(std::cout,32,128,Z_DEFAULT_COMPRESSION);
while ( std::cin )
{
libmaus2::autoarray::AutoArray<char> B(16384);
std::cin.read(B.begin(),B.size());
int64_t const r = std::cin.gcount();
BDP.write(B.begin(),r);
}
BDP.flush();
std::cout.flush();
}
return 0;
#endif
#if 0
{
try
{
libmaus2::lz::BgzfInflateParallel BIP(std::cin /* ,4,16 */);
uint64_t c = 0;
uint64_t b = 0;
uint64_t d = 0;
libmaus2::timing::RealTimeClock rtc; rtc.start();
libmaus2::autoarray::AutoArray<uint8_t> adata(64*1024,false);
while ( (d=BIP.read(reinterpret_cast<char *>(adata.begin()),adata.size())) != 0 )
{
b += d;
if ( ++c % (16*1024) == 0 )
{
std::cerr << c << "\t" << b/(1024.0*1024.0*1024.0) << "\t" << static_cast<double>(b)/(1024.0*1024.0*rtc.getElapsedSeconds()) << " MB/s" << std::endl;
}
}
std::cerr << c << "\t" << b/(1024.0*1024.0*1024.0) << "\t" << static_cast<double>(b)/(1024.0*1024.0*rtc.getElapsedSeconds()) << " MB/s" << std::endl;
std::cerr << "decoded " << b << " bytes in " << rtc.getElapsedSeconds() << " seconds." << std::endl;
}
catch(std::exception const & ex)
{
std::cerr << ex.what() << std::endl;
return EXIT_FAILURE;
}
}
return 0;
#endif
std::cerr << "Testing random data on bgzf...";
testBgzfRandom();
std::cerr << "done." << std::endl;
std::cerr << "Testing mono...";
testBgzfMono();
std::cerr << "done." << std::endl;
::libmaus2::lz::BgzfDeflate<std::ostream> bdefl(std::cout);
char const * str = "Hello, world.\n";
bdefl.write(reinterpret_cast<char const *>(str),strlen(str));
bdefl.flush();
bdefl.write(reinterpret_cast<char const *>(str),strlen(str));
bdefl.flush();
bdefl.addEOFBlock();
return 0;
::libmaus2::lz::BgzfInflateStream SW(std::cin);
::libmaus2::autoarray::AutoArray<char> BB(200,false);
while ( SW.read(BB.begin(),BB.size()) )
{
}
if ( argc < 2 )
return EXIT_FAILURE;
return 0;
#if 0
::libmaus2::lz::GzipHeader GZH(argv[1]);
return 0;
#endif
std::ostringstream ostr;
::libmaus2::autoarray::AutoArray<uint8_t> message = ::libmaus2::util::GetFileSize::readFile(argv[1]);
std::cerr << "Deflating message of length " << message.size() << "...";
::libmaus2::lz::Deflate DEFL(ostr);
DEFL.write ( reinterpret_cast<char const *>(message.begin()), message.size() );
DEFL.flush();
std::cerr << "done." << std::endl;
std::cerr << "Checking output...";
std::istringstream istr(ostr.str());
::libmaus2::lz::Inflate INFL(istr);
int c;
uint64_t i = 0;
while ( (c=INFL.get()) >= 0 )
{
assert ( c == message[i] );
i++;
}
std::cerr << "done." << std::endl;
// std::cerr << "Message size " << message.size() << std::endl;
std::string testfilename = "test";
::libmaus2::lz::BlockDeflate BD(testfilename);
BD.write ( message.begin(), message.size() );
BD.flush();
uint64_t const decpos = message.size() / 3;
::libmaus2::lz::BlockInflate BI(testfilename,decpos);
::libmaus2::autoarray::AutoArray<uint8_t> dmessage (message.size(),false);
uint64_t const red = BI.read(dmessage.begin()+decpos,dmessage.size());
assert ( red == dmessage.size()-decpos );
std::cerr << "(";
for ( uint64_t i = decpos; i < message.size(); ++i )
assert ( message[i] == dmessage[i] );
std::cerr << ")\n";
std::string shortmes1("123456789");
std::string shortmes2("AA");
std::string shortmes3("BB");
std::string shortmes4("CC");
std::string textfile1("test1");
std::string textfile2("test2");
std::string textfile3("test3");
std::string textfile4("test4");
::libmaus2::lz::BlockDeflate BD1(textfile1);
BD1.write ( reinterpret_cast<uint8_t const *>(shortmes1.c_str()), shortmes1.size() );
BD1.flush();
::libmaus2::lz::BlockDeflate BD2(textfile2);
BD2.write ( reinterpret_cast<uint8_t const *>(shortmes2.c_str()), shortmes2.size() );
BD2.flush();
::libmaus2::lz::BlockDeflate BD3(textfile3);
BD3.write ( reinterpret_cast<uint8_t const *>(shortmes3.c_str()), shortmes3.size() );
BD3.flush();
::libmaus2::lz::BlockDeflate BD4(textfile4);
BD4.write ( reinterpret_cast<uint8_t const *>(shortmes4.c_str()), shortmes4.size() );
BD4.flush();
std::vector < std::string > filenames;
filenames.push_back(textfile1);
filenames.push_back(textfile2);
filenames.push_back(textfile3);
filenames.push_back(textfile4);
for ( uint64_t j = 0; j <= 15; ++j )
{
::libmaus2::lz::ConcatBlockInflate CBI(filenames,j);
for ( uint64_t i = 0; i < j; ++i )
std::cerr << ' ';
for ( uint64_t i = 0; i < CBI.n-j; ++i )
std::cerr << (char)CBI.get();
std::cerr << std::endl;
}
return 0;
}