/*!
* Copy constructor
*/
file_name_composer_adapter(file_name_composer_adapter const& that) :
m_Formatter(that.m_Formatter),
m_FormattingStream(m_FileName)
{
m_FormattingStream.exceptions(std::ios_base::badbit | std::ios_base::failbit);
m_FormattingStream.imbue(that.m_FormattingStream.getloc());
}
/*!
* Initializing constructor
*/
explicit file_name_composer_adapter(formatter_type const& formatter, std::locale const& loc = std::locale()) :
m_Formatter(formatter),
m_FormattingStream(m_FileName)
{
m_FormattingStream.exceptions(std::ios_base::badbit | std::ios_base::failbit);
m_FormattingStream.imbue(loc);
}
static bool peekLinearChunk(stream_type & stream, uint64_t const refid, int64_t const chunkid)
{
::libmaus2::bambam::BamIndexLinearChunk LC;
if ( stream.peek() == stream_type::traits_type::eof() )
return false;
stream.read(reinterpret_cast<char *>(&LC),sizeof(::libmaus2::bambam::BamIndexLinearChunk));
stream.clear();
stream.seekg(-static_cast<int64_t>(sizeof(::libmaus2::bambam::BamIndexLinearChunk)),std::ios::cur);
return LC.refid == refid && LC.chunkid == chunkid;
}
static bool peekLinearChunk(stream_type & stream, uint64_t const refid, uint64_t const pos, unsigned int const posshift)
{
::libmaus2::bambam::BamIndexLinearChunk LC;
if ( stream.peek() == stream_type::traits_type::eof() )
return false;
stream.read(reinterpret_cast<char *>(&LC),sizeof(::libmaus2::bambam::BamIndexLinearChunk));
stream.clear();
stream.seekg(-static_cast<int64_t>(sizeof(::libmaus2::bambam::BamIndexLinearChunk)),std::ios::cur);
return (LC.refid == refid) && ((LC.pos >> posshift)==(pos>>posshift));
}
result_type operator() (stream_type& strm, value_type const& value) const
{
strm.flush();
typedef typename stream_type::streambuf_type streambuf_type;
string_type& str = *static_cast< streambuf_type* >(strm.rdbuf())->storage();
char_type buf[std::numeric_limits< unsigned int >::digits10 + 2];
char_type* p = buf;
typedef karma::uint_generator< unsigned int, 10 > uint_gen;
karma::generate(p, uint_gen(), value.line);
str.append(buf, p);
}
void open_next_file(stream_type& res) {
if(files_open_ >= concurrent_files_)
return;
while(paths_cur_ != paths_end_) {
std::string path = *paths_cur_;
++paths_cur_;
res.reset(new file_stream(path.c_str(), *this));
if(res->good())
return;
res.reset();
throw std::runtime_error(err::msg() << "Can't open file '" << path << "'");
}
}
void open_next_pipe(stream_type& res) {
while(!free_pipes_.empty()) {
const char* path = free_pipes_.front();
free_pipes_.pop_front();
res.reset(new pipe_stream(path, *this));
if(res->good()) {
busy_pipes_.insert(path);
return;
}
// The pipe failed to open, so it is not marked as busy. This
// reset will make us forget about this path.
res.reset();
}
}
std::pair<uint64_t,uint64_t> flush(stream_type & out)
{
uint64_t const start = out.tellp();
if ( pc != pa )
{
std::sort(pa,pc);
out.write(reinterpret_cast<char const *>(pa),(pc-pa)*sizeof(element_type));
pc = pa;
}
uint64_t const end = out.tellp();
return std::pair<uint64_t,uint64_t>(start,end);
}
// Called when the timer expires.
// We operate the timer continuously this simplifies the code.
//
void on_timer(error_code ec)
{
if(ec && ec != boost::asio::error::operation_aborted)
return fail("timer", ec);
// Verify that the timer really expired
// since the deadline may have moved.
//
if(timer_.expires_at() <= clock_type::now())
{
// Closing the socket cancels all outstanding
// operations. They will complete with
// boost::asio::error::operation_aborted
//
ws_.next_layer().close(ec);
return;
}
// Wait on the timer
timer_.async_wait(
strand_.wrap(std::bind(
&connection::on_timer,
shared_from_this(),
std::placeholders::_1)));
}
static bool readAlignmentGz(
stream_type & GZ,
::libmaus::bambam::BamAlignment & alignment,
::libmaus::bambam::BamHeader const * bamheader = 0,
bool const validate = true
)
{
/* read alignment block size */
int64_t const bs0 = GZ.get();
int64_t const bs1 = GZ.get();
int64_t const bs2 = GZ.get();
int64_t const bs3 = GZ.get();
if ( bs3 < 0 )
// reached end of file
return false;
/* assemble block size as LE integer */
alignment.blocksize = (bs0 << 0) | (bs1 << 8) | (bs2 << 16) | (bs3 << 24) ;
/* read alignment block */
if ( alignment.blocksize > alignment.D.size() )
alignment.D = ::libmaus::bambam::BamAlignment::D_array_type(alignment.blocksize,false);
GZ.read(reinterpret_cast<char *>(alignment.D.begin()),alignment.blocksize);
if ( static_cast<int64_t>(GZ.gcount()) != static_cast<int64_t>(alignment.blocksize) )
{
::libmaus::exception::LibMausException se;
se.getStream() << "Invalid alignment (EOF in alignment block of length " << alignment.blocksize << ")" << std::endl;
se.finish();
throw se;
}
if ( validate )
{
libmaus_bambam_alignment_validity const validity = bamheader ? alignment.valid(*bamheader) : alignment.valid();
if ( validity != ::libmaus::bambam::libmaus_bambam_alignment_validity_ok )
{
::libmaus::exception::LibMausException se;
se.getStream() << "Invalid alignment: " << validity << std::endl;
se.finish();
throw se;
}
}
return true;
}
CompactFastQHeader(stream_type & stream)
:
blocklen(::libmaus::util::NumberSerialisation::deserialiseNumber(stream)),
numreads(::libmaus::util::NumberSerialisation::deserialiseNumber(stream)),
qbits(stream.get()),
quant(stream)
{
}
static int64_t peekBin(stream_type & stream)
{
::libmaus2::bambam::BamIndexBinChunk BC;
if ( stream.peek() == stream_type::traits_type::eof() )
return -1;
stream.read(
reinterpret_cast<char *>(&BC),
sizeof(::libmaus2::bambam::BamIndexBinChunk)
);
assert ( stream.gcount() == sizeof(::libmaus2::bambam::BamIndexBinChunk) );
stream.clear();
stream.seekg(-static_cast<int64_t>(sizeof(::libmaus2::bambam::BamIndexBinChunk)),std::ios::cur);
return BC.refid;
}
/*!
* The operator generates a file name based on the log record
*/
result_type operator() (record_view const& rec) const
{
boost::log::aux::cleanup_guard< stream_type > cleanup1(m_FormattingStream);
boost::log::aux::cleanup_guard< result_type::string_type > cleanup2(m_FileName);
m_Formatter(rec, m_FormattingStream);
m_FormattingStream.flush();
return result_type(m_FileName);
}
void do_manip(stream_type& strm) const
{
if (error_m != std::ios_base::goodbit)
strm.setstate(error_m);
else
{
std::ios_base::iostate err(error_m);
try
{
(*pf_m)(strm, arg1_m, arg2_m);
}
catch (...)
{
err = handle_error(strm);
}
if (err) strm.setstate(err);
}
}
fcppt::io::basic_scoped_rdbuf<
Ch,
Traits
>::basic_scoped_rdbuf(
stream_type &_source,
stream_type &_receiver
)
:
receiver_(
_receiver
),
old_(
_receiver.rdbuf()
)
{
receiver_.rdbuf(
_source.rdbuf()
);
}
static value_type getLEInteger(stream_type & stream)
{
value_type v = 0;
for ( uint64_t i = 0; i < length; ++i )
if ( stream.peek() == stream_type::traits_type::eof() )
{
libmaus2::exception::LibMausException ex;
ex.getStream() << "Failed to little endian number of length " << length << " in BamIndex::getLEInteger." << std::endl;
ex.finish();
throw ex;
}
else
{
v |= static_cast<value_type>(stream.get()) << (8*i);
}
return v;
}
// Read a message from the websocket stream
void do_read()
{
// Put the read on the timer
timer_.expires_from_now(std::chrono::seconds(15));
// Read a message
ws_.async_read(buffer_,
strand_.wrap(std::bind(
&connection::on_read,
shared_from_this(),
std::placeholders::_1)));
}
// Called after the message read completes
void on_read(error_code ec)
{
// This error means the other side
// closed the websocket stream.
if(ec == websocket::error::closed)
return;
if(ec)
return fail("read", ec);
// Put the write on the timer
timer_.expires_from_now(std::chrono::seconds(15));
// Write the received message back
ws_.binary(ws_.got_binary());
ws_.async_write(buffer_.data(),
strand_.wrap(std::bind(
&connection::on_write,
shared_from_this(),
std::placeholders::_1)));
}
static uint8_t getByte(stream_type & in)
{
int const c = in.get();
if ( c < 0 )
{
::libmaus2::exception::LibMausException se;
se.getStream() << "Unexpected EOF in ::libmaus2::bambam::DecoderBase::getByte()" << std::endl;
se.finish();
throw se;
}
return c;
}
void serialise(stream_type & out) const
{
IHWT->serialise(out);
if ( IHWT->getN() )
for ( int s = IHWT->enctable.minsym; s <= IHWT->enctable.maxsym; ++s )
if ( s > 1 && IHWT->rank(s,IHWT->getN()-1) )
{
assert ( C[s].get() );
C[s]->serialize(out);
}
out.flush();
}
/// Constructor
connection(
server& parent,
tcp::endpoint const& ep,
tcp::socket&& sock)
: log_(parent.log_)
, ep_(ep)
, ws_(std::move(sock))
, timer_(ws_.get_io_service(), (clock_type::time_point::max)())
, strand_(ws_.get_io_service())
, id_([]
{
static std::atomic<std::size_t> n{0};
return ++n;
}())
{
// Invoke the callback for new connections if set.
// This allows the settings on the websocket stream
// to be adjusted. For example to turn compression
// on or off or adjust the read and write buffer sizes.
//
if(parent.mod_)
parent.mod_(ws_);
}
result_type operator() (stream_type& strm, value_type const& value) const
{
if (value.type == attributes::named_scope_entry::function)
{
const char* begin = value.scope_name.c_str();
const char* end = begin + value.scope_name.size();
if (parse_function_name(begin, end, m_include_scope))
{
strm.write(begin, end - begin);
return;
}
}
strm << value.scope_name;
}
result_type operator() (stream_type& strm, value_type const& value) const
{
std::size_t n = value.file_name.size(), i = n;
for (; i > 0; --i)
{
const char c = value.file_name[i - 1];
#if defined(BOOST_WINDOWS)
if (c == '\\')
break;
#endif
if (c == '/')
break;
}
strm.write(value.file_name.c_str() + i, n - i);
}
uint64_t createFinalStream(stream_type & out)
{
flush();
uint64_t p = 0;
p += ::libmaus::util::NumberSerialisation::serialiseNumber(out,symbols); // n
p += root->serialize(out); // huffman code tree
p += ::libmaus::util::NumberSerialisation::serialiseNumber(out,contexts.size()); // number of bit vectors
std::vector<uint64_t> nodeposvec;
for ( uint64_t i = 0; i < contexts.size(); ++i )
{
nodeposvec.push_back(p);
uint64_t const blockswritten = contexts[i]->blockswritten;
uint64_t const datawordswritten = 6*blockswritten;
uint64_t const allwordswritten = 8*blockswritten;
contexts[i].reset();
tmpcnt.closeOutputTempFile(i);
// bits written
p += ::libmaus::serialize::Serialize<uint64_t>::serialize(out,64*datawordswritten);
// auto array header (words written)
p += ::libmaus::serialize::Serialize<uint64_t>::serialize(out,allwordswritten);
//std::string const filename = outputfilenames[i];
//::libmaus::aio::CheckedInputStream istr(filename);
std::istream & istr = tmpcnt.openInputTempFile(i);
// std::ifstream istr(filename.c_str(),std::ios::binary);
// std::cerr << "Copying " << allwordswritten << " from stream " << filename << std::endl;
::libmaus::util::GetFileSize::copy (istr, out, allwordswritten, sizeof(uint64_t));
p += allwordswritten * sizeof(uint64_t);
tmpcnt.closeInputTempFile(i);
// remove(filename.c_str());
}
uint64_t const indexpos = p;
p += ::libmaus::util::NumberSerialisation::serialiseNumberVector(out,nodeposvec);
p += ::libmaus::util::NumberSerialisation::serialiseNumber(out,indexpos);
out.flush();
return p;
}
// Called immediately after the connection is created.
// We keep this separate from the constructor because
// shared_from_this may not be called from constructors.
void run()
{
// Run the timer
on_timer({});
// Put the handshake on the timer
timer_.expires_from_now(std::chrono::seconds(15));
// Read the websocket handshake and send the response
ws_.async_accept_ex(
[](websocket::response_type& res)
{
res.insert(http::field::server, "websocket-server-async");
},
strand_.wrap(std::bind(
&connection::on_accept,
shared_from_this(),
std::placeholders::_1)));
}
bool readBlock(stream_type & stream)
{
state = bgzfinflateblockstate_read_block;
if ( failed() )
return false;
try
{
std::pair<uint64_t,uint64_t> preblockinfo = BgzfInflateBase::readBlock(stream);
blockinfo = ::libmaus2::lz::BgzfInflateInfo(
preblockinfo.first,
preblockinfo.second,
preblockinfo.second ? false : (stream.peek() == stream_type::traits_type::eof())
);
return blockinfo.uncompressed;
}
catch(libmaus2::exception::LibMausException const & lex)
{
libmaus2::exception::LibMausException::unique_ptr_type tex(lex.uclone());
ex = UNIQUE_PTR_MOVE(tex);
return false;
}
catch(std::exception const & lex)
{
libmaus2::exception::LibMausException::unique_ptr_type tex(new libmaus2::exception::LibMausException);
ex = UNIQUE_PTR_MOVE(tex);
ex->getStream() << lex.what();
ex->finish(false);
return false;
}
catch(...)
{
libmaus2::exception::LibMausException::unique_ptr_type tex(new libmaus2::exception::LibMausException);
ex = UNIQUE_PTR_MOVE(tex);
ex->getStream() << "BgzfInflateBlock::readBlock(): unknown exception caught";
ex->finish(false);
return false;
}
}
void term()
{
genc.encode(0);
genc.encode(0);
genc.flush();
SGO.flush();
SGOout.flush();
indexout.clear();
indexout.seekg(0,std::ios::beg);
// uint64_t const indexpos = SGO.getWrittenBytes();
indexout.clear();
indexout.seekg(0,std::ios::beg);
libmaus2::util::GetFileSize::copy(indexout,SGOout,2*sizeof(uint64_t)*indexentries);
libmaus2::util::NumberSerialisation::serialiseNumber(SGOout,indexentries ? prevkey : 0); // highest key in file
libmaus2::util::NumberSerialisation::serialiseNumber(SGOout,indexentries);
SGOout.flush();
}
void init(stream_type & stream)
{
char magic[4];
stream.read(&magic[0],sizeof(magic));
if (
! stream
||
stream.gcount() != 4
||
magic[0] != 'B'
||
magic[1] != 'A'
||
magic[2] != 'I'
||
magic[3] != '\1'
)
{
libmaus2::exception::LibMausException ex;
ex.getStream() << "Failed to read BAI magic BAI\\1." << std::endl;
ex.finish();
throw ex;
}
uint32_t const numref = getLEInteger<stream_type,uint32_t,4>(stream);
refs = libmaus2::autoarray::AutoArray<libmaus2::bambam::BamIndexRef>(numref);
for ( uint64_t i = 0; i < numref; ++i )
{
uint32_t const distbins = getLEInteger<stream_type,uint32_t,4>(stream);
#if 0
std::cerr << "chr " << i << " distbins " << distbins << std::endl;
#endif
if ( distbins )
{
refs[i].bin = libmaus2::autoarray::AutoArray<libmaus2::bambam::BamIndexBin>(distbins,false);
libmaus2::autoarray::AutoArray< std::pair<uint64_t,uint64_t> > pi(distbins,false);
libmaus2::autoarray::AutoArray<libmaus2::bambam::BamIndexBin> prebins(distbins,false);
for ( uint64_t j = 0; j < distbins; ++j )
{
uint32_t const bin = getLEInteger<stream_type,uint32_t,4>(stream);
uint32_t const chunks = getLEInteger<stream_type,uint32_t,4>(stream);
// std::cerr << "chr " << i << " bin " << bin << " chunks " << chunks << std::endl;
prebins[j].bin = bin;
prebins[j].chunks = libmaus2::autoarray::AutoArray<libmaus2::bambam::BamIndexBin::Chunk>(chunks,false);
// read chunks
for ( uint64_t k = 0; k < chunks; ++k )
{
prebins[j].chunks[k].first = getLEInteger<stream_type,uint64_t,8>(stream);
prebins[j].chunks[k].second = getLEInteger<stream_type,uint64_t,8>(stream);
}
pi [ j ] = std::pair<uint64_t,uint64_t>(bin,j);
}
// sort by bin
std::sort(pi.begin(),pi.end());
// move
for ( uint64_t j = 0; j < distbins; ++j )
refs[i].bin[j] = prebins[pi[j].second];
}
uint32_t const lins = getLEInteger<stream_type,uint32_t,4>(stream);
if ( lins )
{
refs[i].lin.intervals = libmaus2::autoarray::AutoArray<uint64_t>(lins,false);
for ( uint64_t j = 0; j < lins; ++j )
refs[i].lin.intervals[j] = getLEInteger<stream_type,uint64_t,8>(stream);
}
}
}
local_precision(stream_type& stream) : pstream(&stream), saved_prec(stream.precision()) {}
~local_precision() { pstream->precision(saved_prec); }