BaseBlockSorting(
std::vector < libmaus2::suffixsort::bwtb3m::MergeStrategyBlock::shared_ptr_type > & rV,
uint64_t const rfreemem,
uint64_t const numthreads,
//! inner node queue
std::deque<libmaus2::suffixsort::bwtb3m::MergeStrategyBlock *> & ritodo,
std::ostream * logstr,
uint64_t const rverbose
)
: V(rV), P(), next(0), freemem(rfreemem), finished(0), freememlock(), itodo(ritodo), threads(numthreads),
verbose(rverbose)
{
for ( uint64_t i = 0; i < V.size(); ++i )
if ( V[i]->directSortSpace() > freemem )
{
libmaus2::exception::LibMausException se;
se.getStream() << "Memory provided is " << freemem << " but "
<< V[i]->directSortSpace() << " are required for sorting block " << i << std::endl;
se.finish();
throw se;
}
for ( uint64_t i = 0; i < numthreads; ++i )
{
BaseBlockSortThread::unique_ptr_type tthreadsi(
new BaseBlockSortThread(i,P,V,next,freemem,finished,freememlock,itodo,pending,logstr,verbose)
);
threads[i] = UNIQUE_PTR_MOVE(tthreadsi);
}
}
::libmaus2::autoarray::AutoArray< std::pair<int64_t,uint64_t> > libmaus2::util::Utf8String::getHistogramAsArray(std::string const & fn)
{
#if defined(_OPENMP)
uint64_t const numthreads = omp_get_max_threads();
#else
uint64_t const numthreads = 1;
#endif
::libmaus2::autoarray::AutoArray<uint64_t> const partstarts = computePartStarts(fn,numthreads);
uint64_t const numparts = partstarts.size()-1;
::libmaus2::parallel::OMPLock lock;
::libmaus2::parallel::PosixMutex mutex;
::libmaus2::util::ExtendingSimpleCountingHash<uint64_t,uint64_t> ESCH(8u);
typedef HistogramThread< ::libmaus2::aio::CheckedInputStream > thread_type;
typedef thread_type::unique_ptr_type thread_ptr_type;
::libmaus2::autoarray::AutoArray< ::libmaus2::aio::CheckedInputStream::unique_ptr_type > getters(numparts);
::libmaus2::autoarray::AutoArray<thread_ptr_type> threads(numparts);
for ( uint64_t i = 0; i < numparts; ++i )
{
::libmaus2::aio::CheckedInputStream::unique_ptr_type tgettersi(
new ::libmaus2::aio::CheckedInputStream(fn)
);
getters[i] = UNIQUE_PTR_MOVE(tgettersi);
getters[i]->setBufferSize(16*1024);
getters[i]->seekg(partstarts[i]);
thread_ptr_type tthreadsi(new thread_type(*getters[i],
partstarts[i+1]-partstarts[i],mutex,ESCH,i));
threads[i] = UNIQUE_PTR_MOVE(tthreadsi);
}
for ( uint64_t i = 0; i < numparts; ++i )
{
threads[i]->join();
threads[i].reset();
}
::libmaus2::autoarray::AutoArray< std::pair<int64_t,uint64_t> > R(ESCH.size(),false);
uint64_t p = 0;
for ( ::libmaus2::util::ExtendingSimpleCountingHash<uint64_t,uint64_t>::key_type const * ita =
ESCH.begin(); ita != ESCH.end(); ++ita )
if ( *ita != ::libmaus2::util::ExtendingSimpleCountingHash<uint64_t,uint64_t>::unused() )
R [ p++ ] = std::pair<int64_t,uint64_t>(*ita,ESCH.getCount(*ita));
std::sort(R.begin(),R.end());
return R;
}
