void
write(const char* c_filename,const ParticlesData& particles,const bool forceCompressed)
{
std::string filename(c_filename);
std::string extension;
bool endsWithGz;
if(!extensionIgnoringGz(filename,extension,endsWithGz)) return;
std::map<std::string,WRITER_FUNCTION>::iterator i=writers().find(extension);
if(i==writers().end()){
std::cerr<<"Partio: No writer defined for extension "<<extension<<std::endl;
return;
}
(*i->second)(c_filename,particles,forceCompressed || endsWithGz);
}
int main(int argc, char *argv[]) {
(void)argc;
(void)argv;
std::cout << "Performance benchmark" << std::endl;
const std::string storage_path = "testStorage";
{
std::cout << "write..." << std::endl;
const size_t chunk_per_storage = 1024 * 10;
const size_t chunk_size = 1024;
const size_t cap_B = 5;
const size_t max_mem_chunks = 100;
if (dariadb::utils::fs::path_exists(storage_path)) {
dariadb::utils::fs::rm(storage_path);
}
dariadb::Time start_time = dariadb::timeutil::current_time();
dariadb::storage::Capacitor::Params cap_param(cap_B, storage_path);
cap_param.max_levels = 11;
auto raw_ptr = new dariadb::storage::Engine(
dariadb::storage::PageManager::Params(storage_path, chunk_per_storage,
chunk_size),
cap_param, dariadb::storage::Engine::Limits(max_mem_chunks));
dariadb::storage::MeasStorage_ptr ms{raw_ptr};
dariadb::IdSet all_id_set;
append_count = 0;
stop_info = false;
std::thread info_thread(show_info, raw_ptr);
std::vector<std::thread> writers(dariadb_bench::total_threads_count);
size_t pos = 0;
for (size_t i = 1; i < dariadb_bench::total_threads_count + 1; i++) {
all_id_set.insert(pos);
std::thread t{dariadb_bench::thread_writer_rnd_stor, dariadb::Id(pos),
dariadb::Time(i), &append_count, raw_ptr};
writers[pos++] = std::move(t);
}
pos = 0;
for (size_t i = 1; i < dariadb_bench::total_threads_count + 1; i++) {
std::thread t = std::move(writers[pos++]);
t.join();
}
stop_info = true;
info_thread.join();
{
std::cout << "full flush..." << std::endl;
auto start = clock();
raw_ptr->flush();
auto elapsed = (((float)clock() - start) / CLOCKS_PER_SEC);
std::cout << "flush time: " << elapsed << std::endl;
}
auto queue_sizes = raw_ptr->queue_size();
std::cout << "\r"
<< " in queue: (p:" << queue_sizes.pages_count << " cap:" << queue_sizes.cola_count << ")"
<< std::endl;
dariadb_bench::readBenchark(all_id_set, ms, 10,start_time, dariadb::timeutil::current_time());
{
std::cout << "read all..." << std::endl;
std::shared_ptr<BenchCallback> clbk{new BenchCallback()};
auto start = clock();
dariadb::storage::QueryInterval qi{
dariadb::IdArray(all_id_set.begin(), all_id_set.end()), 0, start_time,
ms->maxTime() };
ms->readInterval(qi)->readAll(clbk.get());
auto elapsed = (((float)clock() - start) / CLOCKS_PER_SEC);
std::cout << "readed: " << clbk->count << std::endl;
std::cout << "time: " << elapsed << std::endl;
auto expected =
(dariadb_bench::iteration_count * dariadb_bench::total_threads_count);
if (clbk->count != expected) {
std::cout << "expected: " << expected << " get:" << clbk->count << std::endl;
throw MAKE_EXCEPTION("(clbk->count!=(iteration_count*total_threads_count))");
}
}
std::cout << "stoping storage...\n";
ms = nullptr;
}
std::cout << "cleaning...\n";
if (dariadb::utils::fs::path_exists(storage_path)) {
dariadb::utils::fs::rm(storage_path);
}
}
int main(int argc, char *argv[]) {
po::options_description desc("Multithread IO benchmark.\n Allowed options");
desc.add_options()("help", "produce help message")
("mc", po::value<size_t>(&meas2write)->default_value(meas2write), "measurment count")
("ic", po::value<size_t>(&iteration_count)->default_value(iteration_count), "iteration count")
("dyncache", po::value<bool>(&enable_dyn_cache)->default_value(enable_dyn_cache), "enable dynamic cache")
("cache-size", po::value<size_t>(&cache_size)->default_value(cache_size), "cache size")
("cache-pool-size", po::value<size_t>(&cache_pool_size)->default_value(cache_pool_size), "cache pool size")
("thread-count", po::value<size_t>(&thread_count)->default_value(thread_count), "write thread count")
("page-size", po::value<size_t>(&pagesize)->default_value(pagesize), "page size")
("verbose", "verbose ouput")
("dont-remove", "dont remove created storage");
po::variables_map vm;
try {
po::store(po::parse_command_line(argc, argv, desc), vm);
} catch (std::exception &ex) {
logger("Error: " << ex.what());
exit(1);
}
po::notify(vm);
if (vm.count("help")) {
std::cout << desc << std::endl;
return 1;
}
if (vm.count("verbose")) {
verbose = true;
}
if (vm.count("dont-remove")) {
dont_remove = true;
}
makeStorage();
std::thread info_thread(show_info);
logger("threads count: "<<thread_count);
/// writers
std::vector<std::thread> writers(thread_count);
size_t pos = 0;
for (size_t i = 0; i < thread_count; i++) {
std::thread t{ writer, iteration_count };
writers[pos++] = std::move(t);
}
pos = 0;
for (size_t i = 0; i < thread_count; i++) {
std::thread t = std::move(writers[pos++]);
t.join();
}
stop_info = true;
info_thread.join();
/// readers
stop_info = false;
std::thread read_info_thread(show_reads_info);
std::vector<std::thread> readers(thread_count);
pos = 0;
auto reads_per_thread = (iteration_count / ((float)thread_count));
for (size_t i = 0; i < thread_count; i++) {
std::thread t{ reader,i, reads_per_thread*i, reads_per_thread*(i+1) };
readers[pos++] = std::move(t);
}
pos = 0;
for (size_t i = 0; i < thread_count; i++) {
std::thread t = std::move(readers[pos++]);
t.join();
}
stop_info = true;
read_info_thread.join();
ds=nullptr;
if (!dont_remove) {
nkvdb::utils::rm(storage_path);
}
}
int bamsplitmod(libmaus::util::ArgInfo const & arginfo)
{
if ( isatty(STDIN_FILENO) )
{
::libmaus::exception::LibMausException se;
se.getStream() << "Refusing read binary data from terminal, please redirect standard input to pipe or file." << std::endl;
se.finish();
throw se;
}
int const level = libmaus::bambam::BamBlockWriterBaseFactory::checkCompressionLevel(arginfo.getValue<int>("level",getDefaultLevel()));
int const verbose = arginfo.getValue<int>("verbose",getDefaultVerbose());
uint64_t const div = arginfo.getValue<int>("div",getDefaultDiv());
std::string const prefix = arginfo.getUnparsedValue("prefix",getDefaultFilePrefix(arginfo));
if ( ! div )
{
::libmaus::exception::LibMausException se;
se.getStream() << "div cannot be 0." << std::endl;
se.finish();
throw se;
}
libmaus::bambam::BamDecoder bamdec(std::cin);
libmaus::bambam::BamAlignment const & algn = bamdec.getAlignment();
libmaus::bambam::BamHeader const & header = bamdec.getHeader();
::libmaus::bambam::BamHeader::unique_ptr_type uphead(updateHeader(arginfo,header));
libmaus::autoarray::AutoArray<libmaus::aio::CheckedOutputStream::unique_ptr_type> COS(div);
libmaus::autoarray::AutoArray<libmaus::bambam::BamWriter::unique_ptr_type> writers(div);
std::vector < std::string > filenames;
for ( uint64_t i = 0; i < div; ++i )
{
std::ostringstream ostr;
ostr << prefix << "_" << std::setw(6) << std::setfill('0') << i << std::setw(0) << ".bam";
libmaus::aio::CheckedOutputStream::unique_ptr_type tCOS(new libmaus::aio::CheckedOutputStream(ostr.str()));
COS[i] = UNIQUE_PTR_MOVE(tCOS);
libmaus::bambam::BamWriter::unique_ptr_type twriter(new libmaus::bambam::BamWriter(*COS[i],*uphead,level));
writers[i] = UNIQUE_PTR_MOVE(twriter);
}
uint64_t c = 0;
if ( verbose )
{
while ( bamdec.readAlignment() )
{
algn.serialise ( writers [ (c++) % div ] -> getStream() );
if ( ((c) & ((1ull<<20)-1)) == 0 )
std::cerr << "[V] " << c << std::endl;
}
std::cerr << "[V] " << c << std::endl;
}
else
{
while ( bamdec.readAlignment() )
algn.serialise ( writers [ (c++) % div ] -> getStream() );
}
for ( uint64_t i = 0; i < div; ++i )
{
writers[i].reset();
COS[i]->flush();
COS[i].reset();
}
return EXIT_SUCCESS;
}
int main( int, char** )
{
#ifdef LUNCHBOX_USE_OPENMP
const size_t nThreads = lunchbox::OMP::getNThreads() * 3;
#else
const size_t nThreads = 16;
#endif
std::cout << " read, write, push, copy, erase, "
<< " flush/ms, rd, other #threads" << std::endl;
_runSerialTest< std::vector< size_t >, size_t >();
_runSerialTest< Vector_t, size_t >();
std::vector< Reader > readers(nThreads);
std::vector< Writer > writers(nThreads);
std::vector< Pusher > pushers(nThreads);
stage_ = 1;
size_t stage = 0;
for( size_t l = 0; l < nThreads; ++l )
{
readers[l].start();
writers[l].start();
pushers[l].start();
}
lunchbox::sleep( 10 );
for( size_t i = 1; i <= nThreads; i = i<<1 )
for( size_t j = 1; j <= nThreads; j = j<<1 )
{
// concurrent read, write, push
Vector_t vector;
for( size_t k = 0; k < nThreads; ++k )
{
readers[k].vector = k < i ? &vector : 0;
writers[k].vector = k < j ? &vector : 0;
pushers[k].vector = k < j ? &vector : 0;
}
const size_t nextStage = ++stage * STAGESIZE;
_clock.reset();
stage_ = nextStage;
stage_.waitEQ( nextStage + (3 * nThreads) );
TEST( vector.size() >= LOOPSIZE );
// multi-threaded copy
std::vector< Copier > copiers(j);
_clock.reset();
for( size_t k = 0; k < j; ++k )
{
copiers[k].vector = &vector;
copiers[k].start();
}
for( size_t k = 0; k < j; ++k )
copiers[k].join();
for( size_t k = 0; k < vector.size(); ++k )
TEST( vector[k] == k || vector[k] == 0 );
// multi-threaded erase
std::vector< Eraser > erasers(j);
_clock.reset();
for( size_t k = 0; k < j; ++k )
{
erasers[k].vector = &vector;
erasers[k].start();
}
for( size_t k = 0; k < j; ++k )
erasers[k].join();
for( size_t k = 0; k < vector.size(); ++k )
{
if( vector[k] == 0 )
break;
if( k > vector.size() / 2 )
{
TEST( vector[k] > vector[k-1] );
}
else
{
TEST( vector[k] == k );
}
}
// multi-threaded pop_back
const size_t fOps = vector.size();
std::vector< Flusher > flushers(j);
_clock.reset();
for( size_t k = 0; k < j; ++k )
{
flushers[k].vector = &vector;
flushers[k].start();
}
for( size_t k = 0; k < j; ++k )
flushers[k].join();
const float fTime = _clock.getTimef();
TEST( vector.empty( ));
std::cerr << std::setw(11) << float(i*LOOPSIZE)/rTime_ << ", "
<< std::setw(11) << float(j*LOOPSIZE)/wTime_ << ", "
<< std::setw(11) << float(LOOPSIZE)/pTime_ << ", "
<< std::setw(9) << float(j)/cTime_ << ", "
<< std::setw(9) << float(j)/eTime_ << ", "
<< std::setw(9) << float(fOps)/fTime << ", "
<< std::setw(3) << i << ", " << std::setw(3) << j
<< std::endl;
}
stage_ = std::numeric_limits< size_t >::max();
for( size_t k = 0; k < nThreads; ++k )
{
readers[k].join();
writers[k].join();
pushers[k].join();
}
return EXIT_SUCCESS;
}
int main(int argc, char *argv[]) {
(void)argc;
(void)argv;
const size_t id_count = 4;
{
if (dariadb::utils::fs::path_exists(storagePath)) {
dariadb::utils::fs::rm(storagePath);
}
std::thread info_thread(show_info);
dariadb::storage::PageManager::start(
dariadb::storage::PageManager::Params(storagePath, chunks_count, chunks_size));
std::vector<std::thread> writers(dariadb_bench::total_threads_count);
size_t pos = 0;
for (size_t i = 1; i < dariadb_bench::total_threads_count + 1; i++) {
std::thread t{dariadb_bench::thread_writer_rnd_stor, dariadb::Id(pos),
dariadb::Time(i), &append_count,
dariadb::storage::PageManager::instance()};
writers[pos++] = std::move(t);
}
pos = 0;
for (size_t i = 1; i < dariadb_bench::total_threads_count + 1; i++) {
std::thread t = std::move(writers[pos++]);
t.join();
}
stop_info = true;
info_thread.join();
std::random_device r;
std::default_random_engine e1(r());
std::uniform_int_distribution<dariadb::Id> uniform_dist(
dariadb::storage::PageManager::instance()->minTime(),
dariadb::storage::PageManager::instance()->maxTime());
BenchCallback *clbk = new BenchCallback;
dariadb::IdArray ids(id_count);
for (size_t i = 0; i < id_count; ++i) {
ids[i] = i;
}
const size_t runs_count = 100;
auto start = clock();
for (size_t i = 0; i < runs_count; i++) {
auto time_point1 = uniform_dist(e1);
auto time_point2 = uniform_dist(e1);
auto from = std::min(time_point1, time_point2);
auto to = std::max(time_point1, time_point2);
auto link_list = dariadb::storage::PageManager::instance()->chunksByIterval(
dariadb::storage::QueryInterval(ids, 0, from, to));
auto cursor = dariadb::storage::PageManager::instance()->readLinks(link_list);
cursor->readAll(clbk);
assert(clbk->count != 0);
clbk->count = 0;
cursor = nullptr;
}
auto elapsed = ((float)clock() - start) / CLOCKS_PER_SEC;
std::cout << "interval: " << elapsed / runs_count << std::endl;
start = clock();
for (size_t i = 0; i < runs_count; i++) {
auto time_point = uniform_dist(e1);
dariadb::storage::PageManager::instance()->valuesBeforeTimePoint(
dariadb::storage::QueryTimePoint(ids, 0, time_point));
}
elapsed = ((float)clock() - start) / CLOCKS_PER_SEC;
std::cout << "timePoint: " << elapsed / runs_count << std::endl;
delete clbk;
dariadb::storage::PageManager::stop();
if (dariadb::utils::fs::path_exists(storagePath)) {
dariadb::utils::fs::rm(storagePath);
}
}
}
