void stats::reset() { { scoped_mutex_lock ReadLock(read_mutex); //assert(acc_reads == 0); if (acc_reads) STXXL_ERRMSG("Warning: " << acc_reads << " read(s) not yet finished"); reads = 0; volume_read = 0; c_reads = 0; c_volume_read = 0; t_reads = 0; p_reads = 0.0; } { scoped_mutex_lock WriteLock(write_mutex); //assert(acc_writes == 0); if (acc_writes) STXXL_ERRMSG("Warning: " << acc_writes << " write(s) not yet finished"); writes = 0; volume_written = 0; c_writes = 0; c_volume_written = 0; t_writes = 0.0; p_writes = 0.0; } { scoped_mutex_lock IOLock(io_mutex); //assert(acc_ios == 0); if (acc_ios) STXXL_ERRMSG("Warning: " << acc_ios << " io(s) not yet finished"); p_ios = 0.0; } { scoped_mutex_lock WaitLock(wait_mutex); //assert(acc_waits == 0); if (acc_waits) STXXL_ERRMSG("Warning: " << acc_waits << " wait(s) not yet finished"); t_waits = 0.0; p_waits = 0.0; t_wait_read = 0.0; p_wait_read = 0.0; t_wait_write = 0.0; p_wait_write = 0.0; } last_reset = timestamp(); }
bool linuxaio_queue::cancel_request(request_ptr& req) { if (req.empty()) STXXL_THROW_INVALID_ARGUMENT("Empty request canceled disk_queue."); if (post_thread_state() != RUNNING) STXXL_ERRMSG("Request canceled in stopped queue."); if (!dynamic_cast<linuxaio_request*>(req.get())) STXXL_ERRMSG("Non-LinuxAIO request submitted to LinuxAIO queue."); queue_type::iterator pos; { scoped_mutex_lock lock(waiting_mtx); pos = std::find(waiting_requests.begin(), waiting_requests.end(), req _STXXL_FORCE_SEQUENTIAL); if (pos != waiting_requests.end()) { waiting_requests.erase(pos); // polymorphic_downcast to linuxaio_request, // request is canceled, but was not yet posted. dynamic_cast<linuxaio_request*>(req.get())->completed(false, true); num_waiting_requests--; // will never block return true; } } scoped_mutex_lock lock(posted_mtx); return false; }
void linuxaio_queue::add_request(request_ptr& req) { if (req.empty()) STXXL_THROW_INVALID_ARGUMENT("Empty request submitted to disk_queue."); if (post_thread_state() != RUNNING) STXXL_ERRMSG("Request submitted to stopped queue."); if (!dynamic_cast<linuxaio_request*>(req.get())) STXXL_ERRMSG("Non-LinuxAIO request submitted to LinuxAIO queue."); scoped_mutex_lock lock(waiting_mtx); waiting_requests.push_back(req); num_waiting_requests++; }
void request::check_alignment() const { if (m_offset % STXXL_BLOCK_ALIGN != 0) STXXL_ERRMSG("Offset is not aligned: modulo " << STXXL_BLOCK_ALIGN << " = " << m_offset % STXXL_BLOCK_ALIGN); if (m_bytes % STXXL_BLOCK_ALIGN != 0) STXXL_ERRMSG("Size is not a multiple of " << STXXL_BLOCK_ALIGN << ", = " << m_bytes % STXXL_BLOCK_ALIGN); if (unsigned_type(m_buffer) % STXXL_BLOCK_ALIGN != 0) STXXL_ERRMSG("Buffer is not aligned: modulo " << STXXL_BLOCK_ALIGN << " = " << unsigned_type(m_buffer) % STXXL_BLOCK_ALIGN << " (" << m_buffer << ")"); }
int main() { stxxl::config* config = stxxl::config::get_instance(); stxxl::disk_config disk1("/tmp/stxxl-###.tmp", 16 * 1024 * 1024, "syscall autogrow=no"); disk1.unlink_on_open = true; disk1.direct = stxxl::disk_config::DIRECT_OFF; config->add_disk(disk1); vector_type* vector = new vector_type(); try { while (true) vector->push_back(0); } catch (std::exception& e) { STXXL_ERRMSG("Caught exception: " << e.what()); delete vector; // here it will crash in block_manager::delete_block(bid) } STXXL_MSG("Delete done, all is well."); return 0; }
int main(int argc, char* argv[]) { if (argc < 2) { STXXL_MSG("Usage: " << argv[0] << " #log_ins"); return -1; } const int log_nins = atoi(argv[1]); if (log_nins > 31) { STXXL_ERRMSG("This test can't do more than 2^31 operations, you requested 2^" << log_nins); return -1; } btree_type BTree(1024 * 128, 1024 * 128); const stxxl::uint64 nins = 1ULL << log_nins; stxxl::ran32State = (unsigned int)time(NULL); stxxl::vector<int> Values(nins); STXXL_MSG("Generating " << nins << " random values"); stxxl::generate(Values.begin(), Values.end(), rnd_gen(), 4); stxxl::vector<int>::const_iterator it = Values.begin(); STXXL_MSG("Inserting " << nins << " random values into btree"); for ( ; it != Values.end(); ++it) BTree.insert(std::pair<int, double>(*it, double(*it) + 1.0)); STXXL_MSG("Number of elements in btree: " << BTree.size()); STXXL_MSG("Searching " << nins << " existing elements"); stxxl::vector<int>::const_iterator vIt = Values.begin(); for ( ; vIt != Values.end(); ++vIt) { btree_type::iterator bIt = BTree.find(*vIt); STXXL_CHECK(bIt != BTree.end()); STXXL_CHECK(bIt->first == *vIt); } STXXL_MSG("Searching " << nins << " non-existing elements"); stxxl::vector<int>::const_iterator vIt1 = Values.begin(); for ( ; vIt1 != Values.end(); ++vIt1) { btree_type::iterator bIt = BTree.find((*vIt1) + 1); STXXL_CHECK(bIt == BTree.end()); } STXXL_MSG("Test passed."); return 0; }
void wbtl_file::sread(void* buffer, offset_type offset, size_type bytes) { scoped_mutex_lock buffer_lock(buffer_mutex); int cached = -1; offset_type physical_offset; // map logical to physical address { scoped_mutex_lock mapping_lock(mapping_mutex); sortseq::iterator physical = address_mapping.find(offset); if (physical == address_mapping.end()) { STXXL_ERRMSG("wbtl_read: mapping not found: " << FMT_A_S(offset, bytes) << " ==> " << "???"); //STXXL_THROW_ERRNO(io_error, "wbtl_read of unmapped memory"); physical_offset = 0xffffffff; } else { physical_offset = physical->second; } } if (buffer_address[curbuf] <= physical_offset && physical_offset < buffer_address[curbuf] + write_block_size) { // block is in current write buffer assert(physical_offset + bytes <= buffer_address[curbuf] + write_block_size); memcpy(buffer, write_buffer[curbuf] + (physical_offset - buffer_address[curbuf]), bytes); stats::get_instance()->read_cached(bytes); cached = curbuf; } else if (buffer_address[1 - curbuf] <= physical_offset && physical_offset < buffer_address[1 - curbuf] + write_block_size) { // block is in previous write buffer assert(physical_offset + bytes <= buffer_address[1 - curbuf] + write_block_size); memcpy(buffer, write_buffer[1 - curbuf] + (physical_offset - buffer_address[1 - curbuf]), bytes); stats::get_instance()->read_cached(bytes); cached = curbuf; } else if (physical_offset == 0xffffffff) { // block was deleted or never written before char* uninitialized = (char*)malloc(sizeof(char)); memset(buffer, *uninitialized, bytes); free(uninitialized); } else { // block is not cached request_ptr req = storage->aread(buffer, physical_offset, bytes); req->wait(false); } STXXL_VERBOSE_WBTL("wbtl:sread l" << FMT_A_S(offset, bytes) << " @ p" << FMT_A(physical_offset) << " " << std::dec << cached); STXXL_UNUSED(cached); }
void request::check_nref_failed(bool after) { STXXL_ERRMSG("WARNING: serious error, reference to the request is lost " << (after ? "after" : "before") << " serve()" << " nref=" << get_reference_count() << " this=" << this << " offset=" << m_offset << " buffer=" << m_buffer << " bytes=" << m_bytes << " type=" << ((m_type == READ) ? "READ" : "WRITE") << " file=" << m_file << " iotype=" << m_file->io_type() ); }
int main(int argc, char * argv[]) { if (argc < 5) { STXXL_ERRMSG("Usage: " << argv[0] << " D L m seed"); return -1; } int i; const stxxl::int_type D = atoi(argv[1]); const stxxl::int_type L = atoi(argv[2]); const stxxl::int_type m = atoi(argv[3]); stxxl::ran32State = atoi(argv[4]); stxxl::int_type * disks = new stxxl::int_type[L]; stxxl::int_type * prefetch_order = new stxxl::int_type[L]; int * count = new int[D]; for (i = 0; i < D; i++) count[i] = 0; stxxl::random_number<> rnd; for (i = 0; i < L; i++) { disks[i] = rnd(D); count[disks[i]]++; } for (i = 0; i < D; i++) std::cout << "Disk " << i << " has " << count[i] << " blocks" << std::endl; stxxl::compute_prefetch_schedule(disks, disks + L, prefetch_order, m, D); STXXL_MSG("Prefetch order:"); for (i = 0; i < L; ++i) { STXXL_MSG("request " << prefetch_order[i] << " on disk " << disks[prefetch_order[i]]); } STXXL_MSG("Request order:"); for (i = 0; i < L; ++i) { int j; for (j = 0; prefetch_order[j] != i; ++j) ; STXXL_MSG("request " << i << " on disk " << disks[i] << " scheduled as " << j); } delete[] count; delete[] disks; delete[] prefetch_order; }
void stats::_reset_io_wait_time() { #ifndef STXXL_DO_NOT_COUNT_WAIT_TIME { scoped_mutex_lock WaitLock(wait_mutex); //assert(acc_waits == 0); if (acc_waits) STXXL_ERRMSG("Warning: " << acc_waits << " wait(s) not yet finished"); t_waits = 0.0; p_waits = 0.0; } #endif }
// logical address void wbtl_file::discard(offset_type offset, offset_type size) { scoped_mutex_lock mapping_lock(mapping_mutex); sortseq::iterator physical = address_mapping.find(offset); STXXL_VERBOSE_WBTL("wbtl:discard l" << FMT_A_S(offset, size) << " @ p" << FMT_A(physical != address_mapping.end() ? physical->second : 0xffffffff)); if (physical == address_mapping.end()) { // could be OK if the block was never written ... //STXXL_ERRMSG("discard: mapping not found: " << FMT_A_S(offset, size) << " ==> " << "???"); } else { offset_type physical_offset = physical->second; address_mapping.erase(physical); _add_free_region(physical_offset, size); place_map::iterator reverse = reverse_mapping.find(physical_offset); if (reverse == reverse_mapping.end()) { STXXL_ERRMSG("discard: reverse mapping not found: " << FMT_A_S(offset, size) << " ==> " << "???"); } else { assert(offset == (reverse->second).first); reverse_mapping.erase(reverse); } storage->discard(physical_offset, size); } }
int main(int argc, char** argv) { char progsub[256]; if (stxxl::check_library_version() != 0) STXXL_ERRMSG("version mismatch between headers and library"); if (argc > 1) { for (unsigned int i = 0; subtools[i].name; ++i) { if (strcmp(subtools[i].name, argv[1]) == 0) { // replace argv[1] with call string of subtool. snprintf(progsub, sizeof(progsub), "%s %s", argv[0], argv[1]); argv[1] = progsub; return subtools[i].func(argc - 1, argv + 1); } } std::cout << "Unknown subtool '" << argv[1] << "'" << std::endl; } return main_usage(argv[0]); }
void test_all_strategies( stxxl::uint64 data_mem, unsigned memory_to_use, int strategy) { switch (strategy) { case 0: test<T, stxxl::striping, block_size>(data_mem, memory_to_use); break; case 1: test<T, stxxl::SR, block_size>(data_mem, memory_to_use); break; case 2: test<T, stxxl::FR, block_size>(data_mem, memory_to_use); break; case 3: test<T, stxxl::RC, block_size>(data_mem, memory_to_use); break; default: STXXL_ERRMSG("Unknown allocation strategy: " << strategy << ", aborting"); abort(); } }
int main(int argc, char * argv[]) { if (argc < 6) { STXXL_ERRMSG("Usage: " << argv[0] << " <MiB to sort> <MiB to use> <alloc_strategy [0..3]> <blk_size [0..14]> <seed>"); return -1; } #if STXXL_PARALLEL_MULTIWAY_MERGE STXXL_MSG("STXXL_PARALLEL_MULTIWAY_MERGE"); #endif stxxl::uint64 data_mem = stxxl::atoint64(argv[1]) * MB; int sort_mem = atoi(argv[2]) * MB; int strategy = atoi(argv[3]); int block_size = atoi(argv[4]); stxxl::set_seed(strtoul(argv[5], NULL, 10)); STXXL_MSG("Seed " << stxxl::get_next_seed()); stxxl::srandom_number32(); typedef my_type<stxxl::uint64, RECORD_SIZE> my_default_type; switch (block_size) { case 0: test_all_strategies<my_default_type, 128 * 1024>(data_mem, sort_mem, strategy); break; case 1: test_all_strategies<my_default_type, 256 * 1024>(data_mem, sort_mem, strategy); break; case 2: test_all_strategies<my_default_type, 512 * 1024>(data_mem, sort_mem, strategy); break; case 3: test_all_strategies<my_default_type, 1024 * 1024>(data_mem, sort_mem, strategy); break; case 4: test_all_strategies<my_default_type, 2 * 1024 * 1024>(data_mem, sort_mem, strategy); break; case 5: test_all_strategies<my_default_type, 4 * 1024 * 1024>(data_mem, sort_mem, strategy); break; case 6: test_all_strategies<my_default_type, 8 * 1024 * 1024>(data_mem, sort_mem, strategy); break; case 7: test_all_strategies<my_default_type, 16 * 1024 * 1024>(data_mem, sort_mem, strategy); break; case 8: test_all_strategies<my_default_type, 640 * 1024>(data_mem, sort_mem, strategy); break; case 9: test_all_strategies<my_default_type, 768 * 1024>(data_mem, sort_mem, strategy); break; case 10: test_all_strategies<my_default_type, 896 * 1024>(data_mem, sort_mem, strategy); break; case 11: test_all_strategies<my_type<unsigned, 12>, 2 * MB>(data_mem, sort_mem, strategy); break; case 12: test_all_strategies<my_type<unsigned, 12>, 2 * MB + 4096>(data_mem, sort_mem, strategy); break; case 13: test_all_strategies<my_type<unsigned, 20>, 2 * MB + 4096>(data_mem, sort_mem, strategy); break; case 14: test_all_strategies<my_type<unsigned, 8>, 2 * MB>(data_mem, sort_mem, strategy); break; default: STXXL_ERRMSG("Unknown block size: " << block_size << ", aborting"); abort(); } return 0; }
STXXL_BEGIN_NAMESPACE void wincall_file::serve(const request* req) throw (io_error) { scoped_mutex_lock fd_lock(fd_mutex); assert(req->get_file() == this); offset_type offset = req->get_offset(); void* buffer = req->get_buffer(); size_type bytes = req->get_size(); request::request_type type = req->get_type(); if (bytes > 32 * 1024 * 1024) { STXXL_ERRMSG("Using a block size larger than 32 MiB may not work with the " << io_type() << " filetype"); } HANDLE handle = file_des; LARGE_INTEGER desired_pos; desired_pos.QuadPart = offset; if (!SetFilePointerEx(handle, desired_pos, NULL, FILE_BEGIN)) { STXXL_THROW_WIN_LASTERROR(io_error, "SetFilePointerEx in wincall_request::serve()" << " offset=" << offset << " this=" << this << " buffer=" << buffer << " bytes=" << bytes << " type=" << ((type == request::READ) ? "READ" : "WRITE")); } else { stats::scoped_read_write_timer read_write_timer(bytes, type == request::WRITE); if (type == request::READ) { DWORD NumberOfBytesRead = 0; assert(bytes <= std::numeric_limits<DWORD>::max()); if (!ReadFile(handle, buffer, (DWORD)bytes, &NumberOfBytesRead, NULL)) { STXXL_THROW_WIN_LASTERROR(io_error, "ReadFile" << " this=" << this << " offset=" << offset << " buffer=" << buffer << " bytes=" << bytes << " type=" << ((type == request::READ) ? "READ" : "WRITE") << " NumberOfBytesRead= " << NumberOfBytesRead); } else if (NumberOfBytesRead != bytes) { STXXL_THROW_WIN_LASTERROR(io_error, " partial read: missing " << (bytes - NumberOfBytesRead) << " out of " << bytes << " bytes"); } } else { DWORD NumberOfBytesWritten = 0; assert(bytes <= std::numeric_limits<DWORD>::max()); if (!WriteFile(handle, buffer, (DWORD)bytes, &NumberOfBytesWritten, NULL)) { STXXL_THROW_WIN_LASTERROR(io_error, "WriteFile" << " this=" << this << " offset=" << offset << " buffer=" << buffer << " bytes=" << bytes << " type=" << ((type == request::READ) ? "READ" : "WRITE") << " NumberOfBytesWritten= " << NumberOfBytesWritten); } else if (NumberOfBytesWritten != bytes) { STXXL_THROW_WIN_LASTERROR(io_error, " partial write: missing " << (bytes - NumberOfBytesWritten) << " out of " << bytes << " bytes"); } } } }
int main(int argc, char** argv) { stxxl::uint64 volume = 512 * 1024 * 1024; stxxl::uint64 mwmvolume = 50 * 1024 * 1024; stxxl::uint64 iavolume = 10 * 1024 * 1024; unsigned numpbs = 1; unsigned numwbs = 14; stxxl::cmdline_parser cp; cp.set_description("STXXL external array test"); cp.set_author("Thomas Keh <*****@*****.**>"); cp.add_bytes('v', "volume", volume, "Volume to fill into the external array, " "default: 512 MiB, 0 = disable test"); cp.add_bytes('m', "mwmvolume", mwmvolume, "Testing multiway merge of two external arrays - " "the volume of each input array, " "default: 100 MiB, 0 = disable test"); cp.add_bytes('i', "iavolume", iavolume, "Volume to fill into the internal array, " "default: 10 MiB, 0 = disable test"); cp.add_uint('n', "num_prefetch_buffers", numpbs, "Number of prefetch buffer blocks, default: 1"); cp.add_uint('w', "num_write_buffers", numwbs, "Number of write buffer blocks, default: 14"); if (!cp.process(argc, argv)) return EXIT_FAILURE; const size_t block_size = 2 * 1024 * 1024 / sizeof(value_type); if (volume > 0 && volume / sizeof(value_type) < 5 * block_size + 876) { STXXL_ERRMSG("The volume is too small for this test. It must be >= " << (5 * block_size + 876) * sizeof(value_type)); return EXIT_FAILURE; } if (iavolume > 0 && iavolume / sizeof(value_type) < 3) { STXXL_ERRMSG("The internal array volume is too small for this test. " "It must be > " << 3); return EXIT_FAILURE; } STXXL_MEMDUMP(volume); STXXL_MEMDUMP(mwmvolume); STXXL_MEMDUMP(iavolume); STXXL_MEMDUMP(sizeof(value_type)); STXXL_VARDUMP(numpbs); STXXL_VARDUMP(numwbs); bool succ = EXIT_SUCCESS; if (volume > 0) { succ = run_external_array_test(volume) && succ; } if (iavolume > 0) { succ = run_internal_array_test(iavolume) && succ; } if (mwmvolume > 0) { succ = run_multiway_merge<0>(mwmvolume) && succ; succ = run_multiway_merge<1>(mwmvolume) && succ; succ = run_multiway_merge<2>(mwmvolume) && succ; succ = run_multiway_merge<3>(mwmvolume) && succ; succ = run_multiway_merge<4>(mwmvolume) && succ; succ = run_multiway_merge<5>(mwmvolume) && succ; } succ = run_upper_bound_test(3 * ea_type::block_size * sizeof(value_type)) && succ; STXXL_MSG("success = " << succ); return succ; }