void stats::reset()
{
{
scoped_mutex_lock ReadLock(read_mutex);
//assert(acc_reads == 0);
if (acc_reads)
STXXL_ERRMSG("Warning: " << acc_reads <<
" read(s) not yet finished");
reads = 0;
volume_read = 0;
c_reads = 0;
c_volume_read = 0;
t_reads = 0;
p_reads = 0.0;
}
{
scoped_mutex_lock WriteLock(write_mutex);
//assert(acc_writes == 0);
if (acc_writes)
STXXL_ERRMSG("Warning: " << acc_writes <<
" write(s) not yet finished");
writes = 0;
volume_written = 0;
c_writes = 0;
c_volume_written = 0;
t_writes = 0.0;
p_writes = 0.0;
}
{
scoped_mutex_lock IOLock(io_mutex);
//assert(acc_ios == 0);
if (acc_ios)
STXXL_ERRMSG("Warning: " << acc_ios <<
" io(s) not yet finished");
p_ios = 0.0;
}
{
scoped_mutex_lock WaitLock(wait_mutex);
//assert(acc_waits == 0);
if (acc_waits)
STXXL_ERRMSG("Warning: " << acc_waits <<
" wait(s) not yet finished");
t_waits = 0.0;
p_waits = 0.0;
t_wait_read = 0.0;
p_wait_read = 0.0;
t_wait_write = 0.0;
p_wait_write = 0.0;
}
last_reset = timestamp();
}
bool linuxaio_queue::cancel_request(request_ptr& req)
{
if (req.empty())
STXXL_THROW_INVALID_ARGUMENT("Empty request canceled disk_queue.");
if (post_thread_state() != RUNNING)
STXXL_ERRMSG("Request canceled in stopped queue.");
if (!dynamic_cast<linuxaio_request*>(req.get()))
STXXL_ERRMSG("Non-LinuxAIO request submitted to LinuxAIO queue.");
queue_type::iterator pos;
{
scoped_mutex_lock lock(waiting_mtx);
pos = std::find(waiting_requests.begin(), waiting_requests.end(),
req _STXXL_FORCE_SEQUENTIAL);
if (pos != waiting_requests.end())
{
waiting_requests.erase(pos);
// polymorphic_downcast to linuxaio_request,
// request is canceled, but was not yet posted.
dynamic_cast<linuxaio_request*>(req.get())->completed(false, true);
num_waiting_requests--; // will never block
return true;
}
}
scoped_mutex_lock lock(posted_mtx);
return false;
}
void linuxaio_queue::add_request(request_ptr& req)
{
if (req.empty())
STXXL_THROW_INVALID_ARGUMENT("Empty request submitted to disk_queue.");
if (post_thread_state() != RUNNING)
STXXL_ERRMSG("Request submitted to stopped queue.");
if (!dynamic_cast<linuxaio_request*>(req.get()))
STXXL_ERRMSG("Non-LinuxAIO request submitted to LinuxAIO queue.");
scoped_mutex_lock lock(waiting_mtx);
waiting_requests.push_back(req);
num_waiting_requests++;
}
void request::check_alignment() const
{
if (m_offset % STXXL_BLOCK_ALIGN != 0)
STXXL_ERRMSG("Offset is not aligned: modulo " <<
STXXL_BLOCK_ALIGN << " = " << m_offset % STXXL_BLOCK_ALIGN);
if (m_bytes % STXXL_BLOCK_ALIGN != 0)
STXXL_ERRMSG("Size is not a multiple of " <<
STXXL_BLOCK_ALIGN << ", = " << m_bytes % STXXL_BLOCK_ALIGN);
if (unsigned_type(m_buffer) % STXXL_BLOCK_ALIGN != 0)
STXXL_ERRMSG("Buffer is not aligned: modulo " <<
STXXL_BLOCK_ALIGN << " = " << unsigned_type(m_buffer) % STXXL_BLOCK_ALIGN <<
" (" << m_buffer << ")");
}
int main()
{
stxxl::config* config = stxxl::config::get_instance();
stxxl::disk_config disk1("/tmp/stxxl-###.tmp", 16 * 1024 * 1024,
"syscall autogrow=no");
disk1.unlink_on_open = true;
disk1.direct = stxxl::disk_config::DIRECT_OFF;
config->add_disk(disk1);
vector_type* vector = new vector_type();
try {
while (true)
vector->push_back(0);
}
catch (std::exception& e) {
STXXL_ERRMSG("Caught exception: " << e.what());
delete vector; // here it will crash in block_manager::delete_block(bid)
}
STXXL_MSG("Delete done, all is well.");
return 0;
}
int main(int argc, char* argv[])
{
if (argc < 2)
{
STXXL_MSG("Usage: " << argv[0] << " #log_ins");
return -1;
}
const int log_nins = atoi(argv[1]);
if (log_nins > 31) {
STXXL_ERRMSG("This test can't do more than 2^31 operations, you requested 2^" << log_nins);
return -1;
}
btree_type BTree(1024 * 128, 1024 * 128);
const stxxl::uint64 nins = 1ULL << log_nins;
stxxl::ran32State = (unsigned int)time(NULL);
stxxl::vector<int> Values(nins);
STXXL_MSG("Generating " << nins << " random values");
stxxl::generate(Values.begin(), Values.end(), rnd_gen(), 4);
stxxl::vector<int>::const_iterator it = Values.begin();
STXXL_MSG("Inserting " << nins << " random values into btree");
for ( ; it != Values.end(); ++it)
BTree.insert(std::pair<int, double>(*it, double(*it) + 1.0));
STXXL_MSG("Number of elements in btree: " << BTree.size());
STXXL_MSG("Searching " << nins << " existing elements");
stxxl::vector<int>::const_iterator vIt = Values.begin();
for ( ; vIt != Values.end(); ++vIt)
{
btree_type::iterator bIt = BTree.find(*vIt);
STXXL_CHECK(bIt != BTree.end());
STXXL_CHECK(bIt->first == *vIt);
}
STXXL_MSG("Searching " << nins << " non-existing elements");
stxxl::vector<int>::const_iterator vIt1 = Values.begin();
for ( ; vIt1 != Values.end(); ++vIt1)
{
btree_type::iterator bIt = BTree.find((*vIt1) + 1);
STXXL_CHECK(bIt == BTree.end());
}
STXXL_MSG("Test passed.");
return 0;
}
void wbtl_file::sread(void* buffer, offset_type offset, size_type bytes)
{
scoped_mutex_lock buffer_lock(buffer_mutex);
int cached = -1;
offset_type physical_offset;
// map logical to physical address
{
scoped_mutex_lock mapping_lock(mapping_mutex);
sortseq::iterator physical = address_mapping.find(offset);
if (physical == address_mapping.end()) {
STXXL_ERRMSG("wbtl_read: mapping not found: " << FMT_A_S(offset, bytes) << " ==> " << "???");
//STXXL_THROW_ERRNO(io_error, "wbtl_read of unmapped memory");
physical_offset = 0xffffffff;
} else {
physical_offset = physical->second;
}
}
if (buffer_address[curbuf] <= physical_offset &&
physical_offset < buffer_address[curbuf] + write_block_size)
{
// block is in current write buffer
assert(physical_offset + bytes <= buffer_address[curbuf] + write_block_size);
memcpy(buffer, write_buffer[curbuf] + (physical_offset - buffer_address[curbuf]), bytes);
stats::get_instance()->read_cached(bytes);
cached = curbuf;
}
else if (buffer_address[1 - curbuf] <= physical_offset &&
physical_offset < buffer_address[1 - curbuf] + write_block_size)
{
// block is in previous write buffer
assert(physical_offset + bytes <= buffer_address[1 - curbuf] + write_block_size);
memcpy(buffer, write_buffer[1 - curbuf] + (physical_offset - buffer_address[1 - curbuf]), bytes);
stats::get_instance()->read_cached(bytes);
cached = curbuf;
}
else if (physical_offset == 0xffffffff) {
// block was deleted or never written before
char* uninitialized = (char*)malloc(sizeof(char));
memset(buffer, *uninitialized, bytes);
free(uninitialized);
}
else
{
// block is not cached
request_ptr req = storage->aread(buffer, physical_offset, bytes);
req->wait(false);
}
STXXL_VERBOSE_WBTL("wbtl:sread l" << FMT_A_S(offset, bytes) << " @ p" << FMT_A(physical_offset) << " " << std::dec << cached);
STXXL_UNUSED(cached);
}
void request::check_nref_failed(bool after)
{
STXXL_ERRMSG("WARNING: serious error, reference to the request is lost " <<
(after ? "after" : "before") << " serve()" <<
" nref=" << get_reference_count() <<
" this=" << this <<
" offset=" << m_offset <<
" buffer=" << m_buffer <<
" bytes=" << m_bytes <<
" type=" << ((m_type == READ) ? "READ" : "WRITE") <<
" file=" << m_file <<
" iotype=" << m_file->io_type()
);
}
int main(int argc, char * argv[])
{
if (argc < 5)
{
STXXL_ERRMSG("Usage: " << argv[0] << " D L m seed");
return -1;
}
int i;
const stxxl::int_type D = atoi(argv[1]);
const stxxl::int_type L = atoi(argv[2]);
const stxxl::int_type m = atoi(argv[3]);
stxxl::ran32State = atoi(argv[4]);
stxxl::int_type * disks = new stxxl::int_type[L];
stxxl::int_type * prefetch_order = new stxxl::int_type[L];
int * count = new int[D];
for (i = 0; i < D; i++)
count[i] = 0;
stxxl::random_number<> rnd;
for (i = 0; i < L; i++)
{
disks[i] = rnd(D);
count[disks[i]]++;
}
for (i = 0; i < D; i++)
std::cout << "Disk " << i << " has " << count[i] << " blocks" << std::endl;
stxxl::compute_prefetch_schedule(disks, disks + L, prefetch_order, m, D);
STXXL_MSG("Prefetch order:");
for (i = 0; i < L; ++i) {
STXXL_MSG("request " << prefetch_order[i] << " on disk " << disks[prefetch_order[i]]);
}
STXXL_MSG("Request order:");
for (i = 0; i < L; ++i) {
int j;
for (j = 0; prefetch_order[j] != i; ++j) ;
STXXL_MSG("request " << i << " on disk " << disks[i] << " scheduled as " << j);
}
delete[] count;
delete[] disks;
delete[] prefetch_order;
}
void stats::_reset_io_wait_time()
{
#ifndef STXXL_DO_NOT_COUNT_WAIT_TIME
{
scoped_mutex_lock WaitLock(wait_mutex);
//assert(acc_waits == 0);
if (acc_waits)
STXXL_ERRMSG("Warning: " << acc_waits <<
" wait(s) not yet finished");
t_waits = 0.0;
p_waits = 0.0;
}
#endif
}
// logical address
void wbtl_file::discard(offset_type offset, offset_type size)
{
scoped_mutex_lock mapping_lock(mapping_mutex);
sortseq::iterator physical = address_mapping.find(offset);
STXXL_VERBOSE_WBTL("wbtl:discard l" << FMT_A_S(offset, size) << " @ p" << FMT_A(physical != address_mapping.end() ? physical->second : 0xffffffff));
if (physical == address_mapping.end()) {
// could be OK if the block was never written ...
//STXXL_ERRMSG("discard: mapping not found: " << FMT_A_S(offset, size) << " ==> " << "???");
} else {
offset_type physical_offset = physical->second;
address_mapping.erase(physical);
_add_free_region(physical_offset, size);
place_map::iterator reverse = reverse_mapping.find(physical_offset);
if (reverse == reverse_mapping.end()) {
STXXL_ERRMSG("discard: reverse mapping not found: " << FMT_A_S(offset, size) << " ==> " << "???");
} else {
assert(offset == (reverse->second).first);
reverse_mapping.erase(reverse);
}
storage->discard(physical_offset, size);
}
}
int main(int argc, char** argv)
{
char progsub[256];
if (stxxl::check_library_version() != 0)
STXXL_ERRMSG("version mismatch between headers and library");
if (argc > 1)
{
for (unsigned int i = 0; subtools[i].name; ++i)
{
if (strcmp(subtools[i].name, argv[1]) == 0)
{
// replace argv[1] with call string of subtool.
snprintf(progsub, sizeof(progsub), "%s %s", argv[0], argv[1]);
argv[1] = progsub;
return subtools[i].func(argc - 1, argv + 1);
}
}
std::cout << "Unknown subtool '" << argv[1] << "'" << std::endl;
}
return main_usage(argv[0]);
}
void test_all_strategies(
stxxl::uint64 data_mem,
unsigned memory_to_use,
int strategy)
{
switch (strategy)
{
case 0:
test<T, stxxl::striping, block_size>(data_mem, memory_to_use);
break;
case 1:
test<T, stxxl::SR, block_size>(data_mem, memory_to_use);
break;
case 2:
test<T, stxxl::FR, block_size>(data_mem, memory_to_use);
break;
case 3:
test<T, stxxl::RC, block_size>(data_mem, memory_to_use);
break;
default:
STXXL_ERRMSG("Unknown allocation strategy: " << strategy << ", aborting");
abort();
}
}
int main(int argc, char * argv[])
{
if (argc < 6)
{
STXXL_ERRMSG("Usage: " << argv[0] <<
" <MiB to sort> <MiB to use> <alloc_strategy [0..3]> <blk_size [0..14]> <seed>");
return -1;
}
#if STXXL_PARALLEL_MULTIWAY_MERGE
STXXL_MSG("STXXL_PARALLEL_MULTIWAY_MERGE");
#endif
stxxl::uint64 data_mem = stxxl::atoint64(argv[1]) * MB;
int sort_mem = atoi(argv[2]) * MB;
int strategy = atoi(argv[3]);
int block_size = atoi(argv[4]);
stxxl::set_seed(strtoul(argv[5], NULL, 10));
STXXL_MSG("Seed " << stxxl::get_next_seed());
stxxl::srandom_number32();
typedef my_type<stxxl::uint64, RECORD_SIZE> my_default_type;
switch (block_size)
{
case 0:
test_all_strategies<my_default_type, 128 * 1024>(data_mem, sort_mem, strategy);
break;
case 1:
test_all_strategies<my_default_type, 256 * 1024>(data_mem, sort_mem, strategy);
break;
case 2:
test_all_strategies<my_default_type, 512 * 1024>(data_mem, sort_mem, strategy);
break;
case 3:
test_all_strategies<my_default_type, 1024 * 1024>(data_mem, sort_mem, strategy);
break;
case 4:
test_all_strategies<my_default_type, 2 * 1024 * 1024>(data_mem, sort_mem, strategy);
break;
case 5:
test_all_strategies<my_default_type, 4 * 1024 * 1024>(data_mem, sort_mem, strategy);
break;
case 6:
test_all_strategies<my_default_type, 8 * 1024 * 1024>(data_mem, sort_mem, strategy);
break;
case 7:
test_all_strategies<my_default_type, 16 * 1024 * 1024>(data_mem, sort_mem, strategy);
break;
case 8:
test_all_strategies<my_default_type, 640 * 1024>(data_mem, sort_mem, strategy);
break;
case 9:
test_all_strategies<my_default_type, 768 * 1024>(data_mem, sort_mem, strategy);
break;
case 10:
test_all_strategies<my_default_type, 896 * 1024>(data_mem, sort_mem, strategy);
break;
case 11:
test_all_strategies<my_type<unsigned, 12>, 2 * MB>(data_mem, sort_mem, strategy);
break;
case 12:
test_all_strategies<my_type<unsigned, 12>, 2 * MB + 4096>(data_mem, sort_mem, strategy);
break;
case 13:
test_all_strategies<my_type<unsigned, 20>, 2 * MB + 4096>(data_mem, sort_mem, strategy);
break;
case 14:
test_all_strategies<my_type<unsigned, 8>, 2 * MB>(data_mem, sort_mem, strategy);
break;
default:
STXXL_ERRMSG("Unknown block size: " << block_size << ", aborting");
abort();
}
return 0;
}
STXXL_BEGIN_NAMESPACE
void wincall_file::serve(const request* req) throw (io_error)
{
scoped_mutex_lock fd_lock(fd_mutex);
assert(req->get_file() == this);
offset_type offset = req->get_offset();
void* buffer = req->get_buffer();
size_type bytes = req->get_size();
request::request_type type = req->get_type();
if (bytes > 32 * 1024 * 1024) {
STXXL_ERRMSG("Using a block size larger than 32 MiB may not work with the " << io_type() << " filetype");
}
HANDLE handle = file_des;
LARGE_INTEGER desired_pos;
desired_pos.QuadPart = offset;
if (!SetFilePointerEx(handle, desired_pos, NULL, FILE_BEGIN))
{
STXXL_THROW_WIN_LASTERROR(io_error,
"SetFilePointerEx in wincall_request::serve()" <<
" offset=" << offset <<
" this=" << this <<
" buffer=" << buffer <<
" bytes=" << bytes <<
" type=" << ((type == request::READ) ? "READ" : "WRITE"));
}
else
{
stats::scoped_read_write_timer read_write_timer(bytes, type == request::WRITE);
if (type == request::READ)
{
DWORD NumberOfBytesRead = 0;
assert(bytes <= std::numeric_limits<DWORD>::max());
if (!ReadFile(handle, buffer, (DWORD)bytes, &NumberOfBytesRead, NULL))
{
STXXL_THROW_WIN_LASTERROR(io_error,
"ReadFile" <<
" this=" << this <<
" offset=" << offset <<
" buffer=" << buffer <<
" bytes=" << bytes <<
" type=" << ((type == request::READ) ? "READ" : "WRITE") <<
" NumberOfBytesRead= " << NumberOfBytesRead);
}
else if (NumberOfBytesRead != bytes) {
STXXL_THROW_WIN_LASTERROR(io_error, " partial read: missing " << (bytes - NumberOfBytesRead) << " out of " << bytes << " bytes");
}
}
else
{
DWORD NumberOfBytesWritten = 0;
assert(bytes <= std::numeric_limits<DWORD>::max());
if (!WriteFile(handle, buffer, (DWORD)bytes, &NumberOfBytesWritten, NULL))
{
STXXL_THROW_WIN_LASTERROR(io_error,
"WriteFile" <<
" this=" << this <<
" offset=" << offset <<
" buffer=" << buffer <<
" bytes=" << bytes <<
" type=" << ((type == request::READ) ? "READ" : "WRITE") <<
" NumberOfBytesWritten= " << NumberOfBytesWritten);
}
else if (NumberOfBytesWritten != bytes) {
STXXL_THROW_WIN_LASTERROR(io_error, " partial write: missing " << (bytes - NumberOfBytesWritten) << " out of " << bytes << " bytes");
}
}
}
}
int main(int argc, char** argv)
{
stxxl::uint64 volume = 512 * 1024 * 1024;
stxxl::uint64 mwmvolume = 50 * 1024 * 1024;
stxxl::uint64 iavolume = 10 * 1024 * 1024;
unsigned numpbs = 1;
unsigned numwbs = 14;
stxxl::cmdline_parser cp;
cp.set_description("STXXL external array test");
cp.set_author("Thomas Keh <*****@*****.**>");
cp.add_bytes('v', "volume", volume,
"Volume to fill into the external array, "
"default: 512 MiB, 0 = disable test");
cp.add_bytes('m', "mwmvolume", mwmvolume,
"Testing multiway merge of two external arrays - "
"the volume of each input array, "
"default: 100 MiB, 0 = disable test");
cp.add_bytes('i', "iavolume", iavolume,
"Volume to fill into the internal array, "
"default: 10 MiB, 0 = disable test");
cp.add_uint('n', "num_prefetch_buffers", numpbs,
"Number of prefetch buffer blocks, default: 1");
cp.add_uint('w', "num_write_buffers", numwbs,
"Number of write buffer blocks, default: 14");
if (!cp.process(argc, argv))
return EXIT_FAILURE;
const size_t block_size = 2 * 1024 * 1024 / sizeof(value_type);
if (volume > 0 && volume / sizeof(value_type) < 5 * block_size + 876) {
STXXL_ERRMSG("The volume is too small for this test. It must be >= " <<
(5 * block_size + 876) * sizeof(value_type));
return EXIT_FAILURE;
}
if (iavolume > 0 && iavolume / sizeof(value_type) < 3) {
STXXL_ERRMSG("The internal array volume is too small for this test. "
"It must be > " << 3);
return EXIT_FAILURE;
}
STXXL_MEMDUMP(volume);
STXXL_MEMDUMP(mwmvolume);
STXXL_MEMDUMP(iavolume);
STXXL_MEMDUMP(sizeof(value_type));
STXXL_VARDUMP(numpbs);
STXXL_VARDUMP(numwbs);
bool succ = EXIT_SUCCESS;
if (volume > 0) {
succ = run_external_array_test(volume) && succ;
}
if (iavolume > 0) {
succ = run_internal_array_test(iavolume) && succ;
}
if (mwmvolume > 0) {
succ = run_multiway_merge<0>(mwmvolume) && succ;
succ = run_multiway_merge<1>(mwmvolume) && succ;
succ = run_multiway_merge<2>(mwmvolume) && succ;
succ = run_multiway_merge<3>(mwmvolume) && succ;
succ = run_multiway_merge<4>(mwmvolume) && succ;
succ = run_multiway_merge<5>(mwmvolume) && succ;
}
succ = run_upper_bound_test(3 * ea_type::block_size * sizeof(value_type)) && succ;
STXXL_MSG("success = " << succ);
return succ;
}
