コード例 #1
0
void run(char* buffer, file** disks, stxxl::int64 offset, stxxl::int64 length,
         unsigned hdd_blocks, unsigned hdd_bytes, unsigned ssd_blocks, unsigned ssd_bytes, unsigned repeats)
{
    unsigned i, j;
    double begin = timestamp(), end, elapsed;
    request_ptr* reqs = new request_ptr[stxxl::STXXL_MAX(hdd_blocks + ssd_blocks, 1U)];

    struct diskinfo {
        unsigned id;
        unsigned bytes;
        unsigned n;
    };

    diskinfo info[2];

    // HDD
    info[0].id = 0;
    info[0].bytes = hdd_bytes;
    info[0].n = hdd_blocks;

    // SSD
    info[1].id = 1;
    info[1].bytes = ssd_bytes;
    info[1].n = ssd_blocks;

    begin = timestamp();
    double volume = 0;

    for (unsigned repeat = 0; repeat < repeats; ++repeat) {
        int r = 0;
        char* buf = buffer;
        for (i = 0; i < 2; i++)
        {
            for (j = 0; j < info[i].n; j++) {
                unsigned bytes = info[i].bytes;
                stxxl::int64 position = (bytes * (rand() & 0xffff)) % length;
                reqs[r++] = disks[info[i].id]->aread(buf, offset + position, bytes);
                buf += bytes;
                volume += (double)bytes;
            }
        }

        wait_all(reqs, r);
    }

    end = timestamp();
    elapsed = end - begin;

    std::cout << "B_d = " << info[0].bytes << "  B_f = " << info[1].bytes << "  n_d = " << info[0].n << "  n_f = " << info[1].n; //<< std::endl;
    std::cout << " Transferred " << (volume / MB) << " MiB in " << elapsed << " seconds @ " << (volume / MB / elapsed) << " MiB/s" << std::endl;
    delete[] reqs;
}
コード例 #2
0
ファイル: create_files.cpp プロジェクト: songmeixu/stxxl
void watch_times(request_ptr reqs[], unsigned n, double* out)
{
    bool* finished = new bool[n];
    unsigned count = 0;
    unsigned i = 0;
    for (i = 0; i < n; i++)
        finished[i] = false;


    while (count != n)
    {
        usleep(POLL_DELAY);
        i = 0;
        for (i = 0; i < n; i++)
        {
            if (!finished[i])
                if (reqs[i]->poll())
                {
                    finished[i] = true;
                    out[i] = timestamp();
                    count++;
                }
        }
    }
    delete[] finished;
}
コード例 #3
0
    BenchmarkSort(const char* desc, uint64 length, unsigned_type memsize)
    {
        // construct vector
        typedef typename stxxl::VECTOR_GENERATOR<ValueType>::result vector_type;

        uint64 vec_size = stxxl::div_ceil(length, sizeof(ValueType));

        vector_type vec(vec_size);

        // construct random stream

        std::cout << "#!!! running sorting test with " << desc << " = " << sizeof(ValueType) << " bytes."
                  << std::endl;
        {
            std::cout << "# materialize random_stream into vector of size " << vec.size() << std::endl;
            double ts1 = timestamp();

            random_stream rs(vec_size);
            stxxl::stream::materialize(rs, vec.begin(), vec.end());

            double elapsed = timestamp() - ts1;
            output_result(elapsed, vec_size);
        }
        {
            std::cout << "# stxxl::sort vector of size " << vec.size() << std::endl;
            double ts1 = timestamp();

            stxxl::sort(vec.begin(), vec.end(), value_less(), memsize);

            double elapsed = timestamp() - ts1;
            output_result(elapsed, vec_size);
        }
        {
            std::cout << "# stxxl::ksort vector of size " << vec.size() << std::endl;
            double ts1 = timestamp();

            stxxl::ksort(vec.begin(), vec.end(), value_key_second(), memsize);

            double elapsed = timestamp() - ts1;
            output_result(elapsed, vec_size);
        }
        vec.clear();

        {
            std::cout << "# stxxl::stream::sort of size " << vec_size << std::endl;
            double ts1 = timestamp();

            typedef stxxl::stream::sort<random_stream, value_less>
                random_stream_sort_type;

            random_stream stream(vec_size);
            random_stream_sort_type stream_sort(stream, value_less(), memsize);

            stxxl::stream::discard(stream_sort);

            double elapsed = timestamp() - ts1;
            output_result(elapsed, vec_size);
        }

        std::cout << std::endl;
    }
コード例 #4
0
int main()
{
    const stxxl::int64 disk_size = stxxl::int64(1024 * 1024) * 1024 * 40;
    std::cout << sizeof(void *) << std::endl;
    const int block_size = 4 * 1024 * 1024;
    char * buffer = static_cast<char *>(stxxl::aligned_alloc<BLOCK_ALIGN>(block_size));
    memset(buffer, 0, block_size);
    const char * paths[2] = { "/tmp/data1", "/tmp/data2" };
    stxxl::sim_disk_file file1(paths[0], file::CREAT | file::RDWR /* | file::DIRECT */, 0);
    file1.set_size(disk_size);

    stxxl::sim_disk_file file2(paths[1], file::CREAT | file::RDWR /* | file::DIRECT */, 1);
    file2.set_size(disk_size);

    unsigned i = 0;

    stxxl::int64 pos = 0;

    stxxl::request_ptr req;

    STXXL_MSG("Estimated time:" << block_size / double(AVERAGE_SPEED));
    STXXL_MSG("Sequential write");

    for (i = 0; i < 40; i++)
    {
        double begin = timestamp();
        req = file1.awrite(buffer, pos, block_size, stxxl::default_completion_handler());
        req->wait();
        double end = timestamp();

        STXXL_MSG("Pos: " << pos << " block_size:" << block_size << " time:" << (end - begin));
        pos += 1024 * 1024 * 1024;
    }

    double sum = 0.;
    double sum2 = 0.;
    STXXL_MSG("Random write");
    const unsigned int times = 80;
    for (i = 0; i < times; i++)
    {
        stxxl::random_number<> rnd;
        pos = rnd(disk_size / block_size) * block_size;
        double begin = timestamp();
        req = file1.awrite(buffer, pos, block_size, stxxl::default_completion_handler());
        req->wait();
        double diff = timestamp() - begin;

        sum += diff;
        sum2 += diff * diff;

        STXXL_MSG("Pos: " << pos << " block_size:" << block_size << " time:" << (diff));
    }

    sum = sum / double(times);
    sum2 = sum2 / double(times);
    assert(sum2 - sum * sum >= 0.0);
    double err = sqrt(sum2 - sum * sum);
    STXXL_MSG("Standard Deviation: " << err << " s, " << 100. * (err / sum) << " %");

    stxxl::aligned_dealloc<BLOCK_ALIGN>(buffer);

    unlink(paths[0]);
    unlink(paths[1]);

    return 0;
}
コード例 #5
0
ファイル: create_files.cpp プロジェクト: songmeixu/stxxl
int create_files(int argc, char* argv[])
{
    std::vector<std::string> disks_arr;
    stxxl::uint64 offset = 0, length;

    stxxl::cmdline_parser cp;
    cp.add_param_bytes("filesize", "Number of bytes to write to files.", length);
    cp.add_param_stringlist("filename", "Paths to files to write.", disks_arr);

    if (!cp.process(argc, argv))
        return -1;

    stxxl::uint64 endpos = offset + length;

    for (size_t i = 0; i < disks_arr.size(); ++i)
    {
        unlink(disks_arr[i].c_str());
        std::cout << "# Add disk: " << disks_arr[i] << std::endl;
    }

    const size_t ndisks = disks_arr.size();

#if STXXL_WINDOWS
    unsigned buffer_size = 64 * MB;
#else
    unsigned buffer_size = 256 * MB;
#endif
    const unsigned buffer_size_int = buffer_size / sizeof(int);

    unsigned chunks = 2;
    const unsigned chunk_size = buffer_size / chunks;
    const unsigned chunk_size_int = chunk_size / sizeof(int);

    unsigned i = 0, j = 0;

    int* buffer = (int*)stxxl::aligned_alloc<BLOCK_ALIGN>(buffer_size * ndisks);
    file** disks = new file*[ndisks];
    request_ptr* reqs = new request_ptr[ndisks * chunks];
#ifdef WATCH_TIMES
    double* r_finish_times = new double[ndisks];
    double* w_finish_times = new double[ndisks];
#endif

    for (i = 0; i < ndisks * buffer_size_int; i++)
        buffer[i] = i;

    for (i = 0; i < ndisks; i++)
    {
#if STXXL_WINDOWS
 #ifdef RAW_ACCESS
        disks[i] = new stxxl::wincall_file(disks_arr[i],
                                           file::CREAT | file::RDWR | file::DIRECT, i);
 #else
        disks[i] = new stxxl::wincall_file(disks_arr[i],
                                           file::CREAT | file::RDWR, i);
 #endif
#else
 #ifdef RAW_ACCESS
        disks[i] = new stxxl::syscall_file(disks_arr[i],
                                           file::CREAT | file::RDWR | file::DIRECT, i);
 #else
        disks[i] = new stxxl::syscall_file(disks_arr[i],
                                           file::CREAT | file::RDWR, i);
 #endif
#endif
    }

    while (offset < endpos)
    {
        const stxxl::int64 current_block_size = length ? std::min<stxxl::int64>(buffer_size, endpos - offset) : buffer_size;
        const stxxl::int64 current_chunk_size = current_block_size / chunks;

        std::cout << "Disk offset " << std::setw(7) << offset / MB << " MiB: " << std::fixed;

        double begin = timestamp(), end;

#ifndef DO_ONLY_READ
        for (i = 0; i < ndisks; i++)
        {
            for (j = 0; j < chunks; j++)
                reqs[i * chunks + j] =
                    disks[i]->awrite(buffer + buffer_size_int * i + j * chunk_size_int,
                                     offset + j * current_chunk_size,
                                     current_chunk_size,
                                     stxxl::default_completion_handler());
        }

 #ifdef WATCH_TIMES
        watch_times(reqs, ndisks, w_finish_times);
 #else
        wait_all(reqs, ndisks * chunks);
 #endif

        end = timestamp();

#if 0
        std::cout << "WRITE\nDisks: " << ndisks
                  << " \nElapsed time: " << end - begin
                  << " \nThroughput: " << int(double(buffer_size * ndisks) / MB / (end - begin))
                  << " MiB/s \nPer one disk:"
                  << int((buffer_size) / MB / (end - begin)) << " MiB/s"
                  << std::endl;
#endif

 #ifdef WATCH_TIMES
        out_stat(begin, end, w_finish_times, ndisks, disks_arr);
 #endif
        std::cout << std::setw(7) << int(double(current_block_size) / MB / (end - begin)) << " MiB/s,";
#endif


#ifndef NOREAD
        begin = timestamp();

        for (i = 0; i < ndisks; i++)
        {
            for (j = 0; j < chunks; j++)
                reqs[i * chunks + j] = disks[i]->aread(buffer + buffer_size_int * i + j * chunk_size_int,
                                                       offset + j * current_chunk_size,
                                                       current_chunk_size,
                                                       stxxl::default_completion_handler());
        }

 #ifdef WATCH_TIMES
        watch_times(reqs, ndisks, r_finish_times);
 #else
        wait_all(reqs, ndisks * chunks);
 #endif

        end = timestamp();

#if 0
        std::cout << "READ\nDisks: " << ndisks
                  << " \nElapsed time: " << end - begin
                  << " \nThroughput: " << int(double(buffer_size * ndisks) / MB / (end - begin))
                  << " MiB/s \nPer one disk:"
                  << int(double(buffer_size) / MB / (end - begin)) << " MiB/s"
                  << std::endl;
#endif

        std::cout << int(double(current_block_size) / MB / (end - begin)) << " MiB/s" << std::endl;

#ifdef WATCH_TIMES
        out_stat(begin, end, r_finish_times, ndisks, disks_arr);
#endif

        if (CHECK_AFTER_READ) {
            for (int i = 0; unsigned(i) < ndisks * buffer_size_int; i++)
            {
                if (buffer[i] != i)
                {
                    int ibuf = i / buffer_size_int;
                    int pos = i % buffer_size_int;

                    std::cout << "Error on disk " << ibuf << " position " << std::hex << std::setw(8) << offset + pos * sizeof(int)
                              << "  got: " << std::hex << std::setw(8) << buffer[i] << " wanted: " << std::hex << std::setw(8) << i
                              << std::dec << std::endl;

                    i = (ibuf + 1) * buffer_size_int; // jump to next
                }
            }
        }
#else
        std::cout << std::endl;
#endif

        offset += current_block_size;
    }

#ifdef WATCH_TIMES
    delete[] r_finish_times;
    delete[] w_finish_times;
#endif
    delete[] reqs;
    for (i = 0; i < ndisks; i++)
        delete disks[i];
    delete[] disks;
    stxxl::aligned_dealloc<BLOCK_ALIGN>(buffer);

    return 0;
}