Пример #1
0
void aio_testcase(uint64_t block_size, uint64_t total_size_mb, size_t concurrency, bool is_write, bool random_offset)
{
    std::unique_ptr<char[]> buffer(new char[block_size]);
    std::atomic_uint remain_concurrency;
    remain_concurrency = concurrency;
    if (is_write && utils::filesystem::file_exists("temp"))
    {
        utils::filesystem::remove_path("temp");
        dassert(!utils::filesystem::file_exists("temp"), "");
    }
    auto file_handle = dsn_file_open("temp", O_CREAT | O_RDWR, 0666);
    auto total_size_bytes = total_size_mb * 1024 * 1024;
    auto tic = std::chrono::steady_clock::now();

    for (int bytes_written = 0; bytes_written < total_size_bytes; )
    {
        while (true)
        {
            if (remain_concurrency.fetch_sub(1, std::memory_order_acquire) <= 0)
            {
                remain_concurrency.fetch_add(1, std::memory_order_relaxed);
            }
            else
            {
                break;
            }
        }
        auto cb = [&](error_code ec, int sz)
        {
            dassert(ec == ERR_OK && uint64_t(sz) == block_size,
                "ec = %s, sz = %d, block_size = %" PRId64 "",
                ec.to_string(), sz, block_size
                );
            remain_concurrency.fetch_add(1, std::memory_order_relaxed);
        };
        auto offset = random_offset ? dsn_random64(0, total_size_bytes - block_size) : bytes_written;
        if (is_write)
        {
            file::write(file_handle, buffer.get(), block_size, offset, LPC_AIO_TEST, nullptr, cb);
        }
        else
        {
            file::read(file_handle, buffer.get(), block_size, offset, LPC_AIO_TEST, nullptr, cb);
        }

        bytes_written += block_size;
    }
    while (remain_concurrency != concurrency)
    {
        ;
    }
    dsn_file_flush(file_handle);
    auto toc = std::chrono::steady_clock::now();
    auto c*k = dsn_file_close(file_handle);
    EXPECT_EQ(c*k, ERR_OK);

    std::cout << "is_write = " << is_write
        << " random_offset = " << random_offset
        << " block_size = " << block_size
        << " concurrency = " << concurrency
        << " throughput = " << double(total_size_mb) * 1000000 / std::chrono::duration_cast<std::chrono::microseconds>(toc - tic).count() << " mB/s" << std::endl;
}
Пример #2
0
void meta_service::start(const char* data_dir, bool clean_state)
{
    dassert(!_started, "meta service is already started");

    _data_dir = data_dir;
	std::string checkpoint_path = _data_dir + "/checkpoint";
	std::string oplog_path = _data_dir + "/oplog";


    if (clean_state)
    {
        try {
			if (!dsn::utils::filesystem::remove_path(checkpoint_path))
			{
				dassert(false, "Fail to remove file %s.", checkpoint_path.c_str());
			}

			if (!dsn::utils::filesystem::remove_path(oplog_path))
			{
				dassert(false, "Fail to remove file %s.", oplog_path.c_str());
			}
        }
        catch (std::exception& ex)
        {
            ex;
        }
    }
    else
    {
		if (!dsn::utils::filesystem::create_directory(_data_dir))
		{
			dassert(false, "Fail to create directory %s.", _data_dir.c_str());
		}

        if (dsn::utils::filesystem::file_exists(checkpoint_path))
        {
            _state->load(checkpoint_path.c_str());
        }

        if (dsn::utils::filesystem::file_exists(oplog_path))
        {
            replay_log(oplog_path.c_str());
            _state->save(checkpoint_path.c_str());
			if (!dsn::utils::filesystem::remove_path(oplog_path))
			{
				dassert(false, "Fail to remove file %s.", oplog_path.c_str());
			}
        }
    }

    _log = dsn_file_open((_data_dir + "/oplog").c_str(), O_RDWR | O_CREAT, 0666);

    _balancer = new load_balancer(_state);            
    _failure_detector = new meta_server_failure_detector(_state, this);
    
    // TODO: use zookeeper for leader election
    _failure_detector->set_primary(primary_address());

    // make sure the delay is larger than fd.grace to ensure 
    // all machines are in the correct state (assuming connected initially)
    tasking::enqueue(LPC_LBM_START, this, &meta_service::on_load_balance_start, 0, 
        _opts.fd_grace_seconds * 1000);

    auto err = _failure_detector->start(
        _opts.fd_check_interval_seconds,
        _opts.fd_beacon_interval_seconds,
        _opts.fd_lease_seconds,
        _opts.fd_grace_seconds,
        false
        );

    dassert(err == ERR_OK, "FD start failed, err = %s", err.to_string());

    register_rpc_handler(
        RPC_CM_QUERY_NODE_PARTITIONS,
        "RPC_CM_QUERY_NODE_PARTITIONS",
        &meta_service::on_query_configuration_by_node
        );

    register_rpc_handler(
        RPC_CM_QUERY_PARTITION_CONFIG_BY_INDEX,
        "RPC_CM_QUERY_PARTITION_CONFIG_BY_INDEX",
        &meta_service::on_query_configuration_by_index
        );

    register_rpc_handler(
        RPC_CM_UPDATE_PARTITION_CONFIGURATION,
        "RPC_CM_UPDATE_PARTITION_CONFIGURATION",
        &meta_service::on_update_configuration
        );
}
Пример #3
0
void aio_testcase(uint64_t block_size, size_t concurrency, bool is_write, bool shared)
{
    std::unique_ptr<char[]> buffer(new char[block_size]);
    std::vector<dsn_handle_t> files;
    files.resize(concurrency);

    int flag;
    if (is_write)
    {
        flag = O_CREAT | O_RDWR;
        if (shared)
        {
            if (utils::filesystem::file_exists("temp"))
                utils::filesystem::remove_path("temp");
        }
        else
        {
            for (int i = 0; i < concurrency; i++)
            {
                std::stringstream ss;
                ss << "temp." << i;
                auto file = ss.str();
                if (utils::filesystem::file_exists(file))
                    utils::filesystem::remove_path(file);
            }
        }
    }
    else
    {
        flag = O_RDWR;
    }

    if (shared)
    {
        auto file_handle = dsn_file_open("temp", flag, 0666);
        EXPECT_TRUE(file_handle != nullptr);
        for (int i = 0; i < concurrency; i++)
            files[i] = file_handle;
    }
    else
    {
        for (int i = 0; i < concurrency; i++)
        {
            std::stringstream ss;
            ss << "temp." << i;
            auto file = ss.str();
            auto file_handle = dsn_file_open(file.c_str(), flag, 0666);
            EXPECT_TRUE(file_handle != nullptr);
            files[i] = file_handle;
        }
    }
    
    std::atomic<uint64_t> io_count(0);
    std::atomic<uint64_t> cb_flying_count(0);
    volatile bool exit = false;
    std::function<void(int)> cb;
    std::vector<uint64_t> offsets;
    offsets.resize(concurrency);
    
    cb = [&](int index)
    {
        if (!exit)
        {
            auto ioc = io_count++;
            uint64_t offset;
            if (!shared)
            {
                offset = offsets[index];
                offsets[index] += block_size;
            }
            else
            {
                offset = ioc * block_size;
            }

            cb_flying_count++;
            if (is_write)
            {
                file::write(files[index], buffer.get(), (int)block_size, offset,
                    LPC_AIO_TEST, nullptr, [idx = index, &cb, &cb_flying_count](::dsn::error_code code, size_t sz) 
                    {                        
                        if (ERR_OK == code)
                            cb(idx);
                        cb_flying_count--;
                    });                
            }
            else
            {
                file::read(files[index], buffer.get(), (int)block_size, offset,
                    LPC_AIO_TEST, nullptr, [idx = index, &cb, &cb_flying_count](::dsn::error_code code, size_t sz)
                    {                        
                        if (ERR_OK == code)
                            cb(idx);
                        cb_flying_count--;
                    });
            }
        }
    };

    // start
    auto tic = std::chrono::steady_clock::now();
    for (int i = 0; i < concurrency; i++)
    {
        offsets[i] = 0;
        cb(i);
    }

    // run for seconds
    std::this_thread::sleep_for(std::chrono::seconds(10));
    auto ioc = io_count.load();
    auto bytes = ioc * block_size;    
    auto toc = std::chrono::steady_clock::now();
    
    std::cout << "is_write = " << is_write        
        << ", block_size = " << block_size
        << ", shared = " << shared
        << ", concurrency = " << concurrency
        << ", iops = " << (double)ioc / (double)std::chrono::duration_cast<std::chrono::microseconds>(toc - tic).count() * 1000000.0 << " #/s"
        << ", throughput = " << (double)bytes / std::chrono::duration_cast<std::chrono::microseconds>(toc - tic).count() << " mB/s"
        << ", avg_latency = " << (double)std::chrono::duration_cast<std::chrono::microseconds>(toc - tic).count() / (double)(ioc / concurrency) << " us"
        << std::endl;

    // safe exit
    exit = true;

    while (cb_flying_count.load() > 0)
    {
        std::this_thread::sleep_for(std::chrono::milliseconds(1));
    }

    if (shared)
    {
        dsn_file_flush(files[0]);
        auto c*k = dsn_file_close(files[0]);
        EXPECT_EQ(c*k, ERR_OK);
    }
    else
    {
        for (auto& f : files)
        {
            dsn_file_flush(f);
            auto c*k = dsn_file_close(f);
            EXPECT_EQ(c*k, ERR_OK);
        }
    }
}
 error_code meta_state_service_simple::initialize()
 {
     _offset = 0;
     std::string log_path = "meta_state_service.log";
     if (utils::filesystem::file_exists(log_path))
     {
         if (FILE* fd = fopen(log_path.c_str(), "rb"))
         {
             for (;;)
             {
                 log_header header;
                 if (fread(&header, sizeof(log_header), 1, fd) != 1)
                 {
                     break;
                 }
                 if (header.magic != log_header::default_magic)
                 {
                     break;
                 }
                 std::shared_ptr<char> buffer(new char[header.size]);
                 if (fread(buffer.get(), header.size, 1, fd) != 1)
                 {
                     break;
                 }
                 _offset += sizeof(header) + header.size;
                 blob blob_wrapper(buffer, header.size);
                 binary_reader reader(blob_wrapper);
                 int op_type;
                 unmarshall(reader, op_type);
                 switch (static_cast<operation_type>(op_type))
                 {
                 case operation_type::create_node:
                 {
                     std::string node;
                     blob data;
                     create_node_log::parse(reader, node, data);
                     create_node_internal(node, data).end_tracking();
                     break;
                 }
                 case operation_type::delete_node:
                 {
                     std::string node;
                     bool recursively_delete;
                     delete_node_log::parse(reader, node, recursively_delete);
                     delete_node_internal(node, recursively_delete).end_tracking();
                     break;
                 }
                 case operation_type::set_data:
                 {
                     std::string node;
                     blob data;
                     set_data_log::parse(reader, node, data);
                     set_data_internal(node, data).end_tracking();
                     break;
                 }
                 default:
                     //The log is complete but its content is modified by cosmic ray. This is unacceptable
                     dassert(false, "meta state server log corrupted");
                 }   
             }
             fclose(fd);
         }
     }
     _log = dsn_file_open(log_path.c_str(), O_RDWR | O_CREAT | O_BINARY, 0666);
     return ERR_OK;
 }
Пример #5
0
        void nfs_service_impl::on_copy(const ::dsn::service::copy_request& request, ::dsn::rpc_replier< ::dsn::service::copy_response>& reply)
        {
            //dinfo(">>> on call RPC_COPY end, exec RPC_NFS_COPY");

            std::string file_path = dsn::utils::filesystem::path_combine(request.source_dir, request.file_name);
            dsn_handle_t hfile;

            {
                zauto_lock l(_handles_map_lock);
                auto it = _handles_map.find(file_path); // find file handle cache first

                if (it == _handles_map.end()) // not found
                {
                    hfile = dsn_file_open(file_path.c_str(), O_RDONLY | O_BINARY, 0);
                    if (hfile)
                    {
                    
                        file_handle_info_on_server* fh = new file_handle_info_on_server;
                        fh->file_handle = hfile;
                        fh->file_access_count = 1;
                        fh->last_access_time = dsn_now_ms();
                        _handles_map.insert(std::pair<std::string, file_handle_info_on_server*>(file_path, fh));
                    }
                }
                else // found
                {
                    hfile = it->second->file_handle;
                    it->second->file_access_count++;
                    it->second->last_access_time = dsn_now_ms();
                }
            }

            dinfo("nfs: copy file %s [%" PRId64 ", %" PRId64 ")",
                file_path.c_str(),
                request.offset,
                request.offset + request.size
                );

            if (hfile == 0)
            {
                derror("file open failed");
                ::dsn::service::copy_response resp;
                resp.error = ERR_OBJECT_NOT_FOUND;
                reply(resp);
                return;
            }

            callback_para cp(reply);
            cp.bb = blob(
                std::shared_ptr<char>(new char[_opts.nfs_copy_block_bytes], std::default_delete<char[]>{}),
                _opts.nfs_copy_block_bytes);
            cp.dst_dir = std::move(request.dst_dir);
            cp.file_path = std::move(file_path);
            cp.hfile = hfile;
            cp.offset = request.offset;
            cp.size = request.size;

            auto buffer_save = cp.bb.buffer().get();
            file::read(
                hfile,
                buffer_save,
                request.size,
                request.offset,
                LPC_NFS_READ,
                this,
                [this, cp_cap = std::move(cp)] (error_code err, int sz)
                {
                    internal_read_callback(err, sz, std::move(cp_cap));
                }
                );
        }
Пример #6
0
        void nfs_client_impl::continue_write()
        {
            // check write quota
            if (++_concurrent_local_write_count > _opts.max_concurrent_local_writes)
            {
                --_concurrent_local_write_count;
                return;
            }

            // get write
            dsn::ref_ptr<copy_request_ex> reqc;
            while (true)
            {
                {
                    zauto_lock l(_local_writes_lock);
                    if (!_local_writes.empty())
                    {
                        reqc = _local_writes.front();
                        _local_writes.pop();
                    }
                    else
                    {
                        reqc = nullptr;
                        break;
                    }   
                }

                {
                    zauto_lock l(reqc->lock);
                    if (reqc->is_valid)
                        break;
                }
            }

            if (nullptr == reqc)
            {
                --_concurrent_local_write_count;
                return;
            }   

            // real write
            std::string file_path = dsn::utils::filesystem::path_combine(reqc->copy_req.dst_dir, reqc->file_ctx->file_name);
            std::string path = dsn::utils::filesystem::remove_file_name(file_path.c_str());
            if (!dsn::utils::filesystem::create_directory(path))
            {
                dassert(false, "Fail to create directory %s.", path.c_str());
            }

            dsn_handle_t hfile = reqc->file_ctx->file.load();
            if (!hfile)
            {
                zauto_lock l(reqc->file_ctx->user_req->user_req_lock);
                hfile = reqc->file_ctx->file.load();
                if (!hfile)
                {
                    hfile = dsn_file_open(file_path.c_str(), O_RDWR | O_CREAT | O_BINARY, 0666);
                    reqc->file_ctx->file = hfile;
                }
            }

            if (!hfile)
            {
                derror("file open %s failed", file_path.c_str());
                error_code err = ERR_FILE_OPERATION_FAILED;
                handle_completion(reqc->file_ctx->user_req, err);
                --_concurrent_local_write_count;
                continue_write();
                return;
            }

            {
                zauto_lock l(reqc->lock);
                auto& reqc_save = *reqc.get();
                reqc_save.local_write_task = file::write(
                    hfile,
                    reqc_save.response.file_content.data(),
                    reqc_save.response.size,
                    reqc_save.response.offset,
                    LPC_NFS_WRITE,
                    this,
                    [this, reqc_cap = std::move(reqc)] (error_code err, int sz)
                    {
                        local_write_callback(err, sz, std::move(reqc_cap));
                    }
                );
            }
        }
Пример #7
0
void meta_service::start(const char* data_dir, bool clean_state)
{
    dassert(!_started, "meta service is already started");

    _data_dir = data_dir;
	std::string checkpoint_path = _data_dir + "/checkpoint";
	std::string oplog_path = _data_dir + "/oplog";


    if (clean_state)
    {
        try {
			if (!dsn::utils::filesystem::remove_path(checkpoint_path))
			{
				dassert(false, "Fail to remove file %s.", checkpoint_path.c_str());
			}

			if (!dsn::utils::filesystem::remove_path(oplog_path))
			{
				dassert(false, "Fail to remove file %s.", oplog_path.c_str());
			}
        }
        catch (std::exception& ex)
        {
            ex;
        }
    }
    else
    {
		if (!dsn::utils::filesystem::create_directory(_data_dir))
		{
			dassert(false, "Fail to create directory %s.", _data_dir.c_str());
		}

        if (dsn::utils::filesystem::file_exists(checkpoint_path))
        {
            _state->load(checkpoint_path.c_str());
        }

        if (dsn::utils::filesystem::file_exists(oplog_path))
        {
            replay_log(oplog_path.c_str());
            _state->save(checkpoint_path.c_str());
			if (!dsn::utils::filesystem::remove_path(oplog_path))
			{
				dassert(false, "Fail to remove file %s.", oplog_path.c_str());
			}
        }
    }

    _log = dsn_file_open((_data_dir + "/oplog").c_str(), O_RDWR | O_CREAT, 0666);

    _balancer = new load_balancer(_state);            
    _failure_detector = new meta_server_failure_detector(_state, this);
    
    ::dsn::rpc_address primary;
    if (_state->get_meta_server_primary(primary) && primary == primary_address())
    {
        _failure_detector->set_primary(true);
    }   
    else
        _failure_detector->set_primary(false);

    register_rpc_handler(RPC_CM_CALL, "RPC_CM_CALL", &meta_service::on_request);

    // make sure the delay is larger than fd.grace to ensure 
    // all machines are in the correct state (assuming connected initially)
    tasking::enqueue(LPC_LBM_START, this, &meta_service::on_load_balance_start, 0, 
        _opts.fd_grace_seconds * 1000);

    auto err = _failure_detector->start(
        _opts.fd_check_interval_seconds,
        _opts.fd_beacon_interval_seconds,
        _opts.fd_lease_seconds,
        _opts.fd_grace_seconds,
        false
        );

    dassert(err == ERR_OK, "FD start failed, err = %s", err.to_string());
}