void nfs_service_impl::close_file() // release out-of-date file handle
{
zauto_lock l(_handles_map_lock);
for (auto it = _handles_map.begin(); it != _handles_map.end();)
{
auto fptr = it->second;
// not used and expired
if (fptr->file_access_count == 0
&& dsn_now_ms() - fptr->last_access_time > _opts.file_close_expire_time_ms)
{
dinfo("nfs: close file handle %s", it->first.c_str());
it = _handles_map.erase(it);
::dsn::error_code err = dsn_file_close(fptr->file_handle);
dassert(err == ERR_OK, "dsn_file_close failed, err = %s", err.to_string());
delete fptr;
}
else
it++;
}
}
void nfs_client_impl::local_write_callback(error_code err, size_t sz, dsn::ref_ptr<copy_request_ex> reqc)
{
//dassert(reqc->local_write_task == task::get_current_task(), "");
--_concurrent_local_write_count;
// clear all content to release memory quickly
reqc->response.file_content = blob();
continue_write();
bool completed = false;
if (err != ERR_OK)
{
completed = true;
}
else
{
zauto_lock l(reqc->file_ctx->user_req->user_req_lock);
if (++reqc->file_ctx->finished_segments == (int)reqc->file_ctx->copy_requests.size())
{
auto err = dsn_file_close(reqc->file_ctx->file);
dassert(err == ERR_OK, "dsn_file_close failed, err = %s", dsn_error_to_string(err));
reqc->file_ctx->file = static_cast<dsn_handle_t>(0);
reqc->file_ctx->copy_requests.clear();
if (++reqc->file_ctx->user_req->finished_files == (int)reqc->file_ctx->user_req->file_context_map.size())
{
completed = true;
}
}
}
if (completed)
{
handle_completion(reqc->file_ctx->user_req, err);
}
}
void aio_testcase(uint64_t block_size, uint64_t total_size_mb, size_t concurrency, bool is_write, bool random_offset)
{
std::unique_ptr<char[]> buffer(new char[block_size]);
std::atomic_uint remain_concurrency;
remain_concurrency = concurrency;
if (is_write && utils::filesystem::file_exists("temp"))
{
utils::filesystem::remove_path("temp");
dassert(!utils::filesystem::file_exists("temp"), "");
}
auto file_handle = dsn_file_open("temp", O_CREAT | O_RDWR, 0666);
auto total_size_bytes = total_size_mb * 1024 * 1024;
auto tic = std::chrono::steady_clock::now();
for (int bytes_written = 0; bytes_written < total_size_bytes; )
{
while (true)
{
if (remain_concurrency.fetch_sub(1, std::memory_order_acquire) <= 0)
{
remain_concurrency.fetch_add(1, std::memory_order_relaxed);
}
else
{
break;
}
}
auto cb = [&](error_code ec, int sz)
{
dassert(ec == ERR_OK && uint64_t(sz) == block_size,
"ec = %s, sz = %d, block_size = %" PRId64 "",
ec.to_string(), sz, block_size
);
remain_concurrency.fetch_add(1, std::memory_order_relaxed);
};
auto offset = random_offset ? dsn_random64(0, total_size_bytes - block_size) : bytes_written;
if (is_write)
{
file::write(file_handle, buffer.get(), block_size, offset, LPC_AIO_TEST, nullptr, cb);
}
else
{
file::read(file_handle, buffer.get(), block_size, offset, LPC_AIO_TEST, nullptr, cb);
}
bytes_written += block_size;
}
while (remain_concurrency != concurrency)
{
;
}
dsn_file_flush(file_handle);
auto toc = std::chrono::steady_clock::now();
auto c*k = dsn_file_close(file_handle);
EXPECT_EQ(c*k, ERR_OK);
std::cout << "is_write = " << is_write
<< " random_offset = " << random_offset
<< " block_size = " << block_size
<< " concurrency = " << concurrency
<< " throughput = " << double(total_size_mb) * 1000000 / std::chrono::duration_cast<std::chrono::microseconds>(toc - tic).count() << " mB/s" << std::endl;
}
void aio_testcase(uint64_t block_size, size_t concurrency, bool is_write, bool shared)
{
std::unique_ptr<char[]> buffer(new char[block_size]);
std::vector<dsn_handle_t> files;
files.resize(concurrency);
int flag;
if (is_write)
{
flag = O_CREAT | O_RDWR;
if (shared)
{
if (utils::filesystem::file_exists("temp"))
utils::filesystem::remove_path("temp");
}
else
{
for (int i = 0; i < concurrency; i++)
{
std::stringstream ss;
ss << "temp." << i;
auto file = ss.str();
if (utils::filesystem::file_exists(file))
utils::filesystem::remove_path(file);
}
}
}
else
{
flag = O_RDWR;
}
if (shared)
{
auto file_handle = dsn_file_open("temp", flag, 0666);
EXPECT_TRUE(file_handle != nullptr);
for (int i = 0; i < concurrency; i++)
files[i] = file_handle;
}
else
{
for (int i = 0; i < concurrency; i++)
{
std::stringstream ss;
ss << "temp." << i;
auto file = ss.str();
auto file_handle = dsn_file_open(file.c_str(), flag, 0666);
EXPECT_TRUE(file_handle != nullptr);
files[i] = file_handle;
}
}
std::atomic<uint64_t> io_count(0);
std::atomic<uint64_t> cb_flying_count(0);
volatile bool exit = false;
std::function<void(int)> cb;
std::vector<uint64_t> offsets;
offsets.resize(concurrency);
cb = [&](int index)
{
if (!exit)
{
auto ioc = io_count++;
uint64_t offset;
if (!shared)
{
offset = offsets[index];
offsets[index] += block_size;
}
else
{
offset = ioc * block_size;
}
cb_flying_count++;
if (is_write)
{
file::write(files[index], buffer.get(), (int)block_size, offset,
LPC_AIO_TEST, nullptr, [idx = index, &cb, &cb_flying_count](::dsn::error_code code, size_t sz)
{
if (ERR_OK == code)
cb(idx);
cb_flying_count--;
});
}
else
{
file::read(files[index], buffer.get(), (int)block_size, offset,
LPC_AIO_TEST, nullptr, [idx = index, &cb, &cb_flying_count](::dsn::error_code code, size_t sz)
{
if (ERR_OK == code)
cb(idx);
cb_flying_count--;
});
}
}
};
// start
auto tic = std::chrono::steady_clock::now();
for (int i = 0; i < concurrency; i++)
{
offsets[i] = 0;
cb(i);
}
// run for seconds
std::this_thread::sleep_for(std::chrono::seconds(10));
auto ioc = io_count.load();
auto bytes = ioc * block_size;
auto toc = std::chrono::steady_clock::now();
std::cout << "is_write = " << is_write
<< ", block_size = " << block_size
<< ", shared = " << shared
<< ", concurrency = " << concurrency
<< ", iops = " << (double)ioc / (double)std::chrono::duration_cast<std::chrono::microseconds>(toc - tic).count() * 1000000.0 << " #/s"
<< ", throughput = " << (double)bytes / std::chrono::duration_cast<std::chrono::microseconds>(toc - tic).count() << " mB/s"
<< ", avg_latency = " << (double)std::chrono::duration_cast<std::chrono::microseconds>(toc - tic).count() / (double)(ioc / concurrency) << " us"
<< std::endl;
// safe exit
exit = true;
while (cb_flying_count.load() > 0)
{
std::this_thread::sleep_for(std::chrono::milliseconds(1));
}
if (shared)
{
dsn_file_flush(files[0]);
auto c*k = dsn_file_close(files[0]);
EXPECT_EQ(c*k, ERR_OK);
}
else
{
for (auto& f : files)
{
dsn_file_flush(f);
auto c*k = dsn_file_close(f);
EXPECT_EQ(c*k, ERR_OK);
}
}
}
meta_state_service_simple::~meta_state_service_simple()
{
dsn_file_close(_log);
}
void nfs_client_impl::handle_completion(user_request *req, error_code err)
{
{
zauto_lock l(req->user_req_lock);
if (req->is_finished)
return;
req->is_finished = true;
}
for (auto& f : req->file_context_map)
{
for (auto& rc : f.second->copy_requests)
{
::dsn::task_ptr ctask, wtask;
{
zauto_lock l(rc->lock);
rc->is_valid = false;
ctask = rc->remote_copy_task;
wtask = rc->local_write_task;
rc->remote_copy_task = nullptr;
rc->local_write_task = nullptr;
}
if (err != ERR_OK)
{
if (ctask != nullptr)
{
if (ctask->cancel(true))
{
_concurrent_copy_request_count--;
rc->release_ref();
}
}
if (wtask != nullptr)
{
if (wtask->cancel(true))
{
_concurrent_local_write_count--;
}
}
}
}
if (f.second->file)
{
auto err2 = dsn_file_close(f.second->file);
dassert(err2 == ERR_OK, "dsn_file_close failed, err = %s", dsn_error_to_string(err2));
f.second->file = nullptr;
if (f.second->finished_segments != (int)f.second->copy_requests.size())
{
::remove((f.second->user_req->file_size_req.dst_dir
+ f.second->file_name).c_str());
}
f.second->copy_requests.clear();
}
delete f.second;
}
req->file_context_map.clear();
req->nfs_task->enqueue(err, 0);
delete req;
// clear out all canceled requests
if (err != ERR_OK)
{
continue_copy(0);
continue_write();
}
}
