// RPC_SIMPLE_KV_APPEND
void simple_kv_service_impl::on_append(const kv_pair& pr, ::dsn::rpc_replier<int32_t>& reply)
{
{
zauto_lock l(_lock);
auto it = _store.find(pr.key);
if (it != _store.end())
it->second.append(pr.value);
else
_store[pr.key] = pr.value;
}
dinfo("append %s", pr.key.c_str());
reply(0);
}
void failure_detector::register_master(::dsn::rpc_address target)
{
bool setup_timer = false;
uint64_t now = now_ms();
zauto_lock l(_lock);
master_record record(target, now);
auto ret = _masters.insert(std::make_pair(target, record));
if (ret.second)
{
dinfo("register master[%s] successfully", target.to_string());
setup_timer = true;
}
else
{
// active the beacon again in case previously local node is not in target's allow list
if (ret.first->second.rejected)
{
ret.first->second.rejected = false;
setup_timer = true;
}
dinfo("master[%s] already registered", target.to_string());
}
if (setup_timer)
{
ret.first->second.send_beacon_timer = tasking::enqueue_timer(LPC_BEACON_SEND, this,
[this, target]()
{
this->send_beacon(target, now_ms());
},
std::chrono::milliseconds(_beacon_interval_milliseconds)
);
}
}
rpc_address meta_server_failure_detector::get_primary()
{
dsn::utils::auto_lock<zlock> l(_primary_address_lock);
if ( _lock_svc!=nullptr && !_is_primary )
{
uint64_t version;
error_code ec = _lock_svc->query_cache(_primary_lock_id, _lock_owner_id, version);
dinfo("query cache result: err: %s, owner(%s), version(%llu)", ec.to_string(), _lock_owner_id.c_str(), version);
if (ec != ERR_OK)
_primary_address.set_invalid();
else
_primary_address.from_string_ipv4(_lock_owner_id.c_str());
}
return _primary_address;
}
void meta_service::on_list_apps(dsn_message_t req)
{
if (!check_primary(req))
return;
if (!_started)
{
dinfo("list app request, meta server not active");
configuration_list_apps_response response;
response.err = ERR_SERVICE_NOT_ACTIVE;
reply(req, response);
return;
}
_state->list_apps(req);
}
void replica::response_client_message(message_ptr& request, error_code error, decree d/* = invalid_decree*/)
{
if (nullptr == request)
return;
message_ptr resp = request->create_response();
int err = error.get();
resp->writer().write(err);
dassert(error != ERR_SUCCESS, "");
dinfo("handle replication request with rpc_id = %016llx failed, err = %s",
request->header().rpc_id, error.to_string());
rpc::reply(resp);
}
// RPC_SIMPLE_KV_READ
void simple_kv_service_impl::on_read(const std::string& key, ::dsn::service::rpc_replier<std::string>& reply)
{
zauto_lock l(_lock);
auto it = _store.find(key);
if (it == _store.end())
{
reply("");
}
else
{
dinfo("read %s, decree = %lld\n", it->second.c_str(), last_committed_decree());
reply(it->second);
}
}
bool meta_service::check_primary(dsn_message_t req)
{
if (!_failure_detector->is_primary())
{
auto primary = _failure_detector->get_primary();
dinfo("primary address: %s", primary.to_string());
if (!primary.is_invalid())
{
dsn_rpc_forward(req, _failure_detector->get_primary().c_addr());
return false;
}
}
return true;
}
void io_looper::loop_worker()
{
DWORD io_size;
uintptr_t completion_key;
LPOVERLAPPED lolp;
DWORD error;
while (true)
{
BOOL r = ::GetQueuedCompletionStatus(_io_queue, &io_size, &completion_key, &lolp, 1); // 1ms timeout for timers
// everything goes fine
if (r)
{
error = ERROR_SUCCESS;
}
// failed or timeout
else
{
error = ::GetLastError();
if (error == ERROR_ABANDONED_WAIT_0)
{
derror("completion port loop exits");
break;
}
// only possible for timeout
if (NULL == lolp)
{
handle_local_queues();
continue;
}
dinfo("io operation failed in iocp, err = 0x%x", error);
}
if (NON_IO_TASK_NOTIFICATION_KEY == completion_key)
{
handle_local_queues();
}
else
{
io_loop_callback* cb = (io_loop_callback*)completion_key;
(*cb)((int)error, io_size, (uintptr_t)lolp);
}
}
}
// RPC_SIMPLE_KV_READ
void simple_kv_service_impl::on_read(const std::string& key, ::dsn::rpc_replier<std::string>& reply)
{
std::string r;
{
zauto_lock l(_lock);
auto it = _store.find(key);
if (it != _store.end())
{
r = it->second;
}
}
dinfo("read %s", r.c_str());
reply(r);
}
void docker_scheduler::undeploy_docker_unit(void* context, int argc, const char** argv, dsn_cli_reply* reply)
{
auto docker = reinterpret_cast<docker_scheduler*>(context);
if( argc == 3 )
{
std::string name = argv[0];
std::string ldir = argv[1];
std::string rdir = argv[2];
std::function<void(error_code,const std::string&)> cb = [](error_code err,const std::string& err_msg){
dinfo("deploy err %s",err.to_string());
};
docker->delete_containers(name,cb,ldir,rdir);
}
reply->message = "";
reply->size = 0;
reply->context = nullptr;
}
bool failure_detector::unregister_master(::dsn::rpc_address node)
{
zauto_lock l(_lock);
auto it = _masters.find(node);
if (it != _masters.end())
{
it->second.send_beacon_timer->cancel(true);
_masters.erase(it);
dinfo("unregister master[%s] successfully", node.to_string());
return true;
}
else
{
ddebug("unregister master[%s] failed, cannot find it in FD", node.to_string());
return false;
}
}
void disk_engine::complete_io(aio_task* aio, error_code err, uint32_t bytes, int delay_milliseconds)
{
// TODO: failure injection, profiling, throttling
if (err != ERR_OK)
{
dinfo(
"disk operation failure with code %s, err = %s, aio task id = %llx",
aio->spec().name.c_str(),
err.to_string(),
aio->id()
);
}
aio->enqueue(err, bytes);
aio->release_ref(); // added in start_io
_request_count--;
}
IconsGenerator::IconsGenerator(QString outputPath)
: QThread()
, m_outputPath(outputPath)
, m_iconsQueue()
, m_stop(false)
, m_mutex()
, m_waitCondition()
{
// Creating the directory where icons will be saved, if it doesn't exist
QFileInfo dinfo(m_outputPath);
if (!dinfo.exists()) {
if (!QDir().mkpath(m_outputPath)) {
qDebug() << "Cannot create directory" << m_outputPath << "icons generation will probably fail";
}
} else if (!dinfo.isDir() || !dinfo.isWritable()) {
qDebug() << m_outputPath << "is not a directory or is not writable, icons generation will probably fail";
}
}
void VPreviewManager::fetchImageInfoToPreview(const QString &p_text, ImageLinkInfo &p_info)
{
QString surl = fetchImageUrlToPreview(p_text, p_info.m_width, p_info.m_height);
p_info.m_linkShortUrl = surl;
if (surl.isEmpty()) {
p_info.m_linkUrl = surl;
return;
}
const VFile *file = m_editor->getFile();
QString imagePath;
QFileInfo info(file->fetchBasePath(), surl);
if (info.exists()) {
if (info.isNativePath()) {
// Local file.
imagePath = QDir::cleanPath(info.absoluteFilePath());
} else {
imagePath = surl;
}
} else {
QString decodedUrl(surl);
VUtils::decodeUrl(decodedUrl);
QFileInfo dinfo(file->fetchBasePath(), decodedUrl);
if (dinfo.exists()) {
if (dinfo.isNativePath()) {
// Local file.
imagePath = QDir::cleanPath(dinfo.absoluteFilePath());
} else {
imagePath = surl;
}
} else {
QUrl url(surl);
if (url.isLocalFile()) {
imagePath = url.toLocalFile();
} else {
imagePath = url.toString();
}
}
}
p_info.m_linkUrl = imagePath;
}
DataHandler* AnyDimHandler::copy( unsigned short newParentDepth,
unsigned short copyRootDepth,
bool toGlobal, unsigned int n ) const
{
if ( toGlobal ) {
if ( !isGlobal() ) {
cout << "Warning: AnyDimHandler::copy: Cannot copy from nonGlob al to global\n";
return 0;
}
}
// Don't allow copying that would remove an array.
for ( unsigned int i = 0; i < dims_.size(); ++i )
if ( copyRootDepth > dims_[i].depth )
return 0;
if ( n > 1 ) {
DimInfo temp = {n, newParentDepth + 1, 0 };
vector< DimInfo > newDims;
newDims.push_back( temp );
for ( unsigned int i = 0; i < dims_.size(); ++i ) {
newDims.push_back( dims_[i] );
newDims.back().depth += 1 + newParentDepth - copyRootDepth;
}
AnyDimHandler* ret = new AnyDimHandler( dinfo(),
newDims,
1 + pathDepth() + newParentDepth - copyRootDepth, toGlobal );
if ( data_ ) {
ret->assign( data_, end_ - start_ );
}
return ret;
} else {
AnyDimHandler* ret = new AnyDimHandler( this );
if ( !ret->changeDepth(
pathDepth() + 1 + newParentDepth - copyRootDepth
)
) {
delete ret;
return 0;
}
return ret;
}
return 0;
}
void connection_oriented_network::on_client_session_disconnected(rpc_client_session_ptr& s)
{
bool r = false;
{
utils::auto_write_lock l(_clients_lock);
auto it = _clients.find(s->remote_address());
if (it != _clients.end() && it->second.get() == s.get())
{
_clients.erase(it);
r = true;
}
}
if (r)
{
dinfo("client session %s:%d disconnected", s->remote_address().name.c_str(),
static_cast<int>(s->remote_address().port));
}
}
bool perf_counters::remove_counter(const char *full_name)
{
int remain_ref;
{
utils::auto_write_lock l(_lock);
auto it = _counters.find(full_name);
if (it == _counters.end())
return false;
else {
counter_object &c = it->second;
remain_ref = (--c.user_reference);
if (remain_ref == 0) {
_counters.erase(it);
}
}
}
dinfo("performance counter %s is removed, remaining reference (%d)", full_name, remain_ref);
return true;
}
void meta_service::on_query_configuration_by_index(dsn_message_t msg)
{
if (!check_primary(msg))
return;
if (!_started)
{
dinfo("create app request, meta server not active");
configuration_query_by_index_response response;
response.err = ERR_SERVICE_NOT_ACTIVE;
reply(msg, response);
return;
}
configuration_query_by_index_response response;
configuration_query_by_index_request request;
::unmarshall(msg, request);
_state->query_configuration_by_index(request, response);
reply(msg, response);
}
bool failure_detector::switch_master(::dsn::rpc_address from, ::dsn::rpc_address to, uint32_t delay_milliseconds)
{
/* the caller of switch master shoud lock necessarily to protect _masters */
auto it = _masters.find(from);
auto it2 = _masters.find(to);
if (it != _masters.end())
{
if (it2 != _masters.end())
{
dwarn("switch master failed as both are already registered, from[%s], to[%s]",
from.to_string(), to.to_string());
return false;
}
it->second.node = to;
it->second.rejected = false;
it->second.send_beacon_timer->cancel(true);
it->second.send_beacon_timer = tasking::enqueue_timer(LPC_BEACON_SEND, this,
[this, to]()
{
this->send_beacon(to, now_ms());
},
std::chrono::milliseconds(_beacon_interval_milliseconds),
0,
std::chrono::milliseconds(delay_milliseconds)
);
_masters.insert(std::make_pair(to, it->second));
_masters.erase(from);
dinfo("switch master successfully, from[%s], to[%s]",
from.to_string(), to.to_string());
}
else
{
dwarn("switch master failed as from node is not registered yet, from[%s], to[%s]",
from.to_string(), to.to_string());
return false;
}
return true;
}
task_ptr partition_resolver_simple::query_config(int partition_index)
{
dinfo(
"%s.client: start query config, gpid = %d.%d", _app_path.c_str(), _app_id, partition_index);
auto msg = dsn::message_ex::create_request(RPC_CM_QUERY_PARTITION_CONFIG_BY_INDEX);
configuration_query_by_index_request req;
req.app_name = _app_path;
if (partition_index != -1) {
req.partition_indices.push_back(partition_index);
}
marshall(msg, req);
return rpc::call(
_meta_server,
msg,
&_tracker,
[this, partition_index](error_code err, dsn::message_ex *req, dsn::message_ex *resp) {
query_config_reply(err, req, resp, partition_index);
});
}
void connection_oriented_network::on_server_session_disconnected(rpc_server_session_ptr& s)
{
bool r = false;
{
utils::auto_write_lock l(_servers_lock);
auto it = _servers.find(s->remote_address());
if (it != _servers.end() && it->second.get() == s.get())
{
_servers.erase(it);
r = true;
}
}
if (r)
{
dinfo("server session %s:%hu disconnected",
s->remote_address().name.c_str(),
s->remote_address().port
);
}
}
void hpc_rpc_session::do_read(int read_next)
{
utils::auto_lock<utils::ex_lock_nr> l(_send_lock);
while (true)
{
char* ptr = (char*)_parser->read_buffer_ptr((int)read_next);
int remaining = _parser->read_buffer_capacity();
int sz = recv(_socket, ptr, remaining, 0);
int err = errno;
dinfo("(s = %d) call recv on %s:%hu, return %d, err = %s",
_socket,
_remote_addr.name(),
_remote_addr.port(),
sz,
strerror(err)
);
if (sz > 0)
{
message_ex* msg = _parser->get_message_on_receive(sz, read_next);
while (msg != nullptr)
{
this->on_read_completed(msg);
msg = _parser->get_message_on_receive(0, read_next);
}
}
else
{
if (err != EAGAIN && err != EWOULDBLOCK)
{
derror("(s = %d) recv failed, err = %s", _socket, strerror(err));
on_failure();
}
break;
}
}
}
DataHandler* TwoDimHandler::copy(
unsigned short newParentDepth,
unsigned short copyRootDepth,
bool toGlobal, unsigned int n ) const
{
if ( toGlobal ) {
if ( !isGlobal() ) {
cout << "Warning: TwoDimHandler::copy: Cannot copy from nonGlob al to global\n";
return 0;
}
}
if ( copyRootDepth > dims_[0].depth || copyRootDepth >= dims_[1].depth)
return 0;
if ( n > 1 ) {
DimInfo temp = {n, newParentDepth + 1, 0 };
vector< DimInfo > newDims;
newDims.push_back( temp );
newDims.push_back( dims_[0] );
newDims.back().depth += 1 + newParentDepth - copyRootDepth;
newDims.push_back( dims_[1] );
newDims.back().depth += 1 + newParentDepth - copyRootDepth;
AnyDimHandler* ret = new AnyDimHandler( dinfo(),
newDims,
1 + pathDepth() + newParentDepth - copyRootDepth, toGlobal );
if ( data_ ) {
ret->assign( data_, end_ - start_ );
}
return ret;
} else {
TwoDimHandler* ret = new TwoDimHandler( this );
if ( !ret->changeDepth( pathDepth() + 1 + newParentDepth - copyRootDepth ) ) {
delete ret;
return 0;
}
return ret;
}
return 0;
}
void client_net_io::connect()
{
session_state closed_state = SS_CLOSED;
if (_state.compare_exchange_strong(closed_state, SS_CONNECTING))
{
boost::asio::ip::tcp::endpoint ep(
boost::asio::ip::address_v4(ntohl(_remote_addr.ip)), _remote_addr.port);
add_reference();
_socket.async_connect(ep, [this](boost::system::error_code ec)
{
if (!ec)
{
_reconnect_count = 0;
_state = SS_CONNECTED;
dinfo("client session %s:%d connected",
_remote_addr.name.c_str(),
static_cast<int>(_remote_addr.port)
);
set_options();
do_write();
do_read();
}
else
{
derror("network client session connect failed, error = %s",
ec.message().c_str()
);
on_failure();
}
release_reference();
});
}
}
void nfs_service_impl::close_file() // release out-of-date file handle
{
zauto_lock l(_handles_map_lock);
for (auto it = _handles_map.begin(); it != _handles_map.end();)
{
auto fptr = it->second;
// not used and expired
if (fptr->file_access_count == 0
&& dsn_now_ms() - fptr->last_access_time > _opts.file_close_expire_time_ms)
{
dinfo("nfs: close file handle %s", it->first.c_str());
it = _handles_map.erase(it);
::dsn::error_code err = dsn_file_close(fptr->file_handle);
dassert(err == ERR_OK, "dsn_file_close failed, err = %s", err.to_string());
delete fptr;
}
else
it++;
}
}
bool failure_detector::unregister_worker(::dsn::rpc_address node)
{
/*
* callers should use the fd::_lock necessarily
*/
bool ret;
size_t count = _workers.erase(node);
if ( count == 0 )
{
ret = false;
}
else
{
ret = true;
}
dinfo("unregister worker[%s] successfully, removed entry count is %u",
node.to_string(), (uint32_t)count);
return ret;
}
void meta_server_failure_detector::on_ping(const fd::beacon_msg& beacon, ::dsn::rpc_replier<fd::beacon_ack>& reply)
{
fd::beacon_ack ack;
ack.time = beacon.time;
ack.this_node = beacon.to;
ack.allowed = true;
if ( !is_primary() )
{
ack.is_master = false;
ack.primary_node = get_primary();
}
else
{
ack.is_master = true;
ack.primary_node = beacon.to;
failure_detector::on_ping_internal(beacon, ack);
}
dinfo("on_ping, is_master(%s), this_node(%s), primary_node(%s)", ack.is_master?"true":"false",
ack.this_node.to_string(), ack.primary_node.to_string());
reply(ack);
}
// server
hpc_rpc_session::hpc_rpc_session(
socket_t sock,
std::shared_ptr<dsn::message_parser>& parser,
connection_oriented_network& net,
const ::dsn::rpc_address& remote_addr
)
: rpc_session(net, remote_addr, parser),
_socket(sock)
{
dassert(sock != -1, "invalid given socket handle");
_sending_msg = nullptr;
_sending_buffer_start_index = 0;
_looper = nullptr;
memset((void*)&_peer_addr, 0, sizeof(_peer_addr));
_peer_addr.sin_family = AF_INET;
_peer_addr.sin_addr.s_addr = INADDR_ANY;
_peer_addr.sin_port = 0;
socklen_t addr_len = (socklen_t)sizeof(_peer_addr);
if (getpeername(_socket, (struct sockaddr*)&_peer_addr, &addr_len) == -1)
{
dassert(false, "(server) getpeername failed, err = %s", strerror(errno));
}
_ready_event = [this](int err, uint32_t length, uintptr_t lolp_or_events)
{
struct kevent& e = *((struct kevent*)lolp_or_events);
dinfo("(s = %d) (server) epoll for send/recv to %s:%hu, events = %d",
_socket,
_remote_addr.name(),
_remote_addr.port(),
e.filter
);
this->on_send_recv_events_ready(lolp_or_events);
};
}
void failure_detector::send_beacon(::dsn::rpc_address target, uint64_t time)
{
beacon_msg beacon;
beacon.time = time;
beacon.from_addr= primary_address();
beacon.to_addr = target;
dinfo("send ping message, from[%s], to[%s], time[%" PRId64 "]",
beacon.from_addr.to_string(), beacon.to_addr.to_string(), time);
::dsn::rpc::call(
target,
RPC_FD_FAILURE_DETECTOR_PING,
beacon,
this,
[=](error_code err, beacon_ack&& resp)
{
if (err != ::dsn::ERR_OK)
{
beacon_ack ack;
ack.time = beacon.time;
ack.this_node = beacon.to_addr;
ack.primary_node.set_invalid();
ack.is_master = false;
ack.allowed = true;
end_ping(err, ack, nullptr);
}
else
{
end_ping(err, std::move(resp), nullptr);
}
},
0,
std::chrono::milliseconds(_check_interval_milliseconds),
0
);
}
void uri_resolver_manager::setup_resolvers(configuration* config)
{
// [uri-resolver.%resolver-address%]
// factory = %uri-resolver-factory%
// arguments = %uri-resolver-arguments%
std::vector<std::string> sections;
config->get_all_sections(sections);
const int prefix_len = (const int)strlen("uri-resolver.");
for (auto& s : sections)
{
if (s.substr(0, prefix_len) != std::string("uri-resolver."))
continue;
auto resolver_addr = s.substr(prefix_len);
auto it = _resolvers.find(resolver_addr);
if (it != _resolvers.end())
{
dwarn("duplicated uri-resolver definition in config file, for '%s'",
resolver_addr.c_str()
);
continue;
}
auto factory = config->get_string_value(s.c_str(), "factory", "",
"partition-resolver factory name which creates the concrete partition-resolver object");
auto arguments = config->get_string_value(s.c_str(), "arguments", "",
"uri-resolver ctor arguments");
auto resolver = new uri_resolver(resolver_addr.c_str(), factory, arguments);
_resolvers.emplace(resolver_addr, std::shared_ptr<uri_resolver>(resolver));
dinfo("initialize uri-resolver %s", resolver_addr.c_str());
}
}
