void mastermind_t::data::collect_info_loop_impl() {
if (m_logger->verbosity() >= cocaine::logging::info) {
auto current_remote = m_current_remote;
std::ostringstream oss;
oss << "libmastermind: collect_info_loop: begin; current host: ";
if (current_remote.first.empty()) {
oss << "none";
} else {
oss << current_remote.first << ':' << m_current_remote.second;
}
COCAINE_LOG_INFO(m_logger, "%s", oss.str().c_str());
}
auto beg_time = std::chrono::system_clock::now();
{
spent_time_printer_t helper("collect_namespaces_states", m_logger);
collect_namespaces_states();
}
#if 0
{
spent_time_printer_t helper("collect_cache_groups", m_logger);
collect_cache_groups();
}
#endif
{
spent_time_printer_t helper("collect_elliptics_remotes", m_logger);
collect_elliptics_remotes();
}
cache_expire();
generate_fake_caches();
serialize();
auto end_time = std::chrono::system_clock::now();
if (m_logger->verbosity() >= cocaine::logging::info) {
auto current_remote = m_current_remote;
std::ostringstream oss;
oss << "libmastermind: collect_info_loop: end; current host: ";
if (current_remote.first.empty()) {
oss << "none";
} else {
oss << current_remote.first << ':' << m_current_remote.second;
}
oss
<< "; spent time: "
<< std::chrono::duration_cast<std::chrono::milliseconds>(end_time - beg_time).count()
<< " milliseconds";
COCAINE_LOG_INFO(m_logger, "%s", oss.str().c_str());
}
}
std::vector<int> mastermind_t::get_couple(int couple_id, const std::string &ns) {
COCAINE_LOG_INFO(m_data->m_logger, "libmastermind: get_couple: couple_id=%d ns=%s"
, couple_id, ns);
auto cache = m_data->fake_groups_info.copy();
std::vector<int> result;
auto git = cache.get_value().find(couple_id);
if (git == cache.get_value().end()) {
COCAINE_LOG_ERROR(m_data->m_logger
, "libmastermind: get_couple: cannot find couple by the couple_id");
return std::vector<int>();
}
if (git->second.group_status !=
namespace_state_init_t::data_t::couples_t::group_info_t::status_tag::COUPLED) {
COCAINE_LOG_ERROR(m_data->m_logger
, "libmastermind: get_couple: couple status is not COUPLED: %d"
, static_cast<int>(git->second.group_status));
return std::vector<int>();
}
if (git->second.ns != ns) {
COCAINE_LOG_ERROR(m_data->m_logger
, "libmastermind: get_couple: couple belongs to another namespace: %s"
, git->second.ns);
return std::vector<int>();
}
{
std::ostringstream oss;
oss << "libmastermind: get_couple: couple was found: [";
{
const auto &couple = git->second.groups;
for (auto beg = couple.begin(), it = beg, end = couple.end(); it != end; ++it) {
if (beg != it) oss << ", ";
oss << *it;
}
}
oss << "]};";
auto msg = oss.str();
COCAINE_LOG_INFO(m_data->m_logger, "%s", msg.c_str());
}
return git->second.groups;
}
queue_app_context::queue_app_context(cocaine::framework::dispatch_t& dispatch)
: m_id(dispatch.id())
, m_log(dispatch.service_manager()->get_system_logger())
{
//FIXME: pass logger explicitly everywhere
extern void grape_queue_module_set_logger(std::shared_ptr<cocaine::framework::logger_t>);
grape_queue_module_set_logger(m_log);
m_queue.reset(new ioremap::grape::queue(m_id));
m_queue->initialize("queue.conf");
COCAINE_LOG_INFO(m_log, "%s: queue has been successfully configured", m_id.c_str());
// register event handlers
dispatch.on("queue@ping", this, &queue_app_context::process);
dispatch.on("queue@push", this, &queue_app_context::process);
dispatch.on("queue@pop-multi", this, &queue_app_context::process);
dispatch.on("queue@pop-multiple-string", this, &queue_app_context::process);
dispatch.on("queue@pop", this, &queue_app_context::process);
dispatch.on("queue@peek", this, &queue_app_context::process);
dispatch.on("queue@peek-multi", this, &queue_app_context::process);
dispatch.on("queue@ack", this, &queue_app_context::process);
dispatch.on("queue@ack-multi", this, &queue_app_context::process);
dispatch.on("queue@clear", this, &queue_app_context::process);
dispatch.on("queue@stats-clear", this, &queue_app_context::process);
dispatch.on("queue@stats", this, &queue_app_context::process);
}
spent_time_printer_t::~spent_time_printer_t() {
auto end_time = std::chrono::system_clock::now();
COCAINE_LOG_INFO(m_logger, "libmastermind: time spent for \'%s\': %d milliseconds"
, m_handler_name.c_str()
, static_cast<int>(std::chrono::duration_cast<std::chrono::milliseconds>(end_time - m_beg_time).count())
);
}
namespace_state_init_t::data_t
mastermind_t::data::create_namespaces_states(const std::string &name
, const kora::dynamic_t &raw_value) {
namespace_state_init_t::data_t ns_state{name
, kora::config_t(name, raw_value), user_settings_factory};
COCAINE_LOG_INFO(m_logger, "libmastermind: namespace_state: %s", ns_state.extract.c_str());
return ns_state;
}
void mastermind_t::data::collect_info_loop() {
std::unique_lock<std::mutex> lock(m_mutex);
if (m_done) {
COCAINE_LOG_INFO(m_logger, "libmastermind: have to stop immediately");
return;
}
try {
reconnect();
} catch (const std::exception &ex) {
COCAINE_LOG_ERROR(m_logger, "libmastermind: reconnect: %s", ex.what());
}
#if __GNUC_MINOR__ >= 6
auto no_timeout = std::cv_status::no_timeout;
auto timeout = std::cv_status::timeout;
auto tm = timeout;
#else
bool no_timeout = true;
bool timeout = false;
bool tm = timeout;
#endif
COCAINE_LOG_INFO(m_logger, "libmastermind: collect_info_loop: update period is %d", static_cast<int>(m_group_info_update_period));
do {
collect_info_loop_impl();
process_callbacks();
tm = timeout;
do {
tm = m_weight_cache_condition_variable.wait_for(lock,
std::chrono::seconds(
m_group_info_update_period));
} while(tm == no_timeout && m_done == false);
} while(m_done == false);
}
void
mastermind_t::data::collect_namespaces_states() {
try {
auto dynamic_namespaces_states = enqueue("get_namespaces_states").as_object();
for (auto it = dynamic_namespaces_states.begin(), end = dynamic_namespaces_states.end();
it != end; ++it) {
const auto &name = it->first;
try {
if (namespace_state_is_deleted(it->second)) {
std::ostringstream oss;
oss << "libmastermind: namespace \"" << name
<< "\" was detected as deleted ";
if (namespaces_states.remove(name)) {
oss << "and was removed from the cache";
} else {
oss << "but it is not already in the cache";
}
auto msg = oss.str();
COCAINE_LOG_INFO(m_logger, "%s", msg.c_str());
continue;
}
auto ns_state = create_namespaces_states(name, it->second);
// TODO: check new ns_state is better than the old one
// auto old_ns_state = namespaces_states.copy(name);
// if (ns_state is better than old_ns_state) {
// namespaces_states.set(name, ns_state);
// } else {
// throw std::runtime_error("old namespace_state is better than the new one");
// }
namespaces_states.set(name, {std::move(ns_state), std::move(it->second), name});
} catch (const std::exception &ex) {
COCAINE_LOG_ERROR(m_logger
, "libmastermind: cannot update namespace_state for %s: %s"
, name.c_str(), ex.what());
}
}
} catch (const std::exception &ex) {
COCAINE_LOG_ERROR(m_logger
, "libmastermind: cannot process collect_namespaces_states: %s"
, ex.what());
}
}
subscription_t(const std::string& cluster, const std::string& unicorn_prefix, peers_t& peers,
std::chrono::seconds system_weights_ttl, std::chrono::seconds retry_after, api::unicorn_t& unicorn,
executor::owning_asio_t& executor, logging::logger_t& logger)
: cluster_(cluster)
, path_(format("{}/{}/metrics", unicorn_prefix, cluster))
, ttl_(system_weights_ttl)
, retry_after_(retry_after)
, expiration_timer_(executor.asio())
, retry_timer_(executor.asio())
, unicorn_(unicorn)
, peers_(peers)
, logger_(logger) {
subscribe();
schedule_expiration();
COCAINE_LOG_INFO(logger_, "subscribed to metrics by path `{}`", path_);
}
darkmetrics_t::darkmetrics_t(context_t& context, peers_t& peers, const dynamic_t& args)
: unicorn_name_(args.as_object().at("unicorn", dynamic_t::empty_string).as_string())
, unicorn_prefix_(args.as_object().at("unicorn_prefix", dynamic_t::empty_string).as_string())
, unicorn_retry_after_(args.as_object().at("unicorn_retry_after_s", 20U).as_uint())
, enabled_(args.as_object().at("enabled", false).as_bool())
, ttl_(args.as_object().at("ttl_s", 300U).as_uint())
, logger_(context.log("vicodyn/darkmetrics"))
, peers_(peers) {
if (enabled_) {
if (unicorn_name_.empty() || unicorn_prefix_.empty()) {
throw error_t("invalid configuration of darkmetrics: `unicorn` and `unicorn_prefix` are required");
}
unicorn_ = api::unicorn(context, unicorn_name_);
}
COCAINE_LOG_INFO(logger_, "balancing by system weights is {}", enabled_ ? "on" : "off");
}
std::shared_ptr<namespace_state_init_t::data_t>
mastermind_t::data::get_namespace_state(const std::string &name) const {
try {
auto ns_state_cache = namespaces_states.copy(name);
auto result = ns_state_cache.get_shared_value();
if (user_settings_factory && !result->settings.user_settings_ptr) {
throw std::runtime_error("user settings were not initialized");
}
return result;
} catch (const std::exception &ex) {
COCAINE_LOG_INFO(m_logger, "cannot obtain namespace_state for %s: %s"
, name.c_str(), ex.what());
}
return std::shared_ptr<namespace_state_init_t::data_t>();
}
std::vector<int> mastermind_t::get_metabalancer_groups(uint64_t count, const std::string &name_space, uint64_t size) {
try {
auto cache = m_data->namespaces_states.copy(name_space);
if (count != cache.get_value().settings.groups_count) {
throw invalid_groups_count_error();
}
auto couple = cache.get_value().weights.get(size);
{
std::ostringstream oss;
oss
<< "libmastermind: get_metabalancer_groups: request={group-count=" << count
<< ", namespace=" << name_space << ", size=" << size
<< "}; response={"
<< "couple=[";
{
auto &&groups = couple.groups;
for (auto beg = groups.begin(), it = beg, end = groups.end(); it != end; ++it) {
if (beg != it) oss << ", ";
oss << *it;
}
}
oss << "], weight=" << couple.weight << ", free-space=" << couple.memory
<< "};";
auto msg = oss.str();
COCAINE_LOG_INFO(m_data->m_logger, "%s", msg.c_str());
}
return couple.groups;
} catch(const std::system_error &ex) {
COCAINE_LOG_ERROR(
m_data->m_logger,
"libmastermind: cannot obtain couple for: count = %llu; namespace = %s; size = %llu; \"%s\"",
count, name_space.c_str(), size, ex.code().message().c_str());
throw;
}
}
void queue_app_context::process(const std::string &cocaine_event, const std::vector<std::string> &chunks, cocaine::framework::response_ptr response)
{
ioremap::elliptics::exec_context context = ioremap::elliptics::exec_context::from_raw(chunks[0].c_str(), chunks[0].size());
std::string app;
std::string event;
{
char *p = strchr((char*)context.event().c_str(), '@');
app.assign(context.event().c_str(), p - context.event().c_str());
event.assign(p + 1);
}
const std::string action_id = cocaine::format("%s %d, %s", dnet_dump_id_str(context.src_id()->id), context.src_key(), m_id.c_str());
COCAINE_LOG_INFO(m_log, "%s, event: %s, data-size: %ld",
action_id.c_str(),
event.c_str(), context.data().size()
);
if (event == "ping") {
m_queue->final(context, std::string("ok"));
} else if (event == "push") {
void mastermind_t::data::reconnect() {
std::lock_guard<std::mutex> lock(m_reconnect_mutex);
(void) lock;
size_t end = m_next_remote;
size_t index = m_next_remote;
size_t size = m_remotes.size();
do {
auto &remote = m_remotes[index];
try {
COCAINE_LOG_INFO(m_logger,
"libmastermind: reconnect: try to connect to locator %s:%d",
remote.first.c_str(), static_cast<int>(remote.second));
m_app.reset();
m_service_manager = cocaine::framework::service_manager_t::create(
cocaine::framework::service_manager_t::endpoint_t(remote.first, remote.second));
COCAINE_LOG_INFO(m_logger,
"libmastermind: reconnect: connected to locator, getting mastermind service");
auto g = m_service_manager->get_service_async<cocaine::framework::app_service_t>(m_worker_name);
g.wait_for(reconnect_timeout);
if (g.ready() == false){
COCAINE_LOG_ERROR(
m_logger,
"libmastermind: reconnect: cannot get mastermind-service in %d milliseconds from %s:%d",
static_cast<int>(reconnect_timeout.count()),
remote.first.c_str(), static_cast<int>(remote.second));
g = decltype(g)();
m_service_manager.reset();
index = (index + 1) % size;
continue;
}
m_app = g.get();
COCAINE_LOG_INFO(m_logger,
"libmastermind: reconnect: connected to mastermind via locator %s:%d"
, remote.first.c_str(), static_cast<int>(remote.second));
m_current_remote = remote;
m_next_remote = (index + 1) % size;
return;
} catch (const cocaine::framework::service_error_t &ex) {
COCAINE_LOG_ERROR(
m_logger,
"libmastermind: reconnect: service_error: %s; host: %s:%d",
ex.what(), remote.first.c_str(), static_cast<int>(remote.second));
} catch (const std::exception &ex) {
COCAINE_LOG_ERROR(
m_logger,
"libmastermind: reconnect: %s; host: %s:%d",
ex.what(), remote.first.c_str(), static_cast<int>(remote.second));
}
index = (index + 1) % size;
} while (index != end);
m_current_remote = remote_t();
m_app.reset();
m_service_manager.reset();
COCAINE_LOG_ERROR(m_logger, "libmastermind: reconnect: cannot recconect to any host");
throw std::runtime_error("reconnect error: cannot reconnect to any host");
}
