void marathon_dispatcher::handle_api_post(const Json::Value& root)
{
g_logger.log("MESOS_API_POST_EVENT", sinsp_logger::SEV_DEBUG);
std::string uri = get_json_string(root, "uri");
if(uri == "/v2/apps")
{
Json::Value app_obj = root["appDefinition"];
if(!app_obj.empty())
{
std::string app_id = get_json_string(app_obj, "id");
g_logger.log("Adding app [" + app_id + ']', sinsp_logger::SEV_INFO);
marathon_app::ptr_t p_app(new marathon_app(app_id));
m_state.add_or_replace_app(p_app);
g_logger.log("Succesfully added app [" + app_id + ']', sinsp_logger::SEV_INFO);
}
}
}
void marathon_dispatcher::handle_app_terminate(const Json::Value& root)
{
g_logger.log("MESOS_APP_TERMINATED_EVENT", sinsp_logger::SEV_DEBUG);
std::string id = get_json_string(root, "appId");
g_logger.log("Removing app [" + id + ']', sinsp_logger::SEV_INFO);
if(!m_state.remove_app(id))
{
g_logger.log("App [" + id + "] not found.", sinsp_logger::SEV_ERROR);
return;
}
g_logger.log("Succesfully removed app [" + id + ']', sinsp_logger::SEV_INFO);
}
void marathon_dispatcher::handle_status_update(const Json::Value& root)
{
std::string slave_id = get_json_string(root, "slaveId");
std::string app_id = get_json_string(root, "appId");
if(!slave_id.empty() && !app_id.empty())
{
std::string task_status = get_json_string(root, "taskStatus");
if(!task_status.empty())
{
std::string task_id = get_json_string(root, "taskId");
if(!task_id.empty())
{
g_logger.log("App [" + app_id + "], task [" + task_id + "] changed status to " + task_status +
" on slave [" + slave_id + "].\nVersion: " + get_json_string(root, "version") +
", Timestamp: " + get_json_string(root, "version"), sinsp_logger::SEV_INFO);
if(task_status == "TASK_RUNNING")
{
std::string task_name;
std::string::size_type pos = task_id.rfind('.');
if(pos != std::string::npos && pos > 0)
{
task_name = task_id.substr(0, pos);
}
g_logger.log("Handling running notification for task " + task_name + " [" + task_id + ']', sinsp_logger::SEV_INFO);
try
{
std::shared_ptr<mesos_task> t(new mesos_task(task_name, task_id));
t->set_slave_id(slave_id);
t->set_app_id(app_id);
mesos_pair_list label_list;
Json::Value labels = root["labels"];
if(!labels.empty() && labels.isArray())
{
for(const auto& label : labels)
{
std::string key = label["key"].asString();
std::string val = label["value"].asString();
label_list.emplace_back(mesos_pair_list::value_type({key, val}));
}
}
t->add_labels(std::move(label_list));
m_state.add_or_replace_task(m_state.get_framework(m_framework_id), t);
}
catch(std::exception& ex)
{
g_logger.log(ex.what(), sinsp_logger::SEV_ERROR);
return;
}
g_logger.log("Succesfully added or updated task [" + task_id + ']', sinsp_logger::SEV_INFO);
}
else if(task_status == "TASK_FINISHED" || // TERMINAL. The task finished successfully.
task_status == "TASK_FAILED" || // TERMINAL. The task failed to finish successfully.
task_status == "TASK_KILLED" || // TERMINAL. The task was killed by the executor.
task_status == "TASK_LOST" || // TERMINAL. The task failed but can be rescheduled.
task_status == "TASK_ERROR") // TERMINAL. The task description contains an error.
{
std::string msg = get_json_string(root, "message");
std::ostringstream os;
os << "Handling removal notification for task [" << task_id << ']';
if(!msg.empty())
{
os << ", termination message: " << msg;
}
g_logger.log(os.str(), sinsp_logger::SEV_INFO);
try
{
m_state.remove_task(m_state.get_framework(m_framework_id), task_id);
}
catch(std::exception& ex)
{
g_logger.log(ex.what(), sinsp_logger::SEV_ERROR);
return;
}
g_logger.log("Succesfully removed task [" + task_id + ']', sinsp_logger::SEV_INFO);
}
else
{
// Ignored:
// TASK_STAGING; // Initial state. Framework status updates should not use.
// TASK_STARTING;
g_logger.log("Slave [" + slave_id + "], task " + get_json_string(root, "appId") + " (" +
task_id + ") ignored changed status to " + task_status, sinsp_logger::SEV_DEBUG);
}
}
}
}
}
bool k8s_event_handler::handle_component(const Json::Value& json, const msg_data* data)
{
if(m_event_filter)
{
if(m_state)
{
if(data)
{
if((data->m_reason == k8s_component::COMPONENT_ADDED) ||
(data->m_reason == k8s_component::COMPONENT_MODIFIED))
{
g_logger.log("K8s EVENT: handling event.", sinsp_logger::SEV_TRACE);
const Json::Value& involved_object = json["involvedObject"];
if(!involved_object.isNull())
{
bool is_aggregate = (get_json_string(json , "message").find("events with common reason combined") != std::string::npos);
time_t last_ts = get_epoch_utc_seconds(get_json_string(json , "lastTimestamp"));
time_t now_ts = get_epoch_utc_seconds_now();
g_logger.log("K8s EVENT: lastTimestamp=" + std::to_string(last_ts) + ", now_ts=" + std::to_string(now_ts),
sinsp_logger::SEV_TRACE);
if(((last_ts > 0) && (now_ts > 0)) && // we got good timestamps
!is_aggregate && // not an aggregated cached event
((now_ts - last_ts) < 10)) // event not older than 10 seconds
{
const Json::Value& kind = involved_object["kind"];
const Json::Value& event_reason = json["reason"];
g_logger.log("K8s EVENT: involved object and event reason found:" + kind.asString() + '/' + event_reason.asString(),
sinsp_logger::SEV_TRACE);
if(!kind.isNull() && kind.isConvertibleTo(Json::stringValue) &&
!event_reason.isNull() && event_reason.isConvertibleTo(Json::stringValue))
{
bool is_allowed = m_event_filter->allows_all();
std::string type = kind.asString();
if(!is_allowed && !type.empty())
{
std::string reason = event_reason.asString();
is_allowed = m_event_filter->allows_all(type);
if(!is_allowed && !reason.empty())
{
is_allowed = m_event_filter->has(type, reason);
}
}
if(is_allowed)
{
k8s_events& evts = m_state->get_events();
if(evts.size() < sinsp_user_event::max_events_per_cycle())
{
k8s_event_t& evt = m_state->add_component<k8s_events, k8s_event_t>(evts,
data->m_name, data->m_uid, data->m_namespace);
m_state->update_event(evt, json);
m_event_limit_exceeded = false;
if(g_logger.get_severity() >= sinsp_logger::SEV_DEBUG)
{
g_logger.log("K8s EVENT: added event [" + data->m_name + "]. "
"Queued events count=" + std::to_string(evts.size()), sinsp_logger::SEV_DEBUG);
}
}
else if(!m_event_limit_exceeded) // only get in here once per cycle, to send event overflow warning
{
sinsp_user_event::emit_event_overflow("Kubernetes", get_machine_id());
m_event_limit_exceeded = true;
return false;
}
else // event limit exceeded and overflow logged, nothing to do
{
return false;
}
}
else // event not allowed by filter, ignore
{
if(g_logger.get_severity() >= sinsp_logger::SEV_TRACE)
{
g_logger.log("K8s EVENT: filter does not allow {\"" + type + "\", \"{" + event_reason.asString() + "\"} }",
sinsp_logger::SEV_TRACE);
g_logger.log(m_event_filter->to_string(), sinsp_logger::SEV_TRACE);
}
m_event_ignored = true;
return false;
}
}
else
{
g_logger.log("K8s EVENT: event type or involvedObject kind not found.", sinsp_logger::SEV_ERROR);
if(g_logger.get_severity() >= sinsp_logger::SEV_TRACE)
{
g_logger.log(Json::FastWriter().write(json), sinsp_logger::SEV_TRACE);
}
return false;
}
}
else // old event, ignore
{
g_logger.log("K8s EVENT: old event, ignoring: "
", lastTimestamp=" + std::to_string(last_ts) + ", now_ts=" + std::to_string(now_ts),
sinsp_logger::SEV_DEBUG);
m_event_ignored = true;
return false;
}
}
else
{
g_logger.log("K8s EVENT: involvedObject not found.", sinsp_logger::SEV_ERROR);
g_logger.log(Json::FastWriter().write(json), sinsp_logger::SEV_TRACE);
return false;
}
}
else // not ADDED or MODIFIED event, ignore
{
m_event_ignored = true;
return false;
}
}
else
{
g_logger.log("K8s EVENT: msg data is null.", sinsp_logger::SEV_ERROR);
g_logger.log(Json::FastWriter().write(json), sinsp_logger::SEV_TRACE);
return false;
}
}
else
{
g_logger.log("K8s EVENT: state is null.", sinsp_logger::SEV_ERROR);
return false;
}
}
else
{
g_logger.log("K8s EVENT: no filter, K8s events disabled.", sinsp_logger::SEV_TRACE);
return false;
}
return true;
}
void k8s_event_t::update(const Json::Value& item, k8s_state_t& state)
{
#ifndef _WIN32
time_t epoch_time_s = 0;
string event_name;
string description;
severity_t severity = sinsp_logger::SEV_EVT_INFORMATION;
string scope;
tag_map_t tags;
const Json::Value& obj = item["involvedObject"];
//g_logger.log(Json::FastWriter().write(item), sinsp_logger::SEV_TRACE);
if(!obj.isNull())
{
std::string sev = get_json_string(item, "type");
// currently, only "Normal" and "Warning"
severity = sinsp_logger::SEV_EVT_INFORMATION;
if(sev == "Warning") { severity = sinsp_logger::SEV_EVT_WARNING; }
g_logger.log("K8s EVENT : \ncomponent name = " + get_json_string(obj, "name") +
"\nuid = " + get_json_string(obj, "uid") +
"\ntype = " + get_json_string(obj, "kind") +
"\nseverity = " + get_json_string(item, "type") + " (" + std::to_string(severity) + ')', sinsp_logger::SEV_TRACE);
}
else
{
g_logger.log("K8s event: cannot get involved object (null)", sinsp_logger::SEV_ERROR);
}
std::string ts = get_json_string(item , "lastTimestamp");
if(!ts.empty())
{
if((epoch_time_s = get_epoch_utc_seconds(ts)) == (time_t) -1)
{
g_logger.log("K8s event: cannot convert [" + ts + "] to epoch timestamp", sinsp_logger::SEV_ERROR);
}
g_logger.log("K8s EVENT update: time:" + std::to_string(epoch_time_s), sinsp_logger::SEV_DEBUG);
}
else
{
g_logger.log("K8s event: cannot convert time (null, empty or not string)", sinsp_logger::SEV_ERROR);
}
event_name = get_json_string(item , "reason");
const auto& translation = m_name_translation.find(event_name);
if(translation != m_name_translation.end())
{
event_name = translation->second;
}
description = get_json_string(item, "message");
g_logger.log("K8s EVENT message:" + description, sinsp_logger::SEV_DEBUG);
string component_uid = get_json_string(obj, "uid");
if(!component_uid.empty())
{
std::string t;
const k8s_component* comp = state.get_component(component_uid, &t);
if(comp && !t.empty())
{
std::string node_name = comp->get_node_name();
if(!node_name.empty())
{
if(scope.length()) { scope.append(" and "); }
scope.append("kubernetes.node.name=").append(node_name);
}
if(scope.length()) { scope.append(" and "); }
scope.append("kubernetes.").append(t).append(".name=").append(comp->get_name());
const std::string& ns = get_namespace();
if(!ns.empty())
{
scope.append(" and kubernetes.namespace.name=").append(ns);
}
/* no labels for now
for(const auto& label : comp->get_labels())
{
tags[label.first] = label.second;
g_logger.log("EVENT label: [" + label.first + ':' + label.second + ']', sinsp_logger::SEV_DEBUG);
scope.append(" and kubernetes.").append(t).append(".label.").append(label.first).append(1, '=').append(label.second);
}*/
}
else
{
g_logger.log("K8s event: cannot obtain component (UID not found: [" + component_uid + "])", sinsp_logger::SEV_ERROR);
}
}
else
{
g_logger.log("K8s event: cannot obtain tags (UID not retrieved)", sinsp_logger::SEV_ERROR);
}
tags["source"] = "kubernetes";
g_logger.log(sinsp_user_event::to_string(epoch_time_s, std::move(event_name), std::move(description),
std::move(scope), std::move(tags)), severity);
// TODO: sysdig capture?
#endif // _WIN32
}
