void *tc_main(void *ptr) {
netdata_thread_cleanup_push(tc_main_cleanup, ptr);
struct rusage thread;
char command[FILENAME_MAX + 1];
char *words[PLUGINSD_MAX_WORDS] = { NULL };
uint32_t BEGIN_HASH = simple_hash("BEGIN");
uint32_t END_HASH = simple_hash("END");
uint32_t QDISC_HASH = simple_hash("qdisc");
uint32_t CLASS_HASH = simple_hash("class");
uint32_t SENT_HASH = simple_hash("Sent");
uint32_t LENDED_HASH = simple_hash("lended:");
uint32_t TOKENS_HASH = simple_hash("tokens:");
uint32_t SETDEVICENAME_HASH = simple_hash("SETDEVICENAME");
uint32_t SETDEVICEGROUP_HASH = simple_hash("SETDEVICEGROUP");
uint32_t SETCLASSNAME_HASH = simple_hash("SETCLASSNAME");
uint32_t WORKTIME_HASH = simple_hash("WORKTIME");
#ifdef DETACH_PLUGINS_FROM_NETDATA
uint32_t MYPID_HASH = simple_hash("MYPID");
#endif
uint32_t first_hash;
snprintfz(command, TC_LINE_MAX, "%s/tc-qos-helper.sh", netdata_configured_primary_plugins_dir);
char *tc_script = config_get("plugin:tc", "script to run to get tc values", command);
while(!netdata_exit) {
FILE *fp;
struct tc_device *device = NULL;
struct tc_class *class = NULL;
snprintfz(command, TC_LINE_MAX, "exec %s %d", tc_script, localhost->rrd_update_every);
debug(D_TC_LOOP, "executing '%s'", command);
fp = mypopen(command, (pid_t *)&tc_child_pid);
if(unlikely(!fp)) {
error("TC: Cannot popen(\"%s\", \"r\").", command);
goto cleanup;
}
char buffer[TC_LINE_MAX+1] = "";
while(fgets(buffer, TC_LINE_MAX, fp) != NULL) {
if(unlikely(netdata_exit)) break;
buffer[TC_LINE_MAX] = '\0';
// debug(D_TC_LOOP, "TC: read '%s'", buffer);
tc_split_words(buffer, words, PLUGINSD_MAX_WORDS);
if(unlikely(!words[0] || !*words[0])) {
// debug(D_TC_LOOP, "empty line");
continue;
}
// else debug(D_TC_LOOP, "First word is '%s'", words[0]);
first_hash = simple_hash(words[0]);
if(unlikely(device && ((first_hash == CLASS_HASH && strcmp(words[0], "class") == 0) || (first_hash == QDISC_HASH && strcmp(words[0], "qdisc") == 0)))) {
// debug(D_TC_LOOP, "CLASS line on class id='%s', parent='%s', parentid='%s', leaf='%s', leafid='%s'", words[2], words[3], words[4], words[5], words[6]);
char *type = words[1]; // the class/qdisc type: htb, fq_codel, etc
char *id = words[2]; // the class/qdisc major:minor
char *parent = words[3]; // the word 'parent' or 'root'
char *parentid = words[4]; // parentid
char *leaf = words[5]; // the word 'leaf'
char *leafid = words[6]; // leafid
int parent_is_root = 0;
int parent_is_parent = 0;
if(likely(parent)) {
parent_is_parent = !strcmp(parent, "parent");
if(!parent_is_parent)
parent_is_root = !strcmp(parent, "root");
}
if(likely(type && id && (parent_is_root || parent_is_parent))) {
char qdisc = 0;
if(first_hash == QDISC_HASH) {
qdisc = 1;
if(!strcmp(type, "ingress")) {
// we don't want to get the ingress qdisc
// there should be an IFB interface for this
class = NULL;
continue;
}
if(parent_is_parent && parentid) {
// eliminate the minor number from parentid
// why: parentid is the id of the parent class
// but major: is also the id of the parent qdisc
char *s = parentid;
while(*s && *s != ':') s++;
if(*s == ':') s[1] = '\0';
}
}
if(parent_is_root) {
parentid = NULL;
leafid = NULL;
}
else if(!leaf || strcmp(leaf, "leaf") != 0)
leafid = NULL;
char leafbuf[20 + 1] = "";
if(leafid && leafid[strlen(leafid) - 1] == ':') {
strncpyz(leafbuf, leafid, 20 - 1);
strcat(leafbuf, "1");
leafid = leafbuf;
}
class = tc_class_add(device, id, qdisc, parentid, leafid);
}
else {
void *socket_listen_main_single_threaded(void *ptr) {
netdata_thread_cleanup_push(socket_listen_main_single_threaded_cleanup, ptr);
web_server_mode = WEB_SERVER_MODE_SINGLE_THREADED;
web_server_is_multithreaded = 0;
struct web_client *w;
if(!api_sockets.opened)
fatal("LISTENER: no listen sockets available.");
size_t i;
for(i = 0; i < (size_t)FD_SETSIZE ; i++)
single_threaded_clients[i] = NULL;
fd_set ifds, ofds, efds, rifds, rofds, refds;
FD_ZERO (&ifds);
FD_ZERO (&ofds);
FD_ZERO (&efds);
int fdmax = 0;
for(i = 0; i < api_sockets.opened ; i++) {
if (api_sockets.fds[i] < 0 || api_sockets.fds[i] >= (int)FD_SETSIZE)
fatal("LISTENER: Listen socket %d is not ready, or invalid.", api_sockets.fds[i]);
info("Listening on '%s'", (api_sockets.fds_names[i])?api_sockets.fds_names[i]:"UNKNOWN");
FD_SET(api_sockets.fds[i], &ifds);
FD_SET(api_sockets.fds[i], &efds);
if(fdmax < api_sockets.fds[i])
fdmax = api_sockets.fds[i];
}
while(!netdata_exit) {
debug(D_WEB_CLIENT_ACCESS, "LISTENER: single threaded web server waiting (fdmax = %d)...", fdmax);
struct timeval tv = { .tv_sec = 1, .tv_usec = 0 };
rifds = ifds;
rofds = ofds;
refds = efds;
int retval = select(fdmax+1, &rifds, &rofds, &refds, &tv);
if(unlikely(retval == -1)) {
error("LISTENER: select() failed.");
continue;
}
else if(likely(retval)) {
debug(D_WEB_CLIENT_ACCESS, "LISTENER: got something.");
for(i = 0; i < api_sockets.opened ; i++) {
if (FD_ISSET(api_sockets.fds[i], &rifds)) {
debug(D_WEB_CLIENT_ACCESS, "LISTENER: new connection.");
w = web_client_create_on_listenfd(api_sockets.fds[i]);
if(unlikely(!w))
continue;
if(api_sockets.fds_families[i] == AF_UNIX)
web_client_set_unix(w);
else
web_client_set_tcp(w);
if (single_threaded_link_client(w, &ifds, &ofds, &ifds, &fdmax) != 0) {
web_client_release(w);
}
}
}
for(i = 0 ; i <= (size_t)fdmax ; i++) {
if(likely(!FD_ISSET(i, &rifds) && !FD_ISSET(i, &rofds) && !FD_ISSET(i, &refds)))
continue;
w = single_threaded_clients[i];
if(unlikely(!w)) {
// error("no client on slot %zu", i);
continue;
}
if(unlikely(single_threaded_unlink_client(w, &ifds, &ofds, &efds) != 0)) {
// error("failed to unlink client %zu", i);
web_client_release(w);
continue;
}
if (unlikely(FD_ISSET(w->ifd, &refds) || FD_ISSET(w->ofd, &refds))) {
// error("no input on client %zu", i);
web_client_release(w);
continue;
}
if (unlikely(web_client_has_wait_receive(w) && FD_ISSET(w->ifd, &rifds))) {
if (unlikely(web_client_receive(w) < 0)) {
// error("cannot read from client %zu", i);
web_client_release(w);
continue;
}
if (w->mode != WEB_CLIENT_MODE_FILECOPY) {
debug(D_WEB_CLIENT, "%llu: Processing received data.", w->id);
web_client_process_request(w);
}
}
if (unlikely(web_client_has_wait_send(w) && FD_ISSET(w->ofd, &rofds))) {
if (unlikely(web_client_send(w) < 0)) {
// error("cannot send data to client %zu", i);
debug(D_WEB_CLIENT, "%llu: Cannot send data to client. Closing client.", w->id);
web_client_release(w);
continue;
}
}
if(unlikely(single_threaded_link_client(w, &ifds, &ofds, &efds, &fdmax) != 0)) {
// error("failed to link client %zu", i);
web_client_release(w);
}
}
}
else {
debug(D_WEB_CLIENT_ACCESS, "LISTENER: single threaded web server timeout.");
}
}
netdata_thread_cleanup_pop(1);
return NULL;
}
void *proc_diskspace_main(void *ptr) {
netdata_thread_cleanup_push(diskspace_main_cleanup, ptr);
int vdo_cpu_netdata = config_get_boolean("plugin:proc", "netdata server resources", 1);
cleanup_mount_points = config_get_boolean(CONFIG_SECTION_DISKSPACE, "remove charts of unmounted disks" , cleanup_mount_points);
int update_every = (int)config_get_number(CONFIG_SECTION_DISKSPACE, "update every", localhost->rrd_update_every);
if(update_every < localhost->rrd_update_every)
update_every = localhost->rrd_update_every;
check_for_new_mountpoints_every = (int)config_get_number(CONFIG_SECTION_DISKSPACE, "check for new mount points every", check_for_new_mountpoints_every);
if(check_for_new_mountpoints_every < update_every)
check_for_new_mountpoints_every = update_every;
struct rusage thread;
usec_t duration = 0;
usec_t step = update_every * USEC_PER_SEC;
heartbeat_t hb;
heartbeat_init(&hb);
while(!netdata_exit) {
duration = heartbeat_dt_usec(&hb);
/* usec_t hb_dt = */ heartbeat_next(&hb, step);
if(unlikely(netdata_exit)) break;
// --------------------------------------------------------------------------
// this is smart enough not to reload it every time
mountinfo_reload(0);
// --------------------------------------------------------------------------
// disk space metrics
struct mountinfo *mi;
for(mi = disk_mountinfo_root; mi; mi = mi->next) {
if(unlikely(mi->flags & (MOUNTINFO_IS_DUMMY | MOUNTINFO_IS_BIND)))
continue;
do_disk_space_stats(mi, update_every);
if(unlikely(netdata_exit)) break;
}
if(unlikely(netdata_exit)) break;
if(dict_mountpoints)
dictionary_get_all(dict_mountpoints, mount_point_cleanup, NULL);
if(vdo_cpu_netdata) {
static RRDSET *stcpu_thread = NULL, *st_duration = NULL;
static RRDDIM *rd_user = NULL, *rd_system = NULL, *rd_duration = NULL;
// ----------------------------------------------------------------
getrusage(RUSAGE_THREAD, &thread);
if(unlikely(!stcpu_thread)) {
stcpu_thread = rrdset_create_localhost(
"netdata"
, "plugin_diskspace"
, NULL
, "diskspace"
, NULL
, "NetData Disk Space Plugin CPU usage"
, "milliseconds/s"
, "diskspace"
, NULL
, 132020
, update_every
, RRDSET_TYPE_STACKED
);
rd_user = rrddim_add(stcpu_thread, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
rd_system = rrddim_add(stcpu_thread, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
}
else
rrdset_next(stcpu_thread);
rrddim_set_by_pointer(stcpu_thread, rd_user, thread.ru_utime.tv_sec * 1000000ULL + thread.ru_utime.tv_usec);
rrddim_set_by_pointer(stcpu_thread, rd_system, thread.ru_stime.tv_sec * 1000000ULL + thread.ru_stime.tv_usec);
rrdset_done(stcpu_thread);
// ----------------------------------------------------------------
if(unlikely(!st_duration)) {
st_duration = rrdset_create_localhost(
"netdata"
, "plugin_diskspace_dt"
, NULL
, "diskspace"
, NULL
, "NetData Disk Space Plugin Duration"
, "milliseconds/run"
, "diskspace"
, NULL
, 132021
, update_every
, RRDSET_TYPE_AREA
);
rd_duration = rrddim_add(st_duration, "duration", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
}
else
rrdset_next(st_duration);
rrddim_set_by_pointer(st_duration, rd_duration, duration);
rrdset_done(st_duration);
// ----------------------------------------------------------------
if(unlikely(netdata_exit)) break;
}
}
netdata_thread_cleanup_pop(1);
return NULL;
}
void *backends_main(void *ptr) {
netdata_thread_cleanup_push(backends_main_cleanup, ptr);
int default_port = 0;
int sock = -1;
BUFFER *b = buffer_create(1), *response = buffer_create(1);
int (*backend_request_formatter)(BUFFER *, const char *, RRDHOST *, const char *, RRDSET *, RRDDIM *, time_t, time_t, uint32_t) = NULL;
int (*backend_response_checker)(BUFFER *) = NULL;
// ------------------------------------------------------------------------
// collect configuration options
struct timeval timeout = {
.tv_sec = 0,
.tv_usec = 0
};
int enabled = config_get_boolean(CONFIG_SECTION_BACKEND, "enabled", 0);
const char *source = config_get(CONFIG_SECTION_BACKEND, "data source", "average");
const char *type = config_get(CONFIG_SECTION_BACKEND, "type", "graphite");
const char *destination = config_get(CONFIG_SECTION_BACKEND, "destination", "localhost");
backend_prefix = config_get(CONFIG_SECTION_BACKEND, "prefix", "netdata");
const char *hostname = config_get(CONFIG_SECTION_BACKEND, "hostname", localhost->hostname);
backend_update_every = (int)config_get_number(CONFIG_SECTION_BACKEND, "update every", backend_update_every);
int buffer_on_failures = (int)config_get_number(CONFIG_SECTION_BACKEND, "buffer on failures", 10);
long timeoutms = config_get_number(CONFIG_SECTION_BACKEND, "timeout ms", backend_update_every * 2 * 1000);
backend_send_names = config_get_boolean(CONFIG_SECTION_BACKEND, "send names instead of ids", backend_send_names);
charts_pattern = simple_pattern_create(config_get(CONFIG_SECTION_BACKEND, "send charts matching", "*"), NULL, SIMPLE_PATTERN_EXACT);
hosts_pattern = simple_pattern_create(config_get(CONFIG_SECTION_BACKEND, "send hosts matching", "localhost *"), NULL, SIMPLE_PATTERN_EXACT);
// ------------------------------------------------------------------------
// validate configuration options
// and prepare for sending data to our backend
backend_options = backend_parse_data_source(source, backend_options);
if(timeoutms < 1) {
error("BACKEND: invalid timeout %ld ms given. Assuming %d ms.", timeoutms, backend_update_every * 2 * 1000);
timeoutms = backend_update_every * 2 * 1000;
}
timeout.tv_sec = (timeoutms * 1000) / 1000000;
timeout.tv_usec = (timeoutms * 1000) % 1000000;
if(!enabled || backend_update_every < 1)
goto cleanup;
// ------------------------------------------------------------------------
// select the backend type
if(!strcmp(type, "graphite") || !strcmp(type, "graphite:plaintext")) {
default_port = 2003;
backend_response_checker = process_graphite_response;
if((backend_options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_AS_COLLECTED)
backend_request_formatter = format_dimension_collected_graphite_plaintext;
else
backend_request_formatter = format_dimension_stored_graphite_plaintext;
}
else if(!strcmp(type, "opentsdb") || !strcmp(type, "opentsdb:telnet")) {
default_port = 4242;
backend_response_checker = process_opentsdb_response;
if((backend_options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_AS_COLLECTED)
backend_request_formatter = format_dimension_collected_opentsdb_telnet;
else
backend_request_formatter = format_dimension_stored_opentsdb_telnet;
}
else if (!strcmp(type, "json") || !strcmp(type, "json:plaintext")) {
default_port = 5448;
backend_response_checker = process_json_response;
if ((backend_options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_AS_COLLECTED)
backend_request_formatter = format_dimension_collected_json_plaintext;
else
backend_request_formatter = format_dimension_stored_json_plaintext;
}
else {
error("BACKEND: Unknown backend type '%s'", type);
goto cleanup;
}
if(backend_request_formatter == NULL || backend_response_checker == NULL) {
error("BACKEND: backend is misconfigured - disabling it.");
goto cleanup;
}
// ------------------------------------------------------------------------
// prepare the charts for monitoring the backend operation
struct rusage thread;
collected_number
chart_buffered_metrics = 0,
chart_lost_metrics = 0,
chart_sent_metrics = 0,
chart_buffered_bytes = 0,
chart_received_bytes = 0,
chart_sent_bytes = 0,
chart_receptions = 0,
chart_transmission_successes = 0,
chart_transmission_failures = 0,
chart_data_lost_events = 0,
chart_lost_bytes = 0,
chart_backend_reconnects = 0,
chart_backend_latency = 0;
RRDSET *chart_metrics = rrdset_create_localhost("netdata", "backend_metrics", NULL, "backend", NULL, "Netdata Buffered Metrics", "metrics", "backends", NULL, 130600, backend_update_every, RRDSET_TYPE_LINE);
rrddim_add(chart_metrics, "buffered", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
rrddim_add(chart_metrics, "lost", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
rrddim_add(chart_metrics, "sent", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
RRDSET *chart_bytes = rrdset_create_localhost("netdata", "backend_bytes", NULL, "backend", NULL, "Netdata Backend Data Size", "KB", "backends", NULL, 130610, backend_update_every, RRDSET_TYPE_AREA);
rrddim_add(chart_bytes, "buffered", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);
rrddim_add(chart_bytes, "lost", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);
rrddim_add(chart_bytes, "sent", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);
rrddim_add(chart_bytes, "received", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);
RRDSET *chart_ops = rrdset_create_localhost("netdata", "backend_ops", NULL, "backend", NULL, "Netdata Backend Operations", "operations", "backends", NULL, 130630, backend_update_every, RRDSET_TYPE_LINE);
rrddim_add(chart_ops, "write", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
rrddim_add(chart_ops, "discard", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
rrddim_add(chart_ops, "reconnect", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
rrddim_add(chart_ops, "failure", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
rrddim_add(chart_ops, "read", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
/*
* this is misleading - we can only measure the time we need to send data
* this time is not related to the time required for the data to travel to
* the backend database and the time that server needed to process them
*
* issue #1432 and https://www.softlab.ntua.gr/facilities/documentation/unix/unix-socket-faq/unix-socket-faq-2.html
*
RRDSET *chart_latency = rrdset_create_localhost("netdata", "backend_latency", NULL, "backend", NULL, "Netdata Backend Latency", "ms", "backends", NULL, 130620, backend_update_every, RRDSET_TYPE_AREA);
rrddim_add(chart_latency, "latency", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
*/
RRDSET *chart_rusage = rrdset_create_localhost("netdata", "backend_thread_cpu", NULL, "backend", NULL, "NetData Backend Thread CPU usage", "milliseconds/s", "backends", NULL, 130630, backend_update_every, RRDSET_TYPE_STACKED);
rrddim_add(chart_rusage, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
rrddim_add(chart_rusage, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
// ------------------------------------------------------------------------
// prepare the backend main loop
info("BACKEND: configured ('%s' on '%s' sending '%s' data, every %d seconds, as host '%s', with prefix '%s')", type, destination, source, backend_update_every, hostname, backend_prefix);
usec_t step_ut = backend_update_every * USEC_PER_SEC;
time_t after = now_realtime_sec();
int failures = 0;
heartbeat_t hb;
heartbeat_init(&hb);
while(!netdata_exit) {
// ------------------------------------------------------------------------
// Wait for the next iteration point.
heartbeat_next(&hb, step_ut);
time_t before = now_realtime_sec();
debug(D_BACKEND, "BACKEND: preparing buffer for timeframe %lu to %lu", (unsigned long)after, (unsigned long)before);
// ------------------------------------------------------------------------
// add to the buffer the data we need to send to the backend
netdata_thread_disable_cancelability();
size_t count_hosts = 0;
size_t count_charts_total = 0;
size_t count_dims_total = 0;
rrd_rdlock();
RRDHOST *host;
rrdhost_foreach_read(host) {
if(unlikely(!rrdhost_flag_check(host, RRDHOST_FLAG_BACKEND_SEND|RRDHOST_FLAG_BACKEND_DONT_SEND))) {
char *name = (host == localhost)?"localhost":host->hostname;
if (!hosts_pattern || simple_pattern_matches(hosts_pattern, name)) {
rrdhost_flag_set(host, RRDHOST_FLAG_BACKEND_SEND);
info("enabled backend for host '%s'", name);
}
else {
rrdhost_flag_set(host, RRDHOST_FLAG_BACKEND_DONT_SEND);
info("disabled backend for host '%s'", name);
}
}
if(unlikely(!rrdhost_flag_check(host, RRDHOST_FLAG_BACKEND_SEND)))
continue;
rrdhost_rdlock(host);
count_hosts++;
size_t count_charts = 0;
size_t count_dims = 0;
size_t count_dims_skipped = 0;
const char *__hostname = (host == localhost)?hostname:host->hostname;
RRDSET *st;
rrdset_foreach_read(st, host) {
if(likely(backends_can_send_rrdset(backend_options, st))) {
rrdset_rdlock(st);
count_charts++;
RRDDIM *rd;
rrddim_foreach_read(rd, st) {
if (likely(rd->last_collected_time.tv_sec >= after)) {
chart_buffered_metrics += backend_request_formatter(b, backend_prefix, host, __hostname, st, rd, after, before, backend_options);
count_dims++;
}
else {
debug(D_BACKEND, "BACKEND: not sending dimension '%s' of chart '%s' from host '%s', its last data collection (%lu) is not within our timeframe (%lu to %lu)", rd->id, st->id, __hostname, (unsigned long)rd->last_collected_time.tv_sec, (unsigned long)after, (unsigned long)before);
count_dims_skipped++;
}
}
rrdset_unlock(st);
}
}
debug(D_BACKEND, "BACKEND: sending host '%s', metrics of %zu dimensions, of %zu charts. Skipped %zu dimensions.", __hostname, count_dims, count_charts, count_dims_skipped);
count_charts_total += count_charts;
count_dims_total += count_dims;
rrdhost_unlock(host);
}
rrd_unlock();
netdata_thread_enable_cancelability();
debug(D_BACKEND, "BACKEND: buffer has %zu bytes, added metrics for %zu dimensions, of %zu charts, from %zu hosts", buffer_strlen(b), count_dims_total, count_charts_total, count_hosts);
// ------------------------------------------------------------------------
chart_buffered_bytes = (collected_number)buffer_strlen(b);
// reset the monitoring chart counters
chart_received_bytes =
chart_sent_bytes =
chart_sent_metrics =
chart_lost_metrics =
chart_transmission_successes =
chart_transmission_failures =
chart_data_lost_events =
chart_lost_bytes =
chart_backend_reconnects =
chart_backend_latency = 0;
if(unlikely(netdata_exit)) break;
//fprintf(stderr, "\nBACKEND BEGIN:\n%s\nBACKEND END\n", buffer_tostring(b)); // FIXME
//fprintf(stderr, "after = %lu, before = %lu\n", after, before);
// prepare for the next iteration
// to add incrementally data to buffer
after = before;
// ------------------------------------------------------------------------
// if we are connected, receive a response, without blocking
if(likely(sock != -1)) {
errno = 0;
// loop through to collect all data
while(sock != -1 && errno != EWOULDBLOCK) {
buffer_need_bytes(response, 4096);
ssize_t r = recv(sock, &response->buffer[response->len], response->size - response->len, MSG_DONTWAIT);
if(likely(r > 0)) {
// we received some data
response->len += r;
chart_received_bytes += r;
chart_receptions++;
}
else if(r == 0) {
error("BACKEND: '%s' closed the socket", destination);
close(sock);
sock = -1;
}
else {
// failed to receive data
if(errno != EAGAIN && errno != EWOULDBLOCK) {
error("BACKEND: cannot receive data from backend '%s'.", destination);
}
}
}
// if we received data, process them
if(buffer_strlen(response))
backend_response_checker(response);
}
// ------------------------------------------------------------------------
// if we are not connected, connect to a backend server
if(unlikely(sock == -1)) {
usec_t start_ut = now_monotonic_usec();
size_t reconnects = 0;
sock = connect_to_one_of(destination, default_port, &timeout, &reconnects, NULL, 0);
chart_backend_reconnects += reconnects;
chart_backend_latency += now_monotonic_usec() - start_ut;
}
if(unlikely(netdata_exit)) break;
// ------------------------------------------------------------------------
// if we are connected, send our buffer to the backend server
if(likely(sock != -1)) {
size_t len = buffer_strlen(b);
usec_t start_ut = now_monotonic_usec();
int flags = 0;
#ifdef MSG_NOSIGNAL
flags += MSG_NOSIGNAL;
#endif
ssize_t written = send(sock, buffer_tostring(b), len, flags);
chart_backend_latency += now_monotonic_usec() - start_ut;
if(written != -1 && (size_t)written == len) {
// we sent the data successfully
chart_transmission_successes++;
chart_sent_bytes += written;
chart_sent_metrics = chart_buffered_metrics;
// reset the failures count
failures = 0;
// empty the buffer
buffer_flush(b);
}
else {
// oops! we couldn't send (all or some of the) data
error("BACKEND: failed to write data to database backend '%s'. Willing to write %zu bytes, wrote %zd bytes. Will re-connect.", destination, len, written);
chart_transmission_failures++;
if(written != -1)
chart_sent_bytes += written;
// increment the counter we check for data loss
failures++;
// close the socket - we will re-open it next time
close(sock);
sock = -1;
}
}
else {
error("BACKEND: failed to update database backend '%s'", destination);
chart_transmission_failures++;
// increment the counter we check for data loss
failures++;
}
if(failures > buffer_on_failures) {
// too bad! we are going to lose data
chart_lost_bytes += buffer_strlen(b);
error("BACKEND: reached %d backend failures. Flushing buffers to protect this host - this results in data loss on back-end server '%s'", failures, destination);
buffer_flush(b);
failures = 0;
chart_data_lost_events++;
chart_lost_metrics = chart_buffered_metrics;
}
if(unlikely(netdata_exit)) break;
// ------------------------------------------------------------------------
// update the monitoring charts
if(likely(chart_ops->counter_done)) rrdset_next(chart_ops);
rrddim_set(chart_ops, "read", chart_receptions);
rrddim_set(chart_ops, "write", chart_transmission_successes);
rrddim_set(chart_ops, "discard", chart_data_lost_events);
rrddim_set(chart_ops, "failure", chart_transmission_failures);
rrddim_set(chart_ops, "reconnect", chart_backend_reconnects);
rrdset_done(chart_ops);
if(likely(chart_metrics->counter_done)) rrdset_next(chart_metrics);
rrddim_set(chart_metrics, "buffered", chart_buffered_metrics);
rrddim_set(chart_metrics, "lost", chart_lost_metrics);
rrddim_set(chart_metrics, "sent", chart_sent_metrics);
rrdset_done(chart_metrics);
if(likely(chart_bytes->counter_done)) rrdset_next(chart_bytes);
rrddim_set(chart_bytes, "buffered", chart_buffered_bytes);
rrddim_set(chart_bytes, "lost", chart_lost_bytes);
rrddim_set(chart_bytes, "sent", chart_sent_bytes);
rrddim_set(chart_bytes, "received", chart_received_bytes);
rrdset_done(chart_bytes);
/*
if(likely(chart_latency->counter_done)) rrdset_next(chart_latency);
rrddim_set(chart_latency, "latency", chart_backend_latency);
rrdset_done(chart_latency);
*/
getrusage(RUSAGE_THREAD, &thread);
if(likely(chart_rusage->counter_done)) rrdset_next(chart_rusage);
rrddim_set(chart_rusage, "user", thread.ru_utime.tv_sec * 1000000ULL + thread.ru_utime.tv_usec);
rrddim_set(chart_rusage, "system", thread.ru_stime.tv_sec * 1000000ULL + thread.ru_stime.tv_usec);
rrdset_done(chart_rusage);
if(likely(buffer_strlen(b) == 0))
chart_buffered_metrics = 0;
if(unlikely(netdata_exit)) break;
}
cleanup:
if(sock != -1)
close(sock);
buffer_free(b);
buffer_free(response);
netdata_thread_cleanup_pop(1);
return NULL;
}
