/**
* Collects metrics from dispatchers and servers and emits them.
*/
static void *
collector_runner(void *s)
{
int i;
size_t totticks;
size_t totmetrics;
size_t totblackholes;
size_t totqueued;
size_t totstalls;
size_t totdropped;
size_t totsleeps;
size_t ticks;
size_t metrics;
size_t blackholes;
size_t queued;
size_t stalls;
size_t dropped;
size_t sleeps;
time_t now;
time_t nextcycle;
char destbuf[1024]; /* sort of POSIX_MAX_PATH */
char *p;
size_t numaggregators = 0;
aggregator *aggrs = NULL;
server *submission = (server *)s;
server **srvs = NULL;
char metric[METRIC_BUFSIZ];
char *m = NULL;
size_t sizem = 0;
size_t (*s_ticks)(server *);
size_t (*s_metrics)(server *);
size_t (*s_stalls)(server *);
size_t (*s_dropped)(server *);
size_t (*d_ticks)(dispatcher *);
size_t (*d_metrics)(dispatcher *);
size_t (*d_blackholes)(dispatcher *);
size_t (*d_sleeps)(dispatcher *);
size_t (*a_received)(aggregator *);
size_t (*a_sent)(aggregator *);
size_t (*a_dropped)(aggregator *);
/* setup functions to target what the user wants */
if (debug & 2) {
s_ticks = server_get_ticks_sub;
s_metrics = server_get_metrics_sub;
s_stalls = server_get_stalls_sub;
s_dropped = server_get_dropped_sub;
d_ticks = dispatch_get_ticks_sub;
d_metrics = dispatch_get_metrics_sub;
d_blackholes = dispatch_get_blackholes_sub;
d_sleeps = dispatch_get_sleeps_sub;
a_received = aggregator_get_received_sub;
a_sent = aggregator_get_sent_sub;
a_dropped = aggregator_get_dropped_sub;
} else {
s_ticks = server_get_ticks;
s_metrics = server_get_metrics;
s_stalls = server_get_stalls;
s_dropped = server_get_dropped;
d_ticks = dispatch_get_ticks;
d_metrics = dispatch_get_metrics;
d_blackholes = dispatch_get_blackholes;
d_sleeps = dispatch_get_sleeps;
a_received = aggregator_get_received;
a_sent = aggregator_get_sent;
a_dropped = aggregator_get_dropped;
}
#define send(metric) \
if (debug & 1) \
logout("%s", metric); \
else \
server_send(submission, strdup(metric), 1);
nextcycle = time(NULL) + collector_interval;
while (keep_running) {
if (cluster_refresh_pending) {
char *stub = router_getcollectorstub(pending_router);
server **newservers = router_getservers(pending_router);
if (srvs != NULL)
free(srvs);
srvs = newservers;
aggrs = router_getaggregators(pending_router);
numaggregators = aggregator_numaggregators(aggrs);
/* prepare hostname for graphite metrics */
snprintf(metric, sizeof(metric), "%scarbon.relays.%s",
stub == NULL ? "" : stub, relay_hostname);
for (m = metric + strlen("carbon.relays."); *m != '\0'; m++)
if (*m == '.')
*m = '_';
*m++ = '.';
*m = '\0';
sizem = sizeof(metric) - (m - metric);
cluster_refresh_pending = 0;
}
assert(srvs != NULL);
sleep(1);
now = time(NULL);
if (nextcycle > now)
continue;
nextcycle += collector_interval;
totticks = 0;
totmetrics = 0;
totblackholes = 0;
totsleeps = 0;
for (i = 0; dispatchers[i] != NULL; i++) {
totsleeps += sleeps = d_sleeps(dispatchers[i]);
totticks += ticks = d_ticks(dispatchers[i]);
totmetrics += metrics = d_metrics(dispatchers[i]);
totblackholes += blackholes = d_blackholes(dispatchers[i]);
snprintf(m, sizem, "dispatcher%d.metricsReceived %zu %zu\n",
i + 1, metrics, (size_t)now);
send(metric);
snprintf(m, sizem, "dispatcher%d.metricsBlackholed %zu %zu\n",
i + 1, blackholes, (size_t)now);
send(metric);
snprintf(m, sizem, "dispatcher%d.wallTime_us %zu %zu\n",
i + 1, ticks, (size_t)now);
send(metric);
snprintf(m, sizem, "dispatcher%d.sleepTime_us %zu %zu\n",
i + 1, sleeps, (size_t)now);
send(metric);
}
snprintf(m, sizem, "metricsReceived %zu %zu\n",
totmetrics, (size_t)now);
send(metric);
snprintf(m, sizem, "metricsBlackholed %zu %zu\n",
totblackholes, (size_t)now);
send(metric);
snprintf(m, sizem, "dispatch_wallTime_us %zu %zu\n",
totticks, (size_t)now);
send(metric);
snprintf(m, sizem, "dispatch_sleepTime_us %zu %zu\n",
totsleeps, (size_t)now);
send(metric);
#define send_server_metrics(ipbuf, ticks, metrics, queued, stalls, dropped) \
snprintf(m, sizem, "destinations.%s.sent %zu %zu\n", \
ipbuf, metrics, (size_t)now); \
send(metric); \
snprintf(m, sizem, "destinations.%s.queued %zu %zu\n", \
ipbuf, queued, (size_t)now); \
send(metric); \
snprintf(m, sizem, "destinations.%s.stalls %zu %zu\n", \
ipbuf, stalls, (size_t)now); \
send(metric); \
snprintf(m, sizem, "destinations.%s.dropped %zu %zu\n", \
ipbuf, dropped, (size_t)now); \
send(metric); \
snprintf(m, sizem, "destinations.%s.wallTime_us %zu %zu\n", \
ipbuf, ticks, (size_t)now); \
send(metric);
totticks = 0;
totmetrics = 0;
totqueued = 0;
totstalls = 0;
totdropped = 0;
/* exclude internal_submission metrics from the totals to avoid
* artificial doubles due to internal routing details */
ticks = s_ticks(submission);
metrics = s_metrics(submission);
queued = server_get_queue_len(submission);
stalls = s_stalls(submission);
dropped = s_dropped(submission);
send_server_metrics(server_ip(submission),
ticks, metrics, queued, stalls, dropped);
for (i = 0; srvs[i] != NULL; i++) {
switch (server_ctype(srvs[i])) {
case CON_FILE:
case CON_PIPE:
snprintf(destbuf, sizeof(destbuf), "%s",
server_ip(srvs[i]));
break;
case CON_TCP:
snprintf(destbuf, sizeof(destbuf), "%s:%u",
server_ip(srvs[i]), server_port(srvs[i]));
break;
case CON_UDP:
snprintf(destbuf, sizeof(destbuf), "%s:%u-udp",
server_ip(srvs[i]), server_port(srvs[i]));
break;
}
for (p = destbuf; *p != '\0'; p++)
if (*p == '.')
*p = '_';
totticks += ticks = s_ticks(srvs[i]);
totmetrics += metrics = s_metrics(srvs[i]);
totqueued += queued = server_get_queue_len(srvs[i]);
totstalls += stalls = s_stalls(srvs[i]);
totdropped += dropped = s_dropped(srvs[i]);
send_server_metrics(destbuf,
ticks, metrics, queued, stalls, dropped);
}
snprintf(m, sizem, "metricsSent %zu %zu\n",
totmetrics, (size_t)now);
send(metric);
snprintf(m, sizem, "metricsQueued %zu %zu\n",
totqueued, (size_t)now);
send(metric);
snprintf(m, sizem, "metricStalls %zu %zu\n",
totstalls, (size_t)now);
send(metric);
snprintf(m, sizem, "metricsDropped %zu %zu\n",
totdropped, (size_t)now);
send(metric);
snprintf(m, sizem, "server_wallTime_us %zu %zu\n",
totticks, (size_t)now);
send(metric);
snprintf(m, sizem, "connections %zu %zu\n",
dispatch_get_accepted_connections(), (size_t)now);
send(metric);
snprintf(m, sizem, "disconnects %zu %zu\n",
dispatch_get_closed_connections(), (size_t)now);
send(metric);
if (numaggregators > 0) {
snprintf(m, sizem, "aggregators.metricsReceived %zu %zu\n",
a_received(aggrs), (size_t)now);
send(metric);
snprintf(m, sizem, "aggregators.metricsSent %zu %zu\n",
a_sent(aggrs), (size_t)now);
send(metric);
snprintf(m, sizem, "aggregators.metricsDropped %zu %zu\n",
a_dropped(aggrs), (size_t)now);
send(metric);
}
if (debug & 1)
fflush(stdout);
}
if (srvs != NULL)
free(srvs);
return NULL;
}
/**
* Collects metrics from dispatchers and servers and emits them.
*/
static void *
collector_runner(void *s)
{
int i;
size_t totticks;
size_t totmetrics;
size_t totblackholes;
size_t totqueued;
size_t totstalls;
size_t totdropped;
size_t ticks;
size_t metrics;
size_t blackholes;
size_t queued;
size_t stalls;
size_t dropped;
size_t dispatchers_idle;
size_t dispatchers_busy;
time_t now;
time_t nextcycle;
char ipbuf[32];
char *p;
size_t numaggregators = 0;
aggregator *aggrs = NULL;
server *submission = (server *)s;
server **srvs = NULL;
char metric[METRIC_BUFSIZ];
char *m;
size_t sizem = 0;
size_t (*s_ticks)(server *);
size_t (*s_metrics)(server *);
size_t (*s_stalls)(server *);
size_t (*s_dropped)(server *);
size_t (*d_ticks)(dispatcher *);
size_t (*d_metrics)(dispatcher *);
size_t (*d_blackholes)(dispatcher *);
size_t (*a_received)(aggregator *);
size_t (*a_sent)(aggregator *);
size_t (*a_dropped)(aggregator *);
/* prepare hostname for graphite metrics */
snprintf(metric, sizeof(metric), "carbon.relays.%s", relay_hostname);
for (m = metric + strlen("carbon.relays."); *m != '\0'; m++)
if (*m == '.')
*m = '_';
*m++ = '.';
*m = '\0';
sizem = sizeof(metric) - (m - metric);
/* setup functions to target what the user wants */
if (debug & 2) {
s_ticks = server_get_ticks_sub;
s_metrics = server_get_metrics_sub;
s_stalls = server_get_stalls_sub;
s_dropped = server_get_dropped_sub;
d_ticks = dispatch_get_ticks_sub;
d_metrics = dispatch_get_metrics_sub;
d_blackholes = dispatch_get_blackholes_sub;
a_received = aggregator_get_received_sub;
a_sent = aggregator_get_sent_sub;
a_dropped = aggregator_get_dropped_sub;
} else {
s_ticks = server_get_ticks;
s_metrics = server_get_metrics;
s_stalls = server_get_stalls;
s_dropped = server_get_dropped;
d_ticks = dispatch_get_ticks;
d_metrics = dispatch_get_metrics;
d_blackholes = dispatch_get_blackholes;
a_received = aggregator_get_received;
a_sent = aggregator_get_sent;
a_dropped = aggregator_get_dropped;
}
#define send(metric) \
if (debug & 1) \
logout("%s", metric); \
else \
server_send(submission, strdup(metric), 1);
nextcycle = time(NULL) + collector_interval;
while (keep_running) {
if (cluster_refresh_pending) {
server **newservers = router_getservers(pending_clusters);
if (srvs != NULL)
free(srvs);
srvs = newservers;
aggrs = pending_aggrs;
numaggregators = aggregator_numaggregators(aggrs);
cluster_refresh_pending = 0;
}
assert(srvs != NULL);
sleep(1);
now = time(NULL);
if (nextcycle > now)
continue;
nextcycle += collector_interval;
totticks = 0;
totmetrics = 0;
totblackholes = 0;
dispatchers_idle = 0;
dispatchers_busy = 0;
for (i = 0; dispatchers[i] != NULL; i++) {
if (dispatch_busy(dispatchers[i])) {
dispatchers_busy++;
} else {
dispatchers_idle++;
}
totticks += ticks = d_ticks(dispatchers[i]);
totmetrics += metrics = d_metrics(dispatchers[i]);
totblackholes += blackholes = d_blackholes(dispatchers[i]);
snprintf(m, sizem, "dispatcher%d.metricsReceived %zd %zd\n",
i + 1, metrics, (size_t)now);
send(metric);
snprintf(m, sizem, "dispatcher%d.metricsBlackholed %zd %zd\n",
i + 1, blackholes, (size_t)now);
send(metric);
snprintf(m, sizem, "dispatcher%d.wallTime_us %zd %zd\n",
i + 1, ticks, (size_t)now);
send(metric);
}
snprintf(m, sizem, "metricsReceived %zd %zd\n",
totmetrics, (size_t)now);
send(metric);
snprintf(m, sizem, "metricsBlackholed %zd %zd\n",
totblackholes, (size_t)now);
send(metric);
snprintf(m, sizem, "dispatch_wallTime_us %zd %zd\n",
totticks, (size_t)now);
send(metric);
snprintf(m, sizem, "dispatch_busy %zd %zd\n",
dispatchers_busy, (size_t)now);
send(metric);
snprintf(m, sizem, "dispatch_idle %zd %zd\n",
dispatchers_idle, (size_t)now);
send(metric);
totticks = 0;
totmetrics = 0;
totqueued = 0;
totstalls = 0;
totdropped = 0;
for (i = 0; srvs[i] != NULL; i++) {
if (server_ctype(srvs[i]) == CON_PIPE) {
strncpy(ipbuf, "internal", sizeof(ipbuf));
ticks = s_ticks(srvs[i]);
metrics = s_metrics(srvs[i]);
queued = server_get_queue_len(srvs[i]);
stalls = s_stalls(srvs[i]);
dropped = s_dropped(srvs[i]);
} else {
snprintf(ipbuf, sizeof(ipbuf), "%s:%u",
server_ip(srvs[i]), server_port(srvs[i]));
for (p = ipbuf; *p != '\0'; p++)
if (*p == '.')
*p = '_';
totticks += ticks = s_ticks(srvs[i]);
totmetrics += metrics = s_metrics(srvs[i]);
totqueued += queued = server_get_queue_len(srvs[i]);
totstalls += stalls = s_stalls(srvs[i]);
totdropped += dropped = s_dropped(srvs[i]);
}
snprintf(m, sizem, "destinations.%s.sent %zd %zd\n",
ipbuf, metrics, (size_t)now);
send(metric);
snprintf(m, sizem, "destinations.%s.queued %zd %zd\n",
ipbuf, queued, (size_t)now);
send(metric);
snprintf(m, sizem, "destinations.%s.stalls %zd %zd\n",
ipbuf, stalls, (size_t)now);
send(metric);
snprintf(m, sizem, "destinations.%s.dropped %zd %zd\n",
ipbuf, dropped, (size_t)now);
send(metric);
snprintf(m, sizem, "destinations.%s.wallTime_us %zd %zd\n",
ipbuf, ticks, (size_t)now);
send(metric);
}
snprintf(m, sizem, "metricsSent %zd %zd\n",
totmetrics, (size_t)now);
send(metric);
snprintf(m, sizem, "metricsQueued %zd %zd\n",
totqueued, (size_t)now);
send(metric);
snprintf(m, sizem, "metricStalls %zd %zd\n",
totstalls, (size_t)now);
send(metric);
snprintf(m, sizem, "metricsDropped %zd %zd\n",
totdropped, (size_t)now);
send(metric);
snprintf(m, sizem, "server_wallTime_us %zd %zd\n",
totticks, (size_t)now);
send(metric);
snprintf(m, sizem, "connections %zd %zd\n",
dispatch_get_accepted_connections(), (size_t)now);
send(metric);
snprintf(m, sizem, "disconnects %zd %zd\n",
dispatch_get_closed_connections(), (size_t)now);
send(metric);
if (numaggregators > 0) {
snprintf(m, sizem, "aggregators.metricsReceived %zd %zd\n",
a_received(aggrs), (size_t)now);
send(metric);
snprintf(m, sizem, "aggregators.metricsSent %zd %zd\n",
a_sent(aggrs), (size_t)now);
send(metric);
snprintf(m, sizem, "aggregators.metricsDropped %zd %zd\n",
a_dropped(aggrs), (size_t)now);
send(metric);
}
if (debug & 1)
fflush(stdout);
}
if (srvs != NULL)
free(srvs);
return NULL;
}