static void *
collector_writer(void *unused)
{
int i = 0;
size_t queued;
size_t queuesize;
double queueusage;
size_t totdropped;
size_t numaggregators = aggregator_numaggregators();
server **srvs = NULL;
while (keep_running) {
if (cluster_refresh_pending) {
server **newservers = router_getservers(pending_clusters);
if (srvs != NULL)
free(srvs);
srvs = newservers;
cluster_refresh_pending = 0;
}
assert(srvs != NULL);
sleep(1);
i++;
if (i < collector_interval)
continue;
totdropped = 0;
for (i = 0; srvs[i] != NULL; i++) {
queued = server_get_queue_len(srvs[i]);
queuesize = server_get_queue_size(srvs[i]);
totdropped += server_get_dropped(srvs[i]);
queueusage = (double)queued / (double)queuesize;
if (queueusage >= 0.75)
logout("warning: metrics queuing up "
"for %s:%u: %zd metrics (%d%% of queue size)\n",
server_ip(srvs[i]), server_port(srvs[i]), queued,
(int)(queueusage * 100));
}
if (totdropped - lastdropped > 0)
logout("warning: dropped %zd metrics\n", totdropped - lastdropped);
lastdropped = totdropped;
if (numaggregators > 0) {
totdropped = aggregator_get_dropped();
if (totdropped - lastaggrdropped > 0)
logout("warning: aggregator dropped %zd metrics\n",
totdropped - lastaggrdropped);
lastaggrdropped = totdropped;
}
i = 0;
}
return NULL;
}
static void *
collector_writer(void *unused)
{
int i = 0;
size_t queued;
size_t queuesize;
double queueusage;
size_t totdropped;
size_t lastconn = 0;
size_t lastdisc = 0;
size_t numaggregators = 0;
server **srvs = NULL;
aggregator *aggrs = NULL;
while (keep_running) {
if (cluster_refresh_pending) {
server **newservers = router_getservers(pending_router);
if (srvs != NULL)
free(srvs);
srvs = newservers;
aggrs = router_getaggregators(pending_router);
numaggregators = aggregator_numaggregators(aggrs);
cluster_refresh_pending = 0;
}
assert(srvs != NULL);
sleep(1);
if (debug & 1) {
size_t mpsout;
size_t totout;
size_t mpsdrop;
size_t totdrop;
size_t totqueue;
size_t mpsin;
size_t totin;
size_t totconn;
size_t totdisc;
size_t dticks;
size_t dsleeps;
int j;
/* Solaris iostat like output:
metrics in metrics out metrics drop queue conns disconn workr
mps tot mps tot dps tot cur cps tot dps tot act
99999 9999999 99999 9999999 99999 9999999 99999 999 99999 999 99999 99%
*/
if (i % 24 == 0)
printf(" metrics in metrics out metrics drop queue conns disconn workr\n"
" mps tot mps tot dps tot cur cps tot dps tot act\n");
mpsout = totout = 0;
mpsdrop = totdrop = 0;
totqueue = 0;
for (j = 0; srvs[j] != NULL; j++) {
mpsout += server_get_metrics_sub(srvs[j]);
totout += server_get_metrics(srvs[j]);
mpsdrop += server_get_dropped_sub(srvs[j]);
totdrop += server_get_dropped(srvs[j]);
totqueue += server_get_queue_len(srvs[j]);
}
mpsin = totin = 0;
dticks = dsleeps = 0;
for (j = 0; dispatchers[j] != NULL; j++) {
mpsin += dispatch_get_metrics_sub(dispatchers[j]);
totin += dispatch_get_metrics(dispatchers[j]);
dticks += dispatch_get_ticks_sub(dispatchers[j]);
dsleeps += dispatch_get_sleeps_sub(dispatchers[j]);
}
totconn = dispatch_get_accepted_connections();
totdisc = dispatch_get_closed_connections();
printf("%5zu %7zu " /* metrics in */
"%5zu %7zu " /* metrics out */
"%5zu %7zu " /* metrics dropped */
"%5zu " /* queue */
"%3zu %5zu " /* conns */
"%3zu %5zu " /* disconns */
"%2d%%\n", /* workers */
mpsin, totin,
mpsout, totout,
mpsdrop, totdrop,
totqueue,
totconn - lastconn, totconn,
totdisc - lastdisc, totdisc,
(int)(((double)dticks * 100.0) / (double)(dticks + dsleeps))
);
lastconn = totconn;
lastdisc = totdisc;
}
i++;
if (i < collector_interval)
continue;
totdropped = 0;
for (i = 0; srvs[i] != NULL; i++) {
queued = server_get_queue_len(srvs[i]);
queuesize = server_get_queue_size(srvs[i]);
totdropped += server_get_dropped(srvs[i]);
queueusage = (double)queued / (double)queuesize;
if (queueusage >= 0.75)
logout("warning: metrics queuing up "
"for %s:%u: %zu metrics (%d%% of queue size)\n",
server_ip(srvs[i]), server_port(srvs[i]), queued,
(int)(queueusage * 100));
}
if (totdropped - lastdropped > 0)
logout("warning: dropped %zu metrics\n", totdropped - lastdropped);
lastdropped = totdropped;
if (numaggregators > 0) {
totdropped = aggregator_get_dropped(aggrs);
if (totdropped - lastaggrdropped > 0)
logout("warning: aggregator dropped %zu metrics\n",
totdropped - lastaggrdropped);
lastaggrdropped = totdropped;
}
i = 0;
}
return NULL;
}
/**
* Collects metrics from dispatchers and servers and emits them.
*/
static void *
collector_runner(void *s)
{
int i;
size_t totticks;
size_t totmetrics;
size_t totqueued;
size_t totstalls;
size_t totdropped;
size_t ticks;
size_t metrics;
size_t queued;
size_t stalls;
size_t dropped;
size_t dispatchers_idle;
size_t dispatchers_busy;
time_t now;
time_t nextcycle;
char ipbuf[32];
char *p;
size_t numaggregators = aggregator_numaggregators();
server *submission = (server *)s;
server **srvs = NULL;
char metric[METRIC_BUFSIZ];
char *m;
size_t sizem = 0;
/* prepare hostname for graphite metrics */
snprintf(metric, sizeof(metric), "carbon.relays.%s", relay_hostname);
for (m = metric + strlen("carbon.relays."); *m != '\0'; m++)
if (*m == '.')
*m = '_';
*m++ = '.';
*m = '\0';
sizem = sizeof(metric) - (m - metric);
#define send(metric) \
if (debug) \
logout("%s", metric); \
else \
server_send(submission, strdup(metric), 1);
nextcycle = time(NULL) + collector_interval;
while (keep_running) {
if (cluster_refresh_pending) {
server **newservers = router_getservers(pending_clusters);
if (srvs != NULL)
free(srvs);
srvs = newservers;
cluster_refresh_pending = 0;
}
assert(srvs != NULL);
sleep(1);
now = time(NULL);
if (nextcycle > now)
continue;
nextcycle += collector_interval;
totticks = 0;
totmetrics = 0;
dispatchers_idle = 0;
dispatchers_busy = 0;
for (i = 0; dispatchers[i] != NULL; i++) {
if (dispatch_busy(dispatchers[i])) {
dispatchers_busy++;
} else {
dispatchers_idle++;
}
totticks += ticks = dispatch_get_ticks(dispatchers[i]);
totmetrics += metrics = dispatch_get_metrics(dispatchers[i]);
snprintf(m, sizem, "dispatcher%d.metricsReceived %zd %zd\n",
i + 1, metrics, (size_t)now);
send(metric);
snprintf(m, sizem, "dispatcher%d.wallTime_us %zd %zd\n",
i + 1, ticks, (size_t)now);
send(metric);
}
snprintf(m, sizem, "metricsReceived %zd %zd\n",
totmetrics, (size_t)now);
send(metric);
snprintf(m, sizem, "dispatch_wallTime_us %zd %zd\n",
totticks, (size_t)now);
send(metric);
snprintf(m, sizem, "dispatch_busy %zd %zd\n",
dispatchers_busy, (size_t)now);
send(metric);
snprintf(m, sizem, "dispatch_idle %zd %zd\n",
dispatchers_idle, (size_t)now);
send(metric);
totticks = 0;
totmetrics = 0;
totqueued = 0;
totstalls = 0;
totdropped = 0;
for (i = 0; srvs[i] != NULL; i++) {
if (server_ctype(srvs[i]) == CON_PIPE) {
strncpy(ipbuf, "internal", sizeof(ipbuf));
ticks = server_get_ticks(srvs[i]);
metrics = server_get_metrics(srvs[i]);
queued = server_get_queue_len(srvs[i]);
stalls = server_get_stalls(srvs[i]);
dropped = server_get_dropped(srvs[i]);
} else {
snprintf(ipbuf, sizeof(ipbuf), "%s:%u",
server_ip(srvs[i]), server_port(srvs[i]));
for (p = ipbuf; *p != '\0'; p++)
if (*p == '.')
*p = '_';
totticks += ticks = server_get_ticks(srvs[i]);
totmetrics += metrics = server_get_metrics(srvs[i]);
totqueued += queued = server_get_queue_len(srvs[i]);
totstalls += stalls = server_get_stalls(srvs[i]);
totdropped += dropped = server_get_dropped(srvs[i]);
}
snprintf(m, sizem, "destinations.%s.sent %zd %zd\n",
ipbuf, metrics, (size_t)now);
send(metric);
snprintf(m, sizem, "destinations.%s.queued %zd %zd\n",
ipbuf, queued, (size_t)now);
send(metric);
snprintf(m, sizem, "destinations.%s.stalls %zd %zd\n",
ipbuf, stalls, (size_t)now);
send(metric);
snprintf(m, sizem, "destinations.%s.dropped %zd %zd\n",
ipbuf, dropped, (size_t)now);
send(metric);
snprintf(m, sizem, "destinations.%s.wallTime_us %zd %zd\n",
ipbuf, ticks, (size_t)now);
send(metric);
}
snprintf(m, sizem, "metricsSent %zd %zd\n",
totmetrics, (size_t)now);
send(metric);
snprintf(m, sizem, "metricsQueued %zd %zd\n",
totqueued, (size_t)now);
send(metric);
snprintf(m, sizem, "metricStalls %zd %zd\n",
totstalls, (size_t)now);
send(metric);
snprintf(m, sizem, "metricsDropped %zd %zd\n",
totdropped, (size_t)now);
send(metric);
snprintf(m, sizem, "server_wallTime_us %zd %zd\n",
totticks, (size_t)now);
send(metric);
snprintf(m, sizem, "connections %zd %zd\n",
dispatch_get_accepted_connections(), (size_t)now);
send(metric);
snprintf(m, sizem, "disconnects %zd %zd\n",
dispatch_get_closed_connections(), (size_t)now);
send(metric);
if (numaggregators > 0) {
snprintf(m, sizem, "aggregators.metricsReceived %zd %zd\n",
aggregator_get_received(), (size_t)now);
send(metric);
snprintf(m, sizem, "aggregators.metricsSent %zd %zd\n",
aggregator_get_sent(), (size_t)now);
send(metric);
snprintf(m, sizem, "aggregators.metricsDropped %zd %zd\n",
aggregator_get_dropped(), (size_t)now);
send(metric);
}
if (debug)
fflush(stdout);
}
return NULL;
}
