Exemple #1
0
/**
 * Collects metrics from dispatchers and servers and emits them.
 */
static void *
collector_runner(void *s)
{
	int i;
	size_t totticks;
	size_t totmetrics;
	size_t totblackholes;
	size_t totqueued;
	size_t totstalls;
	size_t totdropped;
	size_t totsleeps;
	size_t ticks;
	size_t metrics;
	size_t blackholes;
	size_t queued;
	size_t stalls;
	size_t dropped;
	size_t sleeps;
	time_t now;
	time_t nextcycle;
	char destbuf[1024];  /* sort of POSIX_MAX_PATH */
	char *p;
	size_t numaggregators = 0;
	aggregator *aggrs = NULL;
	server *submission = (server *)s;
	server **srvs = NULL;
	char metric[METRIC_BUFSIZ];
	char *m = NULL;
	size_t sizem = 0;
	size_t (*s_ticks)(server *);
	size_t (*s_metrics)(server *);
	size_t (*s_stalls)(server *);
	size_t (*s_dropped)(server *);
	size_t (*d_ticks)(dispatcher *);
	size_t (*d_metrics)(dispatcher *);
	size_t (*d_blackholes)(dispatcher *);
	size_t (*d_sleeps)(dispatcher *);
	size_t (*a_received)(aggregator *);
	size_t (*a_sent)(aggregator *);
	size_t (*a_dropped)(aggregator *);

	/* setup functions to target what the user wants */
	if (debug & 2) {
		s_ticks = server_get_ticks_sub;
		s_metrics = server_get_metrics_sub;
		s_stalls = server_get_stalls_sub;
		s_dropped = server_get_dropped_sub;
		d_ticks = dispatch_get_ticks_sub;
		d_metrics = dispatch_get_metrics_sub;
		d_blackholes = dispatch_get_blackholes_sub;
		d_sleeps = dispatch_get_sleeps_sub;
		a_received = aggregator_get_received_sub;
		a_sent = aggregator_get_sent_sub;
		a_dropped = aggregator_get_dropped_sub;
	} else {
		s_ticks = server_get_ticks;
		s_metrics = server_get_metrics;
		s_stalls = server_get_stalls;
		s_dropped = server_get_dropped;
		d_ticks = dispatch_get_ticks;
		d_metrics = dispatch_get_metrics;
		d_blackholes = dispatch_get_blackholes;
		d_sleeps = dispatch_get_sleeps;
		a_received = aggregator_get_received;
		a_sent = aggregator_get_sent;
		a_dropped = aggregator_get_dropped;
	}

#define send(metric) \
	if (debug & 1) \
		logout("%s", metric); \
	else \
		server_send(submission, strdup(metric), 1);

	nextcycle = time(NULL) + collector_interval;
	while (keep_running) {
		if (cluster_refresh_pending) {
			char *stub = router_getcollectorstub(pending_router);
			server **newservers = router_getservers(pending_router);
			if (srvs != NULL)
				free(srvs);
			srvs = newservers;
			aggrs = router_getaggregators(pending_router);
			numaggregators = aggregator_numaggregators(aggrs);

			/* prepare hostname for graphite metrics */
			snprintf(metric, sizeof(metric), "%scarbon.relays.%s",
					stub == NULL ? "" : stub, relay_hostname);
			for (m = metric + strlen("carbon.relays."); *m != '\0'; m++)
				if (*m == '.')
					*m = '_';
			*m++ = '.';
			*m = '\0';
			sizem = sizeof(metric) - (m - metric);

			cluster_refresh_pending = 0;
		}
		assert(srvs != NULL);
		sleep(1);
		now = time(NULL);
		if (nextcycle > now)
			continue;
		nextcycle += collector_interval;
		totticks = 0;
		totmetrics = 0;
		totblackholes = 0;
		totsleeps = 0;
		for (i = 0; dispatchers[i] != NULL; i++) {
			totsleeps += sleeps = d_sleeps(dispatchers[i]);
			totticks += ticks = d_ticks(dispatchers[i]);
			totmetrics += metrics = d_metrics(dispatchers[i]);
			totblackholes += blackholes = d_blackholes(dispatchers[i]);
			snprintf(m, sizem, "dispatcher%d.metricsReceived %zu %zu\n",
					i + 1, metrics, (size_t)now);
			send(metric);
			snprintf(m, sizem, "dispatcher%d.metricsBlackholed %zu %zu\n",
					i + 1, blackholes, (size_t)now);
			send(metric);
			snprintf(m, sizem, "dispatcher%d.wallTime_us %zu %zu\n",
					i + 1, ticks, (size_t)now);
			send(metric);
			snprintf(m, sizem, "dispatcher%d.sleepTime_us %zu %zu\n",
					i + 1, sleeps, (size_t)now);
			send(metric);
		}
		snprintf(m, sizem, "metricsReceived %zu %zu\n",
				totmetrics, (size_t)now);
		send(metric);
		snprintf(m, sizem, "metricsBlackholed %zu %zu\n",
				totblackholes, (size_t)now);
		send(metric);
		snprintf(m, sizem, "dispatch_wallTime_us %zu %zu\n",
				totticks, (size_t)now);
		send(metric);
		snprintf(m, sizem, "dispatch_sleepTime_us %zu %zu\n",
				totsleeps, (size_t)now);
		send(metric);

#define send_server_metrics(ipbuf, ticks, metrics, queued, stalls, dropped) \
			snprintf(m, sizem, "destinations.%s.sent %zu %zu\n", \
					ipbuf, metrics, (size_t)now); \
			send(metric); \
			snprintf(m, sizem, "destinations.%s.queued %zu %zu\n", \
					ipbuf, queued, (size_t)now); \
			send(metric); \
			snprintf(m, sizem, "destinations.%s.stalls %zu %zu\n", \
					ipbuf, stalls, (size_t)now); \
			send(metric); \
			snprintf(m, sizem, "destinations.%s.dropped %zu %zu\n", \
					ipbuf, dropped, (size_t)now); \
			send(metric); \
			snprintf(m, sizem, "destinations.%s.wallTime_us %zu %zu\n", \
					ipbuf, ticks, (size_t)now); \
			send(metric);

		totticks = 0;
		totmetrics = 0;
		totqueued = 0;
		totstalls = 0;
		totdropped = 0;

		/* exclude internal_submission metrics from the totals to avoid
		 * artificial doubles due to internal routing details */
		ticks = s_ticks(submission);
		metrics = s_metrics(submission);
		queued = server_get_queue_len(submission);
		stalls = s_stalls(submission);
		dropped = s_dropped(submission);
		send_server_metrics(server_ip(submission),
				ticks, metrics, queued, stalls, dropped);

		for (i = 0; srvs[i] != NULL; i++) {
			switch (server_ctype(srvs[i])) {
				case CON_FILE:
				case CON_PIPE:
					snprintf(destbuf, sizeof(destbuf), "%s",
							server_ip(srvs[i]));
					break;
				case CON_TCP:
					snprintf(destbuf, sizeof(destbuf), "%s:%u",
							server_ip(srvs[i]), server_port(srvs[i]));
					break;
				case CON_UDP:
					snprintf(destbuf, sizeof(destbuf), "%s:%u-udp",
							server_ip(srvs[i]), server_port(srvs[i]));
					break;
			}
			for (p = destbuf; *p != '\0'; p++)
				if (*p == '.')
					*p = '_';

			totticks += ticks = s_ticks(srvs[i]);
			totmetrics += metrics = s_metrics(srvs[i]);
			totqueued += queued = server_get_queue_len(srvs[i]);
			totstalls += stalls = s_stalls(srvs[i]);
			totdropped += dropped = s_dropped(srvs[i]);
			send_server_metrics(destbuf,
					ticks, metrics, queued, stalls, dropped);
		}

		snprintf(m, sizem, "metricsSent %zu %zu\n",
				totmetrics, (size_t)now);
		send(metric);
		snprintf(m, sizem, "metricsQueued %zu %zu\n",
				totqueued, (size_t)now);
		send(metric);
		snprintf(m, sizem, "metricStalls %zu %zu\n",
				totstalls, (size_t)now);
		send(metric);
		snprintf(m, sizem, "metricsDropped %zu %zu\n",
				totdropped, (size_t)now);
		send(metric);
		snprintf(m, sizem, "server_wallTime_us %zu %zu\n",
				totticks, (size_t)now);
		send(metric);
		snprintf(m, sizem, "connections %zu %zu\n",
				dispatch_get_accepted_connections(), (size_t)now);
		send(metric);
		snprintf(m, sizem, "disconnects %zu %zu\n",
				dispatch_get_closed_connections(), (size_t)now);
		send(metric);

		if (numaggregators > 0) {
			snprintf(m, sizem, "aggregators.metricsReceived %zu %zu\n",
					a_received(aggrs), (size_t)now);
			send(metric);
			snprintf(m, sizem, "aggregators.metricsSent %zu %zu\n",
					a_sent(aggrs), (size_t)now);
			send(metric);
			snprintf(m, sizem, "aggregators.metricsDropped %zu %zu\n",
					a_dropped(aggrs), (size_t)now);
			send(metric);
		}

		if (debug & 1)
			fflush(stdout);
	}

	if (srvs != NULL)
		free(srvs);

	return NULL;
}
/**
 * Collects metrics from dispatchers and servers and emits them.
 */
static void *
collector_runner(void *s)
{
	int i;
	size_t totticks;
	size_t totmetrics;
	size_t totblackholes;
	size_t totqueued;
	size_t totstalls;
	size_t totdropped;
	size_t ticks;
	size_t metrics;
	size_t blackholes;
	size_t queued;
	size_t stalls;
	size_t dropped;
	size_t dispatchers_idle;
	size_t dispatchers_busy;
	time_t now;
	time_t nextcycle;
	char ipbuf[32];
	char *p;
	size_t numaggregators = 0;
	aggregator *aggrs = NULL;
	server *submission = (server *)s;
	server **srvs = NULL;
	char metric[METRIC_BUFSIZ];
	char *m;
	size_t sizem = 0;
	size_t (*s_ticks)(server *);
	size_t (*s_metrics)(server *);
	size_t (*s_stalls)(server *);
	size_t (*s_dropped)(server *);
	size_t (*d_ticks)(dispatcher *);
	size_t (*d_metrics)(dispatcher *);
	size_t (*d_blackholes)(dispatcher *);
	size_t (*a_received)(aggregator *);
	size_t (*a_sent)(aggregator *);
	size_t (*a_dropped)(aggregator *);

	/* prepare hostname for graphite metrics */
	snprintf(metric, sizeof(metric), "carbon.relays.%s", relay_hostname);
	for (m = metric + strlen("carbon.relays."); *m != '\0'; m++)
		if (*m == '.')
			*m = '_';
	*m++ = '.';
	*m = '\0';
	sizem = sizeof(metric) - (m - metric);

	/* setup functions to target what the user wants */
	if (debug & 2) {
		s_ticks = server_get_ticks_sub;
		s_metrics = server_get_metrics_sub;
		s_stalls = server_get_stalls_sub;
		s_dropped = server_get_dropped_sub;
		d_ticks = dispatch_get_ticks_sub;
		d_metrics = dispatch_get_metrics_sub;
		d_blackholes = dispatch_get_blackholes_sub;
		a_received = aggregator_get_received_sub;
		a_sent = aggregator_get_sent_sub;
		a_dropped = aggregator_get_dropped_sub;
	} else {
		s_ticks = server_get_ticks;
		s_metrics = server_get_metrics;
		s_stalls = server_get_stalls;
		s_dropped = server_get_dropped;
		d_ticks = dispatch_get_ticks;
		d_metrics = dispatch_get_metrics;
		d_blackholes = dispatch_get_blackholes;
		a_received = aggregator_get_received;
		a_sent = aggregator_get_sent;
		a_dropped = aggregator_get_dropped;
	}

#define send(metric) \
	if (debug & 1) \
		logout("%s", metric); \
	else \
		server_send(submission, strdup(metric), 1);

	nextcycle = time(NULL) + collector_interval;
	while (keep_running) {
		if (cluster_refresh_pending) {
			server **newservers = router_getservers(pending_clusters);
			if (srvs != NULL)
				free(srvs);
			srvs = newservers;
			aggrs = pending_aggrs;
			numaggregators = aggregator_numaggregators(aggrs);
			cluster_refresh_pending = 0;
		}
		assert(srvs != NULL);
		sleep(1);
		now = time(NULL);
		if (nextcycle > now)
			continue;
		nextcycle += collector_interval;
		totticks = 0;
		totmetrics = 0;
		totblackholes = 0;
		dispatchers_idle = 0;
		dispatchers_busy = 0;
		for (i = 0; dispatchers[i] != NULL; i++) {
			if (dispatch_busy(dispatchers[i])) {
				dispatchers_busy++;
			} else {
				dispatchers_idle++;
			}
			totticks += ticks = d_ticks(dispatchers[i]);
			totmetrics += metrics = d_metrics(dispatchers[i]);
			totblackholes += blackholes = d_blackholes(dispatchers[i]);
			snprintf(m, sizem, "dispatcher%d.metricsReceived %zd %zd\n",
					i + 1, metrics, (size_t)now);
			send(metric);
			snprintf(m, sizem, "dispatcher%d.metricsBlackholed %zd %zd\n",
					i + 1, blackholes, (size_t)now);
			send(metric);
			snprintf(m, sizem, "dispatcher%d.wallTime_us %zd %zd\n",
					i + 1, ticks, (size_t)now);
			send(metric);
		}
		snprintf(m, sizem, "metricsReceived %zd %zd\n",
				totmetrics, (size_t)now);
		send(metric);
		snprintf(m, sizem, "metricsBlackholed %zd %zd\n",
				totblackholes, (size_t)now);
		send(metric);
		snprintf(m, sizem, "dispatch_wallTime_us %zd %zd\n",
				totticks, (size_t)now);
		send(metric);
		snprintf(m, sizem, "dispatch_busy %zd %zd\n",
				dispatchers_busy, (size_t)now);
		send(metric);
		snprintf(m, sizem, "dispatch_idle %zd %zd\n",
				dispatchers_idle, (size_t)now);
		send(metric);

		totticks = 0;
		totmetrics = 0;
		totqueued = 0;
		totstalls = 0;
		totdropped = 0;
		for (i = 0; srvs[i] != NULL; i++) {
			if (server_ctype(srvs[i]) == CON_PIPE) {
				strncpy(ipbuf, "internal", sizeof(ipbuf));

				ticks = s_ticks(srvs[i]);
				metrics = s_metrics(srvs[i]);
				queued = server_get_queue_len(srvs[i]);
				stalls = s_stalls(srvs[i]);
				dropped = s_dropped(srvs[i]);
			} else {
				snprintf(ipbuf, sizeof(ipbuf), "%s:%u",
						server_ip(srvs[i]), server_port(srvs[i]));
				for (p = ipbuf; *p != '\0'; p++)
					if (*p == '.')
						*p = '_';

				totticks += ticks = s_ticks(srvs[i]);
				totmetrics += metrics = s_metrics(srvs[i]);
				totqueued += queued = server_get_queue_len(srvs[i]);
				totstalls += stalls = s_stalls(srvs[i]);
				totdropped += dropped = s_dropped(srvs[i]);
			}
			snprintf(m, sizem, "destinations.%s.sent %zd %zd\n",
					ipbuf, metrics, (size_t)now);
			send(metric);
			snprintf(m, sizem, "destinations.%s.queued %zd %zd\n",
					ipbuf, queued, (size_t)now);
			send(metric);
			snprintf(m, sizem, "destinations.%s.stalls %zd %zd\n",
					ipbuf, stalls, (size_t)now);
			send(metric);
			snprintf(m, sizem, "destinations.%s.dropped %zd %zd\n",
					ipbuf, dropped, (size_t)now);
			send(metric);
			snprintf(m, sizem, "destinations.%s.wallTime_us %zd %zd\n",
					ipbuf, ticks, (size_t)now);
			send(metric);
		}

		snprintf(m, sizem, "metricsSent %zd %zd\n",
				totmetrics, (size_t)now);
		send(metric);
		snprintf(m, sizem, "metricsQueued %zd %zd\n",
				totqueued, (size_t)now);
		send(metric);
		snprintf(m, sizem, "metricStalls %zd %zd\n",
				totstalls, (size_t)now);
		send(metric);
		snprintf(m, sizem, "metricsDropped %zd %zd\n",
				totdropped, (size_t)now);
		send(metric);
		snprintf(m, sizem, "server_wallTime_us %zd %zd\n",
				totticks, (size_t)now);
		send(metric);
		snprintf(m, sizem, "connections %zd %zd\n",
				dispatch_get_accepted_connections(), (size_t)now);
		send(metric);
		snprintf(m, sizem, "disconnects %zd %zd\n",
				dispatch_get_closed_connections(), (size_t)now);
		send(metric);

		if (numaggregators > 0) {
			snprintf(m, sizem, "aggregators.metricsReceived %zd %zd\n",
					a_received(aggrs), (size_t)now);
			send(metric);
			snprintf(m, sizem, "aggregators.metricsSent %zd %zd\n",
					a_sent(aggrs), (size_t)now);
			send(metric);
			snprintf(m, sizem, "aggregators.metricsDropped %zd %zd\n",
					a_dropped(aggrs), (size_t)now);
			send(metric);
		}

		if (debug & 1)
			fflush(stdout);
	}

	if (srvs != NULL)
		free(srvs);

	return NULL;
}