static void graphite_run_recvmmsg(struct brubeck_graphite *graphite, int sock) { const unsigned int SIM_PACKETS = graphite->mmsg_count; struct brubeck_server *server = graphite->sampler.server; struct brubeck_graphite_msg msg; struct brubeck_metric *metric; unsigned int i; struct iovec iovecs[SIM_PACKETS]; struct mmsghdr msgs[SIM_PACKETS]; memset(msgs, 0x0, sizeof(msgs)); for (i = 0; i < SIM_PACKETS; ++i) { iovecs[i].iov_base = xmalloc(MAX_PACKET_SIZE); iovecs[i].iov_len = MAX_PACKET_SIZE - 1; msgs[i].msg_hdr.msg_iov = &iovecs[i]; msgs[i].msg_hdr.msg_iovlen = 1; } log_splunk("sampler=graphite event=worker_online syscall=recvmmsg socket=%d", sock); for (;;) { int res = recvmmsg(sock, msgs, SIM_PACKETS, 0, NULL); if (res < 0) { if (errno == EAGAIN || errno == EINTR) continue; log_splunk_errno("sampler=graphite event=failed_read"); brubeck_server_mark_dropped(server); continue; } /* store stats */ brubeck_atomic_add(&server->stats.metrics, SIM_PACKETS); brubeck_atomic_add(&graphite->sampler.inflow, SIM_PACKETS); for (i = 0; i < SIM_PACKETS; ++i) { char *buf = msgs[i].msg_hdr.msg_iov->iov_base; int len = msgs[i].msg_len; if (brubeck_graphite_msg_parse(&msg, buf, len) < 0) { log_splunk("sampler=graphite event=bad_key key='%.*s'", len, buf); brubeck_server_mark_dropped(server); continue; } metric = brubeck_metric_find(server, msg.key, msg.key_len, BRUBECK_MT_GAUGE); if (metric != NULL) brubeck_metric_record(metric, msg.value); } } }
static void statsd_run_recvmmsg(struct brubeck_statsd *statsd, int sock) { const unsigned int SIM_PACKETS = statsd->mmsg_count; struct brubeck_server *server = statsd->sampler.server; struct brubeck_statsd_msg msg; struct brubeck_metric *metric; unsigned int i; struct iovec iovecs[SIM_PACKETS]; struct mmsghdr msgs[SIM_PACKETS]; memset(msgs, 0x0, sizeof(msgs)); for (i = 0; i < SIM_PACKETS; ++i) { iovecs[i].iov_base = xmalloc(MAX_PACKET_SIZE); iovecs[i].iov_len = MAX_PACKET_SIZE; msgs[i].msg_hdr.msg_iov = &iovecs[i]; msgs[i].msg_hdr.msg_iovlen = 1; } log_splunk("sampler=statsd event=worker_online syscall=recvmmsg socket=%d", sock); for (;;) { int res = recvmmsg(sock, msgs, SIM_PACKETS, 0, NULL); if (res == EAGAIN || res == EINTR || res == 0) continue; /* store stats */ brubeck_atomic_add(&server->stats.metrics, SIM_PACKETS); brubeck_atomic_add(&statsd->sampler.inflow, SIM_PACKETS); if (res < 0) { brubeck_server_mark_dropped(server); continue; } for (i = 0; i < SIM_PACKETS; ++i) { char *buf = msgs[i].msg_hdr.msg_iov->iov_base; int len = msgs[i].msg_len; if (brubeck_statsd_msg_parse(&msg, buf, len) < 0) { brubeck_server_mark_dropped(server); log_splunk("sampler=statsd event=packet_drop"); continue; } metric = brubeck_metric_find(server, msg.key, msg.key_len, msg.type); if (metric != NULL) brubeck_metric_record(metric, msg.value); } } }
static void statsd_run_recvmsg(struct brubeck_statsd *statsd, int sock) { struct brubeck_server *server = statsd->sampler.server; struct brubeck_statsd_msg msg; struct brubeck_metric *metric; char buffer[MAX_PACKET_SIZE]; struct sockaddr_in reporter; socklen_t reporter_len = sizeof(reporter); memset(&reporter, 0, reporter_len); log_splunk("sampler=statsd event=worker_online syscall=recvmsg socket=%d", sock); for (;;) { int res = recvfrom(sock, buffer, sizeof(buffer) - 1, 0, (struct sockaddr *)&reporter, &reporter_len); if (res < 0) { if (errno == EAGAIN || errno == EINTR) continue; log_splunk_errno("sampler=statsd event=failed_read from=%s", inet_ntoa(reporter.sin_addr)); brubeck_server_mark_dropped(server); continue; } /* store stats */ brubeck_atomic_inc(&server->stats.metrics); brubeck_atomic_inc(&statsd->sampler.inflow); if (brubeck_statsd_msg_parse(&msg, buffer, (size_t) res) < 0) { if (msg.key_len > 0) buffer[msg.key_len] = ':'; log_splunk("sampler=statsd event=bad_key key='%.*s' from=%s", res, buffer, inet_ntoa(reporter.sin_addr)); brubeck_server_mark_dropped(server); continue; } metric = brubeck_metric_find(server, msg.key, msg.key_len, msg.type); if (metric != NULL) { brubeck_metric_record(metric, msg.value); } } }
struct brubeck_backend * brubeck_carbon_new(struct brubeck_server *server, json_t *settings, int shard_n) { struct brubeck_carbon *carbon = xcalloc(1, sizeof(struct brubeck_carbon)); char *address; const char *global_prefix = carbon_global_prefix; const char *global_count_prefix = carbon_global_count_prefix; const char *prefix_counter = carbon_prefix_counter; const char *prefix_timer = carbon_prefix_timer; const char *prefix_gauge = carbon_prefix_gauge; int port, frequency, pickle, namespacing = 0; int legacy_namespace = 1; json_unpack_or_die(settings, "{s:s, s:i, s?:b, s:i, s?:b, s?:b, s?:s, s?:s, s?:s, s?:s, s?:s}", "address", &address, "port", &port, "pickle", &pickle, "frequency", &frequency, "namespacing", &namespacing, "legacy_namespace", &legacy_namespace, "global_prefix", &global_prefix, "global_count_prefix", &global_count_prefix, "prefix_counter", &prefix_counter, "prefix_timer", &prefix_timer, "prefix_gauge", &prefix_gauge); carbon->backend.type = BRUBECK_BACKEND_CARBON; carbon->backend.shard_n = shard_n; carbon->backend.connect = &carbon_connect; if (pickle) { carbon->backend.sample = &pickle1_each; carbon->backend.flush = &pickle1_flush; carbon->pickler.ptr = malloc(PICKLE_BUFFER_SIZE); pickle1_init(&carbon->pickler); } else { carbon->backend.sample = &plaintext_each; carbon->backend.flush = NULL; } carbon->backend.sample_freq = frequency; carbon->namespacing = namespacing; carbon->legacy_namespace = legacy_namespace; carbon->global_prefix = global_prefix; carbon->global_count_prefix = global_count_prefix; carbon->prefix_counter = prefix_counter; carbon->prefix_timer = prefix_timer; carbon->prefix_gauge = prefix_gauge; carbon->backend.server = server; carbon->out_sock = -1; url_to_inaddr2(&carbon->out_sockaddr, address, port); brubeck_backend_run_threaded((struct brubeck_backend *)carbon); log_splunk("backend=carbon event=started"); return (struct brubeck_backend *)carbon; }
static int carbon_connect(void *backend) { struct brubeck_carbon *self = (struct brubeck_carbon *)backend; if (is_connected(self)) return 0; self->out_sock = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP); if (self->out_sock >= 0) { int rc = connect(self->out_sock, (struct sockaddr *)&self->out_sockaddr, sizeof(self->out_sockaddr)); if (rc == 0) { log_splunk("backend=carbon event=connected"); sock_enlarge_out(self->out_sock); return 0; } close(self->out_sock); self->out_sock = -1; } log_splunk_errno("backend=carbon event=failed_to_connect"); return -1; }
static void statsd_run_recvmsg(struct brubeck_statsd *statsd, int sock) { struct brubeck_server *server = statsd->sampler.server; char *buffer = xmalloc(MAX_PACKET_SIZE); struct sockaddr_in reporter; socklen_t reporter_len = sizeof(reporter); memset(&reporter, 0, reporter_len); log_splunk("sampler=statsd event=worker_online syscall=recvmsg socket=%d", sock); for (;;) { int res = recvfrom(sock, buffer, MAX_PACKET_SIZE - 1, 0, (struct sockaddr *)&reporter, &reporter_len); if (res < 0) { if (errno == EAGAIN || errno == EINTR) continue; log_splunk_errno("sampler=statsd event=failed_read from=%s", inet_ntoa(reporter.sin_addr)); brubeck_stats_inc(server, errors); continue; } brubeck_atomic_inc(&statsd->sampler.inflow); brubeck_statsd_packet_parse(server, buffer, buffer + res); } }
static void load_samplers(struct brubeck_server *server, json_t *samplers) { size_t idx; json_t *s; json_array_foreach(samplers, idx, s) { const char *type = json_string_value(json_object_get(s, "type")); if (type && !strcmp(type, "statsd")) { server->samplers[server->active_samplers++] = brubeck_statsd_new(server, s); } else if (type && !strcmp(type, "statsd-secure")) { server->samplers[server->active_samplers++] = brubeck_statsd_secure_new(server, s); } else { log_splunk("sampler=%s event=invalid_sampler", type); } } }
static void load_backends(struct brubeck_server *server, json_t *backends) { size_t idx; json_t *b; json_array_foreach(backends, idx, b) { const char *type = json_string_value(json_object_get(b, "type")); struct brubeck_backend *backend = NULL; if (type && !strcmp(type, "carbon")) { backend = brubeck_carbon_new(server, b, server->active_backends); server->backends[server->active_backends++] = backend; } else { log_splunk("backend=%s event=invalid_backend", type); } } }
static void dump_all_metrics(struct brubeck_server *server) { FILE *dump = NULL; log_splunk("event=dump_metrics"); if (server->dump_path) dump = fopen(server->dump_path, "w+"); if (!dump) { log_splunk_errno("event=dump_failed"); return; } brubeck_hashtable_foreach(server->metrics, &dump_metric, dump); fclose(dump); }
void brubeck_http_endpoint_init(struct brubeck_server *server, const char *listen) { struct MHD_Daemon *daemon; const char *port = strrchr(listen, ':'); port = port ? port + 1 : listen; daemon = MHD_start_daemon( MHD_USE_SELECT_INTERNALLY, atoi(port), NULL, NULL, &handle_request, server, MHD_OPTION_CONNECTION_TIMEOUT, (unsigned int)10, MHD_OPTION_END); if (!daemon) die("failed to start HTTP endpoint"); log_splunk("event=http_server listen=%s", port); }
static void statsd_run_recvmmsg(struct brubeck_statsd *statsd, int sock) { const unsigned int SIM_PACKETS = statsd->mmsg_count; struct brubeck_server *server = statsd->sampler.server; unsigned int i; struct iovec iovecs[SIM_PACKETS]; struct mmsghdr msgs[SIM_PACKETS]; memset(msgs, 0x0, sizeof(msgs)); for (i = 0; i < SIM_PACKETS; ++i) { iovecs[i].iov_base = xmalloc(MAX_PACKET_SIZE); iovecs[i].iov_len = MAX_PACKET_SIZE - 1; msgs[i].msg_hdr.msg_iov = &iovecs[i]; msgs[i].msg_hdr.msg_iovlen = 1; } log_splunk("sampler=statsd event=worker_online syscall=recvmmsg socket=%d", sock); for (;;) { int res = recvmmsg(sock, msgs, SIM_PACKETS, 0, NULL); if (res < 0) { if (errno == EAGAIN || errno == EINTR) continue; log_splunk_errno("sampler=statsd event=failed_read"); brubeck_stats_inc(server, errors); continue; } /* store stats */ brubeck_atomic_add(&statsd->sampler.inflow, SIM_PACKETS); for (i = 0; i < SIM_PACKETS; ++i) { char *buf = msgs[i].msg_hdr.msg_iov->iov_base; char *end = buf + msgs[i].msg_len; brubeck_statsd_packet_parse(server, buf, end); } } }
struct brubeck_backend * brubeck_carbon_new(struct brubeck_server *server, json_t *settings, int shard_n) { struct brubeck_carbon *carbon = xcalloc(1, sizeof(struct brubeck_carbon)); char *address; int port, frequency, pickle = 0; json_unpack_or_die(settings, "{s:s, s:i, s?:b, s:i}", "address", &address, "port", &port, "pickle", &pickle, "frequency", &frequency); carbon->backend.type = BRUBECK_BACKEND_CARBON; carbon->backend.shard_n = shard_n; carbon->backend.connect = &carbon_connect; if (pickle) { carbon->backend.sample = &pickle1_each; carbon->backend.flush = &pickle1_flush; carbon->pickler.ptr = malloc(PICKLE_BUFFER_SIZE); pickle1_init(&carbon->pickler); } else { carbon->backend.sample = &plaintext_each; carbon->backend.flush = NULL; } carbon->backend.sample_freq = frequency; carbon->backend.server = server; carbon->out_sock = -1; url_to_inaddr2(&carbon->out_sockaddr, address, port); brubeck_backend_run_threaded((struct brubeck_backend *)carbon); log_splunk("backend=carbon event=started"); return (struct brubeck_backend *)carbon; }
void brubeck_statsd_packet_parse(struct brubeck_server *server, char *buffer, char *end) { struct brubeck_statsd_msg msg; struct brubeck_metric *metric; while (buffer < end) { char *stat_end = memchr(buffer, '\n', end - buffer); if (!stat_end) stat_end = end; if (brubeck_statsd_msg_parse(&msg, buffer, stat_end) < 0) { brubeck_stats_inc(server, errors); log_splunk("sampler=statsd event=packet_drop"); } else { brubeck_stats_inc(server, metrics); metric = brubeck_metric_find(server, msg.key, msg.key_len, msg.type); if (metric != NULL) brubeck_metric_record(metric, msg.value, msg.sample_freq); } /* move buf past this stat */ buffer = stat_end + 1; } }
int brubeck_server_run(struct brubeck_server *server) { struct pollfd fds[3]; int nfd = 2; size_t i; memset(fds, 0x0, sizeof(fds)); fds[0].fd = server->fd_signal; fds[0].events = POLLIN; fds[1].fd = server->fd_update; fds[1].events = POLLIN; if (server->fd_expire >= 0) { fds[2].fd = server->fd_expire; fds[2].events = POLLIN; nfd++; } server->running = 1; log_splunk("event=listening"); while (server->running) { if (poll(fds, nfd, -1) < 0) continue; switch (signal_triggered(&fds[0])) { case SIGHUP: gh_log_reopen(); log_splunk("event=reload_log"); break; case SIGUSR2: dump_all_metrics(server); break; case SIGINT: case SIGTERM: server->running = 0; break; } if (timer_elapsed(&fds[1])) { update_flows(server); update_proctitle(server); } if (timer_elapsed(&fds[2])) { log_splunk("event=expire_metrics"); brubeck_hashtable_foreach(server->metrics, &expire_metric, NULL); } } for (i = 0; i < server->active_backends; ++i) pthread_cancel(server->backends[i]->thread); for (i = 0; i < server->active_samplers; ++i) { struct brubeck_sampler *sampler = server->samplers[i]; if (sampler->shutdown) sampler->shutdown(sampler); } log_splunk("event=shutdown"); return 0; }
static void statsd_run_recvmmsg(struct brubeck_statsd *statsd, int sock) { const unsigned int SIM_PACKETS = statsd->mmsg_count; struct brubeck_server *server = statsd->sampler.server; struct brubeck_statsd_msg msg; struct brubeck_metric *metric; unsigned int i; struct iovec iovecs[SIM_PACKETS]; struct mmsghdr msgs[SIM_PACKETS]; memset(msgs, 0x0, sizeof(msgs)); for (i = 0; i < SIM_PACKETS; ++i) { iovecs[i].iov_base = xmalloc(MAX_PACKET_SIZE); iovecs[i].iov_len = MAX_PACKET_SIZE - 1; msgs[i].msg_hdr.msg_iov = &iovecs[i]; msgs[i].msg_hdr.msg_iovlen = 1; } log_splunk("sampler=statsd event=worker_online syscall=recvmmsg socket=%d", sock); for (;;) { int res = recvmmsg(sock, msgs, SIM_PACKETS, 0, NULL); if (res < 0) { if (errno == EAGAIN || errno == EINTR) continue; log_splunk_errno("sampler=statsd event=failed_read"); brubeck_server_mark_dropped(server); continue; } /* store stats */ brubeck_atomic_add(&server->stats.metrics, SIM_PACKETS); brubeck_atomic_add(&statsd->sampler.inflow, SIM_PACKETS); for (i = 0; i < SIM_PACKETS; ++i) { char *buf = msgs[i].msg_hdr.msg_iov->iov_base; int len = msgs[i].msg_len; char *cur; int curLen; int processed = 0; do { cur = memchr(buf,'\n', len); if(cur == NULL){ cur = buf; curLen = len - processed; } else{ curLen = cur - buf; cur = buf; } //log_splunk("msg to be processed ='%.*s'", curLen, cur); if (brubeck_statsd_msg_parse(&msg, cur, curLen) < 0) { if (msg.key_len > 0) buf[msg.key_len] = ':'; log_splunk("sampler=statsd event=bad_key key='%.*s'", curLen, cur); brubeck_server_mark_dropped(server); continue; } //log_splunk("key: %s, value: %s",msg.key, msg.value); metric = brubeck_metric_find(server, msg.key, msg.key_len, msg.type); if (metric != NULL) brubeck_metric_record(metric, msg.value); processed += curLen +1; buf += curLen +1; }while(processed < len); } } }