static int noit_statsd_onload(noit_image_t *self) {
if(!nlerr) nlerr = noit_log_stream_find("error/statsd");
if(!nldeb) nldeb = noit_log_stream_find("debug/statsd");
if(!nlerr) nlerr = noit_error;
if(!nldeb) nldeb = noit_debug;
return 0;
}
static int
noit_lua_general_onload(noit_image_t *self) {
nlerr = noit_log_stream_find("error/lua");
nldeb = noit_log_stream_find("debug/lua");
if(!nlerr) nlerr = noit_stderr;
if(!nldeb) nldeb = noit_debug;
return 0;
}
static int postgres_onload(noit_image_t *self) {
nlerr = noit_log_stream_find("error/postgres");
nldeb = noit_log_stream_find("debug/postgres");
if(!nlerr) nlerr = noit_stderr;
if(!nldeb) nldeb = noit_debug;
eventer_name_callback("http/postgres_drive_session", postgres_drive_session);
return 0;
}
static int test_abort_onload(noit_image_t *self) {
nlerr = noit_log_stream_find("error/test_abort");
nldeb = noit_log_stream_find("debug/test_abort");
if(!nlerr) nlerr = noit_stderr;
if(!nldeb) nldeb = noit_debug;
eventer_name_callback("http/test_abort_drive_session", test_abort_drive_session);
return 0;
}
static int selfcheck_onload(noit_image_t *self) {
nlerr = noit_log_stream_find("error/selfcheck");
nldeb = noit_log_stream_find("debug/selfcheck");
if(!nlerr) nlerr = noit_stderr;
if(!nldeb) nldeb = noit_debug;
eventer_name_callback("selfcheck/selfcheck_log_size", selfcheck_log_size);
return 0;
}
static int dns_onload(noit_image_t *self) {
nlerr = noit_log_stream_find("error/dns");
nldeb = noit_log_stream_find("debug/dns");
if(!nlerr) nlerr = noit_stderr;
if(!nldeb) nldeb = noit_debug;
eventer_name_callback("dns/dns_eventer_callback", dns_eventer_callback);
eventer_name_callback("dns/dns_check_timeout", dns_check_timeout);
eventer_name_callback("dns/dns_invoke_timeouts", dns_invoke_timeouts);
return 0;
}
static int ssh2_onload(noit_image_t *self) {
nlerr = noit_log_stream_find("error/ssh2");
nldeb = noit_log_stream_find("debug/ssh2");
if(!nlerr) nlerr = noit_stderr;
if(!nldeb) nldeb = noit_debug;
eventer_name_callback("http/ssh2_connect_complete", ssh2_connect_complete);
eventer_name_callback("http/ssh2_drive_session", ssh2_drive_session);
return 0;
}
static int
check_test_onload(noit_image_t *self) {
nlerr = noit_log_stream_find("error/checktest");
nldeb = noit_log_stream_find("debug/checktest");
if(!nlerr) nlerr = noit_stderr;
if(!nldeb) nldeb = noit_debug;
noit_skiplist_init(&in_progress);
noit_skiplist_set_compare(&in_progress, check_complete_heap,
check_complete_heap_key);
return 0;
}
void
noit_listener_init(const char *toplevel) {
nlerr = noit_log_stream_find("error/listener");
nldeb = noit_log_stream_find("debug/listener");
if(!nlerr) nlerr = noit_error;
if(!nldeb) nldeb = noit_debug;
eventer_name_callback_ext("noit_listener_acceptor", noit_listener_acceptor,
noit_listener_details, NULL);
eventer_name_callback("noit_listener_accept_ssl", noit_listener_accept_ssl);
eventer_name_callback("control_dispatch", noit_control_dispatch);
noit_listener_reconfig(toplevel);
}
void
noit_console_closure_free(void *vncct) {
noit_console_closure_t ncct = (noit_console_closure_t) vncct;
noit_log_stream_t lf;
noitL(noit_debug, "ncct free(%p)\n", (void *)ncct);
if(ncct->el) el_end(ncct->el);
if(ncct->hist) {
history_end(ncct->hist);
noitL(noit_debug, "ncct free->hist(%p)\n", (void *)ncct->hist);
free(ncct->hist);
}
if(ncct->pty_master >= 0) close(ncct->pty_master);
if(ncct->pty_slave >= 0) close(ncct->pty_slave);
if(ncct->outbuf) free(ncct->outbuf);
if(ncct->telnet) noit_console_telnet_free(ncct->telnet);
noit_hash_destroy(&ncct->userdata, NULL, noit_console_userdata_free);
while(ncct->state_stack) {
noit_console_state_stack_t *tmp;
tmp = ncct->state_stack;
ncct->state_stack = tmp->last;
if(tmp->name) free(tmp->name);
free(tmp);
}
lf = noit_log_stream_find(ncct->feed_path);
noit_log_stream_remove(ncct->feed_path);
if(lf) {
noit_log_stream_free(lf);
}
free(ncct);
}
static void
handle_extra_feeds(noit_check_t *check,
int (*log_f)(noit_log_stream_t ls, noit_check_t *check)) {
noit_log_stream_t ls;
noit_skiplist_node *curr, *next;
const char *feed_name;
if(!check->feeds) return;
curr = next = noit_skiplist_getlist(check->feeds);
while(curr) {
/* We advance next here (before we try to use curr).
* We may need to remove the node we're looking at and that would
* disturb the iterator, so advance in advance. */
noit_skiplist_next(check->feeds, &next);
feed_name = (char *)curr->data;
ls = noit_log_stream_find(feed_name);
if(!ls || log_f(ls, check)) {
noit_check_transient_remove_feed(check, feed_name);
/* noit_skiplisti_remove(check->feeds, curr, free); */
}
curr = next;
}
/* We're done... we may have destroyed the last feed.
* that combined with transience means we should kill the check */
/* noit_check_transient_remove_feed(check, NULL); */
}
noit_log_stream_t
noit_log_stream_new(const char *name, const char *type, const char *path,
void *ctx, noit_hash_table *config) {
noit_log_stream_t ls, saved;
struct _noit_log_stream tmpbuf;
void *vops = NULL;
ls = calloc(1, sizeof(*ls));
ls->name = strdup(name);
ls->path = path ? strdup(path) : NULL;
ls->type = type ? strdup(type) : NULL;
ls->enabled = 1;
ls->config = config;
if(!type)
ls->ops = NULL;
else if(noit_hash_retrieve(&noit_logops, type, strlen(type),
&vops))
ls->ops = vops;
else
goto freebail;
if(ls->ops && ls->ops->openop(ls)) goto freebail;
saved = noit_log_stream_find(name);
if(saved) {
pthread_rwlock_t *lock = saved->lock;
memcpy(&tmpbuf, saved, sizeof(*saved));
memcpy(saved, ls, sizeof(*saved));
memcpy(ls, &tmpbuf, sizeof(*saved));
saved->lock = lock;
ls->lock = NULL;
noit_log_stream_free(ls);
ls = saved;
}
else {
/* We strdup the name *again*. We'going to kansas city shuffle the
* ls later (see memcpy above). However, if don't strdup, then the
* noit_log_stream_free up there will sweep our key right our from
* under us.
*/
if(noit_hash_store(&noit_loggers,
strdup(ls->name), strlen(ls->name), ls) == 0)
goto freebail;
ls->lock = calloc(1, sizeof(*ls->lock));
noit_log_init_rwlock(ls);
}
/* This is for things that don't open on paths */
if(ctx) ls->op_ctx = ctx;
return ls;
freebail:
fprintf(stderr, "Failed to instantiate logger(%s,%s,%s)\n",
name, type ? type : "[null]", path ? path : "[null]");
free(ls->name);
if(ls->path) free(ls->path);
if(ls->type) free(ls->type);
free(ls);
return NULL;
}
static int noit_fq_driver_init(noit_module_generic_t *self) {
if(!nlerr) nlerr = noit_log_stream_find("error/fq_driver");
if(!nlerr) nlerr = noit_error;
stratcon_iep_mq_driver_register("fq", &mq_driver_fq);
register_console_fq_commands();
eventer_add_in_s_us(fq_status_checker, NULL, 0, 0);
return 0;
}
void
stratcon_datastore_core_init() {
static int initialized = 0;
if(initialized) return;
initialized = 1;
ds_err = noit_log_stream_find("error/datastore");
ds_deb = noit_log_stream_find("debug/datastore");
ds_pool_deb = noit_log_stream_find("debug/datastore_pool");
ingest_err = noit_log_stream_find("error/ingest");
if(!ds_err) ds_err = noit_error;
if(!ingest_err) ingest_err = noit_error;
if(!noit_conf_get_string(NULL, "//database/journal/path",
&basejpath)) {
noitL(noit_error, "//database/journal/path is unspecified\n");
exit(-1);
}
}
static int handoff_ingestor_init(noit_module_generic_t *self) {
ds_err = noit_log_stream_find("error/datastore");
ds_deb = noit_log_stream_find("debug/datastore");
ingest_err = noit_log_stream_find("error/ingest");
if(!ds_err) ds_err = noit_error;
if(!ingest_err) ingest_err = noit_error;
if(!noit_conf_get_string(NULL, "/stratcon/database/journal/path",
&basejpath)) {
noitL(noit_error, "/stratcon/database/journal/path is unspecified\n");
exit(-1);
}
noitL(noit_error, "registering /handoff/journals REST endpoint\n");
assert(noit_http_rest_register_auth(
"GET", "/handoff/", "^journals$", handoff_stream,
noit_http_rest_client_cert_auth
) == 0);
return stratcon_datastore_set_ingestor(&handoff_ingestor_api);
}
void cli_log_switches() {
int i;
noit_log_stream_t ls;
for(i=0; i<enable_logs_cnt; i++) {
ls = noit_log_stream_find(enable_logs[i]);
if(!ls) noitL(noit_error, "No such log: '%s'\n", enable_logs[i]);
if(ls && !ls->enabled) {
noitL(noit_error, "Enabling %s\n", enable_logs[i]);
ls->enabled = 1;
}
}
for(i=0; i<disable_logs_cnt; i++) {
ls = noit_log_stream_find(disable_logs[i]);
if(!ls) noitL(noit_error, "No such log: '%s'\n", enable_logs[i]);
if(ls && ls->enabled) {
noitL(noit_error, "Disabling %s\n", disable_logs[i]);
ls->enabled = 0;
}
}
}
static int rest_show_feed(noit_http_rest_closure_t *restc,
int npats, char **pats) {
noit_http_session_ctx *ctx = restc->http_ctx;
const char *err = "unknown error";
const char *jpath_with_sub;
char jlogpath[PATH_MAX], *cp, **subs = NULL;
int nsubs, i;
noit_log_stream_t feed;
jlog_ctx *jctx = NULL;
xmlDocPtr doc = NULL;
xmlNodePtr root = NULL, subnodes;
feed = noit_log_stream_find("feed");
if(!feed) { err = "cannot find feed"; goto error; }
jpath_with_sub = noit_log_stream_get_path(feed);
strlcpy(jlogpath, jpath_with_sub, sizeof(jlogpath));
cp = strchr(jlogpath, '(');
if(cp) *cp = '\0';
jctx = jlog_new(jlogpath);
if((nsubs = jlog_ctx_list_subscribers(jctx, &subs)) == -1) {
err = jlog_ctx_err_string(jctx);
goto error;
}
doc = xmlNewDoc((xmlChar *)"1.0");
root = xmlNewDocNode(doc, NULL, (xmlChar *)"feed", NULL);
xmlDocSetRootElement(doc, root);
subnodes = xmlNewNode(NULL, (xmlChar *)"subscribers");
for(i=0; i<nsubs; i++) {
xmlNewChild(subnodes, NULL, (xmlChar *)"subscriber", (xmlChar *)subs[i]);
}
xmlAddChild(root, subnodes);
noit_http_response_ok(restc->http_ctx, "text/xml");
noit_http_response_xml(restc->http_ctx, doc);
noit_http_response_end(restc->http_ctx);
if(subs) jlog_ctx_list_subscribers_dispose(jctx, subs);
xmlFreeDoc(doc);
jlog_ctx_close(jctx);
return 0;
error:
if(doc) xmlFreeDoc(doc);
if(subs) jlog_ctx_list_subscribers_dispose(jctx, subs);
noit_http_response_server_error(ctx, "text/plain");
noit_http_response_append(ctx, err, strlen(err));
noit_http_response_end(ctx);
if(jctx) jlog_ctx_close(jctx);
return 0;
}
int
noit_rest_eventer_logs(noit_http_rest_closure_t *restc, int n, char **p) {
char *endptr = NULL;
const char *since_s, *last_s;
const char *jsonstr;
char errbuf[128];
unsigned long long since;
int last = 0;
struct json_object *doc;
noit_log_stream_t ls;
noit_http_request *req = noit_http_session_request(restc->http_ctx);
since_s = noit_http_request_querystring(req, "since");
if(since_s) since = strtoull(since_s, &endptr, 10);
last_s = noit_http_request_querystring(req, "last");
if(last_s) last = atoi(last_s);
assert(n==1);
ls = noit_log_stream_find(p[0]);
if(!ls || strcmp(noit_log_stream_get_type(ls),"memory"))
goto not_found;
doc = json_object_new_array();
if(endptr != since_s)
noit_log_memory_lines_since(ls, since, json_spit_log, doc);
else
noit_log_memory_lines(ls, last, json_spit_log, doc);
noit_http_response_ok(restc->http_ctx, "application/json");
jsonstr = json_object_to_json_string(doc);
noit_http_response_append(restc->http_ctx, jsonstr, strlen(jsonstr));
noit_http_response_append(restc->http_ctx, "\n", 1);
json_object_put(doc);
noit_http_response_end(restc->http_ctx);
return 0;
not_found:
doc = json_object_new_object();
snprintf(errbuf, sizeof(errbuf), "log '%s' not found", p[0]);
json_object_object_add(doc, "error", json_object_new_string(errbuf));
jsonstr = json_object_to_json_string(doc);
noit_http_response_not_found(restc->http_ctx, "application/json");
noit_http_response_append(restc->http_ctx, jsonstr, strlen(jsonstr));
noit_http_response_append(restc->http_ctx, "\n", 1);
json_object_put(doc);
noit_http_response_end(restc->http_ctx);
return 0;
}
static int rest_delete_feed(noit_http_rest_closure_t *restc,
int npats, char **pats) {
noit_http_session_ctx *ctx = restc->http_ctx;
const char *err = "unknown error";
const char *jpath_with_sub;
char jlogpath[PATH_MAX], *cp;
int rv;
noit_log_stream_t feed;
jlog_ctx *jctx;
feed = noit_log_stream_find("feed");
if(!feed) { err = "cannot find feed"; goto error; }
jpath_with_sub = noit_log_stream_get_path(feed);
strlcpy(jlogpath, jpath_with_sub, sizeof(jlogpath));
cp = strchr(jlogpath, '(');
if(cp) *cp = '\0';
jctx = jlog_new(jlogpath);
rv = jlog_ctx_remove_subscriber(jctx, pats[0]);
jlog_ctx_close(jctx);
if(rv < 0) {
err = jlog_ctx_err_string(jctx);
goto error;
}
/* removed or note, we should do a sweeping cleanup */
jlog_clean(jlogpath);
if(rv == 0) {
noit_http_response_not_found(ctx, "text/plain");
noit_http_response_end(ctx);
return 0;
}
noit_http_response_standard(ctx, 204, "OK", "text/plain");
noit_http_response_end(ctx);
return 0;
error:
noit_http_response_server_error(ctx, "text/plain");
noit_http_response_append(ctx, err, strlen(err));
noit_http_response_end(ctx);
return 0;
}
static int selfcheck_log_size(eventer_t e, int mask, void *closure,
struct timeval *now) {
selfcheck_info_t *ci = closure;
noit_check_t *check = ci->check;
const char *feedname;
char feedname_buff[128];
noit_log_stream_t feed;
if(mask & (EVENTER_READ | EVENTER_WRITE)) {
/* this case is impossible from the eventer. It is called as
* such on the synchronous completion of the event.
*/
selfcheck_log_results(ci->self, ci->check);
selfcheck_cleanup(ci->self, ci->check);
check->flags &= ~NP_RUNNING;
return 0;
}
switch(mask) {
case EVENTER_ASYNCH_WORK:
/* Check the length of the log */
FETCH_CONFIG_OR(feedname, "feed");
noit_check_interpolate(feedname_buff, sizeof(feedname_buff), feedname,
&ci->attrs, check->config);
feed = noit_log_stream_find(feedname_buff);
if(!feed) ci->logsize = -1;
else ci->logsize = noit_log_stream_size(feed);
ci->timed_out = 0;
return 0;
break;
case EVENTER_ASYNCH_CLEANUP:
/* This sets us up for a completion call. */
e->mask = EVENTER_READ | EVENTER_WRITE;
break;
default:
abort();
}
return 0;
}
static int external_init(noit_module_t *self) {
external_data_t *data;
data = noit_module_get_userdata(self);
if(!data) data = malloc(sizeof(*data));
data->nlerr = noit_log_stream_find("error/external");
data->nldeb = noit_log_stream_find("debug/external");
data->jobq = calloc(1, sizeof(*data->jobq));
eventer_jobq_init(data->jobq, "external");
data->jobq->backq = eventer_default_backq();
eventer_jobq_increase_concurrency(data->jobq);
if(socketpair(AF_UNIX, SOCK_STREAM, 0, data->pipe_n2e) != 0 ||
socketpair(AF_UNIX, SOCK_STREAM, 0, data->pipe_e2n) != 0) {
noitL(noit_error, "external: pipe() failed: %s\n", strerror(errno));
return -1;
}
data->child = fork();
if(data->child == -1) {
/* No child, bail. */
noitL(noit_error, "external: fork() failed: %s\n", strerror(errno));
return -1;
}
/* parent must close the read side of n2e and the write side of e2n */
/* The child must do the opposite */
close(data->pipe_n2e[(data->child == 0) ? 1 : 0]);
close(data->pipe_e2n[(data->child == 0) ? 0 : 1]);
/* Now the parent must set its bits non-blocking, the child need not */
if(data->child != 0) {
/* in the parent */
if(eventer_set_fd_nonblocking(data->pipe_e2n[0]) == -1) {
close(data->pipe_n2e[1]);
close(data->pipe_e2n[0]);
noitL(noit_error,
"external: could not set pipe non-blocking: %s\n",
strerror(errno));
return -1;
}
eventer_t newe;
newe = eventer_alloc();
newe->fd = data->pipe_e2n[0];
newe->mask = EVENTER_READ | EVENTER_EXCEPTION;
newe->callback = external_handler;
newe->closure = self;
eventer_add(newe);
}
else {
const char *user = NULL, *group = NULL;
if(data->options) {
noit_hash_retr_str(data->options, "user", 4, &user);
noit_hash_retr_str(data->options, "group", 4, &group);
}
noit_security_usergroup(user, group, noit_false);
exit(external_child(data));
}
noit_module_set_userdata(self, data);
return 0;
}
int
noit_jlog_handler(eventer_t e, int mask, void *closure,
struct timeval *now) {
eventer_t newe;
pthread_t tid;
pthread_attr_t tattr;
int newmask = EVENTER_READ | EVENTER_EXCEPTION;
acceptor_closure_t *ac = closure;
noit_jlog_closure_t *jcl = ac->service_ctx;
char errbuff[256];
const char *errstr = "unknown error";
if(mask & EVENTER_EXCEPTION || (jcl && jcl->wants_shutdown)) {
int len, nlen;
socket_error:
/* Exceptions cause us to simply snip the connection */
len = strlen(errstr);
nlen = htonl(0 - len);
e->opset->write(e->fd, &nlen, sizeof(nlen), &newmask, e);
e->opset->write(e->fd, errstr, strlen(errstr), &newmask, e);
eventer_remove_fd(e->fd);
e->opset->close(e->fd, &newmask, e);
if(jcl) noit_jlog_closure_free(jcl);
acceptor_closure_free(ac);
return 0;
}
if(!ac->service_ctx) {
noit_log_stream_t ls;
const char *logname, *type;
int first_attempt = 1;
char path[PATH_MAX], subscriber[256], *sub;
jcl = ac->service_ctx = noit_jlog_closure_alloc();
if(!noit_hash_retr_str(ac->config,
"log_transit_feed_name",
strlen("log_transit_feed_name"),
&logname)) {
errstr = "No 'log_transit_feed_name' specified in log_transit.";
noitL(noit_error, "%s\n", errstr);
goto socket_error;
}
ls = noit_log_stream_find(logname);
if(!ls) {
snprintf(errbuff, sizeof(errbuff),
"Could not find log '%s' for log_transit.", logname);
errstr = errbuff;
noitL(noit_error, "%s\n", errstr);
goto socket_error;
}
type = noit_log_stream_get_type(ls);
if(!type || strcmp(type, "jlog")) {
snprintf(errbuff, sizeof(errbuff),
"Log '%s' for log_transit is not a jlog.", logname);
errstr = errbuff;
noitL(noit_error, "%s\n", errstr);
goto socket_error;
}
if(ac->cmd == NOIT_JLOG_DATA_FEED) {
if(!ac->remote_cn) {
errstr = "jlog transit started to unidentified party.";
noitL(noit_error, "%s\n", errstr);
goto socket_error;
}
strlcpy(subscriber, ac->remote_cn, sizeof(subscriber));
jcl->feed_stats = noit_jlog_feed_stats(subscriber);
}
else {
jcl->feed_stats = noit_jlog_feed_stats("~");
snprintf(subscriber, sizeof(subscriber),
"~%07d", noit_atomic_inc32(&tmpfeedcounter));
}
jcl->subscriber = strdup(subscriber);
strlcpy(path, noit_log_stream_get_path(ls), sizeof(path));
sub = strchr(path, '(');
if(sub) {
char *esub = strchr(sub, ')');
if(esub) {
*esub = '\0';
*sub++ = '\0';
}
}
jcl->jlog = jlog_new(path);
if(ac->cmd == NOIT_JLOG_DATA_TEMP_FEED) {
add_sub:
if(jlog_ctx_add_subscriber(jcl->jlog, jcl->subscriber, JLOG_END) == -1) {
snprintf(errbuff, sizeof(errbuff),
"jlog reader[%s] error: %s", jcl->subscriber,
jlog_ctx_err_string(jcl->jlog));
errstr = errbuff;
noitL(noit_error, "%s\n", errstr);
}
}
if(jlog_ctx_open_reader(jcl->jlog, jcl->subscriber) == -1) {
if(sub && !strcmp(sub, "*")) {
if(first_attempt) {
jlog_ctx_close(jcl->jlog);
jcl->jlog = jlog_new(path);
first_attempt = 0;
goto add_sub;
}
}
snprintf(errbuff, sizeof(errbuff),
"jlog reader[%s] error: %s", jcl->subscriber,
jlog_ctx_err_string(jcl->jlog));
errstr = errbuff;
noitL(noit_error, "%s\n", errstr);
goto socket_error;
}
}
/* The jlog stuff is disk I/O and can block us.
* We'll create a new thread to just handle this connection.
*/
eventer_remove_fd(e->fd);
newe = eventer_alloc();
memcpy(newe, e, sizeof(*e));
pthread_attr_init(&tattr);
pthread_attr_setdetachstate(&tattr, PTHREAD_CREATE_DETACHED);
gettimeofday(&jcl->feed_stats->last_connection, NULL);
noit_atomic_inc32(&jcl->feed_stats->connections);
if(pthread_create(&tid, &tattr, noit_jlog_thread_main, newe) == 0) {
return 0;
}
/* Undo our dup */
eventer_free(newe);
/* Creating the thread failed, close it down and deschedule. */
e->opset->close(e->fd, &newmask, e);
return 0;
}