static int example_initiate(noit_module_t *self, noit_check_t *check,
noit_check_t *cause) {
struct example_check_info *ci = check->closure;
const char *limit = "0";
struct timeval now, diff;
BAIL_ON_RUNNING_CHECK(check);
check->flags |= NP_RUNNING;
mtev_hash_retrieve(check->config, "limit", strlen("limit"), (void **)&limit);
ci->limit = atoi(limit);
mtev_gettimeofday(&now, NULL);
sub_timeval(now, check->last_fire_time, &diff);
noit_stats_set_whence(check, &now);
noit_stats_set_duration(check, diff.tv_sec * 1000 + diff.tv_usec / 1000);
noit_stats_set_available(check, NP_AVAILABLE);
noit_stats_set_status(check, "hello world");
if(ci->limit) {
int value = (int)(lrand48() % ci->limit);
noit_stats_set_metric(check, "random", METRIC_INT32, &value);
noit_stats_set_state(check, NP_GOOD);
}
else {
noit_stats_set_metric(check, "random", METRIC_INT32, NULL);
noit_stats_set_state(check, NP_BAD);
}
noit_check_set_stats(check);
check->flags &= ~NP_RUNNING;
return 0;
}
static int noit_rabbimq_connect(iep_thread_driver_t *dr) {
struct amqp_driver *driver = (struct amqp_driver *)dr;
if(!driver->connection) {
int sidx = driver->nconnects++ % driver->nhosts;
struct timeval timeout;
amqp_rpc_reply_t r, *rptr;
mtevL(mtev_error, "AMQP connect: %s:%d\n",
driver->hostname[sidx], driver->port);
BUMPSTAT(connects);
driver->hostidx = sidx;
timeout.tv_sec = driver->heartbeat;
timeout.tv_usec = 0;
driver->sockfd = amqp_open_socket(driver->hostname[sidx], driver->port, &timeout);
if(driver->sockfd < 0) {
mtevL(mtev_error, "AMQP connect failed: %s:%d\n",
driver->hostname[sidx], driver->port);
return -1;
}
if(setsockopt(driver->sockfd, SOL_SOCKET, SO_SNDBUF, &desired_sndbuf, sizeof(desired_sndbuf)) < 0)
mtevL(mtev_debug, "rabbitmq: setsockopt(SO_SNDBUF, %ld) -> %s\n", (long int)desired_sndbuf, strerror(errno));
if(setsockopt(driver->sockfd, SOL_SOCKET, SO_RCVBUF, &desired_rcvbuf, sizeof(desired_rcvbuf)) < 0)
mtevL(mtev_debug, "rabbitmq: setsockopt(SO_RCVBUF, %ld) -> %s\n", (long int)desired_rcvbuf, strerror(errno));
driver->has_error = 0;
driver->connection = amqp_new_connection();
amqp_set_basic_return_cb(driver->connection, noit_rabbitmq_brcb, driver);
amqp_set_sockfd(driver->connection, driver->sockfd);
r = amqp_login(driver->connection,
driver->vhost, 0, 131072, driver->heartbeat,
AMQP_SASL_METHOD_PLAIN,
driver->username, driver->password);
if(r.reply_type != AMQP_RESPONSE_NORMAL) {
mtevL(mtev_error, "AMQP login failed\n");
amqp_connection_close(driver->connection, AMQP_REPLY_SUCCESS);
amqp_destroy_connection(driver->connection);
if(driver->sockfd >= 0) close(driver->sockfd);
driver->sockfd = -1;
driver->connection = NULL;
return -1;
}
amqp_channel_open(driver->connection, 1);
rptr = amqp_get_rpc_reply();
if(rptr->reply_type != AMQP_RESPONSE_NORMAL) {
mtevL(mtev_error, "AMQP channe_open failed\n");
amqp_connection_close(driver->connection, AMQP_REPLY_SUCCESS);
amqp_destroy_connection(driver->connection);
if(driver->sockfd >= 0) close(driver->sockfd);
driver->sockfd = -1;
driver->connection = NULL;
return -1;
}
mtev_gettimeofday(&driver->last_hb, NULL);
return 0;
}
/* 1 means already connected */
return 1;
}
static void ping_icmp_log_results(noit_module_t *self, noit_check_t *check) {
struct check_info *data;
double avail = 0.0, min = MAXFLOAT, max = 0.0, avg = 0.0, cnt;
int avail_needed = 100;
const char *config_val = NULL;
int i, points = 0;
char human_buffer[256];
struct timeval now, duration;
data = (struct check_info *)check->closure;
for(i=0; i<data->expected_count; i++) {
if(data->turnaround[i] >= 0.0) {
points++;
avg += data->turnaround[i];
if(data->turnaround[i] > max) max = data->turnaround[i];
if(data->turnaround[i] < min) min = data->turnaround[i];
}
}
cnt = data->expected_count;
if(points == 0) {
min = 0.0 / 0.0;
max = 0.0 / 0.0;
avg = 0.0 / 0.0;
}
else {
avail = (float)points /cnt;
avg /= (float)points;
}
if(mtev_hash_retr_str(check->config, "avail_needed", strlen("avail_needed"),
&config_val))
avail_needed = atoi(config_val);
snprintf(human_buffer, sizeof(human_buffer),
"cnt=%d,avail=%0.0f,min=%0.4f,max=%0.4f,avg=%0.4f",
(int)cnt, 100.0*avail, min, max, avg);
mtevL(nldeb, "ping_icmp(%s) [%s]\n", check->target_ip, human_buffer);
mtev_gettimeofday(&now, NULL);
sub_timeval(now, check->last_fire_time, &duration);
noit_stats_set_whence(check, &now);
noit_stats_set_duration(check, duration.tv_sec * 1000 + duration.tv_usec / 1000);
noit_stats_set_available(check, (avail > 0.0) ? NP_AVAILABLE : NP_UNAVAILABLE);
noit_stats_set_state(check, (avail < ((float)avail_needed / 100.0)) ? NP_BAD : NP_GOOD);
noit_stats_set_status(check, human_buffer);
noit_stats_set_metric(check, "count",
METRIC_INT32, &data->expected_count);
avail *= 100.0;
noit_stats_set_metric(check, "available", METRIC_DOUBLE, &avail);
noit_stats_set_metric(check, "minimum",
METRIC_DOUBLE, avail > 0.0 ? &min : NULL);
noit_stats_set_metric(check, "maximum",
METRIC_DOUBLE, avail > 0.0 ? &max : NULL);
noit_stats_set_metric(check, "average",
METRIC_DOUBLE, avail > 0.0 ? &avg : NULL);
noit_check_set_stats(check);
}
static int
statsd_submit(noit_module_t *self, noit_check_t *check,
noit_check_t *cause) {
statsd_closure_t *ccl;
struct timeval now, duration;
statsd_mod_config_t *conf;
conf = noit_module_get_userdata(self);
if(!conf->primary_active) conf->check = NULL;
if(0 == memcmp(conf->primary, check->checkid, sizeof(uuid_t))) {
conf->check = check;
if(NOIT_CHECK_DISABLED(check) || NOIT_CHECK_KILLED(check)) {
conf->check = NULL;
return 0;
}
}
/* We are passive, so we don't do anything for transient checks */
if(check->flags & NP_TRANSIENT) return 0;
if(!check->closure) {
ccl = check->closure = calloc(1, sizeof(*ccl));
ccl->self = self;
} else {
// Don't count the first run
char human_buffer[256];
ccl = (statsd_closure_t*)check->closure;
mtev_gettimeofday(&now, NULL);
sub_timeval(now, check->last_fire_time, &duration);
noit_stats_set_whence(check, &now);
noit_stats_set_duration(check, duration.tv_sec * 1000 + duration.tv_usec / 1000);
snprintf(human_buffer, sizeof(human_buffer),
"dur=%ld,run=%d,stats=%d", duration.tv_sec * 1000 + duration.tv_usec / 1000,
check->generation, ccl->stats_count);
mtevL(nldeb, "statsd(%s) [%s]\n", check->target, human_buffer);
// Not sure what to do here
noit_stats_set_available(check, (ccl->stats_count > 0) ?
NP_AVAILABLE : NP_UNAVAILABLE);
noit_stats_set_state(check, (ccl->stats_count > 0) ?
NP_GOOD : NP_BAD);
noit_stats_set_status(check, human_buffer);
if(check->last_fire_time.tv_sec)
noit_check_passive_set_stats(check);
memcpy(&check->last_fire_time, &now, sizeof(duration));
}
ccl->stats_count = 0;
return 0;
}
static void noit_rabbitmq_heartbeat(struct amqp_driver *dr) {
struct timeval n, d;
if(!dr->connection) return;
mtev_gettimeofday(&n, NULL);
sub_timeval(n, dr->last_hb, &d);
if(d.tv_sec >= dr->heartbeat) {
amqp_frame_t f;
f.frame_type = AMQP_FRAME_HEARTBEAT;
f.channel = 0;
amqp_send_frame(dr->connection, &f);
mtevL(mtev_debug, "amqp -> hearbeat\n");
memcpy(&dr->last_hb, &n, sizeof(n));
}
}
void noit_metric_director_init() {
nthreads = eventer_loop_concurrency();
mtevAssert(nthreads > 0);
thread_queues = calloc(sizeof(*thread_queues),nthreads);
check_interests = calloc(sizeof(*check_interests),nthreads);
if(mtev_fq_handle_message_hook_register_available())
mtev_fq_handle_message_hook_register("metric-director", handle_fq_message, NULL);
mtev_log_line_hook_register("metric-director", handle_log_line, NULL);
eventer_t e = eventer_alloc();
e->mask = EVENTER_TIMER;
e->callback = prune_old_dedupe_hashes;
mtev_gettimeofday(&e->whence, NULL);
e->whence.tv_sec += 2;
eventer_add_timed(e);
}
static int
noit_console_show_check(mtev_console_closure_t ncct,
int argc, char **argv,
mtev_console_state_t *state, void *closure) {
int i, cnt;
mtev_conf_t_userdata_t *info;
char xpath[1024];
xmlXPathObjectPtr pobj = NULL;
xmlXPathContextPtr xpath_ctxt = NULL;
mtev_conf_xml_xpath(NULL, &xpath_ctxt);
if(argc > 1) {
nc_printf(ncct, "requires zero or one arguments\n");
return -1;
}
info = mtev_console_userdata_get(ncct, MTEV_CONF_T_USERDATA);
/* We many not be in conf-t mode -- that's fine */
if(noit_console_mkcheck_xpath(xpath, sizeof(xpath), info,
argc ? argv[0] : NULL)) {
nc_printf(ncct, "could not find check '%s'\n", argv[0]);
return -1;
}
pobj = xmlXPathEval((xmlChar *)xpath, xpath_ctxt);
if(!pobj || pobj->type != XPATH_NODESET ||
xmlXPathNodeSetIsEmpty(pobj->nodesetval)) {
nc_printf(ncct, "no checks found\n");
goto out;
}
cnt = xmlXPathNodeSetGetLength(pobj->nodesetval);
if(info && cnt != 1) {
nc_printf(ncct, "Ambiguous check specified\n");
goto out;
}
for(i=0; i<cnt; i++) {
mtev_hash_iter iter = MTEV_HASH_ITER_ZERO;
const char *k;
int klen;
void *data;
uuid_t checkid;
noit_check_t *check;
mtev_hash_table *config;
xmlNodePtr node, anode, mnode = NULL;
char *uuid_conf;
char *module, *value;
node = (mtev_conf_section_t)xmlXPathNodeSetItem(pobj->nodesetval, i);
uuid_conf = (char *)xmlGetProp(node, (xmlChar *)"uuid");
if(!uuid_conf || uuid_parse(uuid_conf, checkid)) {
nc_printf(ncct, "%s has invalid or missing UUID!\n",
(char *)xmlGetNodePath(node) + strlen("/noit"));
continue;
}
nc_printf(ncct, "==== %s ====\n", uuid_conf);
xmlFree(uuid_conf);
#define MYATTR(a,n,b) _mtev_conf_get_string(node, &(n), "@" #a, &(b))
#define INHERIT(a,n,b) \
_mtev_conf_get_string(node, &(n), "ancestor-or-self::node()/@" #a, &(b))
#define SHOW_ATTR(a) do { \
anode = NULL; \
value = NULL; \
INHERIT(a, anode, value); \
nc_attr_show(ncct, #a, node, anode, value); \
if(value != NULL) free(value); \
} while(0)
if(!INHERIT(module, mnode, module)) module = NULL;
if(MYATTR(name, anode, value)) {
nc_printf(ncct, " name: %s\n", value);
free(value);
}
else
nc_printf(ncct, " name: %s [from module]\n", module ? module : "[undef]");
nc_attr_show(ncct, "module", node, mnode, module);
if(module) free(module);
SHOW_ATTR(target);
SHOW_ATTR(seq);
SHOW_ATTR(resolve_rtype);
SHOW_ATTR(period);
SHOW_ATTR(timeout);
SHOW_ATTR(oncheck);
SHOW_ATTR(filterset);
SHOW_ATTR(disable);
/* Print out all the config settings */
config = mtev_conf_get_hash(node, "config");
while(mtev_hash_next(config, &iter, &k, &klen, &data)) {
nc_printf(ncct, " config::%s: %s\n", k, (const char *)data);
}
mtev_hash_destroy(config, free, free);
free(config);
check = noit_poller_lookup(checkid);
if(!check) {
nc_printf(ncct, " ERROR: not in running system\n");
}
else {
int idx = 0;
stats_t *c;
struct timeval *whence;
mtev_hash_table *metrics;
nc_printf(ncct, " target_ip: %s\n", check->target_ip);
nc_printf(ncct, " currently: %08x ", check->flags);
if(NOIT_CHECK_RUNNING(check)) { nc_printf(ncct, "running"); idx++; }
if(NOIT_CHECK_KILLED(check)) nc_printf(ncct, "%skilled", idx++?",":"");
if(!NOIT_CHECK_CONFIGURED(check)) nc_printf(ncct, "%sunconfig", idx++?",":"");
if(NOIT_CHECK_DISABLED(check)) nc_printf(ncct, "%sdisabled", idx++?",":"");
if(!idx) nc_printf(ncct, "idle");
nc_write(ncct, "\n", 1);
if (check->fire_event != NULL) {
struct timeval now, diff;
mtev_gettimeofday(&now, NULL);
sub_timeval(check->fire_event->whence, now, &diff);
nc_printf(ncct, " next run: %0.3f seconds\n",
diff.tv_sec + (diff.tv_usec / 1000000.0));
}
else {
nc_printf(ncct, " next run: unscheduled\n");
}
c = noit_check_get_stats_current(check);
whence = noit_check_stats_whence(c, NULL);
if(whence->tv_sec == 0) {
nc_printf(ncct, " last run: never\n");
}
else {
const char *status;
struct timeval now, *then, diff;
mtev_gettimeofday(&now, NULL);
then = noit_check_stats_whence(c, NULL);
sub_timeval(now, *then, &diff);
nc_printf(ncct, " last run: %0.3f seconds ago\n",
diff.tv_sec + (diff.tv_usec / 1000000.0));
nc_printf(ncct, " availability/state: %s/%s\n",
noit_check_available_string(noit_check_stats_available(c, NULL)),
noit_check_state_string(noit_check_stats_state(c, NULL)));
status = noit_check_stats_status(c, NULL);
nc_printf(ncct, " status: %s\n", status);
nc_printf(ncct, " feeds: %d\n", check->feeds ? check->feeds->size : 0);
}
c = noit_check_get_stats_inprogress(check);
metrics = noit_check_stats_metrics(c);
if(mtev_hash_size(metrics) > 0) {
nc_printf(ncct, " metrics (inprogress):\n");
nc_print_stat_metrics(ncct, check, c);
}
c = noit_check_get_stats_current(check);
metrics = noit_check_stats_metrics(c);
if(mtev_hash_size(metrics)) {
nc_printf(ncct, " metrics (current):\n");
nc_print_stat_metrics(ncct, check, c);
}
c = noit_check_get_stats_previous(check);
metrics = noit_check_stats_metrics(c);
if(mtev_hash_size(metrics) > 0) {
nc_printf(ncct, " metrics (previous):\n");
nc_print_stat_metrics(ncct, check, c);
}
}
}
out:
if(pobj) xmlXPathFreeObject(pobj);
return 0;
}
static interim_journal_t *
interim_journal_get(struct sockaddr *remote, const char *remote_cn_in,
int storagenode_id, const char *fqdn_in) {
void *vhash, *vij;
mtev_hash_table *working_set;
interim_journal_t *ij;
struct timeval now;
char jpath[PATH_MAX];
char remote_str[128];
const char *remote_cn = remote_cn_in ? remote_cn_in : "default";
const char *fqdn = fqdn_in ? fqdn_in : "default";
mtev_convert_sockaddr_to_buff(remote_str, sizeof(remote_str), remote);
if(!*remote_str) strlcpy(remote_str, "default", sizeof(remote_str));
/* Lookup the working set */
if(!mtev_hash_retrieve(&working_sets, remote_cn, strlen(remote_cn), &vhash)) {
working_set = calloc(1, sizeof(*working_set));
mtev_hash_init(working_set);
mtev_hash_store(&working_sets, strdup(remote_cn), strlen(remote_cn),
working_set);
}
else
working_set = vhash;
/* Lookup the interim journal within the working set */
if(!mtev_hash_retrieve(working_set, fqdn, strlen(fqdn), &vij)) {
ij = calloc(1, sizeof(*ij));
mtev_gettimeofday(&now, NULL);
snprintf(jpath, sizeof(jpath), "%s/%s/%s/%d/%08x%08x.tmp",
basejpath, remote_str, remote_cn, storagenode_id,
(unsigned int)now.tv_sec, (unsigned int)now.tv_usec);
ij->remote_str = strdup(remote_str);
ij->remote_cn = strdup(remote_cn);
ij->fqdn = fqdn_in ? strdup(fqdn_in) : NULL;
ij->storagenode_id = storagenode_id;
ij->filename = strdup(jpath);
ij->fd = open(ij->filename, O_RDWR | O_CREAT | O_EXCL, 0640);
if(ij->fd < 0 && errno == ENOENT) {
if(mkdir_for_file(ij->filename, 0750)) {
mtevL(noit_error, "Failed to create dir for '%s': %s\n",
ij->filename, strerror(errno));
exit(-1);
}
ij->fd = open(ij->filename, O_RDWR | O_CREAT | O_EXCL, 0640);
}
if(ij->fd < 0 && errno == EEXIST) {
/* This can only occur if we crash after before checkpointing */
unlink(ij->filename);
ij->fd = open(ij->filename, O_RDWR | O_CREAT | O_EXCL, 0640);
}
if(ij->fd < 0) {
mtevL(noit_error, "Failed to open interim journal '%s': %s\n",
ij->filename, strerror(errno));
exit(-1);
}
mtev_hash_store(working_set, strdup(fqdn), strlen(fqdn), ij);
}
else
ij = vij;
return ij;
}
static int eventer_epoll_impl_loop() {
struct epoll_event *epev;
struct epoll_spec *spec;
spec = eventer_get_spec_for_event(NULL);
epev = malloc(sizeof(*epev) * maxfds);
#ifdef HAVE_SYS_EVENTFD_H
if(spec->event_fd >= 0) {
eventer_t e = eventer_alloc();
e->callback = eventer_epoll_eventfd_read;
e->fd = spec->event_fd;
e->mask = EVENTER_READ;
eventer_add(e);
}
#endif
while(1) {
struct timeval __now, __sleeptime;
int fd_cnt = 0;
__sleeptime = eventer_max_sleeptime;
mtev_gettimeofday(&__now, NULL);
eventer_dispatch_timed(&__now, &__sleeptime);
/* Handle cross_thread dispatches */
eventer_cross_thread_process();
/* Handle recurrent events */
eventer_dispatch_recurrent(&__now);
/* Now we move on to our fd-based events */
do {
fd_cnt = epoll_wait(spec->epoll_fd, epev, maxfds,
__sleeptime.tv_sec * 1000 + __sleeptime.tv_usec / 1000);
} while(fd_cnt < 0 && errno == EINTR);
mtevLT(eventer_deb, &__now, "debug: epoll_wait(%d, [], %d) => %d\n",
spec->epoll_fd, maxfds, fd_cnt);
if(fd_cnt < 0) {
mtevLT(eventer_err, &__now, "epoll_wait: %s\n", strerror(errno));
}
else {
int idx;
/* loop once to clear */
for(idx = 0; idx < fd_cnt; idx++) {
struct epoll_event *ev;
eventer_t e;
int fd, mask = 0;
ev = &epev[idx];
if(ev->events & (EPOLLIN | EPOLLPRI)) mask |= EVENTER_READ;
if(ev->events & (EPOLLOUT)) mask |= EVENTER_WRITE;
if(ev->events & (EPOLLERR|EPOLLHUP)) mask |= EVENTER_EXCEPTION;
fd = ev->data.fd;
e = master_fds[fd].e;
/* It's possible that someone removed the event and freed it
* before we got here.
*/
if(!e) continue;
eventer_epoll_impl_trigger(e, mask);
}
}
}
/* NOTREACHED */
return 0;
}
static void eventer_kqueue_impl_trigger(eventer_t e, int mask) {
ev_lock_state_t lockstate;
struct timeval __now;
int oldmask, newmask;
const char *cbname;
int fd;
u_int64_t start, duration;
int cross_thread = mask & EVENTER_CROSS_THREAD_TRIGGER;
mask = mask & ~(EVENTER_RESERVED);
fd = e->fd;
if(cross_thread) {
if(master_fds[fd].e != NULL) {
mtevL(eventer_deb, "Attempting to trigger already-registered event fd: %d cross thread.\n", fd);
}
/* mtevAssert(master_fds[fd].e == NULL); */
}
if(!pthread_equal(pthread_self(), e->thr_owner)) {
/* If we're triggering across threads, it can't be registered yet */
if(master_fds[fd].e != NULL) {
mtevL(eventer_deb, "Attempting to trigger already-registered event fd: %d cross thread.\n", fd);
}
/* mtevAssert(master_fds[fd].e == NULL); */
eventer_cross_thread_trigger(e,mask);
return;
}
if(master_fds[fd].e == NULL) {
master_fds[fd].e = e;
e->mask = 0;
}
if(e != master_fds[fd].e) return;
lockstate = acquire_master_fd(fd);
if(lockstate == EV_ALREADY_OWNED) return;
mtevAssert(lockstate == EV_OWNED);
mtev_gettimeofday(&__now, NULL);
/* We're going to lie to ourselves. You'd think this should be:
* oldmask = e->mask; However, we just fired with masks[fd], so
* kqueue is clearly looking for all of the events in masks[fd].
* So, we combine them "just to be safe."
*/
oldmask = e->mask | masks[fd];
cbname = eventer_name_for_callback_e(e->callback, e);
mtevLT(eventer_deb, &__now, "kqueue: fire on %d/%x to %s(%p)\n",
fd, masks[fd], cbname?cbname:"???", e->callback);
mtev_memory_begin();
LIBMTEV_EVENTER_CALLBACK_ENTRY((void *)e, (void *)e->callback, (char *)cbname, fd, e->mask, mask);
start = mtev_gethrtime();
newmask = e->callback(e, mask, e->closure, &__now);
duration = mtev_gethrtime() - start;
LIBMTEV_EVENTER_CALLBACK_RETURN((void *)e, (void *)e->callback, (char *)cbname, newmask);
mtev_memory_end();
stats_set_hist_intscale(eventer_callback_latency, duration, -9, 1);
stats_set_hist_intscale(eventer_latency_handle_for_callback(e->callback), duration, -9, 1);
if(newmask) {
if(!pthread_equal(pthread_self(), e->thr_owner)) {
pthread_t tgt = e->thr_owner;
e->thr_owner = pthread_self();
alter_kqueue_mask(e, oldmask, 0);
e->thr_owner = tgt;
mtevL(eventer_deb, "moved event[%p] from t@%llx to t@%llx\n", e, (vpsized_int)pthread_self(), (vpsized_int)tgt);
if(newmask) eventer_cross_thread_trigger(e, newmask & ~(EVENTER_EXCEPTION));
}
else {
if(master_fds[fd].e != e) {
e = master_fds[fd].e;
mtevL(eventer_deb, "%strigger complete [event switched] %d : %x->%x\n", cross_thread ? "[X]" : "", e->fd, master_fds[fd].e->mask, newmask);
} else {
mtevL(eventer_deb, "%strigger complete %d : %x->%x\n", cross_thread ? "[X]" : "", e->fd, oldmask, newmask);
}
alter_kqueue_mask(e, (e->mask == 0 || cross_thread) ? 0 : oldmask, newmask);
/* Set our mask */
e->mask = newmask;
}
}
else {
/*
* Long story long:
* When integrating with a few external event systems, we find
* it difficult to make their use of remove+add as an update
* as it can be recurrent in a single handler call and you cannot
* remove completely from the event system if you are going to
* just update (otherwise the eventer_t in your call stack could
* be stale). What we do is perform a superficial remove, marking
* the mask as 0, but not eventer_remove_fd. Then on an add, if
* we already have an event, we just update the mask (as we
* have not yet returned to the eventer's loop.
* This leaves us in a tricky situation when a remove is called
* and the add doesn't roll in, we return 0 (mask == 0) and hit
* this spot. We have intended to remove the event, but it still
* resides at master_fds[fd].e -- even after we free it.
* So, in the evnet that we return 0 and the event that
* master_fds[fd].e == the event we're about to free... we NULL
* it out.
*/
if(master_fds[fd].e == e) master_fds[fd].e = NULL;
eventer_free(e);
}
release_master_fd(fd, lockstate);
}
void noit_check_resolver_init() {
static int initialized = 0;
int32_t cnt;
mtev_conf_section_t *servers, *searchdomains;
eventer_t e;
DCLOCK();
if(initialized != 0) {
DCUNLOCK();
return;
}
if(dns_init(NULL, 0) < 0)
mtevL(noit_error, "dns initialization failed.\n");
dns_ctx = dns_new(NULL);
if(dns_init(dns_ctx, 0) != 0) {
mtevL(noit_error, "dns initialization failed.\n");
DCUNLOCK();
exit(-1);
}
/* Optional servers */
servers = mtev_conf_get_sections(MTEV_CONF_ROOT, "//resolver//server", &cnt);
if(cnt) {
int i;
char server[128];
dns_add_serv(dns_ctx, NULL); /* reset */
for(i=0;i<cnt;i++) {
if(mtev_conf_get_stringbuf(servers[i], "self::node()",
server, sizeof(server))) {
if(mtev_conf_env_off(servers[i], NULL)) {
mtevL(noit_error, "DNS server %s environmentally ignored.\n", server);
}
else if(dns_add_serv(dns_ctx, server) < 0) {
mtevL(noit_error, "Failed adding DNS server: %s\n", server);
}
}
}
}
mtev_conf_release_sections(servers, cnt);
searchdomains = mtev_conf_get_sections(MTEV_CONF_ROOT, "//resolver//search", &cnt);
if(cnt) {
int i;
char search[128];
dns_add_srch(dns_ctx, NULL); /* reset */
for(i=0;i<cnt;i++) {
if(mtev_conf_get_stringbuf(searchdomains[i], "self::node()",
search, sizeof(search))) {
if(mtev_conf_env_off(searchdomains[i], NULL)) {
mtevL(noit_error, "DNS search %s environmentally ignored.\n", search);
}
else if(dns_add_srch(dns_ctx, search) < 0) {
mtevL(noit_error, "Failed adding DNS search path: %s\n", search);
}
else if(dns_search_flag) dns_search_flag = 0; /* enable search */
}
}
}
mtev_conf_release_sections(searchdomains, cnt);
if(mtev_conf_get_int32(MTEV_CONF_ROOT, "//resolver/@ndots", &cnt))
dns_set_opt(dns_ctx, DNS_OPT_NDOTS, cnt);
if(mtev_conf_get_int32(MTEV_CONF_ROOT, "//resolver/@ntries", &cnt))
dns_set_opt(dns_ctx, DNS_OPT_NTRIES, cnt);
if(mtev_conf_get_int32(MTEV_CONF_ROOT, "//resolver/@timeout", &cnt))
dns_set_opt(dns_ctx, DNS_OPT_TIMEOUT, cnt);
if(dns_open(dns_ctx) < 0) {
mtevL(noit_error, "dns open failed.\n");
DCUNLOCK();
exit(-1);
}
eventer_name_callback("dns_cache_callback", dns_cache_callback);
dns_set_tmcbck(dns_ctx, dns_cache_utm_fn, dns_ctx);
e = eventer_alloc_fd(dns_cache_callback, dns_ctx, dns_sock(dns_ctx),
EVENTER_READ|EVENTER_EXCEPTION);
eventer_add(e);
mtev_skiplist *dc = mtev_skiplist_alloc();
mtev_skiplist_set_compare(dc, name_lookup, name_lookup_k);
mtev_skiplist_add_index(dc, refresh_idx, refresh_idx_k);
ck_pr_barrier();
nc_dns_cache = dc;
/* maybe load it from cache */
if(noit_resolver_cache_load_hook_exists()) {
struct timeval now;
char *key;
void *data;
int len;
mtev_gettimeofday(&now, NULL);
while(noit_resolver_cache_load_hook_invoke(&key, &data, &len) == MTEV_HOOK_CONTINUE) {
dns_cache_node *n;
n = calloc(1, sizeof(*n));
if(dns_cache_node_deserialize(n, data, len) >= 0) {
n->target = strdup(key);
/* if the TTL indicates that it will expire in less than 60 seconds
* (including stuff that should have already expired), then fudge
* the last_updated time to make it expire some random time within
* the next 60 seconds.
*/
if(n->last_needed > now.tv_sec || n->last_updated > now.tv_sec) {
dns_cache_node_free(n);
break; /* impossible */
}
n->last_needed = now.tv_sec;
if(n->last_updated + n->ttl < now.tv_sec + 60) {
int fudge = MIN(60, n->ttl) + 1;
n->last_updated = now.tv_sec - n->ttl + (lrand48() % fudge);
}
mtev_skiplist_insert(nc_dns_cache, n);
n = NULL;
}
else {
mtevL(noit_error, "Failed to deserialize resolver cache record.\n");
}
if(n) dns_cache_node_free(n);
if(key) free(key);
if(data) free(data);
}
}
noit_check_resolver_loop(NULL, 0, NULL, NULL);
register_console_dns_cache_commands();
mtev_hash_init(&etc_hosts_cache);
noit_check_etc_hosts_cache_refresh(NULL, 0, NULL, NULL);
initialized = 1;
DCUNLOCK();
}
int
noit_jlog_handler(eventer_t e, int mask, void *closure,
struct timeval *now) {
eventer_t newe;
pthread_t tid;
pthread_attr_t tattr;
int newmask = EVENTER_READ | EVENTER_EXCEPTION;
acceptor_closure_t *ac = closure;
noit_jlog_closure_t *jcl = ac->service_ctx;
char errbuff[256];
const char *errstr = "unknown error";
if(mask & EVENTER_EXCEPTION || (jcl && jcl->wants_shutdown)) {
int len, nlen;
socket_error:
/* Exceptions cause us to simply snip the connection */
len = strlen(errstr);
nlen = htonl(0 - len);
e->opset->write(e->fd, &nlen, sizeof(nlen), &newmask, e);
e->opset->write(e->fd, errstr, strlen(errstr), &newmask, e);
eventer_remove_fd(e->fd);
e->opset->close(e->fd, &newmask, e);
if(jcl) noit_jlog_closure_free(jcl);
acceptor_closure_free(ac);
return 0;
}
if(!ac->service_ctx) {
mtev_log_stream_t ls;
const char *logname, *type;
int first_attempt = 1;
char path[PATH_MAX], subscriber[256], *sub;
jcl = ac->service_ctx = noit_jlog_closure_alloc();
if(!mtev_hash_retr_str(ac->config,
"log_transit_feed_name",
strlen("log_transit_feed_name"),
&logname)) {
errstr = "No 'log_transit_feed_name' specified in log_transit.";
mtevL(noit_error, "%s\n", errstr);
goto socket_error;
}
ls = mtev_log_stream_find(logname);
if(!ls) {
snprintf(errbuff, sizeof(errbuff),
"Could not find log '%s' for log_transit.", logname);
errstr = errbuff;
mtevL(noit_error, "%s\n", errstr);
goto socket_error;
}
type = mtev_log_stream_get_type(ls);
if(!type || strcmp(type, "jlog")) {
snprintf(errbuff, sizeof(errbuff),
"Log '%s' for log_transit is not a jlog.", logname);
errstr = errbuff;
mtevL(noit_error, "%s\n", errstr);
goto socket_error;
}
if(ac->cmd == NOIT_JLOG_DATA_FEED) {
if(!ac->remote_cn) {
errstr = "jlog transit started to unidentified party.";
mtevL(noit_error, "%s\n", errstr);
goto socket_error;
}
strlcpy(subscriber, ac->remote_cn, sizeof(subscriber));
jcl->feed_stats = noit_jlog_feed_stats(subscriber);
}
else {
jcl->feed_stats = noit_jlog_feed_stats("~");
snprintf(subscriber, sizeof(subscriber),
"~%07d", mtev_atomic_inc32(&tmpfeedcounter));
}
jcl->subscriber = strdup(subscriber);
strlcpy(path, mtev_log_stream_get_path(ls), sizeof(path));
sub = strchr(path, '(');
if(sub) {
char *esub = strchr(sub, ')');
if(esub) {
*esub = '\0';
*sub++ = '\0';
}
}
jcl->jlog = jlog_new(path);
if(ac->cmd == NOIT_JLOG_DATA_TEMP_FEED) {
add_sub:
if(jlog_ctx_add_subscriber(jcl->jlog, jcl->subscriber, JLOG_END) == -1) {
snprintf(errbuff, sizeof(errbuff),
"jlog reader[%s] error: %s", jcl->subscriber,
jlog_ctx_err_string(jcl->jlog));
errstr = errbuff;
mtevL(noit_error, "%s\n", errstr);
}
}
if(jlog_ctx_open_reader(jcl->jlog, jcl->subscriber) == -1) {
if(sub && !strcmp(sub, "*")) {
if(first_attempt) {
jlog_ctx_close(jcl->jlog);
jcl->jlog = jlog_new(path);
first_attempt = 0;
goto add_sub;
}
}
snprintf(errbuff, sizeof(errbuff),
"jlog reader[%s] error: %s", jcl->subscriber,
jlog_ctx_err_string(jcl->jlog));
errstr = errbuff;
mtevL(noit_error, "%s\n", errstr);
goto socket_error;
}
}
/* The jlog stuff is disk I/O and can block us.
* We'll create a new thread to just handle this connection.
*/
eventer_remove_fd(e->fd);
newe = eventer_alloc();
memcpy(newe, e, sizeof(*e));
pthread_attr_init(&tattr);
pthread_attr_setdetachstate(&tattr, PTHREAD_CREATE_DETACHED);
mtev_gettimeofday(&jcl->feed_stats->last_connection, NULL);
mtev_atomic_inc32(&jcl->feed_stats->connections);
if(pthread_create(&tid, &tattr, noit_jlog_thread_main, newe) == 0) {
return 0;
}
/* Undo our dup */
eventer_free(newe);
/* Creating the thread failed, close it down and deschedule. */
e->opset->close(e->fd, &newmask, e);
return 0;
}
void *
noit_jlog_thread_main(void *e_vptr) {
int mask, bytes_read, sleeptime, max_sleeptime;
eventer_t e = e_vptr;
acceptor_closure_t *ac = e->closure;
noit_jlog_closure_t *jcl = ac->service_ctx;
char inbuff[sizeof(jlog_id)];
mtev_memory_init_thread();
eventer_set_fd_blocking(e->fd);
max_sleeptime = DEFAULT_MSECONDS_BETWEEN_BATCHES;
if(ac->cmd == NOIT_JLOG_DATA_TEMP_FEED)
max_sleeptime = DEFAULT_TRANSIENT_MSECONDS_BETWEEN_BATCHES;
sleeptime = max_sleeptime;
while(1) {
jlog_id client_chkpt;
sleeptime = MIN(sleeptime, max_sleeptime);
jlog_get_checkpoint(jcl->jlog, ac->remote_cn, &jcl->chkpt);
jcl->count = jlog_ctx_read_interval(jcl->jlog, &jcl->start, &jcl->finish);
if(jcl->count < 0) {
char idxfile[PATH_MAX];
mtevL(noit_error, "jlog_ctx_read_interval: %s\n",
jlog_ctx_err_string(jcl->jlog));
switch (jlog_ctx_err(jcl->jlog)) {
case JLOG_ERR_FILE_CORRUPT:
case JLOG_ERR_IDX_CORRUPT:
jlog_repair_datafile(jcl->jlog, jcl->start.log);
jlog_repair_datafile(jcl->jlog, jcl->start.log + 1);
mtevL(noit_error,
"jlog reconstructed, deleting corresponding index.\n");
STRSETDATAFILE(jcl->jlog, idxfile, jcl->start.log);
strlcat(idxfile, INDEX_EXT, sizeof(idxfile));
unlink(idxfile);
STRSETDATAFILE(jcl->jlog, idxfile, jcl->start.log + 1);
strlcat(idxfile, INDEX_EXT, sizeof(idxfile));
unlink(idxfile);
goto alldone;
break;
default:
goto alldone;
}
}
if(jcl->count > MAX_ROWS_AT_ONCE) {
/* Artificially set down the range to make the batches a bit easier
* to handle on the stratcond/postgres end.
* However, we must have more data, so drop the sleeptime to 0
*/
jcl->count = MAX_ROWS_AT_ONCE;
jcl->finish.marker = jcl->start.marker + jcl->count;
}
if(jcl->count > 0) {
sleeptime = 0;
if(noit_jlog_push(e, jcl)) {
goto alldone;
}
/* Read our jlog_id accounting for possibly short reads */
bytes_read = 0;
while(bytes_read < sizeof(jlog_id)) {
int len;
if((len = e->opset->read(e->fd, inbuff + bytes_read,
sizeof(jlog_id) - bytes_read,
&mask, e)) <= 0)
goto alldone;
bytes_read += len;
}
memcpy(&client_chkpt, inbuff, sizeof(jlog_id));
/* Fix the endian */
client_chkpt.log = ntohl(client_chkpt.log);
client_chkpt.marker = ntohl(client_chkpt.marker);
if(memcmp(&jcl->chkpt, &client_chkpt, sizeof(jlog_id))) {
mtevL(noit_error,
"client %s submitted invalid checkpoint %u:%u expected %u:%u\n",
ac->remote_cn, client_chkpt.log, client_chkpt.marker,
jcl->chkpt.log, jcl->chkpt.marker);
goto alldone;
}
mtev_gettimeofday(&jcl->feed_stats->last_checkpoint, NULL);
jlog_ctx_read_checkpoint(jcl->jlog, &jcl->chkpt);
}
else {
/* we have nothing to write -- maybe we have no checks configured...
* If this is the case "forever", the remote might disconnect and
* we would never know. Do the painful work of detecting a
* disconnected client.
*/
struct pollfd pfd;
pfd.fd = e->fd;
pfd.events = POLLIN | POLLHUP | POLLRDNORM;
pfd.revents = 0;
if(poll(&pfd, 1, 0) != 0) {
/* normally, we'd recv PEEK|DONTWAIT. However, the client should
* not be writing to us. So, we know we can't have any legitimate
* data on this socket (true even though this is SSL). So, if we're
* here then "shit went wrong"
*/
mtevL(noit_error, "jlog client %s disconnected while idle\n",
ac->remote_cn);
goto alldone;
}
}
if(sleeptime) {
usleep(sleeptime * 1000); /* us -> ms */
}
sleeptime += 1000; /* 1 s */
}
alldone:
e->opset->close(e->fd, &mask, e);
mtev_atomic_dec32(&jcl->feed_stats->connections);
noit_jlog_closure_free(jcl);
acceptor_closure_free(ac);
mtev_memory_maintenance();
return NULL;
}
static void eventer_epoll_impl_trigger(eventer_t e, int mask) {
struct epoll_spec *spec;
struct timeval __now;
int fd, newmask, needs_add = 0;
const char *cbname;
ev_lock_state_t lockstate;
int cross_thread = mask & EVENTER_CROSS_THREAD_TRIGGER;
uint64_t start, duration;
mask = mask & ~(EVENTER_RESERVED);
fd = e->fd;
if(cross_thread) {
if(master_fds[fd].e != NULL) {
mtevL(eventer_deb, "Attempting to trigger already-registered event fd: %d cross thread.\n", fd);
}
/* mtevAssert(master_fds[fd].e == NULL); */
}
if(!pthread_equal(pthread_self(), e->thr_owner)) {
/* If we're triggering across threads, it can't be registered yet */
if(master_fds[fd].e != NULL) {
mtevL(eventer_deb, "Attempting to trigger already-registered event fd: %d cross thread.\n", fd);
}
/* mtevAssert(master_fds[fd].e == NULL); */
eventer_cross_thread_trigger(e,mask);
return;
}
if(master_fds[fd].e == NULL) {
lockstate = acquire_master_fd(fd);
if (lockstate == EV_ALREADY_OWNED) {
/* The incoming triggered event is already owned by this thread.
* This means our floated event completed before the current
* event handler even exited. So it retriggered recursively
* from inside the event handler.
*
* Treat this special case the same as a cross thread trigger
* and just queue this event to be picked up on the next loop
*/
eventer_cross_thread_trigger(e, mask);
return;
}
/*
* If we are readding the event to the master list here, also do the needful
* with the epoll_ctl.
*
* This can happen in cases where some event was floated and the float
* completed so fast that we finished the job in the same thread
* that it started in. Since we `eventer_remove_fd` before we float
* the re-add here should replace the fd in the epoll_ctl.
*/
master_fds[fd].e = e;
e->mask = 0;
struct epoll_event _ev;
memset(&_ev, 0, sizeof(_ev));
_ev.data.fd = fd;
spec = eventer_get_spec_for_event(e);
if(mask & EVENTER_READ) _ev.events |= (EPOLLIN|EPOLLPRI);
if(mask & EVENTER_WRITE) _ev.events |= (EPOLLOUT);
if(mask & EVENTER_EXCEPTION) _ev.events |= (EPOLLERR|EPOLLHUP);
mtevL(eventer_deb, "epoll_ctl(%d, add, %d)\n", spec->epoll_fd, fd);
if (epoll_ctl(spec->epoll_fd, EPOLL_CTL_ADD, fd, &_ev) != 0) {
mtevL(mtev_error, "epoll_ctl(%d, add, %d, %d)\n", spec->epoll_fd, fd, errno);
}
release_master_fd(fd, lockstate);
}
if(e != master_fds[fd].e) {
mtevL(mtev_error, "Incoming event: %p, does not match master list: %p\n", e, master_fds[fd].e);
return;
}
lockstate = acquire_master_fd(fd);
if(lockstate == EV_ALREADY_OWNED) {
mtevL(eventer_deb, "Incoming event: %p already owned by this thread\n", e);
return;
}
mtevAssert(lockstate == EV_OWNED);
mtev_gettimeofday(&__now, NULL);
cbname = eventer_name_for_callback_e(e->callback, e);
spec = eventer_get_spec_for_event(e);
mtevLT(eventer_deb, &__now, "epoll(%d): fire on %d/%x to %s(%p)\n",
spec->epoll_fd, fd, mask, cbname?cbname:"???", e->callback);
mtev_memory_begin();
LIBMTEV_EVENTER_CALLBACK_ENTRY((void *)e, (void *)e->callback, (char *)cbname, fd, e->mask, mask);
start = mtev_gethrtime();
newmask = e->callback(e, mask, e->closure, &__now);
duration = mtev_gethrtime() - start;
LIBMTEV_EVENTER_CALLBACK_RETURN((void *)e, (void *)e->callback, (char *)cbname, newmask);
mtev_memory_end();
stats_set_hist_intscale(eventer_callback_latency, duration, -9, 1);
stats_set_hist_intscale(eventer_latency_handle_for_callback(e->callback), duration, -9, 1);
if(newmask) {
struct epoll_event _ev;
memset(&_ev, 0, sizeof(_ev));
_ev.data.fd = fd;
if(newmask & EVENTER_READ) _ev.events |= (EPOLLIN|EPOLLPRI);
if(newmask & EVENTER_WRITE) _ev.events |= (EPOLLOUT);
if(newmask & EVENTER_EXCEPTION) _ev.events |= (EPOLLERR|EPOLLHUP);
if(master_fds[fd].e == NULL) {
mtevL(mtev_debug, "eventer %s(%p) epoll asked to modify descheduled fd: %d\n",
cbname?cbname:"???", e->callback, fd);
} else {
if(!pthread_equal(pthread_self(), e->thr_owner)) {
pthread_t tgt = e->thr_owner;
e->thr_owner = pthread_self();
spec = eventer_get_spec_for_event(e);
if(e->mask != 0 && !needs_add) {
mtevL(eventer_deb, "epoll_ctl(%d, del, %d)\n", spec->epoll_fd, fd);
if(epoll_ctl(spec->epoll_fd, EPOLL_CTL_DEL, fd, &_ev) != 0) {
mtevFatal(mtev_error,
"epoll_ctl(spec->epoll_fd, EPOLL_CTL_DEL, fd, &_ev) failed; "
"spec->epoll_fd: %d; fd: %d; errno: %d (%s)\n",
spec->epoll_fd, fd, errno, strerror(errno));
}
}
e->thr_owner = tgt;
spec = eventer_get_spec_for_event(e);
mtevL(eventer_deb, "epoll_ctl(%d, add, %d)\n", spec->epoll_fd, fd);
mtevAssert(epoll_ctl(spec->epoll_fd, EPOLL_CTL_ADD, fd, &_ev) == 0);
mtevL(eventer_deb, "epoll(%d) moved event[%p] from t@%d to t@%d\n", spec->epoll_fd, e, (int)pthread_self(), (int)tgt);
}
else {
int epoll_rv;
int epoll_cmd = (e->mask == 0 || needs_add) ? EPOLL_CTL_ADD : EPOLL_CTL_MOD;
spec = eventer_get_spec_for_event(e);
mtevL(eventer_deb, "epoll_ctl(%d, %s, %d)\n", spec->epoll_fd, epoll_cmd == EPOLL_CTL_ADD ? "add" : "mod", fd);
epoll_rv = epoll_ctl(spec->epoll_fd, epoll_cmd, fd, &_ev);
if(epoll_rv != 0 &&
((epoll_cmd == EPOLL_CTL_ADD && errno == EEXIST) ||
(epoll_cmd == EPOLL_CTL_MOD && errno == ENOENT))) {
/* try the other way */
epoll_cmd = (epoll_cmd == EPOLL_CTL_ADD) ? EPOLL_CTL_MOD : EPOLL_CTL_ADD;
mtevL(eventer_deb, "retry epoll_ctl(%d, %s, %d)\n", spec->epoll_fd, epoll_cmd == EPOLL_CTL_ADD ? "add" : "mod", fd);
epoll_rv = epoll_ctl(spec->epoll_fd, epoll_cmd, fd, &_ev);
}
if(epoll_rv != 0) {
const char *cb_name = eventer_name_for_callback_e(e->callback, e);
mtevFatal(mtev_error,
"epoll_ctl(spec->epoll_fd, %s, fd, &_ev) failed; "
"spec->epoll_fd: %d; fd: %d; errno: %d (%s); callback: %s\n",
epoll_cmd == EPOLL_CTL_ADD ? "EPOLL_CTL_ADD" : "EPOLL_CTL_MOD",
spec->epoll_fd, fd, errno, strerror(errno), cb_name ? cb_name : "???");
}
}
}
/* Set our mask */
e->mask = newmask;
}
else {
/* see kqueue implementation for details on the next line */
if(master_fds[fd].e == e) {
/* if newmask == 0 the user has floated the connection. If we get here
* and they have not called `eventer_remove_fd` it is a misuse of mtev.
*
* Check if they are compliant with floats here and remove_fd if they
* forgot to and warn in the log
*/
spec = eventer_get_spec_for_event(e);
struct epoll_event _ev;
memset(&_ev, 0, sizeof(_ev));
_ev.data.fd = fd;
if (epoll_ctl(spec->epoll_fd, EPOLL_CTL_DEL, e->fd, &_ev) == 0) {
mtevL(mtev_error, "WARNING: You forgot to 'eventer_remove_fd()' before returning a mask of zero.\n");
}
master_fds[fd].e = NULL;
}
eventer_free(e);
}
release_master_fd(fd, lockstate);
}
static int ping_icmp_handler(eventer_t e, int mask,
void *closure, struct timeval *now,
u_int8_t family) {
noit_module_t *self = (noit_module_t *)closure;
ping_icmp_data_t *ping_data;
struct check_info *data;
char packet[1500];
int packet_len = sizeof(packet);
union {
struct sockaddr_in in4;
struct sockaddr_in6 in6;
} from;
unsigned int from_len;
struct ping_payload *payload;
if(family != AF_INET && family != AF_INET6) return EVENTER_READ;
ping_data = noit_module_get_userdata(self);
while(1) {
struct ping_session_key k;
int inlen;
u_int8_t iphlen = 0;
void *vcheck;
noit_check_t *check;
struct timeval tt, whence;
from_len = sizeof(from);
inlen = recvfrom(e->fd, packet, packet_len, 0,
(struct sockaddr *)&from, &from_len);
mtev_gettimeofday(now, NULL); /* set it, as we care about accuracy */
if(inlen < 0) {
if(errno == EAGAIN || errno == EINTR) break;
mtevLT(nldeb, now, "ping_icmp recvfrom: %s\n", strerror(errno));
break;
}
if(family == AF_INET) {
struct icmp *icp4;
iphlen = ((struct ip *)packet)->ip_hl << 2;
if((inlen-iphlen) != sizeof(struct icmp)+PING_PAYLOAD_LEN) {
mtevLT(nldeb, now,
"ping_icmp bad size: %d+%d\n", iphlen, inlen-iphlen);
continue;
}
icp4 = (struct icmp *)(packet + iphlen);
payload = (struct ping_payload *)(icp4 + 1);
if(icp4->icmp_type != ICMP_ECHOREPLY) {
mtevLT(nldeb, now, "ping_icmp bad type: %d\n", icp4->icmp_type);
continue;
}
if(icp4->icmp_id != (((vpsized_uint)self) & 0xffff)) {
mtevLT(nldeb, now,
"ping_icmp not sent from this instance (%d:%d) vs. %lu\n",
icp4->icmp_id, ntohs(icp4->icmp_seq),
(unsigned long)(((vpsized_uint)self) & 0xffff));
continue;
}
}
else if(family == AF_INET6) {
struct icmp6_hdr *icp6 = (struct icmp6_hdr *)packet;
if((inlen) != sizeof(struct icmp6_hdr)+PING_PAYLOAD_LEN) {
mtevLT(nldeb, now,
"ping_icmp bad size: %d+%d\n", iphlen, inlen-iphlen);
continue;
}
payload = (struct ping_payload *)(icp6+1);
if(icp6->icmp6_type != ICMP6_ECHO_REPLY) {
mtevLT(nldeb, now, "ping_icmp bad type: %d\n", icp6->icmp6_type);
continue;
}
if(icp6->icmp6_id != (((vpsized_uint)self) & 0xffff)) {
mtevLT(nldeb, now,
"ping_icmp not sent from this instance (%d:%d) vs. %lu\n",
icp6->icmp6_id, ntohs(icp6->icmp6_seq),
(unsigned long)(((vpsized_uint)self) & 0xffff));
continue;
}
}
else {
/* This should be unreachable */
continue;
}
check = NULL;
k.addr_of_check = payload->addr_of_check;
uuid_copy(k.checkid, payload->checkid);
if(mtev_hash_retrieve(ping_data->in_flight,
(const char *)&k, sizeof(k),
&vcheck))
check = vcheck;
/* make sure this check is from this generation! */
if(!check) {
char uuid_str[37];
uuid_unparse_lower(payload->checkid, uuid_str);
mtevLT(nldeb, now,
"ping_icmp response for unknown check '%s'\n", uuid_str);
continue;
}
if((check->generation & 0xffff) != payload->generation) {
mtevLT(nldeb, now,
"ping_icmp response in generation gap\n");
continue;
}
data = (struct check_info *)check->closure;
/* If there is no timeout_event, the check must have completed.
* We have nothing to do. */
if(!data->timeout_event) continue;
/* Sanity check the payload */
if(payload->check_no != data->check_no) continue;
if(payload->check_pack_cnt != data->expected_count) continue;
if(payload->check_pack_no >= data->expected_count) continue;
whence.tv_sec = payload->tv_sec;
whence.tv_usec = payload->tv_usec;
sub_timeval(*now, whence, &tt);
data->turnaround[payload->check_pack_no] =
(float)tt.tv_sec + (float)tt.tv_usec / 1000000.0;
if(ping_icmp_is_complete(self, check)) {
ping_icmp_log_results(self, check);
eventer_remove(data->timeout_event);
free(data->timeout_event->closure);
eventer_free(data->timeout_event);
data->timeout_event = NULL;
check->flags &= ~NP_RUNNING;
k.addr_of_check = (vpsized_uint)check ^ random_num;
uuid_copy(k.checkid, check->checkid);
mtev_hash_delete(ping_data->in_flight, (const char *)&k,
sizeof(k), free, NULL);
}
}
return EVENTER_READ;
}
static void eventer_epoll_impl_trigger(eventer_t e, int mask) {
struct epoll_spec *spec;
struct timeval __now;
int fd, newmask;
const char *cbname;
ev_lock_state_t lockstate;
int cross_thread = mask & EVENTER_CROSS_THREAD_TRIGGER;
int added_to_master_fds = 0;
u_int64_t start, duration;
mask = mask & ~(EVENTER_RESERVED);
fd = e->fd;
if(cross_thread) {
if(master_fds[fd].e != NULL) {
mtevL(eventer_deb, "Attempting to trigger already-registered event fd: %d cross thread.\n", fd);
}
/* mtevAssert(master_fds[fd].e == NULL); */
}
if(!pthread_equal(pthread_self(), e->thr_owner)) {
/* If we're triggering across threads, it can't be registered yet */
if(master_fds[fd].e != NULL) {
mtevL(eventer_deb, "Attempting to trigger already-registered event fd: %d cross thread.\n", fd);
}
/* mtevAssert(master_fds[fd].e == NULL); */
eventer_cross_thread_trigger(e,mask);
return;
}
if(master_fds[fd].e == NULL) {
master_fds[fd].e = e;
e->mask = 0;
added_to_master_fds = 1;
}
if(e != master_fds[fd].e) return;
lockstate = acquire_master_fd(fd);
if(lockstate == EV_ALREADY_OWNED) return;
mtevAssert(lockstate == EV_OWNED);
mtev_gettimeofday(&__now, NULL);
cbname = eventer_name_for_callback_e(e->callback, e);
mtevLT(eventer_deb, &__now, "epoll: fire on %d/%x to %s(%p)\n",
fd, mask, cbname?cbname:"???", e->callback);
mtev_memory_begin();
LIBMTEV_EVENTER_CALLBACK_ENTRY((void *)e, (void *)e->callback, (char *)cbname, fd, e->mask, mask);
start = mtev_gethrtime();
newmask = e->callback(e, mask, e->closure, &__now);
duration = mtev_gethrtime() - start;
LIBMTEV_EVENTER_CALLBACK_RETURN((void *)e, (void *)e->callback, (char *)cbname, newmask);
mtev_memory_end();
stats_set_hist_intscale(eventer_callback_latency, duration, -9, 1);
stats_set_hist_intscale(eventer_latency_handle_for_callback(e->callback), duration, -9, 1);
if(newmask) {
struct epoll_event _ev;
memset(&_ev, 0, sizeof(_ev));
_ev.data.fd = fd;
if(newmask & EVENTER_READ) _ev.events |= (EPOLLIN|EPOLLPRI);
if(newmask & EVENTER_WRITE) _ev.events |= (EPOLLOUT);
if(newmask & EVENTER_EXCEPTION) _ev.events |= (EPOLLERR|EPOLLHUP);
if(master_fds[fd].e == NULL) {
mtevL(mtev_debug, "eventer %s(%p) epoll asked to modify descheduled fd: %d\n",
cbname?cbname:"???", e->callback, fd);
} else {
if(!pthread_equal(pthread_self(), e->thr_owner)) {
pthread_t tgt = e->thr_owner;
e->thr_owner = pthread_self();
spec = eventer_get_spec_for_event(e);
if(! added_to_master_fds && epoll_ctl(spec->epoll_fd, EPOLL_CTL_DEL, fd, &_ev) != 0) {
mtevFatal(mtev_error,
"epoll_ctl(spec->epoll_fd, EPOLL_CTL_DEL, fd, &_ev) failed; "
"spec->epoll_fd: %d; fd: %d; errno: %d (%s)\n",
spec->epoll_fd, fd, errno, strerror(errno));
}
e->thr_owner = tgt;
spec = eventer_get_spec_for_event(e);
mtevAssert(epoll_ctl(spec->epoll_fd, EPOLL_CTL_ADD, fd, &_ev) == 0);
mtevL(eventer_deb, "moved event[%p] from t@%d to t@%d\n", e, (int)pthread_self(), (int)tgt);
}
else {
int epoll_cmd = added_to_master_fds ? EPOLL_CTL_ADD : EPOLL_CTL_MOD;
spec = eventer_get_spec_for_event(e);
if(epoll_ctl(spec->epoll_fd, epoll_cmd, fd, &_ev) != 0) {
const char *cb_name = eventer_name_for_callback_e(e->callback, e);
mtevFatal(mtev_error,
"epoll_ctl(spec->epoll_fd, EPOLL_CTL_MOD, fd, &_ev) failed; "
"spec->epoll_fd: %d; fd: %d; errno: %d (%s); callback: %s\n",
spec->epoll_fd, fd, errno, strerror(errno), cb_name ? cb_name : "???");
}
}
}
/* Set our mask */
e->mask = newmask;
}
else {
/* see kqueue implementation for details on the next line */
if(master_fds[fd].e == e) master_fds[fd].e = NULL;
eventer_free(e);
}
release_master_fd(fd, lockstate);
}
static void
mtev_capabilities_tobuff_json(mtev_capsvc_closure_t *cl, eventer_func_t curr) {
const char **mod_names;
struct utsname utsn;
char vbuff[128];
mtev_hash_table *lc;
mtev_hash_iter iter = MTEV_HASH_ITER_ZERO;
int i, nmods;
struct timeval now;
struct dso_type *t;
struct json_object *doc;
struct json_object *svcs, *bi, *ri, *mods, *feat;
/* fill out capabilities */
/* Create an XML Document */
doc = json_object_new_object();
/* Fill in the document */
mtev_build_version(vbuff, sizeof(vbuff));
json_object_object_add(doc, "version", json_object_new_string(vbuff));
/* Build info */
bi = json_object_new_object();
json_object_object_add(bi, "bitwidth", json_object_new_int(sizeof(void *)*8));
json_object_object_add(bi, "sysname", json_object_new_string(UNAME_S));
json_object_object_add(bi, "nodename", json_object_new_string(UNAME_N));
json_object_object_add(bi, "release", json_object_new_string(UNAME_R));
json_object_object_add(bi, "version", json_object_new_string(UNAME_V));
json_object_object_add(bi, "machine", json_object_new_string(UNAME_M));
json_object_object_add(doc, "unameBuild", bi);
/* Run info */
ri = json_object_new_object();
json_object_object_add(ri, "bitwidth", json_object_new_int(sizeof(void *)*8));
if(uname(&utsn) < 0) {
json_object_object_add(ri, "error", json_object_new_string(strerror(errno)));
} else {
json_object_object_add(ri, "sysname", json_object_new_string(utsn.sysname));
json_object_object_add(ri, "nodename", json_object_new_string(utsn.nodename));
json_object_object_add(ri, "release", json_object_new_string(utsn.release));
json_object_object_add(ri, "version", json_object_new_string(utsn.version));
json_object_object_add(ri, "machine", json_object_new_string(utsn.machine));
}
json_object_object_add(doc, "unameRun", ri);
/* features */
feat = json_object_new_object();
if(mtev_hash_size(&features)) {
mtev_hash_iter iter2 = MTEV_HASH_ITER_ZERO;
while(mtev_hash_adv(&features, &iter2)) {
struct json_object *featnode;
featnode = json_object_new_object();
if(iter2.value.str) json_object_object_add(featnode, "version", json_object_new_string(iter2.value.str));
json_object_object_add(feat, iter2.key.str, featnode);
}
}
json_object_object_add(doc, "features", feat);
/* time (poor man's time check) */
mtev_gettimeofday(&now, NULL);
snprintf(vbuff, sizeof(vbuff), "%llu%03d", (unsigned long long)now.tv_sec,
(int)(now.tv_usec / 1000));
json_object_object_add(doc, "current_time", json_object_new_string(vbuff));
svcs = json_object_new_object();
lc = mtev_listener_commands();
while(mtev_hash_adv(lc, &iter)) {
struct json_object *cnode, *cmds;
char hexcode[11];
const char *name;
eventer_func_t *f = (eventer_func_t *)iter.key.ptr;
mtev_hash_table *sc = (mtev_hash_table *)iter.value.ptr;
mtev_hash_iter sc_iter = MTEV_HASH_ITER_ZERO;
name = eventer_name_for_callback(*f);
cnode = json_object_new_object();
if(iter.klen == 8)
snprintf(hexcode, sizeof(hexcode), "0x%0llx",
(unsigned long long int)(vpsized_uint)**f);
else
snprintf(hexcode, sizeof(hexcode), "0x%0x",
(unsigned int)(vpsized_uint)**f);
json_object_object_add(svcs, hexcode, cnode);
if(name) json_object_object_add(cnode, name, json_object_new_string(name));
cmds = json_object_new_object();
json_object_object_add(cnode, "commands", cmds);
while(mtev_hash_adv(sc, &sc_iter)) {
struct json_object *scnode;
char *name_copy, *version = NULL;
eventer_func_t *f = (eventer_func_t *)sc_iter.value.ptr;
scnode = json_object_new_object();
snprintf(hexcode, sizeof(hexcode), "0x%08x", *((u_int32_t *)sc_iter.key.ptr));
name = eventer_name_for_callback(*f);
name_copy = strdup(name ? name : "[[unknown]]");
version = strchr(name_copy, '/');
if(version) *version++ = '\0';
json_object_object_add(scnode, "name", json_object_new_string(name_copy));
if(version) json_object_object_add(scnode, "version", json_object_new_string(version));
json_object_object_add(cmds, hexcode, scnode);
free(name_copy);
}
}
json_object_object_add(doc, "services", svcs);
mods = json_object_new_object();
#define list_modules_json(func, name) do { \
nmods = func(&mod_names); \
for(i=0; i<nmods; i++) { \
struct json_object *pnode; \
pnode = json_object_new_object(); \
json_object_object_add(pnode, "type", json_object_new_string(name)); \
json_object_object_add(mods, mod_names[i], pnode); \
} \
if(mod_names) free(mod_names); \
} while(0)
for(t = mtev_dso_get_types(); t; t = t->next)
list_modules_json(t->list, t->name);
json_object_object_add(doc, "modules", mods);
/* Write it out to a buffer and copy it for writing */
cl->buff = strdup(json_object_to_json_string(doc));
cl->towrite = strlen(cl->buff);
/* Clean up after ourselves */
json_object_put(doc);
}
static void
mtev_capabilities_tobuff(mtev_capsvc_closure_t *cl, eventer_func_t curr) {
const char **mod_names;
struct utsname utsn;
char vbuff[128], bwstr[4];
mtev_hash_table *lc;
mtev_hash_iter iter = MTEV_HASH_ITER_ZERO;
int i, nmods;
struct timeval now;
struct dso_type *t;
xmlDocPtr xmldoc;
xmlNodePtr root, cmds, bi, ri, mods, feat;
/* fill out capabilities */
/* Create an XML Document */
xmldoc = xmlNewDoc((xmlChar *)"1.0");
root = xmlNewDocNode(xmldoc, NULL, (xmlChar *)capabilities_namespace, NULL);
xmlDocSetRootElement(xmldoc, root);
/* Fill in the document */
mtev_build_version(vbuff, sizeof(vbuff));
xmlNewTextChild(root, NULL, (xmlChar *)"version", (xmlChar *)vbuff);
snprintf(bwstr, sizeof(bwstr), "%d", (int)sizeof(void *)*8);
/* Build info */
bi = xmlNewNode(NULL, (xmlChar *)"unameBuild");
xmlSetProp(bi, (xmlChar *)"bitwidth", (xmlChar *)bwstr);
xmlAddChild(root, bi);
xmlNewTextChild(bi, NULL, (xmlChar *)"sysname", (xmlChar *)UNAME_S);
xmlNewTextChild(bi, NULL, (xmlChar *)"nodename", (xmlChar *)UNAME_N);
xmlNewTextChild(bi, NULL, (xmlChar *)"release", (xmlChar *)UNAME_R);
xmlNewTextChild(bi, NULL, (xmlChar *)"version", (xmlChar *)UNAME_V);
xmlNewTextChild(bi, NULL, (xmlChar *)"machine", (xmlChar *)UNAME_M);
/* Run info */
ri = xmlNewNode(NULL, (xmlChar *)"unameRun");
xmlSetProp(ri, (xmlChar *)"bitwidth", (xmlChar *)bwstr);
xmlAddChild(root, ri);
if(uname(&utsn) < 0) {
xmlNewTextChild(ri, NULL, (xmlChar *)"error", (xmlChar *)strerror(errno));
} else {
xmlNewTextChild(ri, NULL, (xmlChar *)"sysname", (xmlChar *)utsn.sysname);
xmlNewTextChild(ri, NULL, (xmlChar *)"nodename", (xmlChar *)utsn.nodename);
xmlNewTextChild(ri, NULL, (xmlChar *)"release", (xmlChar *)utsn.release);
xmlNewTextChild(ri, NULL, (xmlChar *)"version", (xmlChar *)utsn.version);
xmlNewTextChild(ri, NULL, (xmlChar *)"machine", (xmlChar *)utsn.machine);
}
/* features */
feat = xmlNewNode(NULL, (xmlChar *)"features");
xmlAddChild(root, feat);
if(mtev_hash_size(&features)) {
mtev_hash_iter iter2 = MTEV_HASH_ITER_ZERO;
while(mtev_hash_adv(&features, &iter2)) {
xmlNodePtr featnode;
featnode = xmlNewNode(NULL, (xmlChar *)"feature");
xmlSetProp(featnode, (xmlChar *)"name", (xmlChar *)iter2.key.str);
if(iter2.value.str)
xmlSetProp(featnode, (xmlChar *)"version", (xmlChar *)iter2.value.str);
xmlAddChild(feat, featnode);
}
}
/* time (poor man's time check) */
mtev_gettimeofday(&now, NULL);
snprintf(vbuff, sizeof(vbuff), "%llu.%03d", (unsigned long long)now.tv_sec,
(int)(now.tv_usec / 1000));
xmlNewTextChild(root, NULL, (xmlChar *)"current_time", (xmlChar *)vbuff);
cmds = xmlNewNode(NULL, (xmlChar *)"services");
xmlAddChild(root, cmds);
lc = mtev_listener_commands();
while(mtev_hash_adv(lc, &iter)) {
xmlNodePtr cnode;
char hexcode[11];
const char *name;
eventer_func_t *f = (eventer_func_t *)iter.key.ptr;
mtev_hash_table *sc = (mtev_hash_table *)iter.value.ptr;
mtev_hash_iter sc_iter = MTEV_HASH_ITER_ZERO;
name = eventer_name_for_callback(*f);
cnode = xmlNewNode(NULL, (xmlChar *)"service");
xmlSetProp(cnode, (xmlChar *)"name", name ? (xmlChar *)name : NULL);
if(*f == curr)
xmlSetProp(cnode, (xmlChar *)"connected", (xmlChar *)"true");
xmlAddChild(cmds, cnode);
while(mtev_hash_adv(sc, &sc_iter)) {
xmlNodePtr scnode;
char *name_copy, *version = NULL;
eventer_func_t *f = (eventer_func_t *)sc_iter.value.ptr;
snprintf(hexcode, sizeof(hexcode), "0x%08x", *((u_int32_t *)sc_iter.key.ptr));
name = eventer_name_for_callback(*f);
name_copy = strdup(name ? name : "[[unknown]]");
version = strchr(name_copy, '/');
if(version) *version++ = '\0';
scnode = xmlNewNode(NULL, (xmlChar *)"command");
xmlSetProp(scnode, (xmlChar *)"name", (xmlChar *)name_copy);
if(version)
xmlSetProp(scnode, (xmlChar *)"version", (xmlChar *)version);
xmlSetProp(scnode, (xmlChar *)"code", (xmlChar *)hexcode);
xmlAddChild(cnode, scnode);
free(name_copy);
}
}
mods = xmlNewNode(NULL, (xmlChar *)"modules");
xmlAddChild(root, mods);
#define list_modules(func, name) do { \
nmods = func(&mod_names); \
for(i=0; i<nmods; i++) { \
xmlNodePtr pnode; \
pnode = xmlNewNode(NULL, (xmlChar *)"module"); \
xmlSetProp(pnode, (xmlChar *)"type", (xmlChar *)name); \
xmlSetProp(pnode, (xmlChar *)"name", (xmlChar *)mod_names[i]); \
xmlAddChild(mods, pnode); \
} \
if(mod_names) free(mod_names); \
} while(0)
for(t = mtev_dso_get_types(); t; t = t->next)
list_modules(t->list, t->name);
/* Write it out to a buffer and copy it for writing */
cl->buff = mtev_xmlSaveToBuffer(xmldoc);
cl->towrite = strlen(cl->buff);
/* Clean up after ourselves */
xmlFreeDoc(xmldoc);
}
static int eventer_ports_impl_loop() {
struct timeval __dyna_sleep = { 0, 0 };
struct ports_spec *spec;
spec = eventer_get_spec_for_event(NULL);
while(1) {
struct timeval __now, __sleeptime;
struct timespec __ports_sleeptime;
unsigned int fd_cnt = 0;
int ret;
port_event_t pevents[MAX_PORT_EVENTS];
mtev_gettimeofday(&__now, NULL);
if(compare_timeval(eventer_max_sleeptime, __dyna_sleep) < 0)
__dyna_sleep = eventer_max_sleeptime;
__sleeptime = __dyna_sleep;
eventer_dispatch_timed(&__now, &__sleeptime);
if(compare_timeval(__sleeptime, __dyna_sleep) > 0)
__sleeptime = __dyna_sleep;
/* Handle cross_thread dispatches */
eventer_cross_thread_process();
/* Handle recurrent events */
eventer_dispatch_recurrent(&__now);
/* Now we move on to our fd-based events */
__ports_sleeptime.tv_sec = __sleeptime.tv_sec;
__ports_sleeptime.tv_nsec = __sleeptime.tv_usec * 1000;
fd_cnt = 1;
pevents[0].portev_source = 65535; /* This is impossible */
ret = port_getn(spec->port_fd, pevents, MAX_PORT_EVENTS, &fd_cnt,
&__ports_sleeptime);
spec->wakeup_notify = 0; /* force unlock */
/* The timeout case is a tad complex with ports. -1/ETIME is clearly
* a timeout. However, it i spossible that we got that and fd_cnt isn't
* 0, which means we both timed out and got events... WTF?
*/
if(fd_cnt == 0 ||
(ret == -1 && errno == ETIME && pevents[0].portev_source == 65535))
add_timeval(__dyna_sleep, __dyna_increment, &__dyna_sleep);
if(ret == -1 && (errno != ETIME && errno != EINTR))
mtevLT(eventer_err, &__now, "port_getn: %s\n", strerror(errno));
if(ret < 0)
mtevLT(eventer_deb, &__now, "port_getn: %s\n", strerror(errno));
mtevLT(eventer_deb, &__now, "debug: port_getn(%d, [], %d) => %d\n",
spec->port_fd, fd_cnt, ret);
if(pevents[0].portev_source == 65535) {
/* the impossible still remains, which means our fd_cnt _must_ be 0 */
fd_cnt = 0;
}
if(fd_cnt > 0) {
int idx;
/* Loop a last time to process */
__dyna_sleep.tv_sec = __dyna_sleep.tv_usec = 0; /* reset */
for(idx = 0; idx < fd_cnt; idx++) {
port_event_t *pe;
eventer_t e;
int fd, mask;
pe = &pevents[idx];
if(pe->portev_source != PORT_SOURCE_FD) continue;
fd = (int)pe->portev_object;
mtevAssert((vpsized_int)pe->portev_user == fd);
e = master_fds[fd].e;
/* It's possible that someone removed the event and freed it
* before we got here.... bail out if we're null.
*/
if (!e) continue;
mask = 0;
if(pe->portev_events & (POLLIN | POLLHUP))
mask |= EVENTER_READ;
if(pe->portev_events & (POLLOUT))
mask |= EVENTER_WRITE;
if(pe->portev_events & (POLLERR | POLLHUP | POLLNVAL))
mask |= EVENTER_EXCEPTION;
eventer_ports_impl_trigger(e, mask);
}
}
}
/* NOTREACHED */
return 0;
}
