static lua_module_closure_t *noit_lua_setup_lmc(noit_module_t *mod, const char **obj) {
lua_module_closure_t *lmc;
struct module_conf *mc;
struct module_tls_conf *mtlsc;
mc = mtev_image_get_userdata(&mod->hdr);
if(obj) *obj = mc->object;
lmc = pthread_getspecific(mc->c->key);
if(lmc == NULL) {
int rv;
lmc = calloc(1, sizeof(*lmc));
lmc->pending = calloc(1, sizeof(*lmc->pending));
mtev_hash_init(lmc->pending);
lmc->owner = pthread_self();
lmc->resume = noit_lua_check_resume;
pthread_setspecific(mc->c->key, lmc);
mtevL(nldeb, "lua_state[%s]: new state\n", mod->hdr.name);
lmc->lua_state = mtev_lua_open(mod->hdr.name, lmc,
mc->c->script_dir, mc->c->cpath);
require(lmc->lua_state, rv, noit);
}
mtlsc = __get_module_tls_conf(&mod->hdr);
if(!mtlsc->loaded) {
if(mod->hdr.onload(&mod->hdr) == -1) {
return NULL;
}
mtlsc->loaded = 1;
}
mtevL(nldeb, "lua_state[%s]: %p\n", mod->hdr.name, lmc->lua_state);
return lmc;
}
void mtev_hash_delete_all(mtev_hash_table *h, NoitHashFreeFunc keyfree, NoitHashFreeFunc datafree) {
void *entry = NULL;
ck_hs_iterator_t iterator = CK_HS_ITERATOR_INITIALIZER;
ck_hash_attr_t *data_struct;
if(!h) return;
if(h->u.hs.hf == NULL) {
mtevL(mtev_error, "warning: null hashtable in mtev_hash_delete_all... initializing\n");
mtev_stacktrace(mtev_error);
mtev_hash_init(h);
}
int count = mtev_hash_size(h);
LOCK(h);
while(ck_hs_next(&h->u.hs, &iterator, &entry)) {
data_struct = index_attribute_container((ck_key_t*)entry);
if (data_struct) {
if (keyfree) keyfree(data_struct->key_ptr);
if (datafree) datafree(data_struct->data);
free(data_struct);
}
}
ck_hs_reset_size(&h->u.hs, count);
UNLOCK(h);
}
static int
noit_lua_module_config(noit_module_t *mod,
mtev_hash_table *options) {
struct module_conf *mc;
struct module_tls_conf *mtlsc;
LMC_DECL(L, mod, object);
mc = noit_module_get_userdata(mod);
if(options) {
mtevAssert(mc->options == NULL);
mc->options = calloc(1, sizeof(*mc->options));
mtev_hash_init(mc->options);
mtev_hash_merge_as_dict(mc->options, options);
}
else options = mc->options;
mtlsc = __get_module_tls_conf(&mod->hdr);
if(mtlsc->configured) return mtlsc->configured_return;
SETUP_CALL(L, object, "config", return 0);
noit_lua_setup_module(L, mod);
mtev_lua_hash_to_table(L, options);
lua_pcall(L, 2, 1, 0);
/* If rv == 0, the caller will free options. We've
* already freed options, that would be bad. fudge -> 1 */
RETURN_INT(L, object, "config",
{ mtlsc->configured = 1; mtlsc->configured_return = rv; });
int mtev_hash_size(mtev_hash_table *h) {
if(h->u.hs.hf == NULL) {
mtevL(mtev_error, "warning: null hashtable in mtev_hash_size... initializing\n");
mtev_stacktrace(mtev_error);
mtev_hash_init(h);
}
return ck_hs_count(&h->u.hs);
}
static int noit_fq_driver_init(mtev_dso_generic_t *self) {
if(!nlerr) nlerr = mtev_log_stream_find("error/fq_driver");
if(!nlerr) nlerr = mtev_error;
mtev_hash_init(&filtered_checks_hash);
stratcon_iep_mq_driver_register("fq", &mq_driver_fq);
register_console_fq_commands();
eventer_add_in_s_us(fq_status_checker, NULL, 0, 0);
return 0;
}
void mtev_hash_destroy(mtev_hash_table *h, NoitHashFreeFunc keyfree, NoitHashFreeFunc datafree) {
if(!h) return;
if(h->u.hs.hf == NULL) {
mtevL(mtev_error, "warning: null hashtable in mtev_hash_destroy... initializing\n");
mtev_stacktrace(mtev_error);
mtev_hash_init(h);
}
mtev_hash_delete_all(h, keyfree, datafree);
LOCK(h);
ck_hs_destroy(&h->u.hs);
UNLOCK(h);
mtev_hash_destroy_locks(h);
free(h->u.locks.locks);
}
static void noit_fq_set_filters(mq_command_t *commands, int count) {
int i, j;
if (!global_fq_ctx.filtered_exchange[0]) {
mtevL(mtev_error, "ERROR: trying to set check for filtered exchange when no such exchange exists\n");
return;
}
for (i=0; i<count; i++) {
if (commands[i].action == MQ_ACTION_SET) {
mtev_hash_table *metric_table = calloc(1, sizeof(mtev_hash_table));
mtev_hash_init(metric_table);
for (j=0; j<commands[i].check.metric_count; j++) {
mtev_hash_store(metric_table, strdup(commands[i].check.metrics[j]), strlen(commands[i].check.metrics[j]), NULL);
filtered_metrics_exist = true;
}
mtev_hash_replace(&filtered_checks_hash, strdup(commands[i].check.uuid), strlen(commands[i].check.uuid), metric_table, free, noit_fq_free_metric_hash);
}
else {
if (commands[i].check.metric_count == 0) {
/* Forget the whole check */
if (!mtev_hash_delete(&filtered_checks_hash, commands[i].check.uuid, strlen(commands[i].check.uuid), free, noit_fq_free_metric_hash)) {
mtevL(mtev_error, "failed forgetting check %s - check does not exist\n", commands[i].check.uuid);
}
}
else {
mtev_hash_table *filtered_metrics;
if(mtev_hash_retrieve(&filtered_checks_hash, commands[i].check.uuid, strlen(commands[i].check.uuid), (void**)&filtered_metrics)) {
for (j=0; j<commands[i].check.metric_count; j++) {
if (!mtev_hash_delete(filtered_metrics, commands[i].check.metrics[j], strlen(commands[i].check.metrics[j]),
free, noit_fq_free_metric_hash)) {
mtevL(mtev_error, "failed forgetting metric '%s' for check %s - metric does not exist\n",
commands[i].check.metrics[j], commands[i].check.uuid);
}
}
}
else {
mtevL(mtev_error, "failed forgetting metrics for check %s - check does not exist\n", commands[i].check.uuid);
}
}
}
}
}
int mtev_hash_delete(mtev_hash_table *h, const char *k, int klen,
NoitHashFreeFunc keyfree, NoitHashFreeFunc datafree) {
long hashv;
ck_hash_attr_t *data_struct;
ck_key_t *retrieved_key;
union {
ck_key_t key;
char pad[sizeof(ck_key_t) + ONSTACK_KEY_SIZE];
} onstack_key;
ck_key_t *key = &onstack_key.key;
if(!h) return 0;
if(h->u.hs.hf == NULL) {
mtevL(mtev_error, "warning: null hashtable in mtev_hash_delete... initializing\n");
mtev_stacktrace(mtev_error);
mtev_hash_init(h);
}
if(klen > ONSTACK_KEY_SIZE) key = calloc(1, sizeof(ck_key_t) + klen + 1);
memcpy(key->label, k, klen);
key->label[klen] = 0;
key->len = klen + sizeof(uint32_t);
hashv = CK_HS_HASH(&h->u.hs, hs_hash, key);
LOCK(h);
retrieved_key = ck_hs_remove(&h->u.hs, hashv, key);
UNLOCK(h);
if (retrieved_key) {
data_struct = index_attribute_container(retrieved_key);
if (data_struct) {
if (keyfree) keyfree(data_struct->key_ptr);
if (datafree) datafree(data_struct->data);
free(data_struct);
if(key != &onstack_key.key) free(key);
return 1;
}
}
if(key != &onstack_key.key) free(key);
return 0;
}
/* mtev_hash_next(_str) should not use anything in iter past the
* ck_hs_iterator_t b/c older consumers could have the smaller
* version of the mtev_hash_iter allocated on stack.
*/
int mtev_hash_next(mtev_hash_table *h, mtev_hash_iter *iter,
const char **k, int *klen, void **data) {
void *cursor = NULL;
ck_key_t *key;
ck_hash_attr_t *data_struct;
if(h->u.hs.hf == NULL) {
mtevL(mtev_error, "warning: null hashtable in mtev_hash_next... initializing\n");
mtev_stacktrace(mtev_error);
mtev_hash_init(h);
}
if(!ck_hs_next(&h->u.hs, &iter->iter, &cursor)) return 0;
key = (ck_key_t *)cursor;
data_struct = index_attribute_container(key);
if (data_struct) {
*k = data_struct->key_ptr;
*klen = data_struct->key.len - sizeof(uint32_t);
*data = data_struct->data;
}
return 1;
}
static int ping_icmp_init(noit_module_t *self) {
socklen_t on;
struct protoent *proto;
ping_icmp_data_t *data;
RAND_pseudo_bytes((unsigned char *)&random_num, sizeof(vpsized_uint));
data = malloc(sizeof(*data));
data->in_flight = calloc(1, sizeof(*data->in_flight));
mtev_hash_init(data->in_flight);
data->ipv4_fd = data->ipv6_fd = -1;
if ((proto = getprotobyname("icmp")) == NULL) {
mtevL(noit_error, "Couldn't find 'icmp' protocol\n");
free(data);
return -1;
}
data->ipv4_fd = socket(AF_INET, NE_SOCK_CLOEXEC|SOCK_RAW, proto->p_proto);
if(data->ipv4_fd < 0) {
mtevL(noit_error, "ping_icmp: socket failed: %s\n",
strerror(errno));
}
else {
socklen_t slen = sizeof(on);
if(getsockopt(data->ipv4_fd, SOL_SOCKET, SO_SNDBUF, &on, &slen) == 0) {
if(on <= 0) on = 1024;
while(on < (1 << 20)) {
on <<= 1;
if(setsockopt(data->ipv4_fd, SOL_SOCKET, SO_SNDBUF,
&on, sizeof(on)) != 0) {
on >>= 1;
break;
}
}
mtevL(noit_debug, "ping_icmp: send buffer set to %d\n", on);
}
else
int mtev_hash_replace(mtev_hash_table *h, const char *k, int klen, void *data,
NoitHashFreeFunc keyfree, NoitHashFreeFunc datafree) {
long hashv;
int ret;
void *retrieved_key = NULL;
ck_hash_attr_t *data_struct;
ck_hash_attr_t *attr = calloc(1, sizeof(ck_hash_attr_t) + klen + 1);
if(h->u.hs.hf == NULL) {
mtevL(mtev_error, "warning: null hashtable in mtev_hash_replace... initializing\n");
mtev_stacktrace(mtev_error);
mtev_hash_init(h);
}
memcpy(attr->key.label, k, klen);
attr->key.label[klen] = 0;
attr->key.len = klen + sizeof(uint32_t);
attr->data = data;
attr->key_ptr = (char*)k;
hashv = CK_HS_HASH(&h->u.hs, hs_hash, &attr->key);
LOCK(h);
ret = ck_hs_set(&h->u.hs, hashv, &attr->key, &retrieved_key);
UNLOCK(h);
if (ret) {
if (retrieved_key) {
data_struct = index_attribute_container(retrieved_key);
if (data_struct) {
if (keyfree) keyfree(data_struct->key_ptr);
if (datafree) datafree(data_struct->data);
}
free(data_struct);
}
}
else {
free(attr);
}
return 1;
}
int mtev_hash_store(mtev_hash_table *h, const char *k, int klen, void *data) {
long hashv;
int ret = 0;
ck_hash_attr_t *attr = calloc(1, sizeof(ck_hash_attr_t) + klen + 1);
if(h->u.hs.hf == NULL) {
mtevL(mtev_error, "warning: null hashtable in mtev_hash_store... initializing\n");
mtev_stacktrace(mtev_error);
mtev_hash_init(h);
}
memcpy(attr->key.label, k, klen);
attr->key.label[klen] = 0;
attr->key.len = klen + sizeof(uint32_t);
attr->key_ptr = (char*)k;
attr->data = data;
hashv = CK_HS_HASH(&h->u.hs, hs_hash, &attr->key);
LOCK(h);
ret = ck_hs_put(&h->u.hs, hashv, &attr->key);
UNLOCK(h);
if (!ret) free(attr);
return ret;
}
static interim_journal_t *
interim_journal_get(struct sockaddr *remote, const char *remote_cn_in,
int storagenode_id, const char *fqdn_in) {
void *vhash, *vij;
mtev_hash_table *working_set;
interim_journal_t *ij;
struct timeval now;
char jpath[PATH_MAX];
char remote_str[128];
const char *remote_cn = remote_cn_in ? remote_cn_in : "default";
const char *fqdn = fqdn_in ? fqdn_in : "default";
mtev_convert_sockaddr_to_buff(remote_str, sizeof(remote_str), remote);
if(!*remote_str) strlcpy(remote_str, "default", sizeof(remote_str));
/* Lookup the working set */
if(!mtev_hash_retrieve(&working_sets, remote_cn, strlen(remote_cn), &vhash)) {
working_set = calloc(1, sizeof(*working_set));
mtev_hash_init(working_set);
mtev_hash_store(&working_sets, strdup(remote_cn), strlen(remote_cn),
working_set);
}
else
working_set = vhash;
/* Lookup the interim journal within the working set */
if(!mtev_hash_retrieve(working_set, fqdn, strlen(fqdn), &vij)) {
ij = calloc(1, sizeof(*ij));
mtev_gettimeofday(&now, NULL);
snprintf(jpath, sizeof(jpath), "%s/%s/%s/%d/%08x%08x.tmp",
basejpath, remote_str, remote_cn, storagenode_id,
(unsigned int)now.tv_sec, (unsigned int)now.tv_usec);
ij->remote_str = strdup(remote_str);
ij->remote_cn = strdup(remote_cn);
ij->fqdn = fqdn_in ? strdup(fqdn_in) : NULL;
ij->storagenode_id = storagenode_id;
ij->filename = strdup(jpath);
ij->fd = open(ij->filename, O_RDWR | O_CREAT | O_EXCL, 0640);
if(ij->fd < 0 && errno == ENOENT) {
if(mkdir_for_file(ij->filename, 0750)) {
mtevL(noit_error, "Failed to create dir for '%s': %s\n",
ij->filename, strerror(errno));
exit(-1);
}
ij->fd = open(ij->filename, O_RDWR | O_CREAT | O_EXCL, 0640);
}
if(ij->fd < 0 && errno == EEXIST) {
/* This can only occur if we crash after before checkpointing */
unlink(ij->filename);
ij->fd = open(ij->filename, O_RDWR | O_CREAT | O_EXCL, 0640);
}
if(ij->fd < 0) {
mtevL(noit_error, "Failed to open interim journal '%s': %s\n",
ij->filename, strerror(errno));
exit(-1);
}
mtev_hash_store(working_set, strdup(fqdn), strlen(fqdn), ij);
}
else
ij = vij;
return ij;
}
void
noit_jlog_listener_init_globals(void) {
mtev_hash_init(&feed_stats);
}
void noit_check_resolver_init() {
int cnt;
mtev_conf_section_t *servers, *searchdomains;
eventer_t e;
if(dns_init(NULL, 0) < 0)
mtevL(noit_error, "dns initialization failed.\n");
dns_ctx = dns_new(NULL);
if(dns_init(dns_ctx, 0) != 0) {
mtevL(noit_error, "dns initialization failed.\n");
exit(-1);
}
/* Optional servers */
servers = mtev_conf_get_sections(NULL, "//resolver//server", &cnt);
if(cnt) {
int i;
char server[128];
dns_add_serv(dns_ctx, NULL); /* reset */
for(i=0;i<cnt;i++) {
if(mtev_conf_get_stringbuf(servers[i], "self::node()",
server, sizeof(server))) {
if(dns_add_serv(dns_ctx, server) < 0) {
mtevL(noit_error, "Failed adding DNS server: %s\n", server);
}
}
}
free(servers);
}
searchdomains = mtev_conf_get_sections(NULL, "//resolver//search", &cnt);
if(cnt) {
int i;
char search[128];
dns_add_srch(dns_ctx, NULL); /* reset */
for(i=0;i<cnt;i++) {
if(mtev_conf_get_stringbuf(searchdomains[i], "self::node()",
search, sizeof(search))) {
if(dns_add_srch(dns_ctx, search) < 0) {
mtevL(noit_error, "Failed adding DNS search path: %s\n", search);
}
else if(dns_search_flag) dns_search_flag = 0; /* enable search */
}
}
free(searchdomains);
}
if(mtev_conf_get_int(NULL, "//resolver/@ndots", &cnt))
dns_set_opt(dns_ctx, DNS_OPT_NDOTS, cnt);
if(mtev_conf_get_int(NULL, "//resolver/@ntries", &cnt))
dns_set_opt(dns_ctx, DNS_OPT_NTRIES, cnt);
if(mtev_conf_get_int(NULL, "//resolver/@timeout", &cnt))
dns_set_opt(dns_ctx, DNS_OPT_TIMEOUT, cnt);
if(dns_open(dns_ctx) < 0) {
mtevL(noit_error, "dns open failed.\n");
exit(-1);
}
eventer_name_callback("dns_cache_callback", dns_cache_callback);
dns_set_tmcbck(dns_ctx, dns_cache_utm_fn, dns_ctx);
e = eventer_alloc();
e->mask = EVENTER_READ | EVENTER_EXCEPTION;
e->closure = dns_ctx;
e->callback = dns_cache_callback;
e->fd = dns_sock(dns_ctx);
eventer_add(e);
mtev_skiplist_init(&nc_dns_cache);
mtev_skiplist_set_compare(&nc_dns_cache, name_lookup, name_lookup_k);
mtev_skiplist_add_index(&nc_dns_cache, refresh_idx, refresh_idx_k);
/* maybe load it from cache */
if(noit_resolver_cache_load_hook_exists()) {
struct timeval now;
char *key;
void *data;
int len;
gettimeofday(&now, NULL);
while(noit_resolver_cache_load_hook_invoke(&key, &data, &len) == MTEV_HOOK_CONTINUE) {
dns_cache_node *n;
n = calloc(1, sizeof(*n));
if(dns_cache_node_deserialize(n, data, len) >= 0) {
n->target = strdup(key);
/* if the TTL indicates that it will expire in less than 60 seconds
* (including stuff that should have already expired), then fudge
* the last_updated time to make it expire some random time within
* the next 60 seconds.
*/
if(n->last_needed > now.tv_sec || n->last_updated > now.tv_sec)
break; /* impossible */
n->last_needed = now.tv_sec;
if(n->last_updated + n->ttl < now.tv_sec + 60) {
int fudge = MIN(60, n->ttl) + 1;
n->last_updated = now.tv_sec - n->ttl + (lrand48() % fudge);
}
DCLOCK();
mtev_skiplist_insert(&nc_dns_cache, n);
DCUNLOCK();
n = NULL;
}
else {
mtevL(noit_error, "Failed to deserialize resolver cache record.\n");
}
if(n) dns_cache_node_free(n);
if(key) free(key);
if(data) free(data);
}
}
noit_check_resolver_loop(NULL, 0, NULL, NULL);
register_console_dns_cache_commands();
mtev_hash_init(&etc_hosts_cache);
noit_check_etc_hosts_cache_refresh(NULL, 0, NULL, NULL);
}
void
noit_conf_checks_init_globals(void) {
mtev_hash_init(&check_attrs);
}
static int
mtev_lua_general_init(mtev_dso_generic_t *self) {
const char * const *module;
int (*f)(lua_State *);
lua_general_conf_t *conf = get_config(self);
lua_module_closure_t *lmc = pthread_getspecific(conf->key);
if(lmc) return 0;
if(!lmc) {
lmc = calloc(1, sizeof(*lmc));
lmc->self = self;
mtev_hash_init(&lmc->state_coros);
pthread_setspecific(conf->key, lmc);
}
if(!conf->module || !conf->function) {
mtevL(nlerr, "lua_general cannot be used without module and function config\n");
return -1;
}
lmc->resume = lua_general_resume;
lmc->owner = pthread_self();
lmc->eventer_id = eventer_is_loop(lmc->owner);
lmc->lua_state = mtev_lua_open(self->hdr.name, lmc,
conf->script_dir, conf->cpath);
mtevL(nldeb, "lua_general opening state -> %p\n", lmc->lua_state);
if(lmc->lua_state == NULL) {
mtevL(mtev_error, "lua_general could not add general functions\n");
return -1;
}
luaL_openlib(lmc->lua_state, "mtev", general_lua_funcs, 0);
/* Load some preloads */
for(module = conf->Cpreloads; module && *module; module++) {
int len;
char *symbol = NULL;
len = strlen(*module) + strlen("luaopen_");
symbol = malloc(len+1);
if(!symbol) mtevL(nlerr, "Failed to preload %s: malloc error\n", *module);
else {
snprintf(symbol, len+1, "luaopen_%s", *module);
#ifdef RTLD_DEFAULT
f = dlsym(RTLD_DEFAULT, symbol);
#else
f = dlsym((void *)0, symbol);
#endif
if(!f) mtevL(nlerr, "Failed to preload %s: %s not found\n", *module, symbol);
else f(lmc->lua_state);
}
free(symbol);
}
lmc->pending = calloc(1, sizeof(*lmc->pending));
mtev_hash_init(lmc->pending);
if(conf->booted) return true;
conf->booted = mtev_true;
eventer_add_in_s_us(dispatch_general, self, 0, 0);
if(conf->concurrent) {
int i = 1;
pthread_t tgt, thr;
thr = eventer_choose_owner(i++);
do {
eventer_t e = eventer_in_s_us(dispatch_general, self, 0, 0);
tgt = eventer_choose_owner(i++);
eventer_set_owner(e, tgt);
eventer_add(e);
} while(!pthread_equal(thr,tgt));
}
return 0;
}
void
stratcon_datastore_init_globals(void) {
mtev_hash_init(&working_sets);
}
