static mtev_boolean
check_duplicate(char *payload, size_t payload_len) {
if (dedupe) {
unsigned char *digest = malloc(MD5_DIGEST_LENGTH);
MD5((unsigned char*)payload, payload_len, digest);
uint64_t whence = get_message_time(payload, payload_len);
if(whence > 0) {
mtev_hash_table *hash = get_dedupe_hash(whence);
if (hash) {
int x = mtev_hash_store(hash, (const char *)digest, MD5_DIGEST_LENGTH, (void *)0x1);
if (x == 0) {
/* this is a dupe */
free(digest);
return mtev_true;
}
} else {
free(digest);
}
} else {
free(digest);
}
}
return mtev_false;
}
static mtev_hash_table *
get_dedupe_hash(uint64_t whence)
{
struct hash_and_time *hash_with_time;
mtev_hash_table *hash;
if (mtev_hash_retrieve(&dedupe_hashes, (const char *)&whence, sizeof(whence), (void **)&hash_with_time) == 1) {
hash = &hash_with_time->hash;
} else {
hash_with_time = calloc(1, sizeof(struct hash_and_time));
mtev_hash_init_locks(&hash_with_time->hash, MTEV_HASH_DEFAULT_SIZE, MTEV_HASH_LOCK_MODE_MUTEX);
uint64_t *stored_ts = calloc(1, sizeof(uint64_t));
*stored_ts = whence;
if (mtev_hash_store(&dedupe_hashes, (const char *)stored_ts, sizeof(*stored_ts), hash_with_time) == 0) {
/* ugh, someone beat us */
free(stored_ts);
mtev_hash_destroy(&hash_with_time->hash, NULL, NULL);
free(hash_with_time);
if (mtev_hash_retrieve(&dedupe_hashes, (const char *)&whence, sizeof(whence), (void **)&hash_with_time) == 0) {
return NULL;
}
}
hash = &hash_with_time->hash;
}
hash_with_time->last_touched_s = mtev_gethrtime() / 1000000000;
return hash;
}
void
mtev_capabilities_add_feature(const char *feature, const char *version) {
feature = strdup(feature);
if(version) version = strdup(version);
if(!mtev_hash_store(&features, feature, strlen(feature), (void *)version))
mtevL(mtev_error, "Feature conflict! %s version %s\n",
feature, version ? version : "unpecified");
}
void
noit_adjust_metric_interest(uuid_t id, const char *metric, short cnt) {
int thread_id, icnt;
void *vhash, *vinterests;
mtev_hash_table *level2;
caql_cnt_t *interests;
thread_id = get_my_lane();
/* Get us our UUID->HASH(METRICS) mapping */
if(!mtev_hash_retrieve(&id_level, (const char *)id, UUID_SIZE, &vhash)) {
int found;
uuid_t *copy = malloc(UUID_SIZE);
level2 = calloc(1,sizeof(*level2));
mtev_hash_init_locks(level2, MTEV_HASH_DEFAULT_SIZE, MTEV_HASH_LOCK_MODE_MUTEX);
uuid_copy(*copy, id);
if(!mtev_hash_store(&id_level, (const char *)copy, UUID_SIZE, level2)) {
free(copy);
free(level2);
}
found = mtev_hash_retrieve(&id_level, (const char *)id, UUID_SIZE, &vhash);
mtevAssert(found);
}
level2 = vhash;
if(!mtev_hash_retrieve(level2, metric, strlen(metric), &vinterests)) {
int found;
char *metric_copy;
metric_copy = strdup(metric);
vinterests = calloc(nthreads, sizeof(*interests));
if(!mtev_hash_store(level2, metric_copy, strlen(metric_copy), vinterests)) {
free(metric_copy);
free(vinterests);
}
found = mtev_hash_retrieve(level2, metric, strlen(metric), &vinterests);
mtevAssert(found);
}
interests = vinterests;
/* This is fine because thread_id is only ours */
icnt = interests[thread_id];
icnt += cnt;
if(icnt < 0) icnt = 0;
mtevAssert(icnt <= 0xffff);
interests[thread_id] = icnt;
}
void noit_console_conf_checks_init() {
int i;
for(i=0;i<sizeof(valid_attrs)/sizeof(*valid_attrs);i++) {
mtev_hash_store(&check_attrs,
valid_attrs[i].name, strlen(valid_attrs[i].name),
&valid_attrs[i]);
}
register_console_config_check_commands();
}
jlog_feed_stats_t *
noit_jlog_feed_stats(const char *sub) {
void *vs = NULL;
jlog_feed_stats_t *s = NULL;
if(mtev_hash_retrieve(&feed_stats, sub, strlen(sub), &vs))
return (jlog_feed_stats_t *)vs;
s = calloc(1, sizeof(*s));
s->feed_name = strdup(sub);
mtev_hash_store(&feed_stats, s->feed_name, strlen(s->feed_name), s);
return s;
}
static mtev_hook_return_t
histogram_hook_special_impl(void *closure, noit_check_t *check, stats_t *stats,
const char *metric_name, metric_type_t type, const char *v,
mtev_boolean success) {
void *vht;
histotier *ht;
mtev_hash_table *config, *metrics;
const char *track = "";
mtev_dso_generic_t *self = closure;
struct histogram_config *conf = mtev_image_get_userdata(&self->hdr);
if(success) return MTEV_HOOK_CONTINUE;
config = noit_check_get_module_config(check, histogram_module_id);
if(!config || mtev_hash_size(config) == 0) return MTEV_HOOK_CONTINUE;
mtev_hash_retr_str(config, metric_name, strlen(metric_name), &track);
if(!track || strcmp(track, "add"))
return MTEV_HOOK_CONTINUE;
metrics = noit_check_get_module_metadata(check, histogram_module_id);
if(!metrics) {
metrics = calloc(1, sizeof(*metrics));
noit_check_set_module_metadata(check, histogram_module_id,
metrics, free_hash_o_histotier);
}
if(!mtev_hash_retrieve(metrics, metric_name, strlen(metric_name),
&vht)) {
ht = calloc(1, sizeof(*ht));
vht = ht;
mtev_hash_store(metrics, strdup(metric_name), strlen(metric_name),
vht);
}
else ht = vht;
if(v != NULL) {
/* We expect: H[<float>]=%d */
const char *lhs;
char *endptr;
double bucket;
u_int64_t cnt;
if(v[0] != 'H' || v[1] != '[') return MTEV_HOOK_CONTINUE;
if(NULL == (lhs = strchr(v+2, ']'))) return MTEV_HOOK_CONTINUE;
lhs++;
if(*lhs++ != '=') return MTEV_HOOK_CONTINUE;
bucket = strtod(v+2, &endptr);
if(endptr == v+2) return MTEV_HOOK_CONTINUE;
cnt = strtoull(lhs, &endptr, 10);
if(endptr == lhs) return MTEV_HOOK_CONTINUE;
update_histotier(ht, time(NULL), conf, check, metric_name, bucket, cnt);
}
return MTEV_HOOK_CONTINUE;
}
void *thread_func(void *arg)
{
mtev_hash_table *h = arg;
pthread_t me = pthread_self();
char *key = NULL;
for (int i = 0; i < 100; i++) {
asprintf(&key, "%lu-%d", (unsigned long) me, i);
mtev_hash_store(h, key, strlen(key), key);
}
return NULL;
}
int
noit_check_etc_hosts_cache_refresh(eventer_t e, int mask, void *closure,
struct timeval *now) {
static struct stat last_stat;
struct stat sb;
struct hostent *ent;
int reload = 0;
memset(&sb, 0, sizeof(sb));
stat("/etc/hosts", &sb);
#define CSTAT(f) (sb.f == last_stat.f)
reload = ! (CSTAT(st_dev) && CSTAT(st_ino) && CSTAT(st_mode) && CSTAT(st_uid) &&
CSTAT(st_gid) && CSTAT(st_size) && CSTAT(st_mtime));
memcpy(&last_stat, &sb, sizeof(sb));
if(reload) {
mtev_hash_delete_all(&etc_hosts_cache, free, free);
while(NULL != (ent = gethostent())) {
int i = 0;
char *name = ent->h_name;
while(name) {
void *vnode;
static_host_node *node;
if(!mtev_hash_retrieve(&etc_hosts_cache, name, strlen(name), &vnode)) {
vnode = node = calloc(1, sizeof(*node));
node->target = strdup(name);
mtev_hash_store(&etc_hosts_cache, node->target, strlen(node->target), node);
}
node = vnode;
if(ent->h_addrtype == AF_INET) {
node->has_ip4 = 1;
memcpy(&node->ip4, ent->h_addr_list[0], ent->h_length);
}
if(ent->h_addrtype == AF_INET6) {
node->has_ip6 = 1;
memcpy(&node->ip6, ent->h_addr_list[0], ent->h_length);
}
name = ent->h_aliases[i++];
}
}
endhostent();
mtevL(noit_debug, "reloaded %d /etc/hosts targets\n", mtev_hash_size(&etc_hosts_cache));
}
eventer_add_in_s_us(noit_check_etc_hosts_cache_refresh, NULL, 1, 0);
return 0;
}
static mtev_hook_return_t
histogram_hook_impl(void *closure, noit_check_t *check, stats_t *stats,
metric_t *m) {
void *vht;
histotier *ht;
mtev_hash_table *config, *metrics;
const char *track = "";
mtev_dso_generic_t *self = closure;
struct histogram_config *conf = mtev_image_get_userdata(&self->hdr);
config = noit_check_get_module_config(check, histogram_module_id);
if(!config || mtev_hash_size(config) == 0) return MTEV_HOOK_CONTINUE;
mtev_hash_retr_str(config, m->metric_name, strlen(m->metric_name), &track);
if(!track || strcmp(track, "add"))
return MTEV_HOOK_CONTINUE;
metrics = noit_check_get_module_metadata(check, histogram_module_id);
if(!metrics) {
metrics = calloc(1, sizeof(*metrics));
noit_check_set_module_metadata(check, histogram_module_id,
metrics, free_hash_o_histotier);
}
if(!mtev_hash_retrieve(metrics, m->metric_name, strlen(m->metric_name),
&vht)) {
ht = calloc(1, sizeof(*ht));
vht = ht;
mtev_hash_store(metrics, strdup(m->metric_name), strlen(m->metric_name),
vht);
}
else ht = vht;
if(m->metric_value.vp != NULL) {
#define UPDATE_HISTOTIER(a) update_histotier(ht, time(NULL), conf, check, m->metric_name, *m->metric_value.a, 1)
switch(m->metric_type) {
case METRIC_UINT64:
UPDATE_HISTOTIER(L); break;
case METRIC_INT64:
UPDATE_HISTOTIER(l); break;
case METRIC_UINT32:
UPDATE_HISTOTIER(I); break;
case METRIC_INT32:
UPDATE_HISTOTIER(i); break;
case METRIC_DOUBLE:
UPDATE_HISTOTIER(n); break;
default: /*noop*/
break;
}
}
return MTEV_HOOK_CONTINUE;
}
static ssl_ctx_cache_node *
ssl_ctx_cache_set(ssl_ctx_cache_node *node) {
void *vnode;
mtev_atomic32_t newval;
pthread_mutex_lock(&ssl_ctx_cache_lock);
if(mtev_hash_retrieve(&ssl_ctx_cache, node->key, strlen(node->key),
&vnode)) {
node = vnode;
}
else {
mtev_hash_store(&ssl_ctx_cache, node->key, strlen(node->key), node);
}
newval = mtev_atomic_inc32(&node->refcnt);
pthread_mutex_unlock(&ssl_ctx_cache_lock);
mtevL(eventer_deb, "ssl_ctx_cache->set(%p -> %d)\n", node, newval);
return node;
}
static void noit_fq_set_filters(mq_command_t *commands, int count) {
int i, j;
if (!global_fq_ctx.filtered_exchange[0]) {
mtevL(mtev_error, "ERROR: trying to set check for filtered exchange when no such exchange exists\n");
return;
}
for (i=0; i<count; i++) {
if (commands[i].action == MQ_ACTION_SET) {
mtev_hash_table *metric_table = calloc(1, sizeof(mtev_hash_table));
mtev_hash_init(metric_table);
for (j=0; j<commands[i].check.metric_count; j++) {
mtev_hash_store(metric_table, strdup(commands[i].check.metrics[j]), strlen(commands[i].check.metrics[j]), NULL);
filtered_metrics_exist = true;
}
mtev_hash_replace(&filtered_checks_hash, strdup(commands[i].check.uuid), strlen(commands[i].check.uuid), metric_table, free, noit_fq_free_metric_hash);
}
else {
if (commands[i].check.metric_count == 0) {
/* Forget the whole check */
if (!mtev_hash_delete(&filtered_checks_hash, commands[i].check.uuid, strlen(commands[i].check.uuid), free, noit_fq_free_metric_hash)) {
mtevL(mtev_error, "failed forgetting check %s - check does not exist\n", commands[i].check.uuid);
}
}
else {
mtev_hash_table *filtered_metrics;
if(mtev_hash_retrieve(&filtered_checks_hash, commands[i].check.uuid, strlen(commands[i].check.uuid), (void**)&filtered_metrics)) {
for (j=0; j<commands[i].check.metric_count; j++) {
if (!mtev_hash_delete(filtered_metrics, commands[i].check.metrics[j], strlen(commands[i].check.metrics[j]),
free, noit_fq_free_metric_hash)) {
mtevL(mtev_error, "failed forgetting metric '%s' for check %s - metric does not exist\n",
commands[i].check.metrics[j], commands[i].check.uuid);
}
}
}
else {
mtevL(mtev_error, "failed forgetting metrics for check %s - check does not exist\n", commands[i].check.uuid);
}
}
}
}
}
int mtev_load_image(const char *file, const char *name,
mtev_hash_table *registry,
int (*validate)(mtev_image_t *),
size_t obj_size) {
char module_file[PATH_MAX];
const char *dlsymname;
void *dlhandle = NULL;
void *dlsymbol;
mtev_image_t *obj;
if(file[0] == '/') {
strlcpy(module_file, file, sizeof(module_file));
dlhandle = dlopen(module_file, RTLD_LAZY | RTLD_GLOBAL);
}
else {
char *basepath, *base, *brk;
if(!mtev_conf_get_string(MTEV_CONF_ROOT, "//modules/@directory", &basepath))
basepath = strdup(MTEV_MODULES_DIR);
for (base = strtok_r(basepath, ";:", &brk);
base;
base = strtok_r(NULL, ";:", &brk)) {
snprintf(module_file, sizeof(module_file), "%s/%s.%s",
base, file, MODULEEXT);
dlhandle = dlopen(module_file, RTLD_LAZY | RTLD_GLOBAL);
if(dlhandle) {
mtevL(mtev_debug, "Successfully opened image '%s'\n",
module_file);
break;
} else {
mtevL(mtev_debug, "Tried to open image '%s' but failed: %s\n",
module_file, dlerror());
}
}
free(basepath);
if(!dlhandle) {
snprintf(module_file, sizeof(module_file), "%s/%s.%s",
MTEV_MODULES_DIR, file, MODULEEXT);
dlhandle = dlopen(module_file, RTLD_LAZY | RTLD_GLOBAL);
}
}
if(!dlhandle) {
mtevL(mtev_stderr, "Cannot open image '%s': %s\n",
module_file, dlerror());
return -1;
}
dlsymname = strrchr(name, ':');
if(!dlsymname) dlsymname = name;
else dlsymname++;
dlsymbol = dlsym(dlhandle, dlsymname);
if(!dlsymbol) {
mtevL(mtev_stderr, "Cannot find '%s' in image '%s': %s\n",
dlsymname, module_file, dlerror());
dlclose(dlhandle);
return -1;
}
if(validate(dlsymbol) == -1) {
mtevL(mtev_stderr, "I can't understand module %s\n", name);
dlclose(dlhandle);
return -1;
}
obj = calloc(1, obj_size);
memcpy(obj, dlsymbol, obj_size);
obj->opaque_handle = calloc(1, sizeof(struct __extended_image_data));
if(obj->onload && obj->onload(obj)) {
free(obj->opaque_handle);
free(obj);
dlclose(dlhandle);
return -1;
}
char *namecopy = strdup(name);
if(!mtev_hash_store(registry, namecopy, strlen(namecopy), obj)) {
mtevL(mtev_error, "Attempted to load module %s more than once.\n", name);
dlclose(dlhandle);
free(namecopy);
return -1;
}
((struct __extended_image_data *)obj->opaque_handle)->dlhandle = dlhandle;
return 0;
}
static interim_journal_t *
interim_journal_get(struct sockaddr *remote, const char *remote_cn_in,
int storagenode_id, const char *fqdn_in) {
void *vhash, *vij;
mtev_hash_table *working_set;
interim_journal_t *ij;
struct timeval now;
char jpath[PATH_MAX];
char remote_str[128];
const char *remote_cn = remote_cn_in ? remote_cn_in : "default";
const char *fqdn = fqdn_in ? fqdn_in : "default";
mtev_convert_sockaddr_to_buff(remote_str, sizeof(remote_str), remote);
if(!*remote_str) strlcpy(remote_str, "default", sizeof(remote_str));
/* Lookup the working set */
if(!mtev_hash_retrieve(&working_sets, remote_cn, strlen(remote_cn), &vhash)) {
working_set = calloc(1, sizeof(*working_set));
mtev_hash_store(&working_sets, strdup(remote_cn), strlen(remote_cn),
working_set);
}
else
working_set = vhash;
/* Lookup the interim journal within the working set */
if(!mtev_hash_retrieve(working_set, fqdn, strlen(fqdn), &vij)) {
ij = calloc(1, sizeof(*ij));
gettimeofday(&now, NULL);
snprintf(jpath, sizeof(jpath), "%s/%s/%s/%d/%08x%08x.tmp",
basejpath, remote_str, remote_cn, storagenode_id,
(unsigned int)now.tv_sec, (unsigned int)now.tv_usec);
ij->remote_str = strdup(remote_str);
ij->remote_cn = strdup(remote_cn);
ij->fqdn = fqdn_in ? strdup(fqdn_in) : NULL;
ij->storagenode_id = storagenode_id;
ij->filename = strdup(jpath);
ij->fd = open(ij->filename, O_RDWR | O_CREAT | O_EXCL, 0640);
if(ij->fd < 0 && errno == ENOENT) {
if(mkdir_for_file(ij->filename, 0750)) {
mtevL(noit_error, "Failed to create dir for '%s': %s\n",
ij->filename, strerror(errno));
exit(-1);
}
ij->fd = open(ij->filename, O_RDWR | O_CREAT | O_EXCL, 0640);
}
if(ij->fd < 0 && errno == EEXIST) {
/* This can only occur if we crash after before checkpointing */
unlink(ij->filename);
ij->fd = open(ij->filename, O_RDWR | O_CREAT | O_EXCL, 0640);
}
if(ij->fd < 0) {
mtevL(noit_error, "Failed to open interim journal '%s': %s\n",
ij->filename, strerror(errno));
exit(-1);
}
mtev_hash_store(working_set, strdup(fqdn), strlen(fqdn), ij);
}
else
ij = vij;
return ij;
}
int
noit_filtersets_cull_unused() {
mtev_hash_table active = MTEV_HASH_EMPTY;
char *buffer = NULL;
mtev_conf_section_t *declares;
int i, n_uses = 0, n_declares = 0, removed = 0;
const char *declare_xpath = "//filterset[@name and not (@cull='false')]";
declares = mtev_conf_get_sections(NULL, declare_xpath, &n_declares);
if(declares) {
/* store all unit filtersets used */
for(i=0;i<n_declares;i++) {
if(!buffer) buffer = malloc(128);
if(mtev_conf_get_stringbuf(declares[i], "@name", buffer, 128)) {
if(mtev_hash_store(&active, buffer, strlen(buffer), declares[i])) {
buffer = NULL;
}
else {
void *vnode = NULL;
/* We've just hit a duplicate.... check to see if there's an existing
* entry and if there is, load the latest one and delete the old
* one. */
mtev_hash_retrieve(&active, buffer, strlen(buffer), &vnode);
if (vnode) {
noit_filter_compile_add(declares[i]);
CONF_REMOVE(vnode);
xmlUnlinkNode(vnode);
xmlFreeNode(vnode);
removed++;
if(mtev_hash_replace(&active, buffer, strlen(buffer), declares[i], free, NULL)) {
buffer = NULL;
}
}
}
}
}
if(buffer) free(buffer);
free(declares);
}
n_uses = noit_poller_do(filterset_accum, &active);
if(n_uses > 0 && mtev_hash_size(&active) > 0) {
mtev_hash_iter iter = MTEV_HASH_ITER_ZERO;
const char *filter_name;
int filter_name_len;
void *vnode;
while(mtev_hash_next(&active, &iter, &filter_name, &filter_name_len,
&vnode)) {
if(noit_filter_remove(vnode)) {
CONF_REMOVE(vnode);
xmlUnlinkNode(vnode);
xmlFreeNode(vnode);
removed++;
}
}
}
mtev_hash_destroy(&active, free, NULL);
return removed;
}
