static mtev_boolean check_duplicate(char *payload, size_t payload_len) { if (dedupe) { unsigned char *digest = malloc(MD5_DIGEST_LENGTH); MD5((unsigned char*)payload, payload_len, digest); uint64_t whence = get_message_time(payload, payload_len); if(whence > 0) { mtev_hash_table *hash = get_dedupe_hash(whence); if (hash) { int x = mtev_hash_store(hash, (const char *)digest, MD5_DIGEST_LENGTH, (void *)0x1); if (x == 0) { /* this is a dupe */ free(digest); return mtev_true; } } else { free(digest); } } else { free(digest); } } return mtev_false; }
static mtev_hash_table * get_dedupe_hash(uint64_t whence) { struct hash_and_time *hash_with_time; mtev_hash_table *hash; if (mtev_hash_retrieve(&dedupe_hashes, (const char *)&whence, sizeof(whence), (void **)&hash_with_time) == 1) { hash = &hash_with_time->hash; } else { hash_with_time = calloc(1, sizeof(struct hash_and_time)); mtev_hash_init_locks(&hash_with_time->hash, MTEV_HASH_DEFAULT_SIZE, MTEV_HASH_LOCK_MODE_MUTEX); uint64_t *stored_ts = calloc(1, sizeof(uint64_t)); *stored_ts = whence; if (mtev_hash_store(&dedupe_hashes, (const char *)stored_ts, sizeof(*stored_ts), hash_with_time) == 0) { /* ugh, someone beat us */ free(stored_ts); mtev_hash_destroy(&hash_with_time->hash, NULL, NULL); free(hash_with_time); if (mtev_hash_retrieve(&dedupe_hashes, (const char *)&whence, sizeof(whence), (void **)&hash_with_time) == 0) { return NULL; } } hash = &hash_with_time->hash; } hash_with_time->last_touched_s = mtev_gethrtime() / 1000000000; return hash; }
void mtev_capabilities_add_feature(const char *feature, const char *version) { feature = strdup(feature); if(version) version = strdup(version); if(!mtev_hash_store(&features, feature, strlen(feature), (void *)version)) mtevL(mtev_error, "Feature conflict! %s version %s\n", feature, version ? version : "unpecified"); }
void noit_adjust_metric_interest(uuid_t id, const char *metric, short cnt) { int thread_id, icnt; void *vhash, *vinterests; mtev_hash_table *level2; caql_cnt_t *interests; thread_id = get_my_lane(); /* Get us our UUID->HASH(METRICS) mapping */ if(!mtev_hash_retrieve(&id_level, (const char *)id, UUID_SIZE, &vhash)) { int found; uuid_t *copy = malloc(UUID_SIZE); level2 = calloc(1,sizeof(*level2)); mtev_hash_init_locks(level2, MTEV_HASH_DEFAULT_SIZE, MTEV_HASH_LOCK_MODE_MUTEX); uuid_copy(*copy, id); if(!mtev_hash_store(&id_level, (const char *)copy, UUID_SIZE, level2)) { free(copy); free(level2); } found = mtev_hash_retrieve(&id_level, (const char *)id, UUID_SIZE, &vhash); mtevAssert(found); } level2 = vhash; if(!mtev_hash_retrieve(level2, metric, strlen(metric), &vinterests)) { int found; char *metric_copy; metric_copy = strdup(metric); vinterests = calloc(nthreads, sizeof(*interests)); if(!mtev_hash_store(level2, metric_copy, strlen(metric_copy), vinterests)) { free(metric_copy); free(vinterests); } found = mtev_hash_retrieve(level2, metric, strlen(metric), &vinterests); mtevAssert(found); } interests = vinterests; /* This is fine because thread_id is only ours */ icnt = interests[thread_id]; icnt += cnt; if(icnt < 0) icnt = 0; mtevAssert(icnt <= 0xffff); interests[thread_id] = icnt; }
void noit_console_conf_checks_init() { int i; for(i=0;i<sizeof(valid_attrs)/sizeof(*valid_attrs);i++) { mtev_hash_store(&check_attrs, valid_attrs[i].name, strlen(valid_attrs[i].name), &valid_attrs[i]); } register_console_config_check_commands(); }
jlog_feed_stats_t * noit_jlog_feed_stats(const char *sub) { void *vs = NULL; jlog_feed_stats_t *s = NULL; if(mtev_hash_retrieve(&feed_stats, sub, strlen(sub), &vs)) return (jlog_feed_stats_t *)vs; s = calloc(1, sizeof(*s)); s->feed_name = strdup(sub); mtev_hash_store(&feed_stats, s->feed_name, strlen(s->feed_name), s); return s; }
static mtev_hook_return_t histogram_hook_special_impl(void *closure, noit_check_t *check, stats_t *stats, const char *metric_name, metric_type_t type, const char *v, mtev_boolean success) { void *vht; histotier *ht; mtev_hash_table *config, *metrics; const char *track = ""; mtev_dso_generic_t *self = closure; struct histogram_config *conf = mtev_image_get_userdata(&self->hdr); if(success) return MTEV_HOOK_CONTINUE; config = noit_check_get_module_config(check, histogram_module_id); if(!config || mtev_hash_size(config) == 0) return MTEV_HOOK_CONTINUE; mtev_hash_retr_str(config, metric_name, strlen(metric_name), &track); if(!track || strcmp(track, "add")) return MTEV_HOOK_CONTINUE; metrics = noit_check_get_module_metadata(check, histogram_module_id); if(!metrics) { metrics = calloc(1, sizeof(*metrics)); noit_check_set_module_metadata(check, histogram_module_id, metrics, free_hash_o_histotier); } if(!mtev_hash_retrieve(metrics, metric_name, strlen(metric_name), &vht)) { ht = calloc(1, sizeof(*ht)); vht = ht; mtev_hash_store(metrics, strdup(metric_name), strlen(metric_name), vht); } else ht = vht; if(v != NULL) { /* We expect: H[<float>]=%d */ const char *lhs; char *endptr; double bucket; u_int64_t cnt; if(v[0] != 'H' || v[1] != '[') return MTEV_HOOK_CONTINUE; if(NULL == (lhs = strchr(v+2, ']'))) return MTEV_HOOK_CONTINUE; lhs++; if(*lhs++ != '=') return MTEV_HOOK_CONTINUE; bucket = strtod(v+2, &endptr); if(endptr == v+2) return MTEV_HOOK_CONTINUE; cnt = strtoull(lhs, &endptr, 10); if(endptr == lhs) return MTEV_HOOK_CONTINUE; update_histotier(ht, time(NULL), conf, check, metric_name, bucket, cnt); } return MTEV_HOOK_CONTINUE; }
void *thread_func(void *arg) { mtev_hash_table *h = arg; pthread_t me = pthread_self(); char *key = NULL; for (int i = 0; i < 100; i++) { asprintf(&key, "%lu-%d", (unsigned long) me, i); mtev_hash_store(h, key, strlen(key), key); } return NULL; }
int noit_check_etc_hosts_cache_refresh(eventer_t e, int mask, void *closure, struct timeval *now) { static struct stat last_stat; struct stat sb; struct hostent *ent; int reload = 0; memset(&sb, 0, sizeof(sb)); stat("/etc/hosts", &sb); #define CSTAT(f) (sb.f == last_stat.f) reload = ! (CSTAT(st_dev) && CSTAT(st_ino) && CSTAT(st_mode) && CSTAT(st_uid) && CSTAT(st_gid) && CSTAT(st_size) && CSTAT(st_mtime)); memcpy(&last_stat, &sb, sizeof(sb)); if(reload) { mtev_hash_delete_all(&etc_hosts_cache, free, free); while(NULL != (ent = gethostent())) { int i = 0; char *name = ent->h_name; while(name) { void *vnode; static_host_node *node; if(!mtev_hash_retrieve(&etc_hosts_cache, name, strlen(name), &vnode)) { vnode = node = calloc(1, sizeof(*node)); node->target = strdup(name); mtev_hash_store(&etc_hosts_cache, node->target, strlen(node->target), node); } node = vnode; if(ent->h_addrtype == AF_INET) { node->has_ip4 = 1; memcpy(&node->ip4, ent->h_addr_list[0], ent->h_length); } if(ent->h_addrtype == AF_INET6) { node->has_ip6 = 1; memcpy(&node->ip6, ent->h_addr_list[0], ent->h_length); } name = ent->h_aliases[i++]; } } endhostent(); mtevL(noit_debug, "reloaded %d /etc/hosts targets\n", mtev_hash_size(&etc_hosts_cache)); } eventer_add_in_s_us(noit_check_etc_hosts_cache_refresh, NULL, 1, 0); return 0; }
static mtev_hook_return_t histogram_hook_impl(void *closure, noit_check_t *check, stats_t *stats, metric_t *m) { void *vht; histotier *ht; mtev_hash_table *config, *metrics; const char *track = ""; mtev_dso_generic_t *self = closure; struct histogram_config *conf = mtev_image_get_userdata(&self->hdr); config = noit_check_get_module_config(check, histogram_module_id); if(!config || mtev_hash_size(config) == 0) return MTEV_HOOK_CONTINUE; mtev_hash_retr_str(config, m->metric_name, strlen(m->metric_name), &track); if(!track || strcmp(track, "add")) return MTEV_HOOK_CONTINUE; metrics = noit_check_get_module_metadata(check, histogram_module_id); if(!metrics) { metrics = calloc(1, sizeof(*metrics)); noit_check_set_module_metadata(check, histogram_module_id, metrics, free_hash_o_histotier); } if(!mtev_hash_retrieve(metrics, m->metric_name, strlen(m->metric_name), &vht)) { ht = calloc(1, sizeof(*ht)); vht = ht; mtev_hash_store(metrics, strdup(m->metric_name), strlen(m->metric_name), vht); } else ht = vht; if(m->metric_value.vp != NULL) { #define UPDATE_HISTOTIER(a) update_histotier(ht, time(NULL), conf, check, m->metric_name, *m->metric_value.a, 1) switch(m->metric_type) { case METRIC_UINT64: UPDATE_HISTOTIER(L); break; case METRIC_INT64: UPDATE_HISTOTIER(l); break; case METRIC_UINT32: UPDATE_HISTOTIER(I); break; case METRIC_INT32: UPDATE_HISTOTIER(i); break; case METRIC_DOUBLE: UPDATE_HISTOTIER(n); break; default: /*noop*/ break; } } return MTEV_HOOK_CONTINUE; }
static ssl_ctx_cache_node * ssl_ctx_cache_set(ssl_ctx_cache_node *node) { void *vnode; mtev_atomic32_t newval; pthread_mutex_lock(&ssl_ctx_cache_lock); if(mtev_hash_retrieve(&ssl_ctx_cache, node->key, strlen(node->key), &vnode)) { node = vnode; } else { mtev_hash_store(&ssl_ctx_cache, node->key, strlen(node->key), node); } newval = mtev_atomic_inc32(&node->refcnt); pthread_mutex_unlock(&ssl_ctx_cache_lock); mtevL(eventer_deb, "ssl_ctx_cache->set(%p -> %d)\n", node, newval); return node; }
static void noit_fq_set_filters(mq_command_t *commands, int count) { int i, j; if (!global_fq_ctx.filtered_exchange[0]) { mtevL(mtev_error, "ERROR: trying to set check for filtered exchange when no such exchange exists\n"); return; } for (i=0; i<count; i++) { if (commands[i].action == MQ_ACTION_SET) { mtev_hash_table *metric_table = calloc(1, sizeof(mtev_hash_table)); mtev_hash_init(metric_table); for (j=0; j<commands[i].check.metric_count; j++) { mtev_hash_store(metric_table, strdup(commands[i].check.metrics[j]), strlen(commands[i].check.metrics[j]), NULL); filtered_metrics_exist = true; } mtev_hash_replace(&filtered_checks_hash, strdup(commands[i].check.uuid), strlen(commands[i].check.uuid), metric_table, free, noit_fq_free_metric_hash); } else { if (commands[i].check.metric_count == 0) { /* Forget the whole check */ if (!mtev_hash_delete(&filtered_checks_hash, commands[i].check.uuid, strlen(commands[i].check.uuid), free, noit_fq_free_metric_hash)) { mtevL(mtev_error, "failed forgetting check %s - check does not exist\n", commands[i].check.uuid); } } else { mtev_hash_table *filtered_metrics; if(mtev_hash_retrieve(&filtered_checks_hash, commands[i].check.uuid, strlen(commands[i].check.uuid), (void**)&filtered_metrics)) { for (j=0; j<commands[i].check.metric_count; j++) { if (!mtev_hash_delete(filtered_metrics, commands[i].check.metrics[j], strlen(commands[i].check.metrics[j]), free, noit_fq_free_metric_hash)) { mtevL(mtev_error, "failed forgetting metric '%s' for check %s - metric does not exist\n", commands[i].check.metrics[j], commands[i].check.uuid); } } } else { mtevL(mtev_error, "failed forgetting metrics for check %s - check does not exist\n", commands[i].check.uuid); } } } } }
int mtev_load_image(const char *file, const char *name, mtev_hash_table *registry, int (*validate)(mtev_image_t *), size_t obj_size) { char module_file[PATH_MAX]; const char *dlsymname; void *dlhandle = NULL; void *dlsymbol; mtev_image_t *obj; if(file[0] == '/') { strlcpy(module_file, file, sizeof(module_file)); dlhandle = dlopen(module_file, RTLD_LAZY | RTLD_GLOBAL); } else { char *basepath, *base, *brk; if(!mtev_conf_get_string(MTEV_CONF_ROOT, "//modules/@directory", &basepath)) basepath = strdup(MTEV_MODULES_DIR); for (base = strtok_r(basepath, ";:", &brk); base; base = strtok_r(NULL, ";:", &brk)) { snprintf(module_file, sizeof(module_file), "%s/%s.%s", base, file, MODULEEXT); dlhandle = dlopen(module_file, RTLD_LAZY | RTLD_GLOBAL); if(dlhandle) { mtevL(mtev_debug, "Successfully opened image '%s'\n", module_file); break; } else { mtevL(mtev_debug, "Tried to open image '%s' but failed: %s\n", module_file, dlerror()); } } free(basepath); if(!dlhandle) { snprintf(module_file, sizeof(module_file), "%s/%s.%s", MTEV_MODULES_DIR, file, MODULEEXT); dlhandle = dlopen(module_file, RTLD_LAZY | RTLD_GLOBAL); } } if(!dlhandle) { mtevL(mtev_stderr, "Cannot open image '%s': %s\n", module_file, dlerror()); return -1; } dlsymname = strrchr(name, ':'); if(!dlsymname) dlsymname = name; else dlsymname++; dlsymbol = dlsym(dlhandle, dlsymname); if(!dlsymbol) { mtevL(mtev_stderr, "Cannot find '%s' in image '%s': %s\n", dlsymname, module_file, dlerror()); dlclose(dlhandle); return -1; } if(validate(dlsymbol) == -1) { mtevL(mtev_stderr, "I can't understand module %s\n", name); dlclose(dlhandle); return -1; } obj = calloc(1, obj_size); memcpy(obj, dlsymbol, obj_size); obj->opaque_handle = calloc(1, sizeof(struct __extended_image_data)); if(obj->onload && obj->onload(obj)) { free(obj->opaque_handle); free(obj); dlclose(dlhandle); return -1; } char *namecopy = strdup(name); if(!mtev_hash_store(registry, namecopy, strlen(namecopy), obj)) { mtevL(mtev_error, "Attempted to load module %s more than once.\n", name); dlclose(dlhandle); free(namecopy); return -1; } ((struct __extended_image_data *)obj->opaque_handle)->dlhandle = dlhandle; return 0; }
static interim_journal_t * interim_journal_get(struct sockaddr *remote, const char *remote_cn_in, int storagenode_id, const char *fqdn_in) { void *vhash, *vij; mtev_hash_table *working_set; interim_journal_t *ij; struct timeval now; char jpath[PATH_MAX]; char remote_str[128]; const char *remote_cn = remote_cn_in ? remote_cn_in : "default"; const char *fqdn = fqdn_in ? fqdn_in : "default"; mtev_convert_sockaddr_to_buff(remote_str, sizeof(remote_str), remote); if(!*remote_str) strlcpy(remote_str, "default", sizeof(remote_str)); /* Lookup the working set */ if(!mtev_hash_retrieve(&working_sets, remote_cn, strlen(remote_cn), &vhash)) { working_set = calloc(1, sizeof(*working_set)); mtev_hash_store(&working_sets, strdup(remote_cn), strlen(remote_cn), working_set); } else working_set = vhash; /* Lookup the interim journal within the working set */ if(!mtev_hash_retrieve(working_set, fqdn, strlen(fqdn), &vij)) { ij = calloc(1, sizeof(*ij)); gettimeofday(&now, NULL); snprintf(jpath, sizeof(jpath), "%s/%s/%s/%d/%08x%08x.tmp", basejpath, remote_str, remote_cn, storagenode_id, (unsigned int)now.tv_sec, (unsigned int)now.tv_usec); ij->remote_str = strdup(remote_str); ij->remote_cn = strdup(remote_cn); ij->fqdn = fqdn_in ? strdup(fqdn_in) : NULL; ij->storagenode_id = storagenode_id; ij->filename = strdup(jpath); ij->fd = open(ij->filename, O_RDWR | O_CREAT | O_EXCL, 0640); if(ij->fd < 0 && errno == ENOENT) { if(mkdir_for_file(ij->filename, 0750)) { mtevL(noit_error, "Failed to create dir for '%s': %s\n", ij->filename, strerror(errno)); exit(-1); } ij->fd = open(ij->filename, O_RDWR | O_CREAT | O_EXCL, 0640); } if(ij->fd < 0 && errno == EEXIST) { /* This can only occur if we crash after before checkpointing */ unlink(ij->filename); ij->fd = open(ij->filename, O_RDWR | O_CREAT | O_EXCL, 0640); } if(ij->fd < 0) { mtevL(noit_error, "Failed to open interim journal '%s': %s\n", ij->filename, strerror(errno)); exit(-1); } mtev_hash_store(working_set, strdup(fqdn), strlen(fqdn), ij); } else ij = vij; return ij; }
int noit_filtersets_cull_unused() { mtev_hash_table active = MTEV_HASH_EMPTY; char *buffer = NULL; mtev_conf_section_t *declares; int i, n_uses = 0, n_declares = 0, removed = 0; const char *declare_xpath = "//filterset[@name and not (@cull='false')]"; declares = mtev_conf_get_sections(NULL, declare_xpath, &n_declares); if(declares) { /* store all unit filtersets used */ for(i=0;i<n_declares;i++) { if(!buffer) buffer = malloc(128); if(mtev_conf_get_stringbuf(declares[i], "@name", buffer, 128)) { if(mtev_hash_store(&active, buffer, strlen(buffer), declares[i])) { buffer = NULL; } else { void *vnode = NULL; /* We've just hit a duplicate.... check to see if there's an existing * entry and if there is, load the latest one and delete the old * one. */ mtev_hash_retrieve(&active, buffer, strlen(buffer), &vnode); if (vnode) { noit_filter_compile_add(declares[i]); CONF_REMOVE(vnode); xmlUnlinkNode(vnode); xmlFreeNode(vnode); removed++; if(mtev_hash_replace(&active, buffer, strlen(buffer), declares[i], free, NULL)) { buffer = NULL; } } } } } if(buffer) free(buffer); free(declares); } n_uses = noit_poller_do(filterset_accum, &active); if(n_uses > 0 && mtev_hash_size(&active) > 0) { mtev_hash_iter iter = MTEV_HASH_ITER_ZERO; const char *filter_name; int filter_name_len; void *vnode; while(mtev_hash_next(&active, &iter, &filter_name, &filter_name_len, &vnode)) { if(noit_filter_remove(vnode)) { CONF_REMOVE(vnode); xmlUnlinkNode(vnode); xmlFreeNode(vnode); removed++; } } } mtev_hash_destroy(&active, free, NULL); return removed; }