/* * _cerebrod_listener_initialize * * perform listener initialization */ static void _cerebrod_listener_initialize(void) { int i; Pthread_mutex_lock(&listener_init_lock); if (listener_init) goto out; Pthread_mutex_lock(&listener_fds_lock); for (i = 0; i < conf.listen_message_config_len; i++) { if ((listener_fds[i] = _listener_setup_socket(i)) < 0) CEREBROD_EXIT(("listener fd setup failed")); } Pthread_mutex_unlock(&listener_fds_lock); if (!(clusterlist_handle = clusterlist_module_load())) CEREBROD_EXIT(("clusterlist_module_load")); if ((found_clusterlist_module = clusterlist_module_found(clusterlist_handle)) < 0) CEREBROD_EXIT(("clusterlist_module_found")); if (found_clusterlist_module) { if (clusterlist_module_setup(clusterlist_handle) < 0) CEREBROD_EXIT(("clusterlist_module_setup")); if (conf.debug && conf.listen_debug) { fprintf(stderr, "**************************************\n"); fprintf(stderr, "* Cerebro Clusterlist\n"); fprintf(stderr, "* -----------------------\n"); fprintf(stderr, "* Using Clusterlist: %s\n", clusterlist_module_name(clusterlist_handle)); fprintf(stderr, "**************************************\n"); } } cerebrod_listener_data_initialize(); for (i = 0; i < conf.forward_message_config_len; i++) { /* if the forward destination is local to the machine, don't forward */ if (conf.forward_message_config[i].ip_is_local) continue; if (_forwarding_setup(i) < 0) CEREBROD_EXIT(("forwarding setup failed")); } listener_init++; Pthread_cond_signal(&listener_init_cond); out: Pthread_mutex_unlock(&listener_init_lock); }
void cerebrod_event_update_node_received_time(struct cerebrod_node_data *nd, u_int32_t received_time) { struct cerebrod_event_node_timeout *ntd; #if CEREBRO_DEBUG int rv; #endif /* CEREBRO_DEBUG */ assert(nd); assert(received_time); if (!event_index) return; #if CEREBRO_DEBUG /* Should be called with lock already set */ rv = Pthread_mutex_trylock(&nd->node_data_lock); if (rv != EBUSY) CEREBROD_EXIT(("mutex not locked: rv=%d", rv)); #endif /* CEREBRO_DEBUG */ Pthread_mutex_lock(&event_node_timeout_data_lock); if ((ntd = Hash_find(event_node_timeout_data_index, nd->nodename))) { ntd->last_received_time = received_time; ntd->timeout_occurred = 0; } Pthread_mutex_unlock(&event_node_timeout_data_lock); }
void cerebrod_speaker_data_initialize(void) { #if !WITH_CEREBROD_NO_THREADS Pthread_mutex_lock(&speaker_data_init_lock); #endif /* !WITH_CEREBROD_NO_THREADS */ if (speaker_data_init) goto out; #if !WITH_CEREBROD_NO_THREADS /* * Must lock in this initialization routine, b/c the update thread * in a metric module may call the update state function. */ Pthread_mutex_lock(&metric_list_lock); #endif /* !WITH_CEREBROD_NO_THREADS */ metric_list = List_create((ListDelF)_destroy_speaker_metric_info); if (_setup_metric_modules() < 0) CEREBROD_EXIT(("_setup_metric_modules")); #if !WITH_CEREBROD_NO_THREADS Pthread_mutex_unlock(&metric_list_lock); #endif /* !WITH_CEREBROD_NO_THREADS */ speaker_data_init++; out: #if !WITH_CEREBROD_NO_THREADS Pthread_mutex_unlock(&speaker_data_init_lock); #endif /* !WITH_CEREBROD_NO_THREADS */ ; /* in case !WITH_CEREBRO_NO_THREADS */ }
void cerebrod_rehash(hash_t *old_hash, int *hash_size, int hash_size_increment, int hash_num, pthread_mutex_t *hash_mutex) { hash_t new_hash; assert(old_hash && hash_size && hash_size_increment && hash_num); #if CEREBRO_DEBUG /* Should be called with lock already set */ if (hash_mutex) { int rv = Pthread_mutex_trylock(hash_mutex); if (rv != EBUSY) CEREBROD_EXIT(("mutex not locked: rv=%d", rv)); } #endif /* CEREBRO_DEBUG */ *hash_size += hash_size_increment; new_hash = Hash_create(*hash_size, (hash_key_f)hash_key_string, (hash_cmp_f)strcmp, (hash_del_f)_Free); if (Hash_for_each(*old_hash, _hash_reinsert, &new_hash) != hash_num) CEREBROD_EXIT(("invalid reinsert: hash_num=%d", hash_num)); if (Hash_remove_if(*old_hash, _hash_removeall, NULL) != hash_num) CEREBROD_EXIT(("invalid removeall: hash_num=%d", hash_num)); Hash_destroy(*old_hash); *old_hash = new_hash; }
/* * cerebrod_monitor_modules_update * * Send metric data to the appropriate monitor modules, if necessary. * The struct cerebrod_node_data lock should already be locked. */ void cerebrod_monitor_modules_update(const char *nodename, struct cerebrod_node_data *nd, const char *metric_name, struct cerebrod_message_metric *mm) { struct cerebrod_monitor_module_info *monitor_module; struct cerebrod_monitor_module_list *ml; #if CEREBRO_DEBUG int rv; #endif /* CEREBRO_DEBUG */ assert(nodename && nd && metric_name && mm); if (!monitor_index) return; #if CEREBRO_DEBUG /* Should be called with lock already set */ rv = Pthread_mutex_trylock(&nd->node_data_lock); if (rv != EBUSY) CEREBROD_EXIT(("mutex not locked: rv=%d", rv)); #endif /* CEREBRO_DEBUG */ if ((ml = Hash_find(monitor_index, metric_name))) { ListIterator itr = NULL; Pthread_mutex_lock(&(ml->monitor_list_lock)); itr = List_iterator_create(ml->monitor_list); Pthread_mutex_unlock(&(ml->monitor_list_lock)); while ((monitor_module = list_next(itr))) { Pthread_mutex_lock(&monitor_module->monitor_lock); monitor_module_metric_update(monitor_handle, monitor_module->index, nodename, metric_name, mm->metric_value_type, mm->metric_value_len, mm->metric_value); Pthread_mutex_unlock(&monitor_module->monitor_lock); } Pthread_mutex_lock(&(ml->monitor_list_lock)); List_iterator_destroy(itr); Pthread_mutex_unlock(&(ml->monitor_list_lock)); } }
static void _delete_event_connection_fd(int fd) { struct cerebrod_event_connection_data *ecd; ListIterator eitr; #if CEREBRO_DEBUG int rv; #endif /* CEREBRO_DEBUG */ assert(fd >= 0); #if CEREBRO_DEBUG /* Should be called with lock already set */ rv = Pthread_mutex_trylock(&event_connections_lock); if (rv != EBUSY) CEREBROD_EXIT(("mutex not locked: rv=%d", rv)); #endif /* CEREBRO_DEBUG */ eitr = List_iterator_create(event_connections); while ((ecd = list_next(eitr))) { if (ecd->fd == fd) { List connections; if ((connections = Hash_find(event_connections_index, ecd->event_name))) { ListIterator citr; int *fdPtr; citr = List_iterator_create(connections); while ((fdPtr = list_next(citr))) { if (*fdPtr == fd) { List_delete(citr); break; } } List_iterator_destroy(citr); } List_delete(eitr); break; } } List_iterator_destroy(eitr); }
void cerebrod_speaker_data_get_module_metric_data(struct cerebrod_message *msg, unsigned int *message_len, unsigned int index, int *more_data_to_send) { struct cerebrod_message_metric *mm = NULL; assert(msg && message_len && more_data_to_send); assert(metric_handle && metric_list && metric_list_size); if (!speaker_data_init) CEREBROD_EXIT(("initialization not complete")); *more_data_to_send = 0; #if !WITH_CEREBROD_NO_THREADS Pthread_mutex_lock(&metric_list_lock); #endif /* !WITH_CEREBROD_NO_THREADS */ msg->metrics_len = 0; msg->metrics = Malloc(sizeof(struct cerebrod_message_metric *)*2); memset(msg->metrics, '\0', sizeof(struct cerebrod_message_metric *)*2); /* Since this is the only metric value we're sending, we should * never have more data than can fit in a single UDP packet. */ if ((mm = _get_module_metric_value(index))) { *message_len += CEREBROD_MESSAGE_METRIC_HEADER_LEN; *message_len += mm->metric_value_len; msg->metrics[msg->metrics_len] = mm; msg->metrics_len++; } #if !WITH_CEREBROD_NO_THREADS Pthread_mutex_unlock(&metric_list_lock); #endif /* !WITH_CEREBROD_NO_THREADS */ return; }
void cerebrod_speaker_data_metric_list_sort(void) { #if CEREBRO_DEBUG #if !WITH_CEREBROD_NO_THREADS int rv; #endif /* !WITH_CEREBROD_NO_THREADS */ #endif /* CEREBRO_DEBUG */ assert(metric_list); #if CEREBRO_DEBUG #if !WITH_CEREBROD_NO_THREADS /* Should be called with lock already set */ rv = Pthread_mutex_trylock(&metric_list_lock); if (rv != EBUSY) CEREBROD_EXIT(("mutex not locked: rv=%d", rv)); #endif /* !WITH_CEREBROD_NO_THREADS */ #endif /* CEREBRO_DEBUG */ List_sort(metric_list, _next_call_time_cmp); }
/* * _event_node_timeout_data_add * * Create entries for the event_node_timeout_data list and index * * Returns 0 on success, -1 on error */ static int _event_node_timeout_data_add(const char *nodename, u_int32_t time_now) { struct cerebrod_event_node_timeout_data *ntd; #if CEREBRO_DEBUG int rv; /* Should be called with lock already set */ rv = Pthread_mutex_trylock(&event_node_timeout_data_lock); if (rv != EBUSY) CEREBROD_EXIT(("mutex not locked: rv=%d", rv)); #endif /* CEREBRO_DEBUG */ ntd = (struct cerebrod_event_node_timeout_data *)Malloc(sizeof(struct cerebrod_event_node_timeout_data)); ntd->nodename = (char *)nodename; ntd->last_received_time = time_now; ntd->timeout_occurred = 0; List_append(event_node_timeout_data, ntd); Hash_insert(event_node_timeout_data_index, ntd->nodename, ntd); event_node_timeout_data_index_numnodes++; return 0; }
int cerebrod_reinit_socket(int old_fd, int num, Cerebrod_socket_setup socket_setup, char *msg) { int fd = old_fd; assert(socket_setup && msg); if (errno == EINVAL || errno == EBADF || errno == ENODEV || errno == ENETDOWN || errno == ENETUNREACH || old_fd < 0) { if (!(old_fd < 0)) close(old_fd); /* no-wrapper, make best effort */ if ((fd = socket_setup(num)) < 0) { CEREBROD_DBG(("%s: error re-initializing socket", msg)); /* Wait a bit, so we don't spin */ sleep(CEREBROD_REINITIALIZE_TIME); } else CEREBROD_DBG(("success re-initializing socket")); } else if (errno == EINTR) CEREBROD_DBG(("%s: %s", msg, strerror(errno))); else CEREBROD_EXIT(("%s: %s", msg, strerror(errno))); return fd; }
void cerebrod_event_add_node_timeout_data(struct cerebrod_node_data *nd, u_int32_t received_time) { #if CEREBRO_DEBUG int rv; #endif /* CEREBRO_DEBUG */ assert(nd); assert(received_time); if (!event_index) return; assert(event_node_timeout_data_index); #if CEREBRO_DEBUG /* Should be called with lock already set */ rv = Pthread_mutex_trylock(&listener_data_lock); if (rv != EBUSY) CEREBROD_EXIT(("mutex not locked: rv=%d", rv)); #endif /* CEREBRO_DEBUG */ Pthread_mutex_lock(&event_node_timeout_data_lock); /* Re-hash if our hash is getting too small */ if ((event_node_timeout_data_index_numnodes + 1) > (event_node_timeout_data_index_size*2)) cerebrod_rehash(&event_node_timeout_data_index, &event_node_timeout_data_index_size, EVENT_NODE_TIMEOUT_SIZE_INCREMENT, event_node_timeout_data_index_numnodes, &event_node_timeout_data_lock); _event_node_timeout_data_add(nd->nodename, received_time); event_node_timeout_data_index_numnodes++; Pthread_mutex_unlock(&event_node_timeout_data_lock); }
/* * _setup_metric_modules * * Setup metric modules. Under almost any circumstance, don't return a * -1 error, cerebro can go on without loading metric modules. * * Returns 1 if modules are loaded, 0 if not, -1 on error */ static int _setup_metric_modules(void) { int i; #if CEREBRO_DEBUG #if !WITH_CEREBROD_NO_THREADS int rv; #endif /* !WITH_CEREBROD_NO_THREADS */ #endif /* CEREBRO_DEBUG */ assert(metric_list); #if CEREBRO_DEBUG #if !WITH_CEREBROD_NO_THREADS /* Should be called with lock already set */ rv = Pthread_mutex_trylock(&metric_list_lock); if (rv != EBUSY) CEREBROD_EXIT(("mutex not locked: rv=%d", rv)); #endif /* !WITH_CEREBROD_NO_THREADS */ #endif /* CEREBRO_DEBUG */ if (!(metric_handle = metric_modules_load())) { CEREBROD_DBG(("metric_modules_load")); goto cleanup; } if ((metric_handle_count = metric_modules_count(metric_handle)) < 0) { CEREBROD_DBG(("metric_module_count failed")); goto cleanup; } if (!metric_handle_count) { if (conf.debug && conf.speak_debug) { #if !WITH_CEREBROD_NO_THREADS Pthread_mutex_lock(&debug_output_mutex); #endif /* !WITH_CEREBROD_NO_THREADS */ fprintf(stderr, "**************************************\n"); fprintf(stderr, "* No Metric Modules Found\n"); fprintf(stderr, "**************************************\n"); #if !WITH_CEREBROD_NO_THREADS Pthread_mutex_unlock(&debug_output_mutex); #endif /* !WITH_CEREBROD_NO_THREADS */ } goto cleanup; } for (i = 0; i < metric_handle_count; i++) { struct cerebrod_speaker_metric_info *metric_info; #if !WITH_CEREBROD_NO_THREADS Cerebro_metric_thread_pointer threadPtr; #endif /* !WITH_CEREBROD_NO_THREADS */ char *module_name, *metric_name; int metric_period; u_int32_t metric_flags; module_name = metric_module_name(metric_handle, i); if (conf.metric_module_exclude_len) { int found_exclude = 0; int j; for (j = 0; j < conf.metric_module_exclude_len; j++) { if (!strcasecmp(conf.metric_module_exclude[j], module_name)) { found_exclude++; break; } } if (found_exclude) { if (conf.debug && conf.speak_debug) { #if !WITH_CEREBROD_NO_THREADS Pthread_mutex_lock(&debug_output_mutex); #endif /* !WITH_CEREBROD_NO_THREADS */ fprintf(stderr, "**************************************\n"); fprintf(stderr, "* Skip Metric Module: %s\n", module_name); fprintf(stderr, "**************************************\n"); #if !WITH_CEREBROD_NO_THREADS Pthread_mutex_unlock(&debug_output_mutex); #endif /* !WITH_CEREBROD_NO_THREADS */ } CEREBROD_ERR(("Dropping metric module: %s", module_name)); continue; } } if (conf.debug && conf.speak_debug) { #if !WITH_CEREBROD_NO_THREADS Pthread_mutex_lock(&debug_output_mutex); #endif /* !WITH_CEREBROD_NO_THREADS */ fprintf(stderr, "**************************************\n"); fprintf(stderr, "* Setup Metric Module: %s\n", module_name); fprintf(stderr, "**************************************\n"); #if !WITH_CEREBROD_NO_THREADS Pthread_mutex_unlock(&debug_output_mutex); #endif /* !WITH_CEREBROD_NO_THREADS */ } if (metric_module_setup(metric_handle, i) < 0) { CEREBROD_DBG(("metric_module_setup: %s", module_name)); continue; } if (!(metric_name = metric_module_get_metric_name(metric_handle, i))) { CEREBROD_DBG(("metric_module_get_metric_name: %s", module_name)); metric_module_cleanup(metric_handle, i); continue; } if (metric_module_get_metric_period(metric_handle, i, &metric_period) < 0) { CEREBROD_DBG(("metric_module_get_metric_period: %s", module_name)); metric_module_cleanup(metric_handle, i); continue; } if (metric_module_get_metric_flags(metric_handle, i, &metric_flags) < 0) { CEREBROD_DBG(("metric_module_get_metric_flags: %s", module_name)); metric_module_cleanup(metric_handle, i); continue; } if (metric_flags & CEREBRO_METRIC_MODULE_FLAGS_SEND_ON_PERIOD && metric_period <= 0) { CEREBROD_DBG(("metric module period invalid: %s", module_name)); metric_module_cleanup(metric_handle, i); continue; } if (metric_module_send_message_function_pointer(metric_handle, i, &cerebrod_send_message) < 0) { CEREBROD_DBG(("metric_module_send_message_function_pointer: %s", module_name)); metric_module_cleanup(metric_handle, i); continue; } metric_info = Malloc(sizeof(struct cerebrod_speaker_metric_info)); /* No need to Strdup() the name in this case */ metric_info->metric_name = metric_name; metric_info->metric_origin = CEREBROD_METRIC_SPEAKER_ORIGIN_MODULE; metric_info->metric_period = metric_period; metric_info->metric_flags = metric_flags; metric_info->index = i; /* * If metric period is < 0, it presumably never will be sent * (metric is likely handled by a metric_thread), so set * next_call_time to UINT_MAX. * * If this is a metric that will be piggy-backed on heartbeats, * then initialize next_call_time to 0, so the data is sent on * the first heartbeat * * If this is a metric that will not be piggy-backed on * heartbeats, set the next_call_time to UINT_MAX. Let the * speaker logic decide when packets should be sent. */ if (metric_info->metric_period < 0 || metric_info->metric_flags & CEREBRO_METRIC_MODULE_FLAGS_SEND_ON_PERIOD) metric_info->next_call_time = UINT_MAX; else metric_info->next_call_time = 0; List_append(metric_list, metric_info); metric_list_size++; #if !WITH_CEREBROD_NO_THREADS if ((threadPtr = metric_module_get_metric_thread(metric_handle, i))) { pthread_t thread; pthread_attr_t attr; Pthread_attr_init(&attr); Pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); Pthread_attr_setstacksize(&attr, CEREBROD_THREAD_STACKSIZE); Pthread_create(&thread, &attr, threadPtr, NULL); Pthread_attr_destroy(&attr); } #endif /* !WITH_CEREBROD_NO_THREADS */ } if (!metric_list_size) goto cleanup; cerebrod_speaker_data_metric_list_sort(); return 1; cleanup: if (metric_handle) { /* unload will call module cleanup functions */ metric_modules_unload(metric_handle); metric_handle = NULL; metric_handle_count = 0; } metric_list_size = 0; return 0; }
void cerebrod_event_modules_update(const char *nodename, struct cerebrod_node_data *nd, const char *metric_name, struct cerebrod_message_metric *mm) { struct cerebrod_event_module_info *event_module; struct cerebrod_event_module_list *el; #if CEREBRO_DEBUG int rv; #endif /* CEREBRO_DEBUG */ assert(nodename && nd && metric_name && mm); if (!event_index) return; #if CEREBRO_DEBUG /* Should be called with lock already set */ rv = Pthread_mutex_trylock(&nd->node_data_lock); if (rv != EBUSY) CEREBROD_EXIT(("mutex not locked: rv=%d", rv)); #endif /* CEREBRO_DEBUG */ /* * This function may be called by multiple threads by the listener. * * The event_index is setup at the beginning and is only read, not * written to. However, the lists stored inside the event_index * need to called w/ thread safety (due to the nature of the list * API). * */ if ((el = Hash_find(event_index, metric_name))) { struct cerebro_event *event = NULL; ListIterator itr = NULL; int rv; Pthread_mutex_lock(&(el->event_list_lock)); itr = List_iterator_create(el->event_list); Pthread_mutex_unlock(&(el->event_list_lock)); while ((event_module = list_next(itr))) { Pthread_mutex_lock(&event_module->event_lock); if ((rv = event_module_metric_update(event_handle, event_module->index, nodename, metric_name, mm->metric_value_type, mm->metric_value_len, mm->metric_value, &event)) < 0) { CEREBROD_DBG(("event_module_metric_update")); goto loop_next; } if (rv && event) cerebrod_queue_event(event, event_module->index); loop_next: Pthread_mutex_unlock(&event_module->event_lock); } Pthread_mutex_lock(&(el->event_list_lock)); List_iterator_destroy(itr); Pthread_mutex_unlock(&(el->event_list_lock)); } }
/* * Under almost any circumstance, don't return a -1 error, cerebro can * go on without loading monitor modules. The listener_data_init_lock * should already be set. */ int cerebrod_event_modules_setup(void) { int i, event_module_count, event_index_len, event_index_count = 0; struct cerebrod_event_module_list *el = NULL; #if CEREBRO_DEBUG int rv; #endif /* CEREBRO_DEBUG */ assert(listener_data); #if CEREBRO_DEBUG /* Should be called with lock already set */ rv = Pthread_mutex_trylock(&listener_data_init_lock); if (rv != EBUSY) CEREBROD_EXIT(("mutex not locked: rv=%d", rv)); #endif /* CEREBRO_DEBUG */ if (!conf.event_server) return 0; if (!(event_handle = event_modules_load())) { CEREBROD_DBG(("event_modules_load")); goto cleanup; } if ((event_module_count = event_modules_count(event_handle)) < 0) { CEREBROD_DBG(("event_modules_count")); goto cleanup; } if (!event_module_count) { if (conf.debug && conf.event_server_debug) { Pthread_mutex_lock(&debug_output_mutex); fprintf(stderr, "**************************************\n"); fprintf(stderr, "* No Event Modules Found\n"); fprintf(stderr, "**************************************\n"); Pthread_mutex_unlock(&debug_output_mutex); } goto cleanup; } /* Each event module may want multiple metrics and/or offer multiple * event names. We'll assume there will never be more than 2 per * event module, and that will be enough to avoid all hash * collisions. */ event_index_len = event_module_count * 2; event_index = Hash_create(event_index_len, (hash_key_f)hash_key_string, (hash_cmp_f)strcmp, (hash_del_f)_cerebrod_event_module_list_destroy); event_names = List_create((ListDelF)_Free); event_module_timeouts = List_create((ListDelF)_cerebrod_event_module_timeout_data_destroy); event_module_timeout_index = Hash_create(event_module_count, (hash_key_f)hash_key_string, (hash_cmp_f)strcmp, (hash_del_f)list_destroy); for (i = 0; i < event_module_count; i++) { struct cerebrod_event_module *event_module; char *module_name, *module_metric_names, *module_event_names; char *metricPtr, *metricbuf; char *eventnamePtr, *eventbuf; int timeout; module_name = event_module_name(event_handle, i); if (conf.event_module_exclude_len) { int found_exclude = 0; int j; for (j = 0; j < conf.event_module_exclude_len; j++) { if (!strcasecmp(conf.event_module_exclude[j], module_name)) { found_exclude++; break; } } if (found_exclude) { if (conf.debug && conf.event_server_debug) { Pthread_mutex_lock(&debug_output_mutex); fprintf(stderr, "**************************************\n"); fprintf(stderr, "* Skip Event Module: %s\n", module_name); fprintf(stderr, "**************************************\n"); Pthread_mutex_unlock(&debug_output_mutex); } CEREBROD_ERR(("Dropping event module: %s", module_name)); continue; } } if (conf.debug && conf.event_server_debug) { Pthread_mutex_lock(&debug_output_mutex); fprintf(stderr, "**************************************\n"); fprintf(stderr, "* Setup Event Module: %s\n", module_name); fprintf(stderr, "**************************************\n"); Pthread_mutex_unlock(&debug_output_mutex); } if (event_module_setup(event_handle, i) < 0) { CEREBROD_DBG(("event_module_setup failed: %s", module_name)); continue; } if (!(module_metric_names = event_module_metric_names(event_handle, i)) < 0) { CEREBROD_DBG(("event_module_metric_names failed: %s", module_name)); event_module_cleanup(event_handle, i); continue; } if (!(module_event_names = event_module_event_names(event_handle, i)) < 0) { CEREBROD_DBG(("event_module_event_names failed: %s", module_name)); event_module_cleanup(event_handle, i); continue; } if ((timeout = event_module_timeout_length(event_handle, i)) < 0) { CEREBROD_DBG(("event_module_timeout_length failed: %s", module_name)); event_module_cleanup(event_handle, i); continue; } event_module = Malloc(sizeof(struct cerebrod_event_module_info)); event_module->metric_names = Strdup(module_metric_names); event_module->event_names = Strdup(module_event_names); event_module->index = i; Pthread_mutex_init(&(event_module->event_lock), NULL); /* The monitoring module may support multiple metrics */ metricPtr = strtok_r(event_module->metric_names, ",", &metricbuf); while (metricPtr) { if (!(el = Hash_find(event_index, metricPtr))) { el = (struct cerebrod_event_module_list *)Malloc(sizeof(struct cerebrod_event_module_list)); el->event_list = List_create((ListDelF)_cerebrod_event_module_info_destroy); Pthread_mutex_init(&(el->event_list_lock), NULL); List_append(el->event_list, event_module); Hash_insert(event_index, metricPtr, el); event_index_count++; } else List_append(el->event_list, event_module); metricPtr = strtok_r(NULL, ",", &metricbuf); } /* The monitoring module may support multiple event names */ eventnamePtr = strtok_r(event_module->event_names, ",", &eventbuf); while (eventnamePtr) { if (!list_find_first(event_names, (ListFindF)_cerebrod_name_strcmp, eventnamePtr)) { List_append(event_names, eventnamePtr); if (conf.debug && conf.event_server_debug) { Pthread_mutex_lock(&debug_output_mutex); fprintf(stderr, "**************************************\n"); fprintf(stderr, "* Event Name: %s\n", eventnamePtr); fprintf(stderr, "**************************************\n"); Pthread_mutex_unlock(&debug_output_mutex); } } eventnamePtr = strtok_r(NULL, ",", &eventbuf); } if (timeout) { struct cerebrod_event_module_timeout_data *mtd; List modules_list; if (!(mtd = List_find_first(event_module_timeouts, _event_module_timeout_data_find_callback, &timeout))) { char strbuf[64]; mtd = (struct cerebrod_event_module_timeout_data *)Malloc(sizeof(struct cerebrod_event_module_timeout_data)); mtd->timeout = timeout; snprintf(strbuf, 64, "%d", timeout); mtd->timeout_str = Strdup(strbuf); List_append(event_module_timeouts, mtd); if (timeout < event_module_timeout_min) event_module_timeout_min = timeout; } if (!(modules_list = Hash_find(event_module_timeout_index, mtd->timeout_str))) { modules_list = List_create((ListDelF)NULL); List_append(modules_list, event_module); Hash_insert(event_module_timeout_index, mtd->timeout_str, modules_list); } else List_append(modules_list, event_module); } } List_sort(event_module_timeouts, _event_module_timeout_data_compare); if (!event_index_count) goto cleanup; if (_setup_event_node_timeout_data() < 0) goto cleanup; /* * Since the cerebrod listener is started before any of the event * threads (node_timeout, queue_monitor, server), this must be * created in here (which is called by the listener) to avoid a * possible race of modules creating events before the event_queue * is created. */ event_queue = List_create((ListDelF)cerebrod_event_to_send_destroy); return 1; cleanup: if (event_handle) { event_modules_unload(event_handle); event_handle = NULL; } if (event_index) { Hash_destroy(event_index); event_index = NULL; } if (event_names) { List_destroy(event_names); event_names = NULL; } return 0; }
void * cerebrod_listener(void *arg) { char buf[CEREBRO_MAX_PACKET_LEN]; _cerebrod_listener_initialize(); for (;;) { struct cerebrod_message *msg; char nodename_buf[CEREBRO_MAX_NODENAME_LEN+1]; char nodename_key[CEREBRO_MAX_NODENAME_LEN+1]; struct timeval tv; int in_cluster_flag, i, count; fd_set readfds; int recv_len = 0; int maxfd = 0; FD_ZERO(&readfds); Pthread_mutex_lock(&listener_fds_lock); for (i = 0; i < conf.listen_message_config_len; i++) { if (listener_fds[i] > maxfd) maxfd = listener_fds[i]; FD_SET(listener_fds[i], &readfds); } count = Select(maxfd + 1, &readfds, NULL, NULL, NULL); for (i = 0; i < conf.listen_message_config_len; i++) { if (FD_ISSET(listener_fds[i], &readfds)) { if ((recv_len = recvfrom(listener_fds[i], buf, CEREBRO_MAX_PACKET_LEN, 0, NULL, NULL)) < 0) listener_fds[i] = cerebrod_reinit_socket(listener_fds[i], i, _listener_setup_socket, "listener: recvfrom"); break; } } Pthread_mutex_unlock(&listener_fds_lock); /* No packet read */ if (recv_len <= 0) continue; if (recv_len >= CEREBRO_MAX_PACKET_LEN) { CEREBROD_DBG(("received truncated packet")); continue; } if (_cerebrod_message_check_version(buf, recv_len) < 0) { CEREBROD_DBG(("received invalid version packet")); continue; } if (!(msg = _cerebrod_message_unmarshall(buf, recv_len))) { CEREBROD_DBG(("received unmarshallable packet")); continue; } _cerebrod_message_dump(msg, "Received Message"); /* Guarantee ending '\0' character */ memset(nodename_buf, '\0', CEREBRO_MAX_NODENAME_LEN+1); memcpy(nodename_buf, msg->nodename, CEREBRO_MAX_NODENAME_LEN); if (!strlen(nodename_buf)) { CEREBROD_DBG(("received null nodename")); cerebrod_message_destroy(msg); continue; } if (found_clusterlist_module) { if ((in_cluster_flag = clusterlist_module_node_in_cluster(clusterlist_handle, nodename_buf)) < 0) CEREBROD_EXIT(("clusterlist_module_node_in_cluster: %s", nodename_buf)); /* Second chance, is this data being forwarded from another host */ if (!in_cluster_flag) { if (Hostlist_find(conf.forward_host_accept, nodename_buf) >= 0) in_cluster_flag++; } } else /* must assume it is in the cluster */ /* Note, there is no need to handle 'forward_host_accept' under this case, * since we don't know if it is in the cluster or not anyways. */ in_cluster_flag = 1; if (!in_cluster_flag) { CEREBROD_DBG(("received non-cluster packet: %s", nodename_buf)); cerebrod_message_destroy(msg); continue; } memset(nodename_key, '\0', CEREBRO_MAX_NODENAME_LEN+1); if (found_clusterlist_module) { if (clusterlist_module_get_nodename(clusterlist_handle, nodename_buf, nodename_key, CEREBRO_MAX_NODENAME_LEN+1) < 0) { CEREBROD_DBG(("clusterlist_module_get_nodename: %s", nodename_buf)); cerebrod_message_destroy(msg); continue; } } else memcpy(nodename_key, nodename_buf, CEREBRO_MAX_NODENAME_LEN+1); Gettimeofday(&tv, NULL); cerebrod_listener_data_update(nodename_key, msg, tv.tv_sec); /* Forward data as necessary. Note, there is no need to * marshall data, it should already be marshalled when we * read it earlier. */ for (i = 0; i < conf.forward_message_config_len; i++) { /* if the forward destination is local to the machine, don't forward */ if (conf.forward_message_config[i].ip_is_local) continue; if (!forwarding_info[i].hosts || hostlist_find(forwarding_info[i].hosts, nodename_key) >= 0) { struct sockaddr *addr; struct sockaddr_in msgaddr; unsigned int addrlen; int rv; memset(&msgaddr, '\0', sizeof(struct sockaddr_in)); msgaddr.sin_family = AF_INET; msgaddr.sin_port = htons(conf.forward_message_config[i].destination_port); memcpy(&msgaddr.sin_addr, &conf.forward_message_config[i].ip_in_addr, sizeof(struct in_addr)); addr = (struct sockaddr *)&msgaddr; addrlen = sizeof(struct sockaddr_in); _cerebrod_message_dump(msg, "Forwarding Message"); Pthread_mutex_lock(&forwarding_info[i].lock); if ((rv = sendto(forwarding_info[i].fd, buf, recv_len, 0, addr, addrlen)) != recv_len) { if (rv < 0) forwarding_info[i].fd = cerebrod_reinit_socket(forwarding_info[i].fd, i, _forwarding_setup_socket, "forwarding: sendto"); else CEREBROD_ERR(("sendto: invalid bytes sent")); } Pthread_mutex_unlock(&forwarding_info[i].lock); } } cerebrod_message_destroy(msg); } return NULL; /* NOT REACHED */ }
void cerebrod_speaker_data_get_heartbeat_metric_data(struct cerebrod_message *msg, unsigned int *message_len, int *more_data_to_send) { struct cerebrod_speaker_metric_info *metric_info; ListIterator itr = NULL; struct timeval tv; assert(msg && message_len && more_data_to_send); if (!speaker_data_init) CEREBROD_EXIT(("initialization not complete")); *more_data_to_send = 0; #if !WITH_CEREBROD_NO_THREADS Pthread_mutex_lock(&metric_list_lock); #endif /* !WITH_CEREBROD_NO_THREADS */ /* There may not be any metrics to distribute */ if (!metric_list_size) { msg->metrics_len = 0; msg->metrics = NULL; goto lock_out; } msg->metrics_len = 0; msg->metrics = Malloc(sizeof(struct cerebrod_message_metric *)*(metric_list_size + 1)); memset(msg->metrics, '\0', sizeof(struct cerebrod_message_metric *)*(metric_list_size + 1)); Gettimeofday(&tv, NULL); itr = List_iterator_create(metric_list); while ((metric_info = list_next(itr))) { struct cerebrod_message_metric *mm = NULL; if (tv.tv_sec <= metric_info->next_call_time) break; if (metric_info->metric_origin & CEREBROD_METRIC_SPEAKER_ORIGIN_MODULE && !(metric_info->metric_flags & CEREBRO_METRIC_MODULE_FLAGS_SEND_ON_PERIOD)) mm = _get_module_metric_value(metric_info->index); if (metric_info->metric_origin & CEREBROD_METRIC_SPEAKER_ORIGIN_USERSPACE) mm = _get_userspace_metric_value(metric_info); if (mm) { unsigned int new_len; new_len = *message_len + CEREBROD_MESSAGE_METRIC_HEADER_LEN + mm->metric_value_len; if (new_len >= CEREBRO_MAX_PACKET_LEN) { *more_data_to_send = 1; goto sort_out; } else { *message_len += CEREBROD_MESSAGE_METRIC_HEADER_LEN; *message_len += mm->metric_value_len; msg->metrics[msg->metrics_len] = mm; msg->metrics_len++; } } if (metric_info->metric_origin & CEREBROD_METRIC_SPEAKER_ORIGIN_MODULE) { if (metric_info->metric_period < 0) metric_info->next_call_time = UINT_MAX; /* * Metric period stays at 0 for metrics that need to be * propogated every time */ if (metric_info->metric_period > 0) metric_info->next_call_time = tv.tv_sec + metric_info->metric_period; } if (metric_info->metric_origin & CEREBROD_METRIC_SPEAKER_ORIGIN_USERSPACE) metric_info->next_call_time = UINT_MAX; } sort_out: List_iterator_destroy(itr); cerebrod_speaker_data_metric_list_sort(); lock_out: #if !WITH_CEREBROD_NO_THREADS Pthread_mutex_unlock(&metric_list_lock); #endif /* !WITH_CEREBROD_NO_THREADS */ return; }
void * cerebrod_event_server(void *arg) { int server_fd; _event_server_initialize(); if ((server_fd = _event_server_setup_socket(0)) < 0) CEREBROD_EXIT(("event server fd setup failed")); for (;;) { ListIterator eitr; struct cerebrod_event_connection_data *ecd; struct pollfd *pfds; int pfdslen = 0; int i; /* Note that the list_count won't grow larger after the first * mutex block, b/c the cerebrod_event_queue_monitor thread can * never add to the event_connections. It can only shrink it. */ Pthread_mutex_lock(&event_connections_lock); if (event_connections) pfdslen = List_count(event_connections); Pthread_mutex_unlock(&event_connections_lock); /* The + 1 is b/c of the server_fd. */ pfdslen++; pfds = Malloc(sizeof(struct pollfd) * pfdslen); memset(pfds, '\0', sizeof(struct pollfd) * pfdslen); pfds[0].fd = server_fd; pfds[0].events = POLLIN; pfds[0].revents = 0; /* No 'event_connections' if there are no events */ if (event_connections) { i = 1; Pthread_mutex_lock(&event_connections_lock); eitr = List_iterator_create(event_connections); while ((ecd = list_next(eitr))) { pfds[i].fd = ecd->fd; pfds[i].events = POLLIN; pfds[i].revents = 0; i++; } List_iterator_destroy(eitr); Pthread_mutex_unlock(&event_connections_lock); } Poll(pfds, pfdslen, -1); /* Deal with the server fd first */ if (pfds[0].revents & POLLERR) CEREBROD_DBG(("server_fd POLLERR")); else if (pfds[0].revents & POLLIN) { unsigned int client_addr_len; int fd; struct sockaddr_in client_addr; client_addr_len = sizeof(struct sockaddr_in); if ((fd = accept(server_fd, (struct sockaddr *)&client_addr, &client_addr_len)) < 0) server_fd = cerebrod_reinit_socket(server_fd, 0, _event_server_setup_socket, "event_server: accept"); if (fd >= 0) _event_server_service_connection(fd); } /* Deal with the connecting fds */ for (i = 1; i < pfdslen; i++) { if (pfds[i].revents & POLLERR) { CEREBROD_DBG(("fd = %d POLLERR", pfds[i].fd)); Pthread_mutex_lock(&event_connections_lock); _delete_event_connection_fd(pfds[i].fd); Pthread_mutex_unlock(&event_connections_lock); continue; } if (pfds[i].revents & POLLIN) { char buf[CEREBRO_MAX_PACKET_LEN]; int n; /* We should not expect any actual data. If * we get some, just eat it and move on. * * The common situation is that the client * closes the connection. So we need to delete * our fd. */ n = fd_read_n(pfds[i].fd, buf, CEREBRO_MAX_PACKET_LEN); if (n < 0) CEREBROD_DBG(("fd_read_n = %s", strerror(errno))); if (n <= 0) { if (conf.debug && conf.event_server_debug) { Pthread_mutex_lock(&debug_output_mutex); fprintf(stderr, "**************************************\n"); fprintf(stderr, "* Event Server Close Fd: %d\n", pfds[i].fd); fprintf(stderr, "**************************************\n"); Pthread_mutex_unlock(&debug_output_mutex); } Pthread_mutex_lock(&event_connections_lock); _delete_event_connection_fd(pfds[i].fd); Pthread_mutex_unlock(&event_connections_lock); } } } Free(pfds); } return NULL; /* NOT REACHED */ }
/* * Under almost any circumstance, don't return a -1 error, cerebro can * go on without loading monitor modules. The listener_data_init_lock * should already be set. */ int cerebrod_monitor_modules_setup(void) { int i, monitor_module_count, monitor_index_len, monitor_index_count = 0; struct cerebrod_monitor_module_list *ml = NULL; #if CEREBRO_DEBUG int rv; #endif /* CEREBRO_DEBUG */ #if CEREBRO_DEBUG /* Should be called with lock already set */ rv = Pthread_mutex_trylock(&listener_data_init_lock); if (rv != EBUSY) CEREBROD_EXIT(("mutex not locked: rv=%d", rv)); #endif /* CEREBRO_DEBUG */ if (!(monitor_handle = monitor_modules_load())) { CEREBROD_DBG(("monitor_modules_load")); goto cleanup; } if ((monitor_module_count = monitor_modules_count(monitor_handle)) < 0) { CEREBROD_DBG(("monitor_modules_count")); goto cleanup; } if (!monitor_module_count) { if (conf.debug && conf.listen_debug) { Pthread_mutex_lock(&debug_output_mutex); fprintf(stderr, "**************************************\n"); fprintf(stderr, "* No Monitor Modules Found\n"); fprintf(stderr, "**************************************\n"); Pthread_mutex_unlock(&debug_output_mutex); } goto cleanup; } /* Each monitor module may wish to monitor multiple metrics. We'll * assume there will never be more than 2 metrics per monitor module, and * that will be enough to avoid all hash collisions. */ monitor_index_len = monitor_module_count * 2; monitor_index = Hash_create(monitor_index_len, (hash_key_f)hash_key_string, (hash_cmp_f)strcmp, (hash_del_f)_cerebrod_monitor_module_list_destroy); for (i = 0; i < monitor_module_count; i++) { struct cerebrod_monitor_module_info *monitor_module; char *module_name, *metric_names; char *metricPtr, *metricbuf; module_name = monitor_module_name(monitor_handle, i); if (conf.monitor_module_exclude_len) { int found_exclude = 0; int j; for (j = 0; j < conf.monitor_module_exclude_len; j++) { if (!strcasecmp(conf.monitor_module_exclude[j], module_name)) { found_exclude++; break; } } if (found_exclude) { if (conf.debug && conf.listen_debug) { Pthread_mutex_lock(&debug_output_mutex); fprintf(stderr, "**************************************\n"); fprintf(stderr, "* Skip Monitor Module: %s\n", module_name); fprintf(stderr, "**************************************\n"); Pthread_mutex_unlock(&debug_output_mutex); } CEREBROD_ERR(("Dropping monitor module: %s", module_name)); continue; } } if (conf.debug && conf.listen_debug) { Pthread_mutex_lock(&debug_output_mutex); fprintf(stderr, "**************************************\n"); fprintf(stderr, "* Setup Monitor Module: %s\n", module_name); fprintf(stderr, "**************************************\n"); Pthread_mutex_unlock(&debug_output_mutex); } if (monitor_module_setup(monitor_handle, i) < 0) { CEREBROD_DBG(("monitor_module_setup failed: %s", module_name)); continue; } if (!(metric_names = monitor_module_metric_names(monitor_handle, i)) < 0) { CEREBROD_DBG(("monitor_module_metric_names failed: %s", module_name)); monitor_module_cleanup(monitor_handle, i); continue; } monitor_module = Malloc(sizeof(struct cerebrod_monitor_module_info)); monitor_module->metric_names = Strdup(metric_names); monitor_module->index = i; Pthread_mutex_init(&(monitor_module->monitor_lock), NULL); /* The monitoring module may support multiple metrics */ metricPtr = strtok_r(monitor_module->metric_names, ",", &metricbuf); while (metricPtr) { if (!(ml = Hash_find(monitor_index, metricPtr))) { ml = (struct cerebrod_monitor_module_list *)Malloc(sizeof(struct cerebrod_monitor_module_list)); ml->monitor_list = List_create((ListDelF)_cerebrod_monitor_module_info_destroy); Pthread_mutex_init(&(ml->monitor_list_lock), NULL); List_append(ml->monitor_list, monitor_module); Hash_insert(monitor_index, metricPtr, ml); monitor_index_count++; } else List_append(ml->monitor_list, monitor_module); metricPtr = strtok_r(NULL, ",", &metricbuf); } } if (!monitor_index_count) goto cleanup; return 1; cleanup: if (monitor_handle) { monitor_modules_unload(monitor_handle); monitor_handle = NULL; } if (monitor_index) { Hash_destroy(monitor_index); monitor_index = NULL; } return 0; }