/*
* _cerebrod_listener_initialize
*
* perform listener initialization
*/
static void
_cerebrod_listener_initialize(void)
{
int i;
Pthread_mutex_lock(&listener_init_lock);
if (listener_init)
goto out;
Pthread_mutex_lock(&listener_fds_lock);
for (i = 0; i < conf.listen_message_config_len; i++)
{
if ((listener_fds[i] = _listener_setup_socket(i)) < 0)
CEREBROD_EXIT(("listener fd setup failed"));
}
Pthread_mutex_unlock(&listener_fds_lock);
if (!(clusterlist_handle = clusterlist_module_load()))
CEREBROD_EXIT(("clusterlist_module_load"));
if ((found_clusterlist_module = clusterlist_module_found(clusterlist_handle)) < 0)
CEREBROD_EXIT(("clusterlist_module_found"));
if (found_clusterlist_module)
{
if (clusterlist_module_setup(clusterlist_handle) < 0)
CEREBROD_EXIT(("clusterlist_module_setup"));
if (conf.debug && conf.listen_debug)
{
fprintf(stderr, "**************************************\n");
fprintf(stderr, "* Cerebro Clusterlist\n");
fprintf(stderr, "* -----------------------\n");
fprintf(stderr, "* Using Clusterlist: %s\n",
clusterlist_module_name(clusterlist_handle));
fprintf(stderr, "**************************************\n");
}
}
cerebrod_listener_data_initialize();
for (i = 0; i < conf.forward_message_config_len; i++)
{
/* if the forward destination is local to the machine, don't forward */
if (conf.forward_message_config[i].ip_is_local)
continue;
if (_forwarding_setup(i) < 0)
CEREBROD_EXIT(("forwarding setup failed"));
}
listener_init++;
Pthread_cond_signal(&listener_init_cond);
out:
Pthread_mutex_unlock(&listener_init_lock);
}
void
cerebrod_event_update_node_received_time(struct cerebrod_node_data *nd,
u_int32_t received_time)
{
struct cerebrod_event_node_timeout *ntd;
#if CEREBRO_DEBUG
int rv;
#endif /* CEREBRO_DEBUG */
assert(nd);
assert(received_time);
if (!event_index)
return;
#if CEREBRO_DEBUG
/* Should be called with lock already set */
rv = Pthread_mutex_trylock(&nd->node_data_lock);
if (rv != EBUSY)
CEREBROD_EXIT(("mutex not locked: rv=%d", rv));
#endif /* CEREBRO_DEBUG */
Pthread_mutex_lock(&event_node_timeout_data_lock);
if ((ntd = Hash_find(event_node_timeout_data_index, nd->nodename)))
{
ntd->last_received_time = received_time;
ntd->timeout_occurred = 0;
}
Pthread_mutex_unlock(&event_node_timeout_data_lock);
}
void
cerebrod_speaker_data_initialize(void)
{
#if !WITH_CEREBROD_NO_THREADS
Pthread_mutex_lock(&speaker_data_init_lock);
#endif /* !WITH_CEREBROD_NO_THREADS */
if (speaker_data_init)
goto out;
#if !WITH_CEREBROD_NO_THREADS
/*
* Must lock in this initialization routine, b/c the update thread
* in a metric module may call the update state function.
*/
Pthread_mutex_lock(&metric_list_lock);
#endif /* !WITH_CEREBROD_NO_THREADS */
metric_list = List_create((ListDelF)_destroy_speaker_metric_info);
if (_setup_metric_modules() < 0)
CEREBROD_EXIT(("_setup_metric_modules"));
#if !WITH_CEREBROD_NO_THREADS
Pthread_mutex_unlock(&metric_list_lock);
#endif /* !WITH_CEREBROD_NO_THREADS */
speaker_data_init++;
out:
#if !WITH_CEREBROD_NO_THREADS
Pthread_mutex_unlock(&speaker_data_init_lock);
#endif /* !WITH_CEREBROD_NO_THREADS */
; /* in case !WITH_CEREBRO_NO_THREADS */
}
void
cerebrod_rehash(hash_t *old_hash,
int *hash_size,
int hash_size_increment,
int hash_num,
pthread_mutex_t *hash_mutex)
{
hash_t new_hash;
assert(old_hash && hash_size && hash_size_increment && hash_num);
#if CEREBRO_DEBUG
/* Should be called with lock already set */
if (hash_mutex)
{
int rv = Pthread_mutex_trylock(hash_mutex);
if (rv != EBUSY)
CEREBROD_EXIT(("mutex not locked: rv=%d", rv));
}
#endif /* CEREBRO_DEBUG */
*hash_size += hash_size_increment;
new_hash = Hash_create(*hash_size,
(hash_key_f)hash_key_string,
(hash_cmp_f)strcmp,
(hash_del_f)_Free);
if (Hash_for_each(*old_hash, _hash_reinsert, &new_hash) != hash_num)
CEREBROD_EXIT(("invalid reinsert: hash_num=%d", hash_num));
if (Hash_remove_if(*old_hash, _hash_removeall, NULL) != hash_num)
CEREBROD_EXIT(("invalid removeall: hash_num=%d", hash_num));
Hash_destroy(*old_hash);
*old_hash = new_hash;
}
/*
* cerebrod_monitor_modules_update
*
* Send metric data to the appropriate monitor modules, if necessary.
* The struct cerebrod_node_data lock should already be locked.
*/
void
cerebrod_monitor_modules_update(const char *nodename,
struct cerebrod_node_data *nd,
const char *metric_name,
struct cerebrod_message_metric *mm)
{
struct cerebrod_monitor_module_info *monitor_module;
struct cerebrod_monitor_module_list *ml;
#if CEREBRO_DEBUG
int rv;
#endif /* CEREBRO_DEBUG */
assert(nodename && nd && metric_name && mm);
if (!monitor_index)
return;
#if CEREBRO_DEBUG
/* Should be called with lock already set */
rv = Pthread_mutex_trylock(&nd->node_data_lock);
if (rv != EBUSY)
CEREBROD_EXIT(("mutex not locked: rv=%d", rv));
#endif /* CEREBRO_DEBUG */
if ((ml = Hash_find(monitor_index, metric_name)))
{
ListIterator itr = NULL;
Pthread_mutex_lock(&(ml->monitor_list_lock));
itr = List_iterator_create(ml->monitor_list);
Pthread_mutex_unlock(&(ml->monitor_list_lock));
while ((monitor_module = list_next(itr)))
{
Pthread_mutex_lock(&monitor_module->monitor_lock);
monitor_module_metric_update(monitor_handle,
monitor_module->index,
nodename,
metric_name,
mm->metric_value_type,
mm->metric_value_len,
mm->metric_value);
Pthread_mutex_unlock(&monitor_module->monitor_lock);
}
Pthread_mutex_lock(&(ml->monitor_list_lock));
List_iterator_destroy(itr);
Pthread_mutex_unlock(&(ml->monitor_list_lock));
}
}
static void
_delete_event_connection_fd(int fd)
{
struct cerebrod_event_connection_data *ecd;
ListIterator eitr;
#if CEREBRO_DEBUG
int rv;
#endif /* CEREBRO_DEBUG */
assert(fd >= 0);
#if CEREBRO_DEBUG
/* Should be called with lock already set */
rv = Pthread_mutex_trylock(&event_connections_lock);
if (rv != EBUSY)
CEREBROD_EXIT(("mutex not locked: rv=%d", rv));
#endif /* CEREBRO_DEBUG */
eitr = List_iterator_create(event_connections);
while ((ecd = list_next(eitr)))
{
if (ecd->fd == fd)
{
List connections;
if ((connections = Hash_find(event_connections_index,
ecd->event_name)))
{
ListIterator citr;
int *fdPtr;
citr = List_iterator_create(connections);
while ((fdPtr = list_next(citr)))
{
if (*fdPtr == fd)
{
List_delete(citr);
break;
}
}
List_iterator_destroy(citr);
}
List_delete(eitr);
break;
}
}
List_iterator_destroy(eitr);
}
void
cerebrod_speaker_data_get_module_metric_data(struct cerebrod_message *msg,
unsigned int *message_len,
unsigned int index,
int *more_data_to_send)
{
struct cerebrod_message_metric *mm = NULL;
assert(msg && message_len && more_data_to_send);
assert(metric_handle && metric_list && metric_list_size);
if (!speaker_data_init)
CEREBROD_EXIT(("initialization not complete"));
*more_data_to_send = 0;
#if !WITH_CEREBROD_NO_THREADS
Pthread_mutex_lock(&metric_list_lock);
#endif /* !WITH_CEREBROD_NO_THREADS */
msg->metrics_len = 0;
msg->metrics = Malloc(sizeof(struct cerebrod_message_metric *)*2);
memset(msg->metrics, '\0', sizeof(struct cerebrod_message_metric *)*2);
/* Since this is the only metric value we're sending, we should
* never have more data than can fit in a single UDP packet.
*/
if ((mm = _get_module_metric_value(index)))
{
*message_len += CEREBROD_MESSAGE_METRIC_HEADER_LEN;
*message_len += mm->metric_value_len;
msg->metrics[msg->metrics_len] = mm;
msg->metrics_len++;
}
#if !WITH_CEREBROD_NO_THREADS
Pthread_mutex_unlock(&metric_list_lock);
#endif /* !WITH_CEREBROD_NO_THREADS */
return;
}
void
cerebrod_speaker_data_metric_list_sort(void)
{
#if CEREBRO_DEBUG
#if !WITH_CEREBROD_NO_THREADS
int rv;
#endif /* !WITH_CEREBROD_NO_THREADS */
#endif /* CEREBRO_DEBUG */
assert(metric_list);
#if CEREBRO_DEBUG
#if !WITH_CEREBROD_NO_THREADS
/* Should be called with lock already set */
rv = Pthread_mutex_trylock(&metric_list_lock);
if (rv != EBUSY)
CEREBROD_EXIT(("mutex not locked: rv=%d", rv));
#endif /* !WITH_CEREBROD_NO_THREADS */
#endif /* CEREBRO_DEBUG */
List_sort(metric_list, _next_call_time_cmp);
}
/*
* _event_node_timeout_data_add
*
* Create entries for the event_node_timeout_data list and index
*
* Returns 0 on success, -1 on error
*/
static int
_event_node_timeout_data_add(const char *nodename, u_int32_t time_now)
{
struct cerebrod_event_node_timeout_data *ntd;
#if CEREBRO_DEBUG
int rv;
/* Should be called with lock already set */
rv = Pthread_mutex_trylock(&event_node_timeout_data_lock);
if (rv != EBUSY)
CEREBROD_EXIT(("mutex not locked: rv=%d", rv));
#endif /* CEREBRO_DEBUG */
ntd = (struct cerebrod_event_node_timeout_data *)Malloc(sizeof(struct cerebrod_event_node_timeout_data));
ntd->nodename = (char *)nodename;
ntd->last_received_time = time_now;
ntd->timeout_occurred = 0;
List_append(event_node_timeout_data, ntd);
Hash_insert(event_node_timeout_data_index, ntd->nodename, ntd);
event_node_timeout_data_index_numnodes++;
return 0;
}
int
cerebrod_reinit_socket(int old_fd,
int num,
Cerebrod_socket_setup socket_setup,
char *msg)
{
int fd = old_fd;
assert(socket_setup && msg);
if (errno == EINVAL
|| errno == EBADF
|| errno == ENODEV
|| errno == ENETDOWN
|| errno == ENETUNREACH
|| old_fd < 0)
{
if (!(old_fd < 0))
close(old_fd); /* no-wrapper, make best effort */
if ((fd = socket_setup(num)) < 0)
{
CEREBROD_DBG(("%s: error re-initializing socket", msg));
/* Wait a bit, so we don't spin */
sleep(CEREBROD_REINITIALIZE_TIME);
}
else
CEREBROD_DBG(("success re-initializing socket"));
}
else if (errno == EINTR)
CEREBROD_DBG(("%s: %s", msg, strerror(errno)));
else
CEREBROD_EXIT(("%s: %s", msg, strerror(errno)));
return fd;
}
void
cerebrod_event_add_node_timeout_data(struct cerebrod_node_data *nd,
u_int32_t received_time)
{
#if CEREBRO_DEBUG
int rv;
#endif /* CEREBRO_DEBUG */
assert(nd);
assert(received_time);
if (!event_index)
return;
assert(event_node_timeout_data_index);
#if CEREBRO_DEBUG
/* Should be called with lock already set */
rv = Pthread_mutex_trylock(&listener_data_lock);
if (rv != EBUSY)
CEREBROD_EXIT(("mutex not locked: rv=%d", rv));
#endif /* CEREBRO_DEBUG */
Pthread_mutex_lock(&event_node_timeout_data_lock);
/* Re-hash if our hash is getting too small */
if ((event_node_timeout_data_index_numnodes + 1) > (event_node_timeout_data_index_size*2))
cerebrod_rehash(&event_node_timeout_data_index,
&event_node_timeout_data_index_size,
EVENT_NODE_TIMEOUT_SIZE_INCREMENT,
event_node_timeout_data_index_numnodes,
&event_node_timeout_data_lock);
_event_node_timeout_data_add(nd->nodename, received_time);
event_node_timeout_data_index_numnodes++;
Pthread_mutex_unlock(&event_node_timeout_data_lock);
}
/*
* _setup_metric_modules
*
* Setup metric modules. Under almost any circumstance, don't return a
* -1 error, cerebro can go on without loading metric modules.
*
* Returns 1 if modules are loaded, 0 if not, -1 on error
*/
static int
_setup_metric_modules(void)
{
int i;
#if CEREBRO_DEBUG
#if !WITH_CEREBROD_NO_THREADS
int rv;
#endif /* !WITH_CEREBROD_NO_THREADS */
#endif /* CEREBRO_DEBUG */
assert(metric_list);
#if CEREBRO_DEBUG
#if !WITH_CEREBROD_NO_THREADS
/* Should be called with lock already set */
rv = Pthread_mutex_trylock(&metric_list_lock);
if (rv != EBUSY)
CEREBROD_EXIT(("mutex not locked: rv=%d", rv));
#endif /* !WITH_CEREBROD_NO_THREADS */
#endif /* CEREBRO_DEBUG */
if (!(metric_handle = metric_modules_load()))
{
CEREBROD_DBG(("metric_modules_load"));
goto cleanup;
}
if ((metric_handle_count = metric_modules_count(metric_handle)) < 0)
{
CEREBROD_DBG(("metric_module_count failed"));
goto cleanup;
}
if (!metric_handle_count)
{
if (conf.debug && conf.speak_debug)
{
#if !WITH_CEREBROD_NO_THREADS
Pthread_mutex_lock(&debug_output_mutex);
#endif /* !WITH_CEREBROD_NO_THREADS */
fprintf(stderr, "**************************************\n");
fprintf(stderr, "* No Metric Modules Found\n");
fprintf(stderr, "**************************************\n");
#if !WITH_CEREBROD_NO_THREADS
Pthread_mutex_unlock(&debug_output_mutex);
#endif /* !WITH_CEREBROD_NO_THREADS */
}
goto cleanup;
}
for (i = 0; i < metric_handle_count; i++)
{
struct cerebrod_speaker_metric_info *metric_info;
#if !WITH_CEREBROD_NO_THREADS
Cerebro_metric_thread_pointer threadPtr;
#endif /* !WITH_CEREBROD_NO_THREADS */
char *module_name, *metric_name;
int metric_period;
u_int32_t metric_flags;
module_name = metric_module_name(metric_handle, i);
if (conf.metric_module_exclude_len)
{
int found_exclude = 0;
int j;
for (j = 0; j < conf.metric_module_exclude_len; j++)
{
if (!strcasecmp(conf.metric_module_exclude[j], module_name))
{
found_exclude++;
break;
}
}
if (found_exclude)
{
if (conf.debug && conf.speak_debug)
{
#if !WITH_CEREBROD_NO_THREADS
Pthread_mutex_lock(&debug_output_mutex);
#endif /* !WITH_CEREBROD_NO_THREADS */
fprintf(stderr, "**************************************\n");
fprintf(stderr, "* Skip Metric Module: %s\n", module_name);
fprintf(stderr, "**************************************\n");
#if !WITH_CEREBROD_NO_THREADS
Pthread_mutex_unlock(&debug_output_mutex);
#endif /* !WITH_CEREBROD_NO_THREADS */
}
CEREBROD_ERR(("Dropping metric module: %s", module_name));
continue;
}
}
if (conf.debug && conf.speak_debug)
{
#if !WITH_CEREBROD_NO_THREADS
Pthread_mutex_lock(&debug_output_mutex);
#endif /* !WITH_CEREBROD_NO_THREADS */
fprintf(stderr, "**************************************\n");
fprintf(stderr, "* Setup Metric Module: %s\n", module_name);
fprintf(stderr, "**************************************\n");
#if !WITH_CEREBROD_NO_THREADS
Pthread_mutex_unlock(&debug_output_mutex);
#endif /* !WITH_CEREBROD_NO_THREADS */
}
if (metric_module_setup(metric_handle, i) < 0)
{
CEREBROD_DBG(("metric_module_setup: %s", module_name));
continue;
}
if (!(metric_name = metric_module_get_metric_name(metric_handle, i)))
{
CEREBROD_DBG(("metric_module_get_metric_name: %s", module_name));
metric_module_cleanup(metric_handle, i);
continue;
}
if (metric_module_get_metric_period(metric_handle, i, &metric_period) < 0)
{
CEREBROD_DBG(("metric_module_get_metric_period: %s", module_name));
metric_module_cleanup(metric_handle, i);
continue;
}
if (metric_module_get_metric_flags(metric_handle, i, &metric_flags) < 0)
{
CEREBROD_DBG(("metric_module_get_metric_flags: %s", module_name));
metric_module_cleanup(metric_handle, i);
continue;
}
if (metric_flags & CEREBRO_METRIC_MODULE_FLAGS_SEND_ON_PERIOD
&& metric_period <= 0)
{
CEREBROD_DBG(("metric module period invalid: %s", module_name));
metric_module_cleanup(metric_handle, i);
continue;
}
if (metric_module_send_message_function_pointer(metric_handle, i, &cerebrod_send_message) < 0)
{
CEREBROD_DBG(("metric_module_send_message_function_pointer: %s", module_name));
metric_module_cleanup(metric_handle, i);
continue;
}
metric_info = Malloc(sizeof(struct cerebrod_speaker_metric_info));
/* No need to Strdup() the name in this case */
metric_info->metric_name = metric_name;
metric_info->metric_origin = CEREBROD_METRIC_SPEAKER_ORIGIN_MODULE;
metric_info->metric_period = metric_period;
metric_info->metric_flags = metric_flags;
metric_info->index = i;
/*
* If metric period is < 0, it presumably never will be sent
* (metric is likely handled by a metric_thread), so set
* next_call_time to UINT_MAX.
*
* If this is a metric that will be piggy-backed on heartbeats,
* then initialize next_call_time to 0, so the data is sent on
* the first heartbeat
*
* If this is a metric that will not be piggy-backed on
* heartbeats, set the next_call_time to UINT_MAX. Let the
* speaker logic decide when packets should be sent.
*/
if (metric_info->metric_period < 0
|| metric_info->metric_flags & CEREBRO_METRIC_MODULE_FLAGS_SEND_ON_PERIOD)
metric_info->next_call_time = UINT_MAX;
else
metric_info->next_call_time = 0;
List_append(metric_list, metric_info);
metric_list_size++;
#if !WITH_CEREBROD_NO_THREADS
if ((threadPtr = metric_module_get_metric_thread(metric_handle, i)))
{
pthread_t thread;
pthread_attr_t attr;
Pthread_attr_init(&attr);
Pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
Pthread_attr_setstacksize(&attr, CEREBROD_THREAD_STACKSIZE);
Pthread_create(&thread, &attr, threadPtr, NULL);
Pthread_attr_destroy(&attr);
}
#endif /* !WITH_CEREBROD_NO_THREADS */
}
if (!metric_list_size)
goto cleanup;
cerebrod_speaker_data_metric_list_sort();
return 1;
cleanup:
if (metric_handle)
{
/* unload will call module cleanup functions */
metric_modules_unload(metric_handle);
metric_handle = NULL;
metric_handle_count = 0;
}
metric_list_size = 0;
return 0;
}
void
cerebrod_event_modules_update(const char *nodename,
struct cerebrod_node_data *nd,
const char *metric_name,
struct cerebrod_message_metric *mm)
{
struct cerebrod_event_module_info *event_module;
struct cerebrod_event_module_list *el;
#if CEREBRO_DEBUG
int rv;
#endif /* CEREBRO_DEBUG */
assert(nodename && nd && metric_name && mm);
if (!event_index)
return;
#if CEREBRO_DEBUG
/* Should be called with lock already set */
rv = Pthread_mutex_trylock(&nd->node_data_lock);
if (rv != EBUSY)
CEREBROD_EXIT(("mutex not locked: rv=%d", rv));
#endif /* CEREBRO_DEBUG */
/*
* This function may be called by multiple threads by the listener.
*
* The event_index is setup at the beginning and is only read, not
* written to. However, the lists stored inside the event_index
* need to called w/ thread safety (due to the nature of the list
* API).
*
*/
if ((el = Hash_find(event_index, metric_name)))
{
struct cerebro_event *event = NULL;
ListIterator itr = NULL;
int rv;
Pthread_mutex_lock(&(el->event_list_lock));
itr = List_iterator_create(el->event_list);
Pthread_mutex_unlock(&(el->event_list_lock));
while ((event_module = list_next(itr)))
{
Pthread_mutex_lock(&event_module->event_lock);
if ((rv = event_module_metric_update(event_handle,
event_module->index,
nodename,
metric_name,
mm->metric_value_type,
mm->metric_value_len,
mm->metric_value,
&event)) < 0)
{
CEREBROD_DBG(("event_module_metric_update"));
goto loop_next;
}
if (rv && event)
cerebrod_queue_event(event, event_module->index);
loop_next:
Pthread_mutex_unlock(&event_module->event_lock);
}
Pthread_mutex_lock(&(el->event_list_lock));
List_iterator_destroy(itr);
Pthread_mutex_unlock(&(el->event_list_lock));
}
}
/*
* Under almost any circumstance, don't return a -1 error, cerebro can
* go on without loading monitor modules. The listener_data_init_lock
* should already be set.
*/
int
cerebrod_event_modules_setup(void)
{
int i, event_module_count, event_index_len, event_index_count = 0;
struct cerebrod_event_module_list *el = NULL;
#if CEREBRO_DEBUG
int rv;
#endif /* CEREBRO_DEBUG */
assert(listener_data);
#if CEREBRO_DEBUG
/* Should be called with lock already set */
rv = Pthread_mutex_trylock(&listener_data_init_lock);
if (rv != EBUSY)
CEREBROD_EXIT(("mutex not locked: rv=%d", rv));
#endif /* CEREBRO_DEBUG */
if (!conf.event_server)
return 0;
if (!(event_handle = event_modules_load()))
{
CEREBROD_DBG(("event_modules_load"));
goto cleanup;
}
if ((event_module_count = event_modules_count(event_handle)) < 0)
{
CEREBROD_DBG(("event_modules_count"));
goto cleanup;
}
if (!event_module_count)
{
if (conf.debug && conf.event_server_debug)
{
Pthread_mutex_lock(&debug_output_mutex);
fprintf(stderr, "**************************************\n");
fprintf(stderr, "* No Event Modules Found\n");
fprintf(stderr, "**************************************\n");
Pthread_mutex_unlock(&debug_output_mutex);
}
goto cleanup;
}
/* Each event module may want multiple metrics and/or offer multiple
* event names. We'll assume there will never be more than 2 per
* event module, and that will be enough to avoid all hash
* collisions.
*/
event_index_len = event_module_count * 2;
event_index = Hash_create(event_index_len,
(hash_key_f)hash_key_string,
(hash_cmp_f)strcmp,
(hash_del_f)_cerebrod_event_module_list_destroy);
event_names = List_create((ListDelF)_Free);
event_module_timeouts = List_create((ListDelF)_cerebrod_event_module_timeout_data_destroy);
event_module_timeout_index = Hash_create(event_module_count,
(hash_key_f)hash_key_string,
(hash_cmp_f)strcmp,
(hash_del_f)list_destroy);
for (i = 0; i < event_module_count; i++)
{
struct cerebrod_event_module *event_module;
char *module_name, *module_metric_names, *module_event_names;
char *metricPtr, *metricbuf;
char *eventnamePtr, *eventbuf;
int timeout;
module_name = event_module_name(event_handle, i);
if (conf.event_module_exclude_len)
{
int found_exclude = 0;
int j;
for (j = 0; j < conf.event_module_exclude_len; j++)
{
if (!strcasecmp(conf.event_module_exclude[j], module_name))
{
found_exclude++;
break;
}
}
if (found_exclude)
{
if (conf.debug && conf.event_server_debug)
{
Pthread_mutex_lock(&debug_output_mutex);
fprintf(stderr, "**************************************\n");
fprintf(stderr, "* Skip Event Module: %s\n", module_name);
fprintf(stderr, "**************************************\n");
Pthread_mutex_unlock(&debug_output_mutex);
}
CEREBROD_ERR(("Dropping event module: %s", module_name));
continue;
}
}
if (conf.debug && conf.event_server_debug)
{
Pthread_mutex_lock(&debug_output_mutex);
fprintf(stderr, "**************************************\n");
fprintf(stderr, "* Setup Event Module: %s\n", module_name);
fprintf(stderr, "**************************************\n");
Pthread_mutex_unlock(&debug_output_mutex);
}
if (event_module_setup(event_handle, i) < 0)
{
CEREBROD_DBG(("event_module_setup failed: %s", module_name));
continue;
}
if (!(module_metric_names = event_module_metric_names(event_handle, i)) < 0)
{
CEREBROD_DBG(("event_module_metric_names failed: %s", module_name));
event_module_cleanup(event_handle, i);
continue;
}
if (!(module_event_names = event_module_event_names(event_handle, i)) < 0)
{
CEREBROD_DBG(("event_module_event_names failed: %s", module_name));
event_module_cleanup(event_handle, i);
continue;
}
if ((timeout = event_module_timeout_length(event_handle, i)) < 0)
{
CEREBROD_DBG(("event_module_timeout_length failed: %s", module_name));
event_module_cleanup(event_handle, i);
continue;
}
event_module = Malloc(sizeof(struct cerebrod_event_module_info));
event_module->metric_names = Strdup(module_metric_names);
event_module->event_names = Strdup(module_event_names);
event_module->index = i;
Pthread_mutex_init(&(event_module->event_lock), NULL);
/* The monitoring module may support multiple metrics */
metricPtr = strtok_r(event_module->metric_names, ",", &metricbuf);
while (metricPtr)
{
if (!(el = Hash_find(event_index, metricPtr)))
{
el = (struct cerebrod_event_module_list *)Malloc(sizeof(struct cerebrod_event_module_list));
el->event_list = List_create((ListDelF)_cerebrod_event_module_info_destroy);
Pthread_mutex_init(&(el->event_list_lock), NULL);
List_append(el->event_list, event_module);
Hash_insert(event_index, metricPtr, el);
event_index_count++;
}
else
List_append(el->event_list, event_module);
metricPtr = strtok_r(NULL, ",", &metricbuf);
}
/* The monitoring module may support multiple event names */
eventnamePtr = strtok_r(event_module->event_names, ",", &eventbuf);
while (eventnamePtr)
{
if (!list_find_first(event_names,
(ListFindF)_cerebrod_name_strcmp,
eventnamePtr))
{
List_append(event_names, eventnamePtr);
if (conf.debug && conf.event_server_debug)
{
Pthread_mutex_lock(&debug_output_mutex);
fprintf(stderr, "**************************************\n");
fprintf(stderr, "* Event Name: %s\n", eventnamePtr);
fprintf(stderr, "**************************************\n");
Pthread_mutex_unlock(&debug_output_mutex);
}
}
eventnamePtr = strtok_r(NULL, ",", &eventbuf);
}
if (timeout)
{
struct cerebrod_event_module_timeout_data *mtd;
List modules_list;
if (!(mtd = List_find_first(event_module_timeouts,
_event_module_timeout_data_find_callback,
&timeout)))
{
char strbuf[64];
mtd = (struct cerebrod_event_module_timeout_data *)Malloc(sizeof(struct cerebrod_event_module_timeout_data));
mtd->timeout = timeout;
snprintf(strbuf, 64, "%d", timeout);
mtd->timeout_str = Strdup(strbuf);
List_append(event_module_timeouts, mtd);
if (timeout < event_module_timeout_min)
event_module_timeout_min = timeout;
}
if (!(modules_list = Hash_find(event_module_timeout_index,
mtd->timeout_str)))
{
modules_list = List_create((ListDelF)NULL);
List_append(modules_list, event_module);
Hash_insert(event_module_timeout_index,
mtd->timeout_str,
modules_list);
}
else
List_append(modules_list, event_module);
}
}
List_sort(event_module_timeouts, _event_module_timeout_data_compare);
if (!event_index_count)
goto cleanup;
if (_setup_event_node_timeout_data() < 0)
goto cleanup;
/*
* Since the cerebrod listener is started before any of the event
* threads (node_timeout, queue_monitor, server), this must be
* created in here (which is called by the listener) to avoid a
* possible race of modules creating events before the event_queue
* is created.
*/
event_queue = List_create((ListDelF)cerebrod_event_to_send_destroy);
return 1;
cleanup:
if (event_handle)
{
event_modules_unload(event_handle);
event_handle = NULL;
}
if (event_index)
{
Hash_destroy(event_index);
event_index = NULL;
}
if (event_names)
{
List_destroy(event_names);
event_names = NULL;
}
return 0;
}
void *
cerebrod_listener(void *arg)
{
char buf[CEREBRO_MAX_PACKET_LEN];
_cerebrod_listener_initialize();
for (;;)
{
struct cerebrod_message *msg;
char nodename_buf[CEREBRO_MAX_NODENAME_LEN+1];
char nodename_key[CEREBRO_MAX_NODENAME_LEN+1];
struct timeval tv;
int in_cluster_flag, i, count;
fd_set readfds;
int recv_len = 0;
int maxfd = 0;
FD_ZERO(&readfds);
Pthread_mutex_lock(&listener_fds_lock);
for (i = 0; i < conf.listen_message_config_len; i++)
{
if (listener_fds[i] > maxfd)
maxfd = listener_fds[i];
FD_SET(listener_fds[i], &readfds);
}
count = Select(maxfd + 1, &readfds, NULL, NULL, NULL);
for (i = 0; i < conf.listen_message_config_len; i++)
{
if (FD_ISSET(listener_fds[i], &readfds))
{
if ((recv_len = recvfrom(listener_fds[i],
buf,
CEREBRO_MAX_PACKET_LEN,
0,
NULL,
NULL)) < 0)
listener_fds[i] = cerebrod_reinit_socket(listener_fds[i],
i,
_listener_setup_socket,
"listener: recvfrom");
break;
}
}
Pthread_mutex_unlock(&listener_fds_lock);
/* No packet read */
if (recv_len <= 0)
continue;
if (recv_len >= CEREBRO_MAX_PACKET_LEN)
{
CEREBROD_DBG(("received truncated packet"));
continue;
}
if (_cerebrod_message_check_version(buf, recv_len) < 0)
{
CEREBROD_DBG(("received invalid version packet"));
continue;
}
if (!(msg = _cerebrod_message_unmarshall(buf, recv_len)))
{
CEREBROD_DBG(("received unmarshallable packet"));
continue;
}
_cerebrod_message_dump(msg, "Received Message");
/* Guarantee ending '\0' character */
memset(nodename_buf, '\0', CEREBRO_MAX_NODENAME_LEN+1);
memcpy(nodename_buf, msg->nodename, CEREBRO_MAX_NODENAME_LEN);
if (!strlen(nodename_buf))
{
CEREBROD_DBG(("received null nodename"));
cerebrod_message_destroy(msg);
continue;
}
if (found_clusterlist_module)
{
if ((in_cluster_flag = clusterlist_module_node_in_cluster(clusterlist_handle,
nodename_buf)) < 0)
CEREBROD_EXIT(("clusterlist_module_node_in_cluster: %s", nodename_buf));
/* Second chance, is this data being forwarded from another host */
if (!in_cluster_flag)
{
if (Hostlist_find(conf.forward_host_accept, nodename_buf) >= 0)
in_cluster_flag++;
}
}
else
/* must assume it is in the cluster */
/* Note, there is no need to handle 'forward_host_accept' under this case,
* since we don't know if it is in the cluster or not anyways.
*/
in_cluster_flag = 1;
if (!in_cluster_flag)
{
CEREBROD_DBG(("received non-cluster packet: %s", nodename_buf));
cerebrod_message_destroy(msg);
continue;
}
memset(nodename_key, '\0', CEREBRO_MAX_NODENAME_LEN+1);
if (found_clusterlist_module)
{
if (clusterlist_module_get_nodename(clusterlist_handle,
nodename_buf,
nodename_key,
CEREBRO_MAX_NODENAME_LEN+1) < 0)
{
CEREBROD_DBG(("clusterlist_module_get_nodename: %s", nodename_buf));
cerebrod_message_destroy(msg);
continue;
}
}
else
memcpy(nodename_key, nodename_buf, CEREBRO_MAX_NODENAME_LEN+1);
Gettimeofday(&tv, NULL);
cerebrod_listener_data_update(nodename_key, msg, tv.tv_sec);
/* Forward data as necessary. Note, there is no need to
* marshall data, it should already be marshalled when we
* read it earlier.
*/
for (i = 0; i < conf.forward_message_config_len; i++)
{
/* if the forward destination is local to the machine, don't forward */
if (conf.forward_message_config[i].ip_is_local)
continue;
if (!forwarding_info[i].hosts
|| hostlist_find(forwarding_info[i].hosts, nodename_key) >= 0)
{
struct sockaddr *addr;
struct sockaddr_in msgaddr;
unsigned int addrlen;
int rv;
memset(&msgaddr, '\0', sizeof(struct sockaddr_in));
msgaddr.sin_family = AF_INET;
msgaddr.sin_port = htons(conf.forward_message_config[i].destination_port);
memcpy(&msgaddr.sin_addr,
&conf.forward_message_config[i].ip_in_addr,
sizeof(struct in_addr));
addr = (struct sockaddr *)&msgaddr;
addrlen = sizeof(struct sockaddr_in);
_cerebrod_message_dump(msg, "Forwarding Message");
Pthread_mutex_lock(&forwarding_info[i].lock);
if ((rv = sendto(forwarding_info[i].fd,
buf,
recv_len,
0,
addr,
addrlen)) != recv_len)
{
if (rv < 0)
forwarding_info[i].fd = cerebrod_reinit_socket(forwarding_info[i].fd,
i,
_forwarding_setup_socket,
"forwarding: sendto");
else
CEREBROD_ERR(("sendto: invalid bytes sent"));
}
Pthread_mutex_unlock(&forwarding_info[i].lock);
}
}
cerebrod_message_destroy(msg);
}
return NULL; /* NOT REACHED */
}
void
cerebrod_speaker_data_get_heartbeat_metric_data(struct cerebrod_message *msg,
unsigned int *message_len,
int *more_data_to_send)
{
struct cerebrod_speaker_metric_info *metric_info;
ListIterator itr = NULL;
struct timeval tv;
assert(msg && message_len && more_data_to_send);
if (!speaker_data_init)
CEREBROD_EXIT(("initialization not complete"));
*more_data_to_send = 0;
#if !WITH_CEREBROD_NO_THREADS
Pthread_mutex_lock(&metric_list_lock);
#endif /* !WITH_CEREBROD_NO_THREADS */
/* There may not be any metrics to distribute */
if (!metric_list_size)
{
msg->metrics_len = 0;
msg->metrics = NULL;
goto lock_out;
}
msg->metrics_len = 0;
msg->metrics = Malloc(sizeof(struct cerebrod_message_metric *)*(metric_list_size + 1));
memset(msg->metrics, '\0', sizeof(struct cerebrod_message_metric *)*(metric_list_size + 1));
Gettimeofday(&tv, NULL);
itr = List_iterator_create(metric_list);
while ((metric_info = list_next(itr)))
{
struct cerebrod_message_metric *mm = NULL;
if (tv.tv_sec <= metric_info->next_call_time)
break;
if (metric_info->metric_origin & CEREBROD_METRIC_SPEAKER_ORIGIN_MODULE
&& !(metric_info->metric_flags & CEREBRO_METRIC_MODULE_FLAGS_SEND_ON_PERIOD))
mm = _get_module_metric_value(metric_info->index);
if (metric_info->metric_origin & CEREBROD_METRIC_SPEAKER_ORIGIN_USERSPACE)
mm = _get_userspace_metric_value(metric_info);
if (mm)
{
unsigned int new_len;
new_len = *message_len + CEREBROD_MESSAGE_METRIC_HEADER_LEN + mm->metric_value_len;
if (new_len >= CEREBRO_MAX_PACKET_LEN)
{
*more_data_to_send = 1;
goto sort_out;
}
else
{
*message_len += CEREBROD_MESSAGE_METRIC_HEADER_LEN;
*message_len += mm->metric_value_len;
msg->metrics[msg->metrics_len] = mm;
msg->metrics_len++;
}
}
if (metric_info->metric_origin & CEREBROD_METRIC_SPEAKER_ORIGIN_MODULE)
{
if (metric_info->metric_period < 0)
metric_info->next_call_time = UINT_MAX;
/*
* Metric period stays at 0 for metrics that need to be
* propogated every time
*/
if (metric_info->metric_period > 0)
metric_info->next_call_time = tv.tv_sec + metric_info->metric_period;
}
if (metric_info->metric_origin & CEREBROD_METRIC_SPEAKER_ORIGIN_USERSPACE)
metric_info->next_call_time = UINT_MAX;
}
sort_out:
List_iterator_destroy(itr);
cerebrod_speaker_data_metric_list_sort();
lock_out:
#if !WITH_CEREBROD_NO_THREADS
Pthread_mutex_unlock(&metric_list_lock);
#endif /* !WITH_CEREBROD_NO_THREADS */
return;
}
void *
cerebrod_event_server(void *arg)
{
int server_fd;
_event_server_initialize();
if ((server_fd = _event_server_setup_socket(0)) < 0)
CEREBROD_EXIT(("event server fd setup failed"));
for (;;)
{
ListIterator eitr;
struct cerebrod_event_connection_data *ecd;
struct pollfd *pfds;
int pfdslen = 0;
int i;
/* Note that the list_count won't grow larger after the first
* mutex block, b/c the cerebrod_event_queue_monitor thread can
* never add to the event_connections. It can only shrink it.
*/
Pthread_mutex_lock(&event_connections_lock);
if (event_connections)
pfdslen = List_count(event_connections);
Pthread_mutex_unlock(&event_connections_lock);
/* The + 1 is b/c of the server_fd. */
pfdslen++;
pfds = Malloc(sizeof(struct pollfd) * pfdslen);
memset(pfds, '\0', sizeof(struct pollfd) * pfdslen);
pfds[0].fd = server_fd;
pfds[0].events = POLLIN;
pfds[0].revents = 0;
/* No 'event_connections' if there are no events */
if (event_connections)
{
i = 1;
Pthread_mutex_lock(&event_connections_lock);
eitr = List_iterator_create(event_connections);
while ((ecd = list_next(eitr)))
{
pfds[i].fd = ecd->fd;
pfds[i].events = POLLIN;
pfds[i].revents = 0;
i++;
}
List_iterator_destroy(eitr);
Pthread_mutex_unlock(&event_connections_lock);
}
Poll(pfds, pfdslen, -1);
/* Deal with the server fd first */
if (pfds[0].revents & POLLERR)
CEREBROD_DBG(("server_fd POLLERR"));
else if (pfds[0].revents & POLLIN)
{
unsigned int client_addr_len;
int fd;
struct sockaddr_in client_addr;
client_addr_len = sizeof(struct sockaddr_in);
if ((fd = accept(server_fd,
(struct sockaddr *)&client_addr,
&client_addr_len)) < 0)
server_fd = cerebrod_reinit_socket(server_fd,
0,
_event_server_setup_socket,
"event_server: accept");
if (fd >= 0)
_event_server_service_connection(fd);
}
/* Deal with the connecting fds */
for (i = 1; i < pfdslen; i++)
{
if (pfds[i].revents & POLLERR)
{
CEREBROD_DBG(("fd = %d POLLERR", pfds[i].fd));
Pthread_mutex_lock(&event_connections_lock);
_delete_event_connection_fd(pfds[i].fd);
Pthread_mutex_unlock(&event_connections_lock);
continue;
}
if (pfds[i].revents & POLLIN)
{
char buf[CEREBRO_MAX_PACKET_LEN];
int n;
/* We should not expect any actual data. If
* we get some, just eat it and move on.
*
* The common situation is that the client
* closes the connection. So we need to delete
* our fd.
*/
n = fd_read_n(pfds[i].fd,
buf,
CEREBRO_MAX_PACKET_LEN);
if (n < 0)
CEREBROD_DBG(("fd_read_n = %s", strerror(errno)));
if (n <= 0)
{
if (conf.debug && conf.event_server_debug)
{
Pthread_mutex_lock(&debug_output_mutex);
fprintf(stderr, "**************************************\n");
fprintf(stderr, "* Event Server Close Fd: %d\n", pfds[i].fd);
fprintf(stderr, "**************************************\n");
Pthread_mutex_unlock(&debug_output_mutex);
}
Pthread_mutex_lock(&event_connections_lock);
_delete_event_connection_fd(pfds[i].fd);
Pthread_mutex_unlock(&event_connections_lock);
}
}
}
Free(pfds);
}
return NULL; /* NOT REACHED */
}
/*
* Under almost any circumstance, don't return a -1 error, cerebro can
* go on without loading monitor modules. The listener_data_init_lock
* should already be set.
*/
int
cerebrod_monitor_modules_setup(void)
{
int i, monitor_module_count, monitor_index_len, monitor_index_count = 0;
struct cerebrod_monitor_module_list *ml = NULL;
#if CEREBRO_DEBUG
int rv;
#endif /* CEREBRO_DEBUG */
#if CEREBRO_DEBUG
/* Should be called with lock already set */
rv = Pthread_mutex_trylock(&listener_data_init_lock);
if (rv != EBUSY)
CEREBROD_EXIT(("mutex not locked: rv=%d", rv));
#endif /* CEREBRO_DEBUG */
if (!(monitor_handle = monitor_modules_load()))
{
CEREBROD_DBG(("monitor_modules_load"));
goto cleanup;
}
if ((monitor_module_count = monitor_modules_count(monitor_handle)) < 0)
{
CEREBROD_DBG(("monitor_modules_count"));
goto cleanup;
}
if (!monitor_module_count)
{
if (conf.debug && conf.listen_debug)
{
Pthread_mutex_lock(&debug_output_mutex);
fprintf(stderr, "**************************************\n");
fprintf(stderr, "* No Monitor Modules Found\n");
fprintf(stderr, "**************************************\n");
Pthread_mutex_unlock(&debug_output_mutex);
}
goto cleanup;
}
/* Each monitor module may wish to monitor multiple metrics. We'll
* assume there will never be more than 2 metrics per monitor module, and
* that will be enough to avoid all hash collisions.
*/
monitor_index_len = monitor_module_count * 2;
monitor_index = Hash_create(monitor_index_len,
(hash_key_f)hash_key_string,
(hash_cmp_f)strcmp,
(hash_del_f)_cerebrod_monitor_module_list_destroy);
for (i = 0; i < monitor_module_count; i++)
{
struct cerebrod_monitor_module_info *monitor_module;
char *module_name, *metric_names;
char *metricPtr, *metricbuf;
module_name = monitor_module_name(monitor_handle, i);
if (conf.monitor_module_exclude_len)
{
int found_exclude = 0;
int j;
for (j = 0; j < conf.monitor_module_exclude_len; j++)
{
if (!strcasecmp(conf.monitor_module_exclude[j], module_name))
{
found_exclude++;
break;
}
}
if (found_exclude)
{
if (conf.debug && conf.listen_debug)
{
Pthread_mutex_lock(&debug_output_mutex);
fprintf(stderr, "**************************************\n");
fprintf(stderr, "* Skip Monitor Module: %s\n", module_name);
fprintf(stderr, "**************************************\n");
Pthread_mutex_unlock(&debug_output_mutex);
}
CEREBROD_ERR(("Dropping monitor module: %s", module_name));
continue;
}
}
if (conf.debug && conf.listen_debug)
{
Pthread_mutex_lock(&debug_output_mutex);
fprintf(stderr, "**************************************\n");
fprintf(stderr, "* Setup Monitor Module: %s\n", module_name);
fprintf(stderr, "**************************************\n");
Pthread_mutex_unlock(&debug_output_mutex);
}
if (monitor_module_setup(monitor_handle, i) < 0)
{
CEREBROD_DBG(("monitor_module_setup failed: %s", module_name));
continue;
}
if (!(metric_names = monitor_module_metric_names(monitor_handle, i)) < 0)
{
CEREBROD_DBG(("monitor_module_metric_names failed: %s", module_name));
monitor_module_cleanup(monitor_handle, i);
continue;
}
monitor_module = Malloc(sizeof(struct cerebrod_monitor_module_info));
monitor_module->metric_names = Strdup(metric_names);
monitor_module->index = i;
Pthread_mutex_init(&(monitor_module->monitor_lock), NULL);
/* The monitoring module may support multiple metrics */
metricPtr = strtok_r(monitor_module->metric_names, ",", &metricbuf);
while (metricPtr)
{
if (!(ml = Hash_find(monitor_index, metricPtr)))
{
ml = (struct cerebrod_monitor_module_list *)Malloc(sizeof(struct cerebrod_monitor_module_list));
ml->monitor_list = List_create((ListDelF)_cerebrod_monitor_module_info_destroy);
Pthread_mutex_init(&(ml->monitor_list_lock), NULL);
List_append(ml->monitor_list, monitor_module);
Hash_insert(monitor_index, metricPtr, ml);
monitor_index_count++;
}
else
List_append(ml->monitor_list, monitor_module);
metricPtr = strtok_r(NULL, ",", &metricbuf);
}
}
if (!monitor_index_count)
goto cleanup;
return 1;
cleanup:
if (monitor_handle)
{
monitor_modules_unload(monitor_handle);
monitor_handle = NULL;
}
if (monitor_index)
{
Hash_destroy(monitor_index);
monitor_index = NULL;
}
return 0;
}
