/*
* _forwarding_setup
*
* 'index' is the index of the message config and forwarding info.
*
* Returns 0 success, -1 on error
*/
static int
_forwarding_setup(int index)
{
hostlist_iterator_t itr = NULL;
char *node;
int rv = -1;
assert(index >= 0 && index < conf.forward_message_config_len);
/* We require a separate hostlist here b/c the hosts input by the
* user and/or received by the remote hosts need to be mapped to a
* single hostname.
*/
if ((forwarding_info[index].fd = _forwarding_setup_socket(index)) < 0)
goto cleanup;
if (conf.forward_message_config[index].hosts)
{
forwarding_info[index].hosts = Hostlist_create(NULL);
itr = Hostlist_iterator_create(conf.forward_message_config[index].hosts);
while ((node = Hostlist_next(itr)))
{
char nodebuf[CEREBRO_MAX_NODENAME_LEN+1];
char *nodeptr;
if (found_clusterlist_module)
{
if (clusterlist_module_get_nodename(clusterlist_handle,
node,
nodebuf,
CEREBRO_MAX_NODENAME_LEN+1) < 0)
{
CEREBROD_DBG(("clusterlist_module_get_nodename: %s", nodebuf));
Hostlist_destroy(forwarding_info[index].hosts);
goto cleanup;
}
nodeptr = nodebuf;
}
else
nodeptr = node;
Hostlist_push(forwarding_info[index].hosts, nodeptr);
free(node);
}
}
else
forwarding_info[index].hosts = NULL;
Pthread_mutex_init(&(forwarding_info[index].lock), NULL);
rv = 0;
cleanup:
if (itr)
Hostlist_iterator_destroy(itr);
return rv;
}
/*
* _get_userspace_metric_value
*
* Get the metric value data supplied by a userspace program
*
* Returns message metric data on success, NULL otherwise
*/
static struct cerebrod_message_metric *
_get_userspace_metric_value(struct cerebrod_speaker_metric_info *metric_info)
{
struct cerebrod_message_metric *mm = NULL;
u_int32_t mtype, mlen;
void *mvalue;
assert(metric_info);
#if CEREBRO_DEBUG
if (metric_info->next_call_time)
CEREBROD_DBG(("Unexpected next_call_time"));
#endif /* CEREBRO_DEBUG */
mtype = metric_info->metric_value_type;
mlen = metric_info->metric_value_len;
mvalue = metric_info->metric_value;
if (check_data_type_len_value(mtype, mlen, mvalue) < 0)
goto cleanup;
mm = Malloc(sizeof(struct cerebrod_message_metric));
memset(mm, '\0', sizeof(struct cerebrod_message_metric));
/* need not overflow */
strncpy(mm->metric_name,
metric_info->metric_name,
CEREBRO_MAX_METRIC_NAME_LEN);
mm->metric_value_type = metric_info->metric_value_type;
mm->metric_value_len = metric_info->metric_value_len;
if (mm->metric_value_len)
{
mm->metric_value = Malloc(metric_info->metric_value_len);
memcpy(mm->metric_value,
metric_info->metric_value,
metric_info->metric_value_len);
}
else
mm->metric_value = NULL;
return mm;
cleanup:
if (mm)
{
if (mm->metric_value)
Free(mm->metric_value);
Free(mm);
}
return NULL;
}
int
cerebrod_reinit_socket(int old_fd,
int num,
Cerebrod_socket_setup socket_setup,
char *msg)
{
int fd = old_fd;
assert(socket_setup && msg);
if (errno == EINVAL
|| errno == EBADF
|| errno == ENODEV
|| errno == ENETDOWN
|| errno == ENETUNREACH
|| old_fd < 0)
{
if (!(old_fd < 0))
close(old_fd); /* no-wrapper, make best effort */
if ((fd = socket_setup(num)) < 0)
{
CEREBROD_DBG(("%s: error re-initializing socket", msg));
/* Wait a bit, so we don't spin */
sleep(CEREBROD_REINITIALIZE_TIME);
}
else
CEREBROD_DBG(("success re-initializing socket"));
}
else if (errno == EINTR)
CEREBROD_DBG(("%s: %s", msg, strerror(errno)));
else
CEREBROD_EXIT(("%s: %s", msg, strerror(errno)));
return fd;
}
/*
* _event_server_request_check_version
*
* Check that the version is correct prior to unmarshalling
*
* Returns 0 if version is correct, -1 if not
*/
static int
_event_server_request_check_version(const char *buf,
unsigned int buflen,
int32_t *version)
{
assert(buflen >= sizeof(int32_t) && version);
if (!Unmarshall_int32(version, buf, buflen))
{
CEREBROD_DBG(("version could not be unmarshalled"));
return -1;
}
if (*version != CEREBRO_EVENT_SERVER_PROTOCOL_VERSION)
return -1;
return 0;
}
void
cerebrod_event_modules_update(const char *nodename,
struct cerebrod_node_data *nd,
const char *metric_name,
struct cerebrod_message_metric *mm)
{
struct cerebrod_event_module_info *event_module;
struct cerebrod_event_module_list *el;
#if CEREBRO_DEBUG
int rv;
#endif /* CEREBRO_DEBUG */
assert(nodename && nd && metric_name && mm);
if (!event_index)
return;
#if CEREBRO_DEBUG
/* Should be called with lock already set */
rv = Pthread_mutex_trylock(&nd->node_data_lock);
if (rv != EBUSY)
CEREBROD_EXIT(("mutex not locked: rv=%d", rv));
#endif /* CEREBRO_DEBUG */
/*
* This function may be called by multiple threads by the listener.
*
* The event_index is setup at the beginning and is only read, not
* written to. However, the lists stored inside the event_index
* need to called w/ thread safety (due to the nature of the list
* API).
*
*/
if ((el = Hash_find(event_index, metric_name)))
{
struct cerebro_event *event = NULL;
ListIterator itr = NULL;
int rv;
Pthread_mutex_lock(&(el->event_list_lock));
itr = List_iterator_create(el->event_list);
Pthread_mutex_unlock(&(el->event_list_lock));
while ((event_module = list_next(itr)))
{
Pthread_mutex_lock(&event_module->event_lock);
if ((rv = event_module_metric_update(event_handle,
event_module->index,
nodename,
metric_name,
mm->metric_value_type,
mm->metric_value_len,
mm->metric_value,
&event)) < 0)
{
CEREBROD_DBG(("event_module_metric_update"));
goto loop_next;
}
if (rv && event)
cerebrod_queue_event(event, event_module->index);
loop_next:
Pthread_mutex_unlock(&event_module->event_lock);
}
Pthread_mutex_lock(&(el->event_list_lock));
List_iterator_destroy(itr);
Pthread_mutex_unlock(&(el->event_list_lock));
}
}
/*
* Under almost any circumstance, don't return a -1 error, cerebro can
* go on without loading monitor modules. The listener_data_init_lock
* should already be set.
*/
int
cerebrod_event_modules_setup(void)
{
int i, event_module_count, event_index_len, event_index_count = 0;
struct cerebrod_event_module_list *el = NULL;
#if CEREBRO_DEBUG
int rv;
#endif /* CEREBRO_DEBUG */
assert(listener_data);
#if CEREBRO_DEBUG
/* Should be called with lock already set */
rv = Pthread_mutex_trylock(&listener_data_init_lock);
if (rv != EBUSY)
CEREBROD_EXIT(("mutex not locked: rv=%d", rv));
#endif /* CEREBRO_DEBUG */
if (!conf.event_server)
return 0;
if (!(event_handle = event_modules_load()))
{
CEREBROD_DBG(("event_modules_load"));
goto cleanup;
}
if ((event_module_count = event_modules_count(event_handle)) < 0)
{
CEREBROD_DBG(("event_modules_count"));
goto cleanup;
}
if (!event_module_count)
{
if (conf.debug && conf.event_server_debug)
{
Pthread_mutex_lock(&debug_output_mutex);
fprintf(stderr, "**************************************\n");
fprintf(stderr, "* No Event Modules Found\n");
fprintf(stderr, "**************************************\n");
Pthread_mutex_unlock(&debug_output_mutex);
}
goto cleanup;
}
/* Each event module may want multiple metrics and/or offer multiple
* event names. We'll assume there will never be more than 2 per
* event module, and that will be enough to avoid all hash
* collisions.
*/
event_index_len = event_module_count * 2;
event_index = Hash_create(event_index_len,
(hash_key_f)hash_key_string,
(hash_cmp_f)strcmp,
(hash_del_f)_cerebrod_event_module_list_destroy);
event_names = List_create((ListDelF)_Free);
event_module_timeouts = List_create((ListDelF)_cerebrod_event_module_timeout_data_destroy);
event_module_timeout_index = Hash_create(event_module_count,
(hash_key_f)hash_key_string,
(hash_cmp_f)strcmp,
(hash_del_f)list_destroy);
for (i = 0; i < event_module_count; i++)
{
struct cerebrod_event_module *event_module;
char *module_name, *module_metric_names, *module_event_names;
char *metricPtr, *metricbuf;
char *eventnamePtr, *eventbuf;
int timeout;
module_name = event_module_name(event_handle, i);
if (conf.event_module_exclude_len)
{
int found_exclude = 0;
int j;
for (j = 0; j < conf.event_module_exclude_len; j++)
{
if (!strcasecmp(conf.event_module_exclude[j], module_name))
{
found_exclude++;
break;
}
}
if (found_exclude)
{
if (conf.debug && conf.event_server_debug)
{
Pthread_mutex_lock(&debug_output_mutex);
fprintf(stderr, "**************************************\n");
fprintf(stderr, "* Skip Event Module: %s\n", module_name);
fprintf(stderr, "**************************************\n");
Pthread_mutex_unlock(&debug_output_mutex);
}
CEREBROD_ERR(("Dropping event module: %s", module_name));
continue;
}
}
if (conf.debug && conf.event_server_debug)
{
Pthread_mutex_lock(&debug_output_mutex);
fprintf(stderr, "**************************************\n");
fprintf(stderr, "* Setup Event Module: %s\n", module_name);
fprintf(stderr, "**************************************\n");
Pthread_mutex_unlock(&debug_output_mutex);
}
if (event_module_setup(event_handle, i) < 0)
{
CEREBROD_DBG(("event_module_setup failed: %s", module_name));
continue;
}
if (!(module_metric_names = event_module_metric_names(event_handle, i)) < 0)
{
CEREBROD_DBG(("event_module_metric_names failed: %s", module_name));
event_module_cleanup(event_handle, i);
continue;
}
if (!(module_event_names = event_module_event_names(event_handle, i)) < 0)
{
CEREBROD_DBG(("event_module_event_names failed: %s", module_name));
event_module_cleanup(event_handle, i);
continue;
}
if ((timeout = event_module_timeout_length(event_handle, i)) < 0)
{
CEREBROD_DBG(("event_module_timeout_length failed: %s", module_name));
event_module_cleanup(event_handle, i);
continue;
}
event_module = Malloc(sizeof(struct cerebrod_event_module_info));
event_module->metric_names = Strdup(module_metric_names);
event_module->event_names = Strdup(module_event_names);
event_module->index = i;
Pthread_mutex_init(&(event_module->event_lock), NULL);
/* The monitoring module may support multiple metrics */
metricPtr = strtok_r(event_module->metric_names, ",", &metricbuf);
while (metricPtr)
{
if (!(el = Hash_find(event_index, metricPtr)))
{
el = (struct cerebrod_event_module_list *)Malloc(sizeof(struct cerebrod_event_module_list));
el->event_list = List_create((ListDelF)_cerebrod_event_module_info_destroy);
Pthread_mutex_init(&(el->event_list_lock), NULL);
List_append(el->event_list, event_module);
Hash_insert(event_index, metricPtr, el);
event_index_count++;
}
else
List_append(el->event_list, event_module);
metricPtr = strtok_r(NULL, ",", &metricbuf);
}
/* The monitoring module may support multiple event names */
eventnamePtr = strtok_r(event_module->event_names, ",", &eventbuf);
while (eventnamePtr)
{
if (!list_find_first(event_names,
(ListFindF)_cerebrod_name_strcmp,
eventnamePtr))
{
List_append(event_names, eventnamePtr);
if (conf.debug && conf.event_server_debug)
{
Pthread_mutex_lock(&debug_output_mutex);
fprintf(stderr, "**************************************\n");
fprintf(stderr, "* Event Name: %s\n", eventnamePtr);
fprintf(stderr, "**************************************\n");
Pthread_mutex_unlock(&debug_output_mutex);
}
}
eventnamePtr = strtok_r(NULL, ",", &eventbuf);
}
if (timeout)
{
struct cerebrod_event_module_timeout_data *mtd;
List modules_list;
if (!(mtd = List_find_first(event_module_timeouts,
_event_module_timeout_data_find_callback,
&timeout)))
{
char strbuf[64];
mtd = (struct cerebrod_event_module_timeout_data *)Malloc(sizeof(struct cerebrod_event_module_timeout_data));
mtd->timeout = timeout;
snprintf(strbuf, 64, "%d", timeout);
mtd->timeout_str = Strdup(strbuf);
List_append(event_module_timeouts, mtd);
if (timeout < event_module_timeout_min)
event_module_timeout_min = timeout;
}
if (!(modules_list = Hash_find(event_module_timeout_index,
mtd->timeout_str)))
{
modules_list = List_create((ListDelF)NULL);
List_append(modules_list, event_module);
Hash_insert(event_module_timeout_index,
mtd->timeout_str,
modules_list);
}
else
List_append(modules_list, event_module);
}
}
List_sort(event_module_timeouts, _event_module_timeout_data_compare);
if (!event_index_count)
goto cleanup;
if (_setup_event_node_timeout_data() < 0)
goto cleanup;
/*
* Since the cerebrod listener is started before any of the event
* threads (node_timeout, queue_monitor, server), this must be
* created in here (which is called by the listener) to avoid a
* possible race of modules creating events before the event_queue
* is created.
*/
event_queue = List_create((ListDelF)cerebrod_event_to_send_destroy);
return 1;
cleanup:
if (event_handle)
{
event_modules_unload(event_handle);
event_handle = NULL;
}
if (event_index)
{
Hash_destroy(event_index);
event_index = NULL;
}
if (event_names)
{
List_destroy(event_names);
event_names = NULL;
}
return 0;
}
void *
cerebrod_listener(void *arg)
{
char buf[CEREBRO_MAX_PACKET_LEN];
_cerebrod_listener_initialize();
for (;;)
{
struct cerebrod_message *msg;
char nodename_buf[CEREBRO_MAX_NODENAME_LEN+1];
char nodename_key[CEREBRO_MAX_NODENAME_LEN+1];
struct timeval tv;
int in_cluster_flag, i, count;
fd_set readfds;
int recv_len = 0;
int maxfd = 0;
FD_ZERO(&readfds);
Pthread_mutex_lock(&listener_fds_lock);
for (i = 0; i < conf.listen_message_config_len; i++)
{
if (listener_fds[i] > maxfd)
maxfd = listener_fds[i];
FD_SET(listener_fds[i], &readfds);
}
count = Select(maxfd + 1, &readfds, NULL, NULL, NULL);
for (i = 0; i < conf.listen_message_config_len; i++)
{
if (FD_ISSET(listener_fds[i], &readfds))
{
if ((recv_len = recvfrom(listener_fds[i],
buf,
CEREBRO_MAX_PACKET_LEN,
0,
NULL,
NULL)) < 0)
listener_fds[i] = cerebrod_reinit_socket(listener_fds[i],
i,
_listener_setup_socket,
"listener: recvfrom");
break;
}
}
Pthread_mutex_unlock(&listener_fds_lock);
/* No packet read */
if (recv_len <= 0)
continue;
if (recv_len >= CEREBRO_MAX_PACKET_LEN)
{
CEREBROD_DBG(("received truncated packet"));
continue;
}
if (_cerebrod_message_check_version(buf, recv_len) < 0)
{
CEREBROD_DBG(("received invalid version packet"));
continue;
}
if (!(msg = _cerebrod_message_unmarshall(buf, recv_len)))
{
CEREBROD_DBG(("received unmarshallable packet"));
continue;
}
_cerebrod_message_dump(msg, "Received Message");
/* Guarantee ending '\0' character */
memset(nodename_buf, '\0', CEREBRO_MAX_NODENAME_LEN+1);
memcpy(nodename_buf, msg->nodename, CEREBRO_MAX_NODENAME_LEN);
if (!strlen(nodename_buf))
{
CEREBROD_DBG(("received null nodename"));
cerebrod_message_destroy(msg);
continue;
}
if (found_clusterlist_module)
{
if ((in_cluster_flag = clusterlist_module_node_in_cluster(clusterlist_handle,
nodename_buf)) < 0)
CEREBROD_EXIT(("clusterlist_module_node_in_cluster: %s", nodename_buf));
/* Second chance, is this data being forwarded from another host */
if (!in_cluster_flag)
{
if (Hostlist_find(conf.forward_host_accept, nodename_buf) >= 0)
in_cluster_flag++;
}
}
else
/* must assume it is in the cluster */
/* Note, there is no need to handle 'forward_host_accept' under this case,
* since we don't know if it is in the cluster or not anyways.
*/
in_cluster_flag = 1;
if (!in_cluster_flag)
{
CEREBROD_DBG(("received non-cluster packet: %s", nodename_buf));
cerebrod_message_destroy(msg);
continue;
}
memset(nodename_key, '\0', CEREBRO_MAX_NODENAME_LEN+1);
if (found_clusterlist_module)
{
if (clusterlist_module_get_nodename(clusterlist_handle,
nodename_buf,
nodename_key,
CEREBRO_MAX_NODENAME_LEN+1) < 0)
{
CEREBROD_DBG(("clusterlist_module_get_nodename: %s", nodename_buf));
cerebrod_message_destroy(msg);
continue;
}
}
else
memcpy(nodename_key, nodename_buf, CEREBRO_MAX_NODENAME_LEN+1);
Gettimeofday(&tv, NULL);
cerebrod_listener_data_update(nodename_key, msg, tv.tv_sec);
/* Forward data as necessary. Note, there is no need to
* marshall data, it should already be marshalled when we
* read it earlier.
*/
for (i = 0; i < conf.forward_message_config_len; i++)
{
/* if the forward destination is local to the machine, don't forward */
if (conf.forward_message_config[i].ip_is_local)
continue;
if (!forwarding_info[i].hosts
|| hostlist_find(forwarding_info[i].hosts, nodename_key) >= 0)
{
struct sockaddr *addr;
struct sockaddr_in msgaddr;
unsigned int addrlen;
int rv;
memset(&msgaddr, '\0', sizeof(struct sockaddr_in));
msgaddr.sin_family = AF_INET;
msgaddr.sin_port = htons(conf.forward_message_config[i].destination_port);
memcpy(&msgaddr.sin_addr,
&conf.forward_message_config[i].ip_in_addr,
sizeof(struct in_addr));
addr = (struct sockaddr *)&msgaddr;
addrlen = sizeof(struct sockaddr_in);
_cerebrod_message_dump(msg, "Forwarding Message");
Pthread_mutex_lock(&forwarding_info[i].lock);
if ((rv = sendto(forwarding_info[i].fd,
buf,
recv_len,
0,
addr,
addrlen)) != recv_len)
{
if (rv < 0)
forwarding_info[i].fd = cerebrod_reinit_socket(forwarding_info[i].fd,
i,
_forwarding_setup_socket,
"forwarding: sendto");
else
CEREBROD_ERR(("sendto: invalid bytes sent"));
}
Pthread_mutex_unlock(&forwarding_info[i].lock);
}
}
cerebrod_message_destroy(msg);
}
return NULL; /* NOT REACHED */
}
/*
* _cerebrod_message_unmarshall
*
* unmarshall contents of a message packet buffer and
* return in an allocated message
*
* Returns message data on success, NULL on error
*/
static struct cerebrod_message *
_cerebrod_message_unmarshall(const char *buf, unsigned int buflen)
{
struct cerebrod_message *msg = NULL;
struct cerebrod_message_metric *mm = NULL;
unsigned int size;
char *bufPtr;
int i, n, bufPtrlen, c = 0;
assert(buf);
msg = Malloc(sizeof(struct cerebrod_message));
memset(msg, '\0', sizeof(struct cerebrod_message));
if (!(n = Unmarshall_int32(&(msg->version), buf + c, buflen - c)))
goto cleanup;
c += n;
bufPtr = msg->nodename;
bufPtrlen = sizeof(msg->nodename);
if (!(n = Unmarshall_buffer(bufPtr, bufPtrlen, buf + c, buflen - c)))
goto cleanup;
c += n;
if (!(n = Unmarshall_u_int32(&(msg->metrics_len), buf + c, buflen - c)))
goto cleanup;
c += n;
/* If no metrics in this packet, just return with the header */
if (!msg->metrics_len)
{
if (buflen != CEREBROD_MESSAGE_HEADER_LEN)
{
CEREBROD_DBG(("invalid packet length for no metrics"));
goto cleanup;
}
msg->metrics = NULL;
return msg;
}
size = sizeof(struct cerebrod_message_metric *)*(msg->metrics_len + 1);
msg->metrics = Malloc(size);
memset(msg->metrics, '\0', size);
for (i = 0; i < msg->metrics_len; i++)
{
char *mname;
int mnamelen;
mm = Malloc(sizeof(struct cerebrod_message_metric));
memset(mm, '\0', sizeof(struct cerebrod_message_metric));
mname = mm->metric_name;
mnamelen = sizeof(mm->metric_name);
if (!(n = Unmarshall_buffer(mname, mnamelen, buf + c, buflen - c)))
goto cleanup;
c += n;
if ((n = unmarshall_data_type_len(&(mm->metric_value_type),
&(mm->metric_value_len),
buf + c,
buflen - c,
NULL)) < 0)
goto cleanup;
c += n;
if (check_data_type_len(mm->metric_value_type, mm->metric_value_len) < 0)
goto cleanup;
mm->metric_value = NULL;
if (mm->metric_value_len)
{
mm->metric_value = Malloc(mm->metric_value_len);
if ((n = unmarshall_data_value(mm->metric_value_type,
mm->metric_value_len,
mm->metric_value,
mm->metric_value_len,
buf + c,
buflen - c,
NULL)) < 0)
goto cleanup;
c += n;
}
msg->metrics[i] = mm;
}
mm = NULL;
return msg;
cleanup:
if (mm)
{
if (mm->metric_value)
Free(mm->metric_value);
Free(mm);
}
if (msg)
{
if (msg->metrics)
{
i = 0;
while (msg->metrics[i])
{
if (msg->metrics[i]->metric_value)
Free(msg->metrics[i]->metric_value);
Free(msg->metrics[i]);
i++;
}
Free(msg->metrics);
}
Free(msg);
}
return NULL;
}
void *
cerebrod_event_queue_monitor(void *arg)
{
List temp_event_queue;
_event_queue_monitor_initialize();
/* Don't bother if there isn't an event queue (i.e. no event modules) */
if (!event_queue)
return NULL;
temp_event_queue = List_create((ListDelF)cerebrod_event_to_send_destroy);
/*
* achu: The listener and thus event update initialization is
* started after this thread is started. So the and event_index may
* not be set up the first time this loop is reached.
*
* However, it must be set after the condition is signaled, b/c the
* listener (and thus event update code) and event node timeout
* thread begin after the listener is setup.
*
* Thus, we put the event_queue assert inside the loop.
*/
for (;;)
{
struct cerebrod_event_to_send *ets;
ListIterator eitr;
ListIterator titr;
Pthread_mutex_lock(&event_queue_lock);
assert(event_queue);
while (list_count(event_queue) == 0)
Pthread_cond_wait(&event_queue_cond, &event_queue_lock);
/* Debug dumping in the below loop can race with the debug
* dumping from the listener, b/c of racing on the
* event_queue_lock. To avoid this race, we copy the data off
* the event_queue, so the event_queue_lock can be freed up.
*/
eitr = List_iterator_create(event_queue);
while ((ets = list_next(eitr)))
{
List_append(temp_event_queue, ets);
List_remove(eitr);
}
List_iterator_destroy(eitr);
Pthread_mutex_unlock(&event_queue_lock);
titr = List_iterator_create(temp_event_queue);
while ((ets = list_next(titr)))
{
List connections;
_event_dump(ets->event);
Pthread_mutex_lock(&event_connections_lock);
if ((connections = Hash_find(event_connections_index, ets->event_name)))
{
char buf[CEREBRO_MAX_PACKET_LEN];
int elen;
if ((elen = _event_marshall(ets->event,
buf,
CEREBRO_MAX_PACKET_LEN)) > 0)
{
ListIterator citr;
int *fd;
citr = List_iterator_create(connections);
while ((fd = list_next(citr)))
{
if (fd_write_n(*fd, buf, elen) < 0)
{
CEREBROD_DBG(("fd_write_n: %s", strerror(errno)));
if (errno == EPIPE
|| errno == EINVAL
|| errno == EBADF
|| errno == ENODEV
|| errno == ENETDOWN
|| errno == ENETUNREACH)
{
if (conf.event_server_debug)
{
Pthread_mutex_lock(&debug_output_mutex);
fprintf(stderr, "**************************************\n");
fprintf(stderr, "* Event Connection Died: errno = %d\n", errno);
fprintf(stderr, "**************************************\n");
Pthread_mutex_unlock(&debug_output_mutex);
}
List_delete(citr);
}
continue;
}
}
List_iterator_destroy(citr);
}
}
Pthread_mutex_unlock(&event_connections_lock);
List_delete(titr);
}
List_iterator_destroy(titr);
}
List_destroy(temp_event_queue);
return NULL; /* NOT REACHED */
}
void *
cerebrod_event_server(void *arg)
{
int server_fd;
_event_server_initialize();
if ((server_fd = _event_server_setup_socket(0)) < 0)
CEREBROD_EXIT(("event server fd setup failed"));
for (;;)
{
ListIterator eitr;
struct cerebrod_event_connection_data *ecd;
struct pollfd *pfds;
int pfdslen = 0;
int i;
/* Note that the list_count won't grow larger after the first
* mutex block, b/c the cerebrod_event_queue_monitor thread can
* never add to the event_connections. It can only shrink it.
*/
Pthread_mutex_lock(&event_connections_lock);
if (event_connections)
pfdslen = List_count(event_connections);
Pthread_mutex_unlock(&event_connections_lock);
/* The + 1 is b/c of the server_fd. */
pfdslen++;
pfds = Malloc(sizeof(struct pollfd) * pfdslen);
memset(pfds, '\0', sizeof(struct pollfd) * pfdslen);
pfds[0].fd = server_fd;
pfds[0].events = POLLIN;
pfds[0].revents = 0;
/* No 'event_connections' if there are no events */
if (event_connections)
{
i = 1;
Pthread_mutex_lock(&event_connections_lock);
eitr = List_iterator_create(event_connections);
while ((ecd = list_next(eitr)))
{
pfds[i].fd = ecd->fd;
pfds[i].events = POLLIN;
pfds[i].revents = 0;
i++;
}
List_iterator_destroy(eitr);
Pthread_mutex_unlock(&event_connections_lock);
}
Poll(pfds, pfdslen, -1);
/* Deal with the server fd first */
if (pfds[0].revents & POLLERR)
CEREBROD_DBG(("server_fd POLLERR"));
else if (pfds[0].revents & POLLIN)
{
unsigned int client_addr_len;
int fd;
struct sockaddr_in client_addr;
client_addr_len = sizeof(struct sockaddr_in);
if ((fd = accept(server_fd,
(struct sockaddr *)&client_addr,
&client_addr_len)) < 0)
server_fd = cerebrod_reinit_socket(server_fd,
0,
_event_server_setup_socket,
"event_server: accept");
if (fd >= 0)
_event_server_service_connection(fd);
}
/* Deal with the connecting fds */
for (i = 1; i < pfdslen; i++)
{
if (pfds[i].revents & POLLERR)
{
CEREBROD_DBG(("fd = %d POLLERR", pfds[i].fd));
Pthread_mutex_lock(&event_connections_lock);
_delete_event_connection_fd(pfds[i].fd);
Pthread_mutex_unlock(&event_connections_lock);
continue;
}
if (pfds[i].revents & POLLIN)
{
char buf[CEREBRO_MAX_PACKET_LEN];
int n;
/* We should not expect any actual data. If
* we get some, just eat it and move on.
*
* The common situation is that the client
* closes the connection. So we need to delete
* our fd.
*/
n = fd_read_n(pfds[i].fd,
buf,
CEREBRO_MAX_PACKET_LEN);
if (n < 0)
CEREBROD_DBG(("fd_read_n = %s", strerror(errno)));
if (n <= 0)
{
if (conf.debug && conf.event_server_debug)
{
Pthread_mutex_lock(&debug_output_mutex);
fprintf(stderr, "**************************************\n");
fprintf(stderr, "* Event Server Close Fd: %d\n", pfds[i].fd);
fprintf(stderr, "**************************************\n");
Pthread_mutex_unlock(&debug_output_mutex);
}
Pthread_mutex_lock(&event_connections_lock);
_delete_event_connection_fd(pfds[i].fd);
Pthread_mutex_unlock(&event_connections_lock);
}
}
}
Free(pfds);
}
return NULL; /* NOT REACHED */
}
/*
* Under almost any circumstance, don't return a -1 error, cerebro can
* go on without loading monitor modules. The listener_data_init_lock
* should already be set.
*/
int
cerebrod_monitor_modules_setup(void)
{
int i, monitor_module_count, monitor_index_len, monitor_index_count = 0;
struct cerebrod_monitor_module_list *ml = NULL;
#if CEREBRO_DEBUG
int rv;
#endif /* CEREBRO_DEBUG */
#if CEREBRO_DEBUG
/* Should be called with lock already set */
rv = Pthread_mutex_trylock(&listener_data_init_lock);
if (rv != EBUSY)
CEREBROD_EXIT(("mutex not locked: rv=%d", rv));
#endif /* CEREBRO_DEBUG */
if (!(monitor_handle = monitor_modules_load()))
{
CEREBROD_DBG(("monitor_modules_load"));
goto cleanup;
}
if ((monitor_module_count = monitor_modules_count(monitor_handle)) < 0)
{
CEREBROD_DBG(("monitor_modules_count"));
goto cleanup;
}
if (!monitor_module_count)
{
if (conf.debug && conf.listen_debug)
{
Pthread_mutex_lock(&debug_output_mutex);
fprintf(stderr, "**************************************\n");
fprintf(stderr, "* No Monitor Modules Found\n");
fprintf(stderr, "**************************************\n");
Pthread_mutex_unlock(&debug_output_mutex);
}
goto cleanup;
}
/* Each monitor module may wish to monitor multiple metrics. We'll
* assume there will never be more than 2 metrics per monitor module, and
* that will be enough to avoid all hash collisions.
*/
monitor_index_len = monitor_module_count * 2;
monitor_index = Hash_create(monitor_index_len,
(hash_key_f)hash_key_string,
(hash_cmp_f)strcmp,
(hash_del_f)_cerebrod_monitor_module_list_destroy);
for (i = 0; i < monitor_module_count; i++)
{
struct cerebrod_monitor_module_info *monitor_module;
char *module_name, *metric_names;
char *metricPtr, *metricbuf;
module_name = monitor_module_name(monitor_handle, i);
if (conf.monitor_module_exclude_len)
{
int found_exclude = 0;
int j;
for (j = 0; j < conf.monitor_module_exclude_len; j++)
{
if (!strcasecmp(conf.monitor_module_exclude[j], module_name))
{
found_exclude++;
break;
}
}
if (found_exclude)
{
if (conf.debug && conf.listen_debug)
{
Pthread_mutex_lock(&debug_output_mutex);
fprintf(stderr, "**************************************\n");
fprintf(stderr, "* Skip Monitor Module: %s\n", module_name);
fprintf(stderr, "**************************************\n");
Pthread_mutex_unlock(&debug_output_mutex);
}
CEREBROD_ERR(("Dropping monitor module: %s", module_name));
continue;
}
}
if (conf.debug && conf.listen_debug)
{
Pthread_mutex_lock(&debug_output_mutex);
fprintf(stderr, "**************************************\n");
fprintf(stderr, "* Setup Monitor Module: %s\n", module_name);
fprintf(stderr, "**************************************\n");
Pthread_mutex_unlock(&debug_output_mutex);
}
if (monitor_module_setup(monitor_handle, i) < 0)
{
CEREBROD_DBG(("monitor_module_setup failed: %s", module_name));
continue;
}
if (!(metric_names = monitor_module_metric_names(monitor_handle, i)) < 0)
{
CEREBROD_DBG(("monitor_module_metric_names failed: %s", module_name));
monitor_module_cleanup(monitor_handle, i);
continue;
}
monitor_module = Malloc(sizeof(struct cerebrod_monitor_module_info));
monitor_module->metric_names = Strdup(metric_names);
monitor_module->index = i;
Pthread_mutex_init(&(monitor_module->monitor_lock), NULL);
/* The monitoring module may support multiple metrics */
metricPtr = strtok_r(monitor_module->metric_names, ",", &metricbuf);
while (metricPtr)
{
if (!(ml = Hash_find(monitor_index, metricPtr)))
{
ml = (struct cerebrod_monitor_module_list *)Malloc(sizeof(struct cerebrod_monitor_module_list));
ml->monitor_list = List_create((ListDelF)_cerebrod_monitor_module_info_destroy);
Pthread_mutex_init(&(ml->monitor_list_lock), NULL);
List_append(ml->monitor_list, monitor_module);
Hash_insert(monitor_index, metricPtr, ml);
monitor_index_count++;
}
else
List_append(ml->monitor_list, monitor_module);
metricPtr = strtok_r(NULL, ",", &metricbuf);
}
}
if (!monitor_index_count)
goto cleanup;
return 1;
cleanup:
if (monitor_handle)
{
monitor_modules_unload(monitor_handle);
monitor_handle = NULL;
}
if (monitor_index)
{
Hash_destroy(monitor_index);
monitor_index = NULL;
}
return 0;
}
/*
* _get_module_metric_value
*
* Get the metric value data from a module
*
* Returns message metric data on success, NULL otherwise
*/
static struct cerebrod_message_metric *
_get_module_metric_value(unsigned int index)
{
struct cerebrod_message_metric *mm = NULL;
char *metric_name;
u_int32_t mtype, mlen;
void *mvalue = NULL;
assert(index < metric_handle_count);
mm = Malloc(sizeof(struct cerebrod_message_metric));
memset(mm, '\0', sizeof(struct cerebrod_message_metric));
if (!(metric_name = metric_module_get_metric_name(metric_handle, index)))
{
CEREBROD_DBG(("metric_module_get_metric_name: %d", index));
goto cleanup;
}
/* need not overflow */
strncpy(mm->metric_name, metric_name, CEREBRO_MAX_METRIC_NAME_LEN);
if (metric_module_get_metric_value(metric_handle,
index,
&mtype,
&mlen,
&mvalue) < 0)
{
CEREBROD_DBG(("metric_module_get_metric_value: %d", index));
goto cleanup;
}
if (mtype == CEREBRO_DATA_VALUE_TYPE_STRING
&& mlen > CEREBRO_MAX_DATA_STRING_LEN)
{
CEREBROD_DBG(("truncate metric string: %d", mlen));
mlen = CEREBRO_MAX_DATA_STRING_LEN;
}
if (mtype == CEREBRO_DATA_VALUE_TYPE_STRING && !mlen)
{
CEREBROD_DBG(("adjusting metric type to none"));
mtype = CEREBRO_DATA_VALUE_TYPE_NONE;
}
if (check_data_type_len_value(mtype, mlen, mvalue) < 0)
goto cleanup;
mm->metric_value_type = mtype;
mm->metric_value_len = mlen;
if (mm->metric_value_len)
{
mm->metric_value = Malloc(mm->metric_value_len);
memcpy(mm->metric_value, mvalue, mm->metric_value_len);
}
else
mm->metric_value = NULL;
metric_module_destroy_metric_value(metric_handle, index, mvalue);
return mm;
cleanup:
if (mvalue)
metric_module_destroy_metric_value(metric_handle, index, mvalue);
if (mm)
{
if (mm->metric_value)
Free(mm->metric_value);
Free(mm);
}
return NULL;
}
/*
* _setup_metric_modules
*
* Setup metric modules. Under almost any circumstance, don't return a
* -1 error, cerebro can go on without loading metric modules.
*
* Returns 1 if modules are loaded, 0 if not, -1 on error
*/
static int
_setup_metric_modules(void)
{
int i;
#if CEREBRO_DEBUG
#if !WITH_CEREBROD_NO_THREADS
int rv;
#endif /* !WITH_CEREBROD_NO_THREADS */
#endif /* CEREBRO_DEBUG */
assert(metric_list);
#if CEREBRO_DEBUG
#if !WITH_CEREBROD_NO_THREADS
/* Should be called with lock already set */
rv = Pthread_mutex_trylock(&metric_list_lock);
if (rv != EBUSY)
CEREBROD_EXIT(("mutex not locked: rv=%d", rv));
#endif /* !WITH_CEREBROD_NO_THREADS */
#endif /* CEREBRO_DEBUG */
if (!(metric_handle = metric_modules_load()))
{
CEREBROD_DBG(("metric_modules_load"));
goto cleanup;
}
if ((metric_handle_count = metric_modules_count(metric_handle)) < 0)
{
CEREBROD_DBG(("metric_module_count failed"));
goto cleanup;
}
if (!metric_handle_count)
{
if (conf.debug && conf.speak_debug)
{
#if !WITH_CEREBROD_NO_THREADS
Pthread_mutex_lock(&debug_output_mutex);
#endif /* !WITH_CEREBROD_NO_THREADS */
fprintf(stderr, "**************************************\n");
fprintf(stderr, "* No Metric Modules Found\n");
fprintf(stderr, "**************************************\n");
#if !WITH_CEREBROD_NO_THREADS
Pthread_mutex_unlock(&debug_output_mutex);
#endif /* !WITH_CEREBROD_NO_THREADS */
}
goto cleanup;
}
for (i = 0; i < metric_handle_count; i++)
{
struct cerebrod_speaker_metric_info *metric_info;
#if !WITH_CEREBROD_NO_THREADS
Cerebro_metric_thread_pointer threadPtr;
#endif /* !WITH_CEREBROD_NO_THREADS */
char *module_name, *metric_name;
int metric_period;
u_int32_t metric_flags;
module_name = metric_module_name(metric_handle, i);
if (conf.metric_module_exclude_len)
{
int found_exclude = 0;
int j;
for (j = 0; j < conf.metric_module_exclude_len; j++)
{
if (!strcasecmp(conf.metric_module_exclude[j], module_name))
{
found_exclude++;
break;
}
}
if (found_exclude)
{
if (conf.debug && conf.speak_debug)
{
#if !WITH_CEREBROD_NO_THREADS
Pthread_mutex_lock(&debug_output_mutex);
#endif /* !WITH_CEREBROD_NO_THREADS */
fprintf(stderr, "**************************************\n");
fprintf(stderr, "* Skip Metric Module: %s\n", module_name);
fprintf(stderr, "**************************************\n");
#if !WITH_CEREBROD_NO_THREADS
Pthread_mutex_unlock(&debug_output_mutex);
#endif /* !WITH_CEREBROD_NO_THREADS */
}
CEREBROD_ERR(("Dropping metric module: %s", module_name));
continue;
}
}
if (conf.debug && conf.speak_debug)
{
#if !WITH_CEREBROD_NO_THREADS
Pthread_mutex_lock(&debug_output_mutex);
#endif /* !WITH_CEREBROD_NO_THREADS */
fprintf(stderr, "**************************************\n");
fprintf(stderr, "* Setup Metric Module: %s\n", module_name);
fprintf(stderr, "**************************************\n");
#if !WITH_CEREBROD_NO_THREADS
Pthread_mutex_unlock(&debug_output_mutex);
#endif /* !WITH_CEREBROD_NO_THREADS */
}
if (metric_module_setup(metric_handle, i) < 0)
{
CEREBROD_DBG(("metric_module_setup: %s", module_name));
continue;
}
if (!(metric_name = metric_module_get_metric_name(metric_handle, i)))
{
CEREBROD_DBG(("metric_module_get_metric_name: %s", module_name));
metric_module_cleanup(metric_handle, i);
continue;
}
if (metric_module_get_metric_period(metric_handle, i, &metric_period) < 0)
{
CEREBROD_DBG(("metric_module_get_metric_period: %s", module_name));
metric_module_cleanup(metric_handle, i);
continue;
}
if (metric_module_get_metric_flags(metric_handle, i, &metric_flags) < 0)
{
CEREBROD_DBG(("metric_module_get_metric_flags: %s", module_name));
metric_module_cleanup(metric_handle, i);
continue;
}
if (metric_flags & CEREBRO_METRIC_MODULE_FLAGS_SEND_ON_PERIOD
&& metric_period <= 0)
{
CEREBROD_DBG(("metric module period invalid: %s", module_name));
metric_module_cleanup(metric_handle, i);
continue;
}
if (metric_module_send_message_function_pointer(metric_handle, i, &cerebrod_send_message) < 0)
{
CEREBROD_DBG(("metric_module_send_message_function_pointer: %s", module_name));
metric_module_cleanup(metric_handle, i);
continue;
}
metric_info = Malloc(sizeof(struct cerebrod_speaker_metric_info));
/* No need to Strdup() the name in this case */
metric_info->metric_name = metric_name;
metric_info->metric_origin = CEREBROD_METRIC_SPEAKER_ORIGIN_MODULE;
metric_info->metric_period = metric_period;
metric_info->metric_flags = metric_flags;
metric_info->index = i;
/*
* If metric period is < 0, it presumably never will be sent
* (metric is likely handled by a metric_thread), so set
* next_call_time to UINT_MAX.
*
* If this is a metric that will be piggy-backed on heartbeats,
* then initialize next_call_time to 0, so the data is sent on
* the first heartbeat
*
* If this is a metric that will not be piggy-backed on
* heartbeats, set the next_call_time to UINT_MAX. Let the
* speaker logic decide when packets should be sent.
*/
if (metric_info->metric_period < 0
|| metric_info->metric_flags & CEREBRO_METRIC_MODULE_FLAGS_SEND_ON_PERIOD)
metric_info->next_call_time = UINT_MAX;
else
metric_info->next_call_time = 0;
List_append(metric_list, metric_info);
metric_list_size++;
#if !WITH_CEREBROD_NO_THREADS
if ((threadPtr = metric_module_get_metric_thread(metric_handle, i)))
{
pthread_t thread;
pthread_attr_t attr;
Pthread_attr_init(&attr);
Pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
Pthread_attr_setstacksize(&attr, CEREBROD_THREAD_STACKSIZE);
Pthread_create(&thread, &attr, threadPtr, NULL);
Pthread_attr_destroy(&attr);
}
#endif /* !WITH_CEREBROD_NO_THREADS */
}
if (!metric_list_size)
goto cleanup;
cerebrod_speaker_data_metric_list_sort();
return 1;
cleanup:
if (metric_handle)
{
/* unload will call module cleanup functions */
metric_modules_unload(metric_handle);
metric_handle = NULL;
metric_handle_count = 0;
}
metric_list_size = 0;
return 0;
}
