void *
cerebrod_listener(void *arg)
{
char buf[CEREBRO_MAX_PACKET_LEN];
_cerebrod_listener_initialize();
for (;;)
{
struct cerebrod_message *msg;
char nodename_buf[CEREBRO_MAX_NODENAME_LEN+1];
char nodename_key[CEREBRO_MAX_NODENAME_LEN+1];
struct timeval tv;
int in_cluster_flag, i, count;
fd_set readfds;
int recv_len = 0;
int maxfd = 0;
FD_ZERO(&readfds);
Pthread_mutex_lock(&listener_fds_lock);
for (i = 0; i < conf.listen_message_config_len; i++)
{
if (listener_fds[i] > maxfd)
maxfd = listener_fds[i];
FD_SET(listener_fds[i], &readfds);
}
count = Select(maxfd + 1, &readfds, NULL, NULL, NULL);
for (i = 0; i < conf.listen_message_config_len; i++)
{
if (FD_ISSET(listener_fds[i], &readfds))
{
if ((recv_len = recvfrom(listener_fds[i],
buf,
CEREBRO_MAX_PACKET_LEN,
0,
NULL,
NULL)) < 0)
listener_fds[i] = cerebrod_reinit_socket(listener_fds[i],
i,
_listener_setup_socket,
"listener: recvfrom");
break;
}
}
Pthread_mutex_unlock(&listener_fds_lock);
/* No packet read */
if (recv_len <= 0)
continue;
if (recv_len >= CEREBRO_MAX_PACKET_LEN)
{
CEREBROD_DBG(("received truncated packet"));
continue;
}
if (_cerebrod_message_check_version(buf, recv_len) < 0)
{
CEREBROD_DBG(("received invalid version packet"));
continue;
}
if (!(msg = _cerebrod_message_unmarshall(buf, recv_len)))
{
CEREBROD_DBG(("received unmarshallable packet"));
continue;
}
_cerebrod_message_dump(msg, "Received Message");
/* Guarantee ending '\0' character */
memset(nodename_buf, '\0', CEREBRO_MAX_NODENAME_LEN+1);
memcpy(nodename_buf, msg->nodename, CEREBRO_MAX_NODENAME_LEN);
if (!strlen(nodename_buf))
{
CEREBROD_DBG(("received null nodename"));
cerebrod_message_destroy(msg);
continue;
}
if (found_clusterlist_module)
{
if ((in_cluster_flag = clusterlist_module_node_in_cluster(clusterlist_handle,
nodename_buf)) < 0)
CEREBROD_EXIT(("clusterlist_module_node_in_cluster: %s", nodename_buf));
/* Second chance, is this data being forwarded from another host */
if (!in_cluster_flag)
{
if (Hostlist_find(conf.forward_host_accept, nodename_buf) >= 0)
in_cluster_flag++;
}
}
else
/* must assume it is in the cluster */
/* Note, there is no need to handle 'forward_host_accept' under this case,
* since we don't know if it is in the cluster or not anyways.
*/
in_cluster_flag = 1;
if (!in_cluster_flag)
{
CEREBROD_DBG(("received non-cluster packet: %s", nodename_buf));
cerebrod_message_destroy(msg);
continue;
}
memset(nodename_key, '\0', CEREBRO_MAX_NODENAME_LEN+1);
if (found_clusterlist_module)
{
if (clusterlist_module_get_nodename(clusterlist_handle,
nodename_buf,
nodename_key,
CEREBRO_MAX_NODENAME_LEN+1) < 0)
{
CEREBROD_DBG(("clusterlist_module_get_nodename: %s", nodename_buf));
cerebrod_message_destroy(msg);
continue;
}
}
else
memcpy(nodename_key, nodename_buf, CEREBRO_MAX_NODENAME_LEN+1);
Gettimeofday(&tv, NULL);
cerebrod_listener_data_update(nodename_key, msg, tv.tv_sec);
/* Forward data as necessary. Note, there is no need to
* marshall data, it should already be marshalled when we
* read it earlier.
*/
for (i = 0; i < conf.forward_message_config_len; i++)
{
/* if the forward destination is local to the machine, don't forward */
if (conf.forward_message_config[i].ip_is_local)
continue;
if (!forwarding_info[i].hosts
|| hostlist_find(forwarding_info[i].hosts, nodename_key) >= 0)
{
struct sockaddr *addr;
struct sockaddr_in msgaddr;
unsigned int addrlen;
int rv;
memset(&msgaddr, '\0', sizeof(struct sockaddr_in));
msgaddr.sin_family = AF_INET;
msgaddr.sin_port = htons(conf.forward_message_config[i].destination_port);
memcpy(&msgaddr.sin_addr,
&conf.forward_message_config[i].ip_in_addr,
sizeof(struct in_addr));
addr = (struct sockaddr *)&msgaddr;
addrlen = sizeof(struct sockaddr_in);
_cerebrod_message_dump(msg, "Forwarding Message");
Pthread_mutex_lock(&forwarding_info[i].lock);
if ((rv = sendto(forwarding_info[i].fd,
buf,
recv_len,
0,
addr,
addrlen)) != recv_len)
{
if (rv < 0)
forwarding_info[i].fd = cerebrod_reinit_socket(forwarding_info[i].fd,
i,
_forwarding_setup_socket,
"forwarding: sendto");
else
CEREBROD_ERR(("sendto: invalid bytes sent"));
}
Pthread_mutex_unlock(&forwarding_info[i].lock);
}
}
cerebrod_message_destroy(msg);
}
return NULL; /* NOT REACHED */
}
void *
cerebrod_event_server(void *arg)
{
int server_fd;
_event_server_initialize();
if ((server_fd = _event_server_setup_socket(0)) < 0)
CEREBROD_EXIT(("event server fd setup failed"));
for (;;)
{
ListIterator eitr;
struct cerebrod_event_connection_data *ecd;
struct pollfd *pfds;
int pfdslen = 0;
int i;
/* Note that the list_count won't grow larger after the first
* mutex block, b/c the cerebrod_event_queue_monitor thread can
* never add to the event_connections. It can only shrink it.
*/
Pthread_mutex_lock(&event_connections_lock);
if (event_connections)
pfdslen = List_count(event_connections);
Pthread_mutex_unlock(&event_connections_lock);
/* The + 1 is b/c of the server_fd. */
pfdslen++;
pfds = Malloc(sizeof(struct pollfd) * pfdslen);
memset(pfds, '\0', sizeof(struct pollfd) * pfdslen);
pfds[0].fd = server_fd;
pfds[0].events = POLLIN;
pfds[0].revents = 0;
/* No 'event_connections' if there are no events */
if (event_connections)
{
i = 1;
Pthread_mutex_lock(&event_connections_lock);
eitr = List_iterator_create(event_connections);
while ((ecd = list_next(eitr)))
{
pfds[i].fd = ecd->fd;
pfds[i].events = POLLIN;
pfds[i].revents = 0;
i++;
}
List_iterator_destroy(eitr);
Pthread_mutex_unlock(&event_connections_lock);
}
Poll(pfds, pfdslen, -1);
/* Deal with the server fd first */
if (pfds[0].revents & POLLERR)
CEREBROD_DBG(("server_fd POLLERR"));
else if (pfds[0].revents & POLLIN)
{
unsigned int client_addr_len;
int fd;
struct sockaddr_in client_addr;
client_addr_len = sizeof(struct sockaddr_in);
if ((fd = accept(server_fd,
(struct sockaddr *)&client_addr,
&client_addr_len)) < 0)
server_fd = cerebrod_reinit_socket(server_fd,
0,
_event_server_setup_socket,
"event_server: accept");
if (fd >= 0)
_event_server_service_connection(fd);
}
/* Deal with the connecting fds */
for (i = 1; i < pfdslen; i++)
{
if (pfds[i].revents & POLLERR)
{
CEREBROD_DBG(("fd = %d POLLERR", pfds[i].fd));
Pthread_mutex_lock(&event_connections_lock);
_delete_event_connection_fd(pfds[i].fd);
Pthread_mutex_unlock(&event_connections_lock);
continue;
}
if (pfds[i].revents & POLLIN)
{
char buf[CEREBRO_MAX_PACKET_LEN];
int n;
/* We should not expect any actual data. If
* we get some, just eat it and move on.
*
* The common situation is that the client
* closes the connection. So we need to delete
* our fd.
*/
n = fd_read_n(pfds[i].fd,
buf,
CEREBRO_MAX_PACKET_LEN);
if (n < 0)
CEREBROD_DBG(("fd_read_n = %s", strerror(errno)));
if (n <= 0)
{
if (conf.debug && conf.event_server_debug)
{
Pthread_mutex_lock(&debug_output_mutex);
fprintf(stderr, "**************************************\n");
fprintf(stderr, "* Event Server Close Fd: %d\n", pfds[i].fd);
fprintf(stderr, "**************************************\n");
Pthread_mutex_unlock(&debug_output_mutex);
}
Pthread_mutex_lock(&event_connections_lock);
_delete_event_connection_fd(pfds[i].fd);
Pthread_mutex_unlock(&event_connections_lock);
}
}
}
Free(pfds);
}
return NULL; /* NOT REACHED */
}
