void * cerebrod_listener(void *arg) { char buf[CEREBRO_MAX_PACKET_LEN]; _cerebrod_listener_initialize(); for (;;) { struct cerebrod_message *msg; char nodename_buf[CEREBRO_MAX_NODENAME_LEN+1]; char nodename_key[CEREBRO_MAX_NODENAME_LEN+1]; struct timeval tv; int in_cluster_flag, i, count; fd_set readfds; int recv_len = 0; int maxfd = 0; FD_ZERO(&readfds); Pthread_mutex_lock(&listener_fds_lock); for (i = 0; i < conf.listen_message_config_len; i++) { if (listener_fds[i] > maxfd) maxfd = listener_fds[i]; FD_SET(listener_fds[i], &readfds); } count = Select(maxfd + 1, &readfds, NULL, NULL, NULL); for (i = 0; i < conf.listen_message_config_len; i++) { if (FD_ISSET(listener_fds[i], &readfds)) { if ((recv_len = recvfrom(listener_fds[i], buf, CEREBRO_MAX_PACKET_LEN, 0, NULL, NULL)) < 0) listener_fds[i] = cerebrod_reinit_socket(listener_fds[i], i, _listener_setup_socket, "listener: recvfrom"); break; } } Pthread_mutex_unlock(&listener_fds_lock); /* No packet read */ if (recv_len <= 0) continue; if (recv_len >= CEREBRO_MAX_PACKET_LEN) { CEREBROD_DBG(("received truncated packet")); continue; } if (_cerebrod_message_check_version(buf, recv_len) < 0) { CEREBROD_DBG(("received invalid version packet")); continue; } if (!(msg = _cerebrod_message_unmarshall(buf, recv_len))) { CEREBROD_DBG(("received unmarshallable packet")); continue; } _cerebrod_message_dump(msg, "Received Message"); /* Guarantee ending '\0' character */ memset(nodename_buf, '\0', CEREBRO_MAX_NODENAME_LEN+1); memcpy(nodename_buf, msg->nodename, CEREBRO_MAX_NODENAME_LEN); if (!strlen(nodename_buf)) { CEREBROD_DBG(("received null nodename")); cerebrod_message_destroy(msg); continue; } if (found_clusterlist_module) { if ((in_cluster_flag = clusterlist_module_node_in_cluster(clusterlist_handle, nodename_buf)) < 0) CEREBROD_EXIT(("clusterlist_module_node_in_cluster: %s", nodename_buf)); /* Second chance, is this data being forwarded from another host */ if (!in_cluster_flag) { if (Hostlist_find(conf.forward_host_accept, nodename_buf) >= 0) in_cluster_flag++; } } else /* must assume it is in the cluster */ /* Note, there is no need to handle 'forward_host_accept' under this case, * since we don't know if it is in the cluster or not anyways. */ in_cluster_flag = 1; if (!in_cluster_flag) { CEREBROD_DBG(("received non-cluster packet: %s", nodename_buf)); cerebrod_message_destroy(msg); continue; } memset(nodename_key, '\0', CEREBRO_MAX_NODENAME_LEN+1); if (found_clusterlist_module) { if (clusterlist_module_get_nodename(clusterlist_handle, nodename_buf, nodename_key, CEREBRO_MAX_NODENAME_LEN+1) < 0) { CEREBROD_DBG(("clusterlist_module_get_nodename: %s", nodename_buf)); cerebrod_message_destroy(msg); continue; } } else memcpy(nodename_key, nodename_buf, CEREBRO_MAX_NODENAME_LEN+1); Gettimeofday(&tv, NULL); cerebrod_listener_data_update(nodename_key, msg, tv.tv_sec); /* Forward data as necessary. Note, there is no need to * marshall data, it should already be marshalled when we * read it earlier. */ for (i = 0; i < conf.forward_message_config_len; i++) { /* if the forward destination is local to the machine, don't forward */ if (conf.forward_message_config[i].ip_is_local) continue; if (!forwarding_info[i].hosts || hostlist_find(forwarding_info[i].hosts, nodename_key) >= 0) { struct sockaddr *addr; struct sockaddr_in msgaddr; unsigned int addrlen; int rv; memset(&msgaddr, '\0', sizeof(struct sockaddr_in)); msgaddr.sin_family = AF_INET; msgaddr.sin_port = htons(conf.forward_message_config[i].destination_port); memcpy(&msgaddr.sin_addr, &conf.forward_message_config[i].ip_in_addr, sizeof(struct in_addr)); addr = (struct sockaddr *)&msgaddr; addrlen = sizeof(struct sockaddr_in); _cerebrod_message_dump(msg, "Forwarding Message"); Pthread_mutex_lock(&forwarding_info[i].lock); if ((rv = sendto(forwarding_info[i].fd, buf, recv_len, 0, addr, addrlen)) != recv_len) { if (rv < 0) forwarding_info[i].fd = cerebrod_reinit_socket(forwarding_info[i].fd, i, _forwarding_setup_socket, "forwarding: sendto"); else CEREBROD_ERR(("sendto: invalid bytes sent")); } Pthread_mutex_unlock(&forwarding_info[i].lock); } } cerebrod_message_destroy(msg); } return NULL; /* NOT REACHED */ }
void * cerebrod_event_server(void *arg) { int server_fd; _event_server_initialize(); if ((server_fd = _event_server_setup_socket(0)) < 0) CEREBROD_EXIT(("event server fd setup failed")); for (;;) { ListIterator eitr; struct cerebrod_event_connection_data *ecd; struct pollfd *pfds; int pfdslen = 0; int i; /* Note that the list_count won't grow larger after the first * mutex block, b/c the cerebrod_event_queue_monitor thread can * never add to the event_connections. It can only shrink it. */ Pthread_mutex_lock(&event_connections_lock); if (event_connections) pfdslen = List_count(event_connections); Pthread_mutex_unlock(&event_connections_lock); /* The + 1 is b/c of the server_fd. */ pfdslen++; pfds = Malloc(sizeof(struct pollfd) * pfdslen); memset(pfds, '\0', sizeof(struct pollfd) * pfdslen); pfds[0].fd = server_fd; pfds[0].events = POLLIN; pfds[0].revents = 0; /* No 'event_connections' if there are no events */ if (event_connections) { i = 1; Pthread_mutex_lock(&event_connections_lock); eitr = List_iterator_create(event_connections); while ((ecd = list_next(eitr))) { pfds[i].fd = ecd->fd; pfds[i].events = POLLIN; pfds[i].revents = 0; i++; } List_iterator_destroy(eitr); Pthread_mutex_unlock(&event_connections_lock); } Poll(pfds, pfdslen, -1); /* Deal with the server fd first */ if (pfds[0].revents & POLLERR) CEREBROD_DBG(("server_fd POLLERR")); else if (pfds[0].revents & POLLIN) { unsigned int client_addr_len; int fd; struct sockaddr_in client_addr; client_addr_len = sizeof(struct sockaddr_in); if ((fd = accept(server_fd, (struct sockaddr *)&client_addr, &client_addr_len)) < 0) server_fd = cerebrod_reinit_socket(server_fd, 0, _event_server_setup_socket, "event_server: accept"); if (fd >= 0) _event_server_service_connection(fd); } /* Deal with the connecting fds */ for (i = 1; i < pfdslen; i++) { if (pfds[i].revents & POLLERR) { CEREBROD_DBG(("fd = %d POLLERR", pfds[i].fd)); Pthread_mutex_lock(&event_connections_lock); _delete_event_connection_fd(pfds[i].fd); Pthread_mutex_unlock(&event_connections_lock); continue; } if (pfds[i].revents & POLLIN) { char buf[CEREBRO_MAX_PACKET_LEN]; int n; /* We should not expect any actual data. If * we get some, just eat it and move on. * * The common situation is that the client * closes the connection. So we need to delete * our fd. */ n = fd_read_n(pfds[i].fd, buf, CEREBRO_MAX_PACKET_LEN); if (n < 0) CEREBROD_DBG(("fd_read_n = %s", strerror(errno))); if (n <= 0) { if (conf.debug && conf.event_server_debug) { Pthread_mutex_lock(&debug_output_mutex); fprintf(stderr, "**************************************\n"); fprintf(stderr, "* Event Server Close Fd: %d\n", pfds[i].fd); fprintf(stderr, "**************************************\n"); Pthread_mutex_unlock(&debug_output_mutex); } Pthread_mutex_lock(&event_connections_lock); _delete_event_connection_fd(pfds[i].fd); Pthread_mutex_unlock(&event_connections_lock); } } } Free(pfds); } return NULL; /* NOT REACHED */ }