static void threadpool_thread_create(threadpool_t *pool) {
pthread_t tid;
pthread_attr_t attr;
Pthread_attr_init(&attr);
Pthread_attr_setstacksize(&attr, pool->thread_stack_size);
Pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
Pthread_create(&tid, &attr, thread_loop, pool);
Pthread_attr_destroy(&attr);
}
/*
* _setup_metric_modules
*
* Setup metric modules. Under almost any circumstance, don't return a
* -1 error, cerebro can go on without loading metric modules.
*
* Returns 1 if modules are loaded, 0 if not, -1 on error
*/
static int
_setup_metric_modules(void)
{
int i;
#if CEREBRO_DEBUG
#if !WITH_CEREBROD_NO_THREADS
int rv;
#endif /* !WITH_CEREBROD_NO_THREADS */
#endif /* CEREBRO_DEBUG */
assert(metric_list);
#if CEREBRO_DEBUG
#if !WITH_CEREBROD_NO_THREADS
/* Should be called with lock already set */
rv = Pthread_mutex_trylock(&metric_list_lock);
if (rv != EBUSY)
CEREBRO_EXIT(("mutex not locked: rv=%d", rv));
#endif /* !WITH_CEREBROD_NO_THREADS */
#endif /* CEREBRO_DEBUG */
if (!(metric_handle = metric_modules_load()))
{
CEREBRO_DBG(("metric_modules_load"));
goto cleanup;
}
if ((metric_handle_count = metric_modules_count(metric_handle)) < 0)
{
CEREBRO_DBG(("metric_module_count failed"));
goto cleanup;
}
if (!metric_handle_count)
{
#if CEREBRO_DEBUG
if (conf.debug && conf.speak_debug)
{
#if !WITH_CEREBROD_NO_THREADS
Pthread_mutex_lock(&debug_output_mutex);
#endif /* !WITH_CEREBROD_NO_THREADS */
fprintf(stderr, "**************************************\n");
fprintf(stderr, "* No Metric Modules Found\n");
fprintf(stderr, "**************************************\n");
#if !WITH_CEREBROD_NO_THREADS
Pthread_mutex_unlock(&debug_output_mutex);
#endif /* !WITH_CEREBROD_NO_THREADS */
}
#endif /* CEREBRO_DEBUG */
goto cleanup;
}
for (i = 0; i < metric_handle_count; i++)
{
struct cerebrod_speaker_metric_info *metric_info;
#if !WITH_CEREBROD_NO_THREADS
Cerebro_metric_thread_pointer threadPtr;
#endif /* !WITH_CEREBROD_NO_THREADS */
char *module_name, *metric_name;
int metric_period;
u_int32_t metric_flags;
module_name = metric_module_name(metric_handle, i);
if (conf.metric_module_exclude_len)
{
int found_exclude = 0;
int j;
for (j = 0; j < conf.metric_module_exclude_len; j++)
{
if (!strcasecmp(conf.metric_module_exclude[j], module_name))
{
found_exclude++;
break;
}
}
if (found_exclude)
{
#if CEREBRO_DEBUG
if (conf.debug && conf.speak_debug)
{
#if !WITH_CEREBROD_NO_THREADS
Pthread_mutex_lock(&debug_output_mutex);
#endif /* !WITH_CEREBROD_NO_THREADS */
fprintf(stderr, "**************************************\n");
fprintf(stderr, "* Skip Metric Module: %s\n", module_name);
fprintf(stderr, "**************************************\n");
#if !WITH_CEREBROD_NO_THREADS
Pthread_mutex_unlock(&debug_output_mutex);
#endif /* !WITH_CEREBROD_NO_THREADS */
}
#endif /* CEREBRO_DEBUG */
CEREBRO_ERR(("Dropping metric module: %s", module_name));
continue;
}
}
#if CEREBRO_DEBUG
if (conf.debug && conf.speak_debug)
{
#if !WITH_CEREBROD_NO_THREADS
Pthread_mutex_lock(&debug_output_mutex);
#endif /* !WITH_CEREBROD_NO_THREADS */
fprintf(stderr, "**************************************\n");
fprintf(stderr, "* Setup Metric Module: %s\n", module_name);
fprintf(stderr, "**************************************\n");
#if !WITH_CEREBROD_NO_THREADS
Pthread_mutex_unlock(&debug_output_mutex);
#endif /* !WITH_CEREBROD_NO_THREADS */
}
#endif /* CEREBRO_DEBUG */
if (metric_module_setup(metric_handle, i) < 0)
{
CEREBRO_DBG(("metric_module_setup: %s", module_name));
continue;
}
if (!(metric_name = metric_module_get_metric_name(metric_handle, i)))
{
CEREBRO_DBG(("metric_module_get_metric_name: %s", module_name));
metric_module_cleanup(metric_handle, i);
continue;
}
if (metric_module_get_metric_period(metric_handle, i, &metric_period) < 0)
{
CEREBRO_DBG(("metric_module_get_metric_period: %s", module_name));
metric_module_cleanup(metric_handle, i);
continue;
}
if (metric_module_get_metric_flags(metric_handle, i, &metric_flags) < 0)
{
CEREBRO_DBG(("metric_module_get_metric_flags: %s", module_name));
metric_module_cleanup(metric_handle, i);
continue;
}
if (metric_flags & CEREBRO_METRIC_MODULE_FLAGS_SEND_ON_PERIOD
&& metric_period <= 0)
{
CEREBRO_DBG(("metric module period invalid: %s", module_name));
metric_module_cleanup(metric_handle, i);
continue;
}
if (metric_module_send_message_function_pointer(metric_handle, i, &cerebrod_send_message) < 0)
{
CEREBRO_DBG(("metric_module_send_message_function_pointer: %s", module_name));
metric_module_cleanup(metric_handle, i);
continue;
}
metric_info = Malloc(sizeof(struct cerebrod_speaker_metric_info));
/* No need to Strdup() the name in this case */
metric_info->metric_name = metric_name;
metric_info->metric_origin = CEREBROD_METRIC_SPEAKER_ORIGIN_MODULE;
metric_info->metric_period = metric_period;
metric_info->metric_flags = metric_flags;
metric_info->index = i;
/*
* If metric period is < 0, it presumably never will be sent
* (metric is likely handled by a metric_thread), so set
* next_call_time to UINT_MAX.
*
* If this is a metric that will be piggy-backed on heartbeats,
* then initialize next_call_time to 0, so the data is sent on
* the first heartbeat
*
* If this is a metric that will not be piggy-backed on
* heartbeats, set the next_call_time to UINT_MAX. Let the
* speaker logic decide when packets should be sent.
*/
if (metric_info->metric_period < 0
|| metric_info->metric_flags & CEREBRO_METRIC_MODULE_FLAGS_SEND_ON_PERIOD)
metric_info->next_call_time = UINT_MAX;
else
metric_info->next_call_time = 0;
List_append(metric_list, metric_info);
metric_list_size++;
#if !WITH_CEREBROD_NO_THREADS
if ((threadPtr = metric_module_get_metric_thread(metric_handle, i)))
{
pthread_t thread;
pthread_attr_t attr;
Pthread_attr_init(&attr);
Pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
Pthread_attr_setstacksize(&attr, CEREBROD_THREAD_STACKSIZE);
Pthread_create(&thread, &attr, threadPtr, NULL);
Pthread_attr_destroy(&attr);
}
#endif /* !WITH_CEREBROD_NO_THREADS */
}
if (!metric_list_size)
goto cleanup;
cerebrod_speaker_data_metric_list_sort();
return 1;
cleanup:
if (metric_handle)
{
/* unload will call module cleanup functions */
metric_modules_unload(metric_handle);
metric_handle = NULL;
metric_handle_count = 0;
}
metric_list_size = 0;
return 0;
}
/*
* _event_server_service_connection
*
* Service a connection from a client to receive event packets. Use
* wrapper functions minimally, b/c we want to return errors to the
* user instead of exitting with errors.
*
*/
static void
_event_server_service_connection(int fd)
{
int recv_len;
struct cerebro_event_server_request req;
struct cerebrod_event_connection_data *ecd = NULL;
char buf[CEREBRO_MAX_PACKET_LEN];
char event_name_buf[CEREBRO_MAX_EVENT_NAME_LEN+1];
char *event_name_ptr = NULL;
int32_t version;
int *fdptr = NULL;
List connections = NULL;
assert(fd >= 0);
memset(&req, '\0', sizeof(struct cerebro_event_server_request));
if ((recv_len = receive_data(fd,
CEREBRO_EVENT_SERVER_REQUEST_PACKET_LEN,
buf,
CEREBRO_MAX_PACKET_LEN,
CEREBRO_EVENT_SERVER_PROTOCOL_CLIENT_TIMEOUT_LEN,
NULL)) < 0)
goto cleanup;
if (recv_len < sizeof(version))
goto cleanup;
if (_event_server_request_check_version(buf, recv_len, &version) < 0)
{
_event_server_err_only_response(fd,
version,
CEREBRO_EVENT_SERVER_PROTOCOL_ERR_VERSION_INVALID);
goto cleanup;
}
if (recv_len != CEREBRO_EVENT_SERVER_REQUEST_PACKET_LEN)
{
_event_server_err_only_response(fd,
version,
CEREBRO_EVENT_SERVER_PROTOCOL_ERR_PACKET_INVALID);
goto cleanup;
}
if (_event_server_request_unmarshall(&req, buf, recv_len) < 0)
{
_event_server_err_only_response(fd,
version,
CEREBRO_EVENT_SERVER_PROTOCOL_ERR_PACKET_INVALID);
goto cleanup;
}
_event_server_request_dump(&req);
/* Guarantee ending '\0' character */
memset(event_name_buf, '\0', CEREBRO_MAX_EVENT_NAME_LEN+1);
memcpy(event_name_buf, req.event_name, CEREBRO_MAX_EVENT_NAME_LEN);
if (!strlen(event_name_buf))
{
_event_server_err_only_response(fd,
req.version,
CEREBRO_EVENT_SERVER_PROTOCOL_ERR_EVENT_INVALID);
goto cleanup;
}
/* Is it the special event-names request */
if (!strcmp(event_name_buf, CEREBRO_EVENT_NAMES))
{
pthread_t thread;
pthread_attr_t attr;
int *arg;
Pthread_attr_init(&attr);
Pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
Pthread_attr_setstacksize(&attr, CEREBROD_THREAD_STACKSIZE);
arg = Malloc(sizeof(int));
*arg = fd;
Pthread_create(&thread,
&attr,
_respond_with_event_names,
(void *)arg);
Pthread_attr_destroy(&attr);
return;
}
if (!event_names)
{
_event_server_err_only_response(fd,
req.version,
CEREBRO_EVENT_SERVER_PROTOCOL_ERR_EVENT_INVALID);
goto cleanup;
}
/* Event names is not changeable - so no need for a lock */
if (!(event_name_ptr = list_find_first(event_names,
_event_names_compare,
event_name_buf)))
{
_event_server_err_only_response(fd,
req.version,
CEREBRO_EVENT_SERVER_PROTOCOL_ERR_EVENT_INVALID);
goto cleanup;
}
if (!(ecd = (struct cerebrod_event_connection_data *)malloc(sizeof(struct cerebrod_event_connection_data))))
{
CEREBROD_ERR(("malloc: %s", strerror(errno)));
_event_server_err_only_response(fd,
req.version,
CEREBRO_EVENT_SERVER_PROTOCOL_ERR_INTERNAL_ERROR);
goto cleanup;
}
ecd->event_name = event_name_ptr;
ecd->fd = fd;
if (!(fdptr = (int *)malloc(sizeof(int))))
{
CEREBROD_ERR(("malloc: %s", strerror(errno)));
_event_server_err_only_response(fd,
req.version,
CEREBRO_EVENT_SERVER_PROTOCOL_ERR_INTERNAL_ERROR);
goto cleanup;
}
*fdptr = fd;
Pthread_mutex_lock(&event_connections_lock);
if (!list_append(event_connections, ecd))
{
CEREBROD_ERR(("list_append: %s", strerror(errno)));
_event_server_err_only_response(fd,
req.version,
CEREBRO_EVENT_SERVER_PROTOCOL_ERR_INTERNAL_ERROR);
goto cleanup;
}
if (!(connections = Hash_find(event_connections_index,
ecd->event_name)))
{
if (!(connections = list_create((ListDelF)free)))
{
CEREBROD_ERR(("list_create: %s", strerror(errno)));
_event_server_err_only_response(fd,
req.version,
CEREBRO_EVENT_SERVER_PROTOCOL_ERR_INTERNAL_ERROR);
goto cleanup;
}
if (!Hash_insert(event_connections_index, ecd->event_name, connections))
{
CEREBROD_ERR(("Hash_insert: %s", strerror(errno)));
_event_server_err_only_response(fd,
req.version,
CEREBRO_EVENT_SERVER_PROTOCOL_ERR_INTERNAL_ERROR);
list_destroy(connections);
goto cleanup;
}
}
if (!list_append(connections, fdptr))
{
CEREBROD_ERR(("list_append: %s", strerror(errno)));
_event_server_err_only_response(fd,
req.version,
CEREBRO_EVENT_SERVER_PROTOCOL_ERR_INTERNAL_ERROR);
goto cleanup;
}
Pthread_mutex_unlock(&event_connections_lock);
/* Clear this pointer so we know it's stored away in a list */
fdptr = NULL;
_event_server_err_only_response(fd,
req.version,
CEREBRO_EVENT_SERVER_PROTOCOL_ERR_SUCCESS);
return;
cleanup:
if (ecd)
free(ecd);
if (fdptr)
free(fdptr);
/* ignore potential error, we're in the error path already */
close(fd);
return;
}
int
main(int argc, char **argv)
{
cerebro_err_init(argv[0]);
cerebro_err_set_flags(CEREBRO_ERROR_STDERR | CEREBRO_ERROR_SYSLOG);
cerebrod_config_setup(argc, argv);
#if CEREBRO_DEBUG
if (!conf.debug)
{
cerebrod_daemon_init();
cerebro_err_set_flags(CEREBRO_ERROR_SYSLOG);
}
else
cerebro_err_set_flags(CEREBRO_ERROR_STDERR);
#else /* !CEREBRO_DEBUG */
cerebrod_daemon_init();
cerebro_err_set_flags(CEREBRO_ERROR_SYSLOG);
#endif /* !CEREBRO_DEBUG */
/* Call after daemonization, since daemonization closes currently
* open fds
*/
openlog(argv[0], LOG_ODELAY | LOG_PID, LOG_DAEMON);
#if !WITH_CEREBROD_SPEAKER_ONLY
/* Start metric server before the listener begins receiving data. */
if (conf.metric_server)
{
pthread_t thread;
pthread_attr_t attr;
Pthread_attr_init(&attr);
Pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
Pthread_attr_setstacksize(&attr, CEREBROD_THREAD_STACKSIZE);
Pthread_create(&thread, &attr, cerebrod_metric_server, NULL);
Pthread_attr_destroy(&attr);
/* Wait for initialization to complete */
Pthread_mutex_lock(&metric_server_init_lock);
while (!metric_server_init)
Pthread_cond_wait(&metric_server_init_cond, &metric_server_init_lock);
Pthread_mutex_unlock(&metric_server_init_lock);
}
/* Start listening server before speaker so that listener
* can receive packets from a later created speaker
*/
if (conf.listen)
{
int i;
for (i = 0; i < conf.listen_threads; i++)
{
pthread_t thread;
pthread_attr_t attr;
Pthread_attr_init(&attr);
Pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
Pthread_attr_setstacksize(&attr, CEREBROD_THREAD_STACKSIZE);
Pthread_create(&thread, &attr, cerebrod_listener, NULL);
Pthread_attr_destroy(&attr);
}
/* Wait for initialization to complete */
Pthread_mutex_lock(&listener_init_lock);
while (!listener_init)
Pthread_cond_wait(&listener_init_cond, &listener_init_lock);
Pthread_mutex_unlock(&listener_init_lock);
}
/* Start all the event server, queue monitor, and node timeout
* threads after the listener thread, since they use data created by
* the listener thread.
*/
if (conf.event_server)
{
pthread_t thread;
pthread_attr_t attr;
Pthread_attr_init(&attr);
Pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
Pthread_attr_setstacksize(&attr, CEREBROD_THREAD_STACKSIZE);
Pthread_create(&thread, &attr, cerebrod_event_queue_monitor, NULL);
Pthread_attr_destroy(&attr);
/* Wait for initialization to complete */
Pthread_mutex_lock(&event_queue_monitor_init_lock);
while (!event_queue_monitor_init)
Pthread_cond_wait(&event_queue_monitor_init_cond, &event_queue_monitor_init_lock);
Pthread_mutex_unlock(&event_queue_monitor_init_lock);
Pthread_attr_init(&attr);
Pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
Pthread_attr_setstacksize(&attr, CEREBROD_THREAD_STACKSIZE);
Pthread_create(&thread, &attr, cerebrod_event_server, NULL);
Pthread_attr_destroy(&attr);
/* Wait for initialization to complete */
Pthread_mutex_lock(&event_server_init_lock);
while (!event_server_init)
Pthread_cond_wait(&event_server_init_cond, &event_server_init_lock);
Pthread_mutex_unlock(&event_server_init_lock);
Pthread_attr_init(&attr);
Pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
Pthread_attr_setstacksize(&attr, CEREBROD_THREAD_STACKSIZE);
Pthread_create(&thread, &attr, cerebrod_event_node_timeout_monitor, NULL);
Pthread_attr_destroy(&attr);
/* Wait for initialization to complete */
Pthread_mutex_lock(&event_node_timeout_monitor_init_lock);
while (!event_node_timeout_monitor_init)
Pthread_cond_wait(&event_node_timeout_monitor_init_cond,
&event_node_timeout_monitor_init_lock);
Pthread_mutex_unlock(&event_node_timeout_monitor_init_lock);
}
/* Start metric controller - see comments at speaker below */
if (conf.metric_controller)
{
pthread_t thread;
pthread_attr_t attr;
Pthread_attr_init(&attr);
Pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
Pthread_attr_setstacksize(&attr, CEREBROD_THREAD_STACKSIZE);
Pthread_create(&thread, &attr, cerebrod_metric_controller, NULL);
Pthread_attr_destroy(&attr);
/* Wait for initialization to complete */
Pthread_mutex_lock(&metric_controller_init_lock);
while (!metric_controller_init)
Pthread_cond_wait(&metric_controller_init_cond,
&metric_controller_init_lock);
Pthread_mutex_unlock(&metric_controller_init_lock);
}
#endif /* !WITH_CEREBROD_SPEAKER_ONLY */
/* Start speaker
*
* It may make more logical sense to start the metric controller
* after the speaker since metric data cannot be propogated until
* after the speaker has finished being setup. We run the speaker
* last b/c it is the common case. Most machines (particularly
* compute nodes in a cluster) will only speak, and do nothing else.
* By having the speaker last, it does not need to run in a thread.
* We run it out of "main" instead to minimize memory usage by
* not needing to start the speaker in a thread.
*/
if (conf.speak)
cerebrod_speaker(NULL);
/* If speak is set, we do not reach this point */
for (;;)
sleep(INT_MAX);
return 0; /* NOT REACHED */
}
/*************************************************
* Function: Pthread_attr_set()
* Description: 设置线程属性包裹函数
* 默认线程属性：
* 1.不继承创建线程的调度策略和参数,使用设置的参数
* 2.系统范围内争抢CPU
* 3.线程采用SCHED_RR调度算法
* 4.线程分离属性
* 5.线程优先级:在调度策略的优先级minPriority~maxPriority范围内
* 6.线程栈大小属性：最小为16k
* Input: *attr---线程属性结构
* priority---线程优先级
* stacksize---线程栈大小
* Output: *attr---线程属性结构
* Return: 0/error
*************************************************/
int Pthread_attr_set(pthread_attr_t *attr, int priority, size_t stacksize)
{
int rval;
if(attr == AII_NULL)
{
return -1;
}
/*初始化线程属性结构*/
rval = Pthread_attr_init(attr);
if (rval != 0)
{
return rval;
}
/*不继承创建线程的调度策略和参数*/
rval = Pthread_attr_setinheritsched(attr, PTHREAD_EXPLICIT_SCHED);
if (rval != 0)
{
Pthread_attr_destroy(attr);
return rval;
}
/*系统范围内争抢CPU*/
rval = Pthread_attr_setscope(attr, PTHREAD_SCOPE_SYSTEM);
if (rval != 0)
{
Pthread_attr_destroy(attr);
return rval;
}
/*线程采用SCHED_RR调度算法*/
rval = Pthread_attr_setschedpolicy(attr, SCHED_RR);
if (rval != 0)
{
Pthread_attr_destroy(attr);
return rval;
}
/*设置线程分离属性:线程结束后，由系统自动回收资源*/
rval = Pthread_attr_setdetachstate(attr, PTHREAD_CREATE_DETACHED);
if (rval != 0)
{
Pthread_attr_destroy(attr);
return rval;
}
/*设置线程优先级属性*/
rval = Pthread_attr_setschedpriority(attr, priority);
if (rval != 0)
{
Pthread_attr_destroy(attr);
return rval;
}
/*设置线程栈大小属性*/
rval = Pthread_attr_setstacksize(attr, stacksize);
if (rval != 0)
{
Pthread_attr_destroy(attr);
return rval;
}
return 0;
}
