/* ------------------------------------------------------------------------- */
static void
thread_pool_init_pool(struct thread_pool_t* pool,
uint32_t num_threads,
uint32_t job_slots)
{
int i;
printf("[threadpool] Pool initialising on thread 0x%lx\n",
(intptr_t)pthread_self());
/* set struct memory to a defined state */
memset(pool, 0, sizeof(struct thread_pool_t));
/*
* Set number of threads to create - if num_threads is 0, set it to the
* number of CPU cores present
*/
if(num_threads)
pool->num_threads = num_threads;
else
pool->num_threads = get_number_of_cores();
/* allocate num_threads workers */
pool->worker = (struct thread_pool_worker_t*)
MALLOC(sizeof(*pool->worker) * pool->num_threads);
memset(pool->worker, 0, sizeof(*pool->worker) * pool->num_threads);
/* initialise workers */
for(i = 0; i != pool->num_threads; ++i)
{
thread_pool_init_worker(&pool->worker[i], job_slots);
pool->worker[i].pool = pool;
}
/* conditional variable and mutex for when workers go to sleep */
pthread_spin_init(&pool->queue_lock, PTHREAD_PROCESS_PRIVATE);
pthread_mutex_init(&pool->mutex, NULL);
pthread_cond_init(&pool->job_finished_cv, NULL);
/* launches all worker threads */
thread_pool_resume(pool);
}
/* ------------------------------------------------------------------------- */
static void
thread_pool_init_pool(struct thread_pool_t* pool, int num_threads)
{
pthread_attr_t attr;
int i;
char thread_self_str[sizeof(int)*8+3];
sprintf(thread_self_str, "0x%lx", (intptr_t)pthread_self());
llog(LOG_INFO, NULL, 2, "Thread pool initialising on thread ", thread_self_str);
/* init jobs */
unordered_vector_init_vector(&pool->jobs, sizeof(struct thread_pool_job_t));
/* set pool to active, so threads know to not exit */
pool->active = 1;
/* for portability, explicitely create threads in a joinable state */
pthread_attr_init(&attr);
pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
/* initialise mutex and conditional variables */
pthread_cond_init(&pool->cv, NULL);
pthread_mutex_init(&pool->mutex, NULL);
/* set number of threads to create - if num_threads is 0, set it to the number of CPU cores present */
if(num_threads)
pool->num_threads = num_threads;
else
pool->num_threads = get_number_of_cores();
/* launch threads and make them idle */
pool->thread = (pthread_t*)MALLOC(pool->num_threads * sizeof(pthread_t));
for(i = 0; i != pool->num_threads; ++i)
pthread_create(&pool->thread[i], &attr, (void*(*)(void*))thread_pool_worker, pool);
/* clean up */
pthread_attr_destroy(&attr);
}
/****** binding_support/get_topology() ***********************************
* NAME
* get_topology() -- Creates the topology string for the current host.
*
* SYNOPSIS
* bool get_topology(char** topology, int* length)
*
* FUNCTION
* Creates the topology string for the current host. When created,
* it has to be freed from outside.
*
* INPUTS
* char** topology - The topology string for the current host.
* int* length - The length of the topology string.
*
* RESULT
* bool - when true the topology string could be generated (and memory
* is allocated otherwise false
*
* NOTES
* MT-NOTE: get_topology() is MT safe
*
*******************************************************************************/
bool get_topology(char** topology, int* length)
{
bool success = false;
if (HAVE_HWLOC) {
/* initialize length of topology string */
(*length) = 0;
/* check if topology is supported via hwloc */
if (has_topology_information()) {
int num_sockets;
/* topology string */
dstring d_topology = DSTRING_INIT;
/* build the topology string */
if ((num_sockets = get_number_of_sockets())) {
int num_cores, ctr_cores, ctr_sockets, ctr_threads;
char* s = "S"; /* socket */
char* c = "C"; /* core */
char* t = "T"; /* thread */
for (ctr_sockets = 0; ctr_sockets < num_sockets; ctr_sockets++) {
/* append new socket */
sge_dstring_append_char(&d_topology, *s);
(*length)++;
/* for each socket get the number of cores */
if ((num_cores = get_number_of_cores(ctr_sockets))) {
/* for thread counting */
int* proc_ids = NULL;
int number_of_threads = 0;
/* check each core */
for (ctr_cores = 0; ctr_cores < num_cores; ctr_cores++) {
sge_dstring_append_char(&d_topology, *c);
(*length)++;
/* check if the core has threads */
if (get_processor_ids(ctr_sockets, ctr_cores, &proc_ids,
&number_of_threads)
&& number_of_threads > 1) {
/* print the threads */
for (ctr_threads = 0; ctr_threads < number_of_threads;
ctr_threads++) {
sge_dstring_append_char(&d_topology, *t);
(*length)++;
}
}
sge_free(&proc_ids);
}
}
} /* for each socket */
if ((*length) != 0) {
/* convert d_topolgy into topology */
(*length)++; /* we need `\0` at the end */
/* copy element */
(*topology) = sge_strdup(NULL, sge_dstring_get_string(&d_topology));
success = true;
}
sge_dstring_free(&d_topology);
}
}
}
return success;
}
/****** shepherd_binding/binding_set_striding() *************************************
* NAME
* binding_set_striding() -- Binds current process to cores.
*
* SYNOPSIS
* bool binding_set_striding(int first_socket, int first_core, int
* number_of_cores, int offset, int stepsize)
*
* FUNCTION
* Performs a core binding for the calling process according to the
* 'striding' strategy. The first core used is specified by first_socket
* (beginning with 0) and first_core (beginning with 0). If first_core is
* greater than available cores on first_socket, the next socket is examined
* and first_core is reduced by the skipped cores. If the first_core could
* not be found on system (because it was to high) no binding will be done.
*
* If the first core was choosen the next one is defined by the step size 'n'
* which is incremented to the first core found. If the socket has not the
* core (because it was the last core of the socket for example) the next
* socket is examined.
*
* If the system is out of cores and there are still some cores to select
* (because of the number_of_cores parameter) no core binding will be performed.
*
* INPUTS
* int first_socket - first socket to begin with
* int first_core - first core to start with
* int number_of_cores - total number of cores to be used
* int offset - core offset for first core (increments first core used)
* int stepsize - step size
* int type - type of binding (set or env or pe)
*
* RESULT
* bool - Returns true if the binding was performed, otherwise false.
*
* NOTES
* MT-NOTE: binding_set_striding() is MT safe
*
*******************************************************************************/
static bool
binding_set_striding(int first_socket, int first_core, int number_of_cores,
int offset, int stepsize, const binding_type_t type)
{
/* n := take every n-th core */
bool bound = false;
#if HAVE_HWLOC
dstring error = DSTRING_INIT;
if (has_core_binding() == true) {
sge_dstring_free(&error);
/* bitmask for processors to turn on and off */
hwloc_bitmap_t cpuset = hwloc_bitmap_alloc();
/* when library offers architecture:
- get virtual processor ids in the following manner:
* on socket "first_socket" choose core number "first_core + offset"
* then add n: if core is not available go to next socket
* ...
*/
if (has_topology_information()) {
/* number of cores set in processor binding mask */
int cores_set = 0;
/* next socket to use */
int next_socket = first_socket;
/* next core to use */
int next_core = first_core + offset;
/* maximal number of sockets on this system */
int max_number_of_sockets = get_number_of_sockets();
hwloc_obj_t core;
/* check if we are already out of range */
if (next_socket >= max_number_of_sockets) {
shepherd_trace("binding_set_striding: already out of sockets");
hwloc_bitmap_free(cpuset);
return false;
}
while (get_number_of_cores(next_socket) <= next_core) {
/* move on to next socket - could be that we have to deal only with cores
instead of <socket><core> tuples */
next_core -= get_number_of_cores(next_socket);
next_socket++;
if (next_socket >= max_number_of_sockets) {
/* we are out of sockets - we do nothing */
shepherd_trace("binding_set_striding: first core: out of sockets");
hwloc_bitmap_free(cpuset);
return false;
}
}
core = hwloc_get_obj_below_by_type(sge_hwloc_topology,
HWLOC_OBJ_SOCKET, next_socket,
HWLOC_OBJ_CORE, next_core);
hwloc_bitmap_or(cpuset, cpuset, core->cpuset);
/* collect the rest of the processor ids */
for (cores_set = 1; cores_set < number_of_cores; cores_set++) {
/* calculate next_core number */
next_core += stepsize;
/* check if we are already out of range */
if (next_socket >= max_number_of_sockets) {
shepherd_trace("binding_set_striding: out of sockets");
hwloc_bitmap_free(cpuset);
return false;
}
while (get_number_of_cores(next_socket) <= next_core) {
/* move on to next socket - could be that we have to deal only with cores
instead of <socket><core> tuples */
next_core -= get_number_of_cores(next_socket);
next_socket++;
if (next_socket >= max_number_of_sockets) {
/* we are out of sockets - we do nothing */
shepherd_trace("binding_set_striding: out of sockets!");
hwloc_bitmap_free(cpuset);
return false;
}
core = hwloc_get_obj_below_by_type(sge_hwloc_topology,
HWLOC_OBJ_SOCKET,
next_socket,
HWLOC_OBJ_CORE,
next_core);
hwloc_bitmap_or(cpuset, cpuset, core->cpuset);
}
} /* collecting processor ids */
if (type == BINDING_TYPE_PE) {
/* rankfile is created: do nothing */
} else if (type == BINDING_TYPE_ENV) {
/* set the environment variable */
if (create_binding_env(cpuset) == true) {
shepherd_trace("binding_set_striding: SGE_BINDING env var created");
} else {
shepherd_trace("binding_set_striding: problems while creating SGE_BINDING env");
}
} else {
/* bind process to mask */
if (bind_process_to_mask(cpuset) == true) {
/* there was an error while binding */
bound = true;
}
}
hwloc_bitmap_free(cpuset);
} else {
/* setting bitmask without topology information which could
not be right? */
shepherd_trace("binding_set_striding: bitmask without topology information");
return false;
}
} else {
/* has no core binding feature */
sge_dstring_free(&error);
return false;
}
#endif /* HAVE_HWLOC */
return bound;
}
/****** shepherd_binding/binding_set_linear() ***************************************
* NAME
* binding_set_linear() -- Bind current process linear to chunk of cores.
*
* SYNOPSIS
* bool binding_set_linear(int first_socket, int first_core, int
* number_of_cores, int offset)
*
* FUNCTION
* Binds current process (shepherd) to a set of cores. All processes
* started by the current process inherit the core binding.
*
* The core binding is done in a linear manner, that means that
* the process is bound to 'number_of_cores' cores using one core
* after another starting at socket 'first_socket' (usually 0) and
* core = 'first_core' (usually 0) + 'offset'. If the core number
* is higher than the number of cores which are provided by socket
* 'first_socket' then the next socket is taken (the core number
* defines how many cores are skiped).
*
* INPUTS
* int first_socket - The first socket (starting at 0) to bind to.
* int first_core - The first core to bind.
* int number_of_cores - The number_of_cores of cores to bind to.
* int offset - The user specified core number offset.
* binding_type_t type - The type of binding ONLY FOR EXECD ( set | env | pe )
*
* RESULT
* bool - true if binding for current process was done, false if not
*
* NOTES
* MT-NOTE: binding_set_linear() is not MT safe
*
*******************************************************************************/
static bool binding_set_linear(int first_socket, int first_core,
int number_of_cores, int offset, const binding_type_t type)
{
#if HAVE_HWLOC
/* sets bitmask in a linear manner */
/* first core is on exclusive host 0 */
/* first core could be set from scheduler */
/* offset is the first core to start with (makes sense only with
exclusive host) */
dstring error = DSTRING_INIT;
if (has_core_binding() == true) {
/* bitmask for processors to turn on and off */
hwloc_bitmap_t cpuset = hwloc_bitmap_alloc();
if (has_topology_information()) {
/* number of cores set in processor binding mask */
int cores_set;
/* next socket to use */
int next_socket = first_socket;
/* the number of cores of the next socket */
int socket_number_of_cores;
/* next core to use */
int next_core = first_core + offset;
/* maximal number of sockets on this system */
int max_number_of_sockets = get_number_of_sockets();
hwloc_obj_t this_core;
/* strategy: go to the first_socket and the first_core + offset and
fill up socket and go to the next one. */
/* TODO maybe better to search for using a core exclusively? */
while (get_number_of_cores(next_socket) <= next_core) {
/* TODO which kind of warning when first socket does not
offer this? */
/* move on to next socket - could be that we have to deal
only with cores instead of <socket><core> tuples */
next_core -= get_number_of_cores(next_socket);
next_socket++;
if (next_socket >= max_number_of_sockets) {
/* we are out of sockets - we do nothing */
hwloc_bitmap_free(cpuset);
return false;
}
}
this_core =
hwloc_get_obj_below_by_type(sge_hwloc_topology,
HWLOC_OBJ_SOCKET, next_socket,
HWLOC_OBJ_CORE, next_core);
hwloc_bitmap_or(cpuset, cpuset, this_core->cpuset);
/* collect the other processor ids with the strategy */
for (cores_set = 1; cores_set < number_of_cores; cores_set++) {
next_core++;
/* jump to next socket when it is needed */
/* maybe the next socket could offer 0 cores (I can't see when,
but just to be sure) */
while ((socket_number_of_cores = get_number_of_cores(next_socket))
<= next_core) {
next_socket++;
next_core = next_core - socket_number_of_cores;
if (next_socket >= max_number_of_sockets) {
/* we are out of sockets - we do nothing */
hwloc_bitmap_free(cpuset);
return false;
}
}
this_core =
hwloc_get_obj_below_by_type(sge_hwloc_topology,
HWLOC_OBJ_SOCKET, next_socket,
HWLOC_OBJ_CORE, next_core);
hwloc_bitmap_or(cpuset, cpuset, this_core->cpuset);
}
/* check what to do with the processor ids (set, env or pe) */
if (type == BINDING_TYPE_PE) {
/* is done outside */
} else if (type == BINDING_TYPE_ENV) {
/* set the environment variable */
/* this does not show up in "environment" file !!! */
if (create_binding_env(cpuset) == true) {
shepherd_trace("binding_set_linear: SGE_BINDING env var created");
} else {
shepherd_trace("binding_set_linear: problems while creating SGE_BINDING env");
}
} else {
/* bind SET process to mask */
if (bind_process_to_mask(cpuset) == false) {
/* there was an error while binding */
hwloc_bitmap_free(cpuset);
return false;
}
}
hwloc_bitmap_free(cpuset);
} else {
/* TODO DG strategy without topology information but with
working library? */
shepherd_trace("binding_set_linear: no information about topology");
return false;
}
} else {
shepherd_trace("binding_set_linear: binding not supported: %s",
sge_dstring_get_string(&error));
sge_dstring_free(&error);
}
#endif /* HAVE_HWLOC */
return true;
}
