/*
* Tests for correct handling of -m and --socket-mem flags
*/
static int
test_memory_flags(void)
{
#ifdef RTE_EXEC_ENV_BSDAPP
/* BSD target doesn't support prefixes at this point */
const char * prefix = "";
#else
char prefix[PATH_MAX], tmp[PATH_MAX];
if (get_current_prefix(tmp, sizeof(tmp)) == NULL) {
printf("Error - unable to get current prefix!\n");
return -1;
}
snprintf(prefix, sizeof(prefix), "--file-prefix=%s", tmp);
#endif
/* valid -m flag and mp flag */
const char *argv0[] = {prgname, prefix, mp_flag, "-c", "10",
"-n", "2", "-m", DEFAULT_MEM_SIZE};
/* valid -m flag */
const char *argv1[] = {prgname, "-c", "10", "-n", "2",
"--file-prefix=" memtest, "-m", DEFAULT_MEM_SIZE};
/* invalid (zero) --socket-mem flag */
const char *argv2[] = {prgname, "-c", "10", "-n", "2",
"--file-prefix=" memtest, "--socket-mem=0,0,0,0"};
/* invalid (incomplete) --socket-mem flag */
const char *argv3[] = {prgname, "-c", "10", "-n", "2",
"--file-prefix=" memtest, "--socket-mem=2,2,"};
/* invalid (mixed with invalid data) --socket-mem flag */
const char *argv4[] = {prgname, "-c", "10", "-n", "2",
"--file-prefix=" memtest, "--socket-mem=2,2,Fred"};
/* invalid (with numeric value as last character) --socket-mem flag */
const char *argv5[] = {prgname, "-c", "10", "-n", "2",
"--file-prefix=" memtest, "--socket-mem=2,2,Fred0"};
/* invalid (with empty socket) --socket-mem flag */
const char *argv6[] = {prgname, "-c", "10", "-n", "2",
"--file-prefix=" memtest, "--socket-mem=2,,2"};
/* invalid (null) --socket-mem flag */
const char *argv7[] = {prgname, "-c", "10", "-n", "2",
"--file-prefix=" memtest, "--socket-mem="};
/* valid --socket-mem specified together with -m flag */
const char *argv8[] = {prgname, "-c", "10", "-n", "2",
"--file-prefix=" memtest, "-m", DEFAULT_MEM_SIZE, "--socket-mem=2,2"};
/* construct an invalid socket mask with 2 megs on each socket plus
* extra 2 megs on socket that doesn't exist on current system */
char invalid_socket_mem[SOCKET_MEM_STRLEN];
char buf[SOCKET_MEM_STRLEN]; /* to avoid copying string onto itself */
#ifdef RTE_EXEC_ENV_BSDAPP
int i, num_sockets = 1;
#else
int i, num_sockets = get_number_of_sockets();
#endif
if (num_sockets <= 0 || num_sockets > RTE_MAX_NUMA_NODES) {
printf("Error - cannot get number of sockets!\n");
return -1;
}
snprintf(invalid_socket_mem, sizeof(invalid_socket_mem), "--socket-mem=");
/* add one extra socket */
for (i = 0; i < num_sockets + 1; i++) {
snprintf(buf, sizeof(buf), "%s%s", invalid_socket_mem, DEFAULT_MEM_SIZE);
snprintf(invalid_socket_mem, sizeof(invalid_socket_mem), "%s", buf);
if (num_sockets + 1 - i > 1) {
snprintf(buf, sizeof(buf), "%s,", invalid_socket_mem);
snprintf(invalid_socket_mem, sizeof(invalid_socket_mem), "%s", buf);
}
}
/* construct a valid socket mask with 2 megs on each existing socket */
char valid_socket_mem[SOCKET_MEM_STRLEN];
snprintf(valid_socket_mem, sizeof(valid_socket_mem), "--socket-mem=");
/* add one extra socket */
for (i = 0; i < num_sockets; i++) {
snprintf(buf, sizeof(buf), "%s%s", valid_socket_mem, DEFAULT_MEM_SIZE);
snprintf(valid_socket_mem, sizeof(valid_socket_mem), "%s", buf);
if (num_sockets - i > 1) {
snprintf(buf, sizeof(buf), "%s,", valid_socket_mem);
snprintf(valid_socket_mem, sizeof(valid_socket_mem), "%s", buf);
}
}
/* invalid --socket-mem flag (with extra socket) */
const char *argv9[] = {prgname, "-c", "10", "-n", "2",
"--file-prefix=" memtest, invalid_socket_mem};
/* valid --socket-mem flag */
const char *argv10[] = {prgname, "-c", "10", "-n", "2",
"--file-prefix=" memtest, valid_socket_mem};
if (launch_proc(argv0) != 0) {
printf("Error - secondary process failed with valid -m flag !\n");
return -1;
}
#ifdef RTE_EXEC_ENV_BSDAPP
/* no other tests are applicable to BSD */
return 0;
#endif
if (launch_proc(argv1) != 0) {
printf("Error - process failed with valid -m flag!\n");
return -1;
}
#ifdef RTE_LIBRTE_XEN_DOM0
return 0;
#endif
if (launch_proc(argv2) == 0) {
printf("Error - process run ok with invalid (zero) --socket-mem!\n");
return -1;
}
if (launch_proc(argv3) == 0) {
printf("Error - process run ok with invalid "
"(incomplete) --socket-mem!\n");
return -1;
}
if (launch_proc(argv4) == 0) {
printf("Error - process run ok with invalid "
"(mixed with invalid input) --socket-mem!\n");
return -1;
}
if (launch_proc(argv5) == 0) {
printf("Error - process run ok with invalid "
"(mixed with invalid input with a numeric value as "
"last character) --socket-mem!\n");
return -1;
}
if (launch_proc(argv6) == 0) {
printf("Error - process run ok with invalid "
"(with empty socket) --socket-mem!\n");
return -1;
}
if (launch_proc(argv7) == 0) {
printf("Error - process run ok with invalid (null) --socket-mem!\n");
return -1;
}
if (launch_proc(argv8) == 0) {
printf("Error - process run ok with --socket-mem and -m specified!\n");
return -1;
}
if (launch_proc(argv9) == 0) {
printf("Error - process run ok with extra socket in --socket-mem!\n");
return -1;
}
if (launch_proc(argv10) != 0) {
printf("Error - process failed with valid --socket-mem!\n");
return -1;
}
return 0;
}
/****** shepherd_binding/binding_set_striding() *************************************
* NAME
* binding_set_striding() -- Binds current process to cores.
*
* SYNOPSIS
* bool binding_set_striding(int first_socket, int first_core, int
* number_of_cores, int offset, int stepsize)
*
* FUNCTION
* Performs a core binding for the calling process according to the
* 'striding' strategy. The first core used is specified by first_socket
* (beginning with 0) and first_core (beginning with 0). If first_core is
* greater than available cores on first_socket, the next socket is examined
* and first_core is reduced by the skipped cores. If the first_core could
* not be found on system (because it was to high) no binding will be done.
*
* If the first core was choosen the next one is defined by the step size 'n'
* which is incremented to the first core found. If the socket has not the
* core (because it was the last core of the socket for example) the next
* socket is examined.
*
* If the system is out of cores and there are still some cores to select
* (because of the number_of_cores parameter) no core binding will be performed.
*
* INPUTS
* int first_socket - first socket to begin with
* int first_core - first core to start with
* int number_of_cores - total number of cores to be used
* int offset - core offset for first core (increments first core used)
* int stepsize - step size
* int type - type of binding (set or env or pe)
*
* RESULT
* bool - Returns true if the binding was performed, otherwise false.
*
* NOTES
* MT-NOTE: binding_set_striding() is MT safe
*
*******************************************************************************/
static bool
binding_set_striding(int first_socket, int first_core, int number_of_cores,
int offset, int stepsize, const binding_type_t type)
{
/* n := take every n-th core */
bool bound = false;
#if HAVE_HWLOC
dstring error = DSTRING_INIT;
if (has_core_binding() == true) {
sge_dstring_free(&error);
/* bitmask for processors to turn on and off */
hwloc_bitmap_t cpuset = hwloc_bitmap_alloc();
/* when library offers architecture:
- get virtual processor ids in the following manner:
* on socket "first_socket" choose core number "first_core + offset"
* then add n: if core is not available go to next socket
* ...
*/
if (has_topology_information()) {
/* number of cores set in processor binding mask */
int cores_set = 0;
/* next socket to use */
int next_socket = first_socket;
/* next core to use */
int next_core = first_core + offset;
/* maximal number of sockets on this system */
int max_number_of_sockets = get_number_of_sockets();
hwloc_obj_t core;
/* check if we are already out of range */
if (next_socket >= max_number_of_sockets) {
shepherd_trace("binding_set_striding: already out of sockets");
hwloc_bitmap_free(cpuset);
return false;
}
while (get_number_of_cores(next_socket) <= next_core) {
/* move on to next socket - could be that we have to deal only with cores
instead of <socket><core> tuples */
next_core -= get_number_of_cores(next_socket);
next_socket++;
if (next_socket >= max_number_of_sockets) {
/* we are out of sockets - we do nothing */
shepherd_trace("binding_set_striding: first core: out of sockets");
hwloc_bitmap_free(cpuset);
return false;
}
}
core = hwloc_get_obj_below_by_type(sge_hwloc_topology,
HWLOC_OBJ_SOCKET, next_socket,
HWLOC_OBJ_CORE, next_core);
hwloc_bitmap_or(cpuset, cpuset, core->cpuset);
/* collect the rest of the processor ids */
for (cores_set = 1; cores_set < number_of_cores; cores_set++) {
/* calculate next_core number */
next_core += stepsize;
/* check if we are already out of range */
if (next_socket >= max_number_of_sockets) {
shepherd_trace("binding_set_striding: out of sockets");
hwloc_bitmap_free(cpuset);
return false;
}
while (get_number_of_cores(next_socket) <= next_core) {
/* move on to next socket - could be that we have to deal only with cores
instead of <socket><core> tuples */
next_core -= get_number_of_cores(next_socket);
next_socket++;
if (next_socket >= max_number_of_sockets) {
/* we are out of sockets - we do nothing */
shepherd_trace("binding_set_striding: out of sockets!");
hwloc_bitmap_free(cpuset);
return false;
}
core = hwloc_get_obj_below_by_type(sge_hwloc_topology,
HWLOC_OBJ_SOCKET,
next_socket,
HWLOC_OBJ_CORE,
next_core);
hwloc_bitmap_or(cpuset, cpuset, core->cpuset);
}
} /* collecting processor ids */
if (type == BINDING_TYPE_PE) {
/* rankfile is created: do nothing */
} else if (type == BINDING_TYPE_ENV) {
/* set the environment variable */
if (create_binding_env(cpuset) == true) {
shepherd_trace("binding_set_striding: SGE_BINDING env var created");
} else {
shepherd_trace("binding_set_striding: problems while creating SGE_BINDING env");
}
} else {
/* bind process to mask */
if (bind_process_to_mask(cpuset) == true) {
/* there was an error while binding */
bound = true;
}
}
hwloc_bitmap_free(cpuset);
} else {
/* setting bitmask without topology information which could
not be right? */
shepherd_trace("binding_set_striding: bitmask without topology information");
return false;
}
} else {
/* has no core binding feature */
sge_dstring_free(&error);
return false;
}
#endif /* HAVE_HWLOC */
return bound;
}
/****** binding_support/get_topology() ***********************************
* NAME
* get_topology() -- Creates the topology string for the current host.
*
* SYNOPSIS
* bool get_topology(char** topology, int* length)
*
* FUNCTION
* Creates the topology string for the current host. When created,
* it has to be freed from outside.
*
* INPUTS
* char** topology - The topology string for the current host.
* int* length - The length of the topology string.
*
* RESULT
* bool - when true the topology string could be generated (and memory
* is allocated otherwise false
*
* NOTES
* MT-NOTE: get_topology() is MT safe
*
*******************************************************************************/
bool get_topology(char** topology, int* length)
{
bool success = false;
if (HAVE_HWLOC) {
/* initialize length of topology string */
(*length) = 0;
/* check if topology is supported via hwloc */
if (has_topology_information()) {
int num_sockets;
/* topology string */
dstring d_topology = DSTRING_INIT;
/* build the topology string */
if ((num_sockets = get_number_of_sockets())) {
int num_cores, ctr_cores, ctr_sockets, ctr_threads;
char* s = "S"; /* socket */
char* c = "C"; /* core */
char* t = "T"; /* thread */
for (ctr_sockets = 0; ctr_sockets < num_sockets; ctr_sockets++) {
/* append new socket */
sge_dstring_append_char(&d_topology, *s);
(*length)++;
/* for each socket get the number of cores */
if ((num_cores = get_number_of_cores(ctr_sockets))) {
/* for thread counting */
int* proc_ids = NULL;
int number_of_threads = 0;
/* check each core */
for (ctr_cores = 0; ctr_cores < num_cores; ctr_cores++) {
sge_dstring_append_char(&d_topology, *c);
(*length)++;
/* check if the core has threads */
if (get_processor_ids(ctr_sockets, ctr_cores, &proc_ids,
&number_of_threads)
&& number_of_threads > 1) {
/* print the threads */
for (ctr_threads = 0; ctr_threads < number_of_threads;
ctr_threads++) {
sge_dstring_append_char(&d_topology, *t);
(*length)++;
}
}
sge_free(&proc_ids);
}
}
} /* for each socket */
if ((*length) != 0) {
/* convert d_topolgy into topology */
(*length)++; /* we need `\0` at the end */
/* copy element */
(*topology) = sge_strdup(NULL, sge_dstring_get_string(&d_topology));
success = true;
}
sge_dstring_free(&d_topology);
}
}
}
return success;
}
/****** shepherd_binding/binding_set_linear() ***************************************
* NAME
* binding_set_linear() -- Bind current process linear to chunk of cores.
*
* SYNOPSIS
* bool binding_set_linear(int first_socket, int first_core, int
* number_of_cores, int offset)
*
* FUNCTION
* Binds current process (shepherd) to a set of cores. All processes
* started by the current process inherit the core binding.
*
* The core binding is done in a linear manner, that means that
* the process is bound to 'number_of_cores' cores using one core
* after another starting at socket 'first_socket' (usually 0) and
* core = 'first_core' (usually 0) + 'offset'. If the core number
* is higher than the number of cores which are provided by socket
* 'first_socket' then the next socket is taken (the core number
* defines how many cores are skiped).
*
* INPUTS
* int first_socket - The first socket (starting at 0) to bind to.
* int first_core - The first core to bind.
* int number_of_cores - The number_of_cores of cores to bind to.
* int offset - The user specified core number offset.
* binding_type_t type - The type of binding ONLY FOR EXECD ( set | env | pe )
*
* RESULT
* bool - true if binding for current process was done, false if not
*
* NOTES
* MT-NOTE: binding_set_linear() is not MT safe
*
*******************************************************************************/
static bool binding_set_linear(int first_socket, int first_core,
int number_of_cores, int offset, const binding_type_t type)
{
#if HAVE_HWLOC
/* sets bitmask in a linear manner */
/* first core is on exclusive host 0 */
/* first core could be set from scheduler */
/* offset is the first core to start with (makes sense only with
exclusive host) */
dstring error = DSTRING_INIT;
if (has_core_binding() == true) {
/* bitmask for processors to turn on and off */
hwloc_bitmap_t cpuset = hwloc_bitmap_alloc();
if (has_topology_information()) {
/* number of cores set in processor binding mask */
int cores_set;
/* next socket to use */
int next_socket = first_socket;
/* the number of cores of the next socket */
int socket_number_of_cores;
/* next core to use */
int next_core = first_core + offset;
/* maximal number of sockets on this system */
int max_number_of_sockets = get_number_of_sockets();
hwloc_obj_t this_core;
/* strategy: go to the first_socket and the first_core + offset and
fill up socket and go to the next one. */
/* TODO maybe better to search for using a core exclusively? */
while (get_number_of_cores(next_socket) <= next_core) {
/* TODO which kind of warning when first socket does not
offer this? */
/* move on to next socket - could be that we have to deal
only with cores instead of <socket><core> tuples */
next_core -= get_number_of_cores(next_socket);
next_socket++;
if (next_socket >= max_number_of_sockets) {
/* we are out of sockets - we do nothing */
hwloc_bitmap_free(cpuset);
return false;
}
}
this_core =
hwloc_get_obj_below_by_type(sge_hwloc_topology,
HWLOC_OBJ_SOCKET, next_socket,
HWLOC_OBJ_CORE, next_core);
hwloc_bitmap_or(cpuset, cpuset, this_core->cpuset);
/* collect the other processor ids with the strategy */
for (cores_set = 1; cores_set < number_of_cores; cores_set++) {
next_core++;
/* jump to next socket when it is needed */
/* maybe the next socket could offer 0 cores (I can't see when,
but just to be sure) */
while ((socket_number_of_cores = get_number_of_cores(next_socket))
<= next_core) {
next_socket++;
next_core = next_core - socket_number_of_cores;
if (next_socket >= max_number_of_sockets) {
/* we are out of sockets - we do nothing */
hwloc_bitmap_free(cpuset);
return false;
}
}
this_core =
hwloc_get_obj_below_by_type(sge_hwloc_topology,
HWLOC_OBJ_SOCKET, next_socket,
HWLOC_OBJ_CORE, next_core);
hwloc_bitmap_or(cpuset, cpuset, this_core->cpuset);
}
/* check what to do with the processor ids (set, env or pe) */
if (type == BINDING_TYPE_PE) {
/* is done outside */
} else if (type == BINDING_TYPE_ENV) {
/* set the environment variable */
/* this does not show up in "environment" file !!! */
if (create_binding_env(cpuset) == true) {
shepherd_trace("binding_set_linear: SGE_BINDING env var created");
} else {
shepherd_trace("binding_set_linear: problems while creating SGE_BINDING env");
}
} else {
/* bind SET process to mask */
if (bind_process_to_mask(cpuset) == false) {
/* there was an error while binding */
hwloc_bitmap_free(cpuset);
return false;
}
}
hwloc_bitmap_free(cpuset);
} else {
/* TODO DG strategy without topology information but with
working library? */
shepherd_trace("binding_set_linear: no information about topology");
return false;
}
} else {
shepherd_trace("binding_set_linear: binding not supported: %s",
sge_dstring_get_string(&error));
sge_dstring_free(&error);
}
#endif /* HAVE_HWLOC */
return true;
}
