int bind_cpu(int cpu) {
struct bitmask *nodemask = numa_parse_nodestring("0");
struct bitmask *cpumask = numa_allocate_cpumask();
numa_bind(nodemask);
cpumask = numa_bitmask_clearall(cpumask);
return numa_sched_setaffinity( getpid(), numa_bitmask_setbit(cpumask, cpu-1) );
}
int bind2node(int core_id) {
char node_str[8];
if (core_id < 0 || numa_available() == -1)
return -1;
snprintf(node_str, sizeof(node_str), "%u", numa_node_of_cpu(core_id));
numa_bind(numa_parse_nodestring(node_str));
return 0;
}
static struct bitmask *numactl_parse_nodestring(char *s)
{
static char *last;
if (s[0] == 's' && !strcmp(s, "same")) {
if (!last)
usage_msg("same needs previous node specification");
s = last;
} else {
last = s;
}
return numa_parse_nodestring(s);
}
static int fill_bandwidth_values_from_enviroment(int* bandwidth, int bandwidth_len, char *hbw_nodes_env)
{
struct bitmask *hbw_nodes_bm = numa_parse_nodestring(hbw_nodes_env);
if (!hbw_nodes_bm) {
log_err("Wrong MEMKIND_HBW_NODES environment value.");
return MEMKIND_ERROR_ENVIRON;
}
else {
assign_arbitrary_bandwidth_values(bandwidth, bandwidth_len, hbw_nodes_bm);
numa_bitmask_free(hbw_nodes_bm);
}
return 0;
}
/*
* Function: _get_numa_nodes
* Description:
* Returns a count of the NUMA nodes that the application is running on.
*
* Returns an array of NUMA nodes that the application is running on.
*
*
* IN char* path -- The path to the directory containing the files containing
* information about NUMA nodes.
*
* OUT *cnt -- The number of NUMA nodes in the array
* OUT **numa_array -- An integer array containing the NUMA nodes.
* This array must be xfreed by the caller.
*
* RETURN
* 0 on success and -1 on failure.
*/
static int _get_numa_nodes(char *path, int *cnt, int32_t **numa_array) {
struct bitmask *bm;
int i, index, rc = 0;
int lsz;
size_t sz;
char buffer[PATH_MAX];
FILE *f = NULL;
char *lin = NULL;
rc = snprintf(buffer, sizeof(buffer), "%s/%s", path, "mems");
if (rc < 0) {
CRAY_ERR("snprintf failed. Return code: %d", rc);
}
f = fopen(buffer, "r");
if (f == NULL ) {
CRAY_ERR("Failed to open file %s: %m", buffer);
return -1;
}
lsz = getline(&lin, &sz, f);
if (lsz > 0) {
if (lin[strlen(lin) - 1] == '\n') {
lin[strlen(lin) - 1] = '\0';
}
bm = numa_parse_nodestring(lin);
if (bm == NULL ) {
CRAY_ERR("Error numa_parse_nodestring:"
" Invalid node string: %s", lin);
free(lin);
return SLURM_ERROR;
}
} else {
CRAY_ERR("Reading %s failed", buffer);
return SLURM_ERROR;
}
free(lin);
*cnt = numa_bitmask_weight(bm);
if (*cnt == 0) {
CRAY_ERR("No NUMA Nodes found");
return -1;
}
if (debug_flags & DEBUG_FLAG_TASK) {
info("Bitmask %#lx size: %lu sizeof(*(bm->maskp)): %zd"
" weight: %u",
*(bm->maskp), bm->size, sizeof(*(bm->maskp)), *cnt);
}
*numa_array = xmalloc(*cnt * sizeof(int32_t));
index = 0;
for (i = 0; i < bm->size; i++) {
if (*(bm->maskp) & ((long unsigned) 1 << i)) {
if (debug_flags & DEBUG_FLAG_TASK) {
info("(%s: %d: %s) NUMA Node %d is present",
THIS_FILE, __LINE__, __FUNCTION__, i);
}
(*numa_array)[index++] = i;
}
}
numa_free_nodemask(bm);
return 0;
}
static void memkind_hbw_closest_numanode_init(void)
{
struct memkind_hbw_closest_numanode_t *g =
&memkind_hbw_closest_numanode_g;
int *bandwidth = NULL;
int num_unique = 0;
int high_bandwidth = 0;
int node;
struct bandwidth_nodes_t *bandwidth_nodes = NULL;
char *hbw_nodes_env;
struct bitmask *hbw_nodes_bm;
g->num_cpu = numa_num_configured_cpus();
g->closest_numanode = (int *)je_malloc(sizeof(int) * g->num_cpu);
bandwidth = (int *)je_malloc(sizeof(int) * NUMA_NUM_NODES);
if (!(g->closest_numanode && bandwidth)) {
g->init_err = MEMKIND_ERROR_MALLOC;
}
if (!g->init_err) {
hbw_nodes_env = getenv("MEMKIND_HBW_NODES");
if (hbw_nodes_env) {
hbw_nodes_bm = numa_parse_nodestring(hbw_nodes_env);
if (!hbw_nodes_bm) {
g->init_err = MEMKIND_ERROR_ENVIRON;
}
else {
for (node = 0; node < NUMA_NUM_NODES; ++node) {
if (numa_bitmask_isbitset(hbw_nodes_bm, node)) {
bandwidth[node] = 2;
}
else {
bandwidth[node] = 1;
}
}
numa_bitmask_free(hbw_nodes_bm);
}
}
else {
g->init_err = parse_node_bandwidth(NUMA_NUM_NODES, bandwidth,
MEMKIND_BANDWIDTH_PATH);
}
}
if (!g->init_err) {
g->init_err = create_bandwidth_nodes(NUMA_NUM_NODES, bandwidth,
&num_unique, &bandwidth_nodes);
}
if (!g->init_err) {
if (num_unique == 1) {
g->init_err = MEMKIND_ERROR_UNAVAILABLE;
}
}
if (!g->init_err) {
high_bandwidth = bandwidth_nodes[num_unique-1].bandwidth;
g->init_err = set_closest_numanode(num_unique, bandwidth_nodes,
high_bandwidth, g->num_cpu,
g->closest_numanode);
}
if (bandwidth_nodes) {
je_free(bandwidth_nodes);
}
if (bandwidth) {
je_free(bandwidth);
}
if (g->init_err) {
if (g->closest_numanode) {
je_free(g->closest_numanode);
g->closest_numanode = NULL;
}
}
}
int main(int argc, char *argv[])
{
char *p;
int option;
struct timespec result;
unsigned long bytes;
double duration, mbytes;
struct bitmask *from;
struct bitmask *to;
pagesize = getpagesize();
/* Command line processing */
opterr = 1;
cmd = argv[0];
while ((option = getopt(argc, argv, optstr)) != EOF)
switch (option) {
case 'h' :
case '?' :
usage();
break;
case 'v' :
verbose++;
break;
case 'p' :
pages = strtoul(optarg, &p, 0);
if (p == optarg || *p)
usage();
break;
}
if (!argv[optind])
usage();
if (verbose > 1)
printf("numa_max_node = %d\n", numa_max_node());
numa_exit_on_error = 1;
from = numa_parse_nodestring(argv[optind]);
if (!from) {
printf ("<%s> is invalid\n", argv[optind]);
exit(1);
}
if (errno) {
perror("from mask");
exit(1);
}
if (verbose)
printmask("From", from);
if (!argv[optind+1])
usage();
to = numa_parse_nodestring(argv[optind+1]);
if (!to) {
printf ("<%s> is invalid\n", argv[optind+1]);
exit(1);
}
if (errno) {
perror("to mask");
exit(1);
}
if (verbose)
printmask("To", to);
bytes = pages * pagesize;
if (verbose)
printf("Allocating %lu pages of %lu bytes of memory\n",
pages, pagesize);
memory = memalign(pagesize, bytes);
if (!memory) {
printf("Out of Memory\n");
exit(2);
}
if (mbind(memory, bytes, MPOL_BIND, from->maskp, from->size, 0) < 0)
numa_error("mbind");
if (verbose)
printf("Dirtying memory....\n");
for (p = memory; p <= memory + bytes; p += pagesize)
*p = 1;
if (verbose)
printf("Starting test\n");
displaymap();
clock_gettime(CLOCK_REALTIME, &start);
if (mbind(memory, bytes, MPOL_BIND, to->maskp, to->size, MPOL_MF_MOVE) <0)
numa_error("memory move");
clock_gettime(CLOCK_REALTIME, &end);
displaymap();
result.tv_sec = end.tv_sec - start.tv_sec;
result.tv_nsec = end.tv_nsec - start.tv_nsec;
if (result.tv_nsec < 0) {
result.tv_sec--;
result.tv_nsec += 1000000000;
}
if (result.tv_nsec >= 1000000000) {
result.tv_sec++;
result.tv_nsec -= 1000000000;
}
duration = result.tv_sec + result.tv_nsec / 1000000000.0;
mbytes = bytes / (1024*1024.0);
printf("%1.1f Mbyte migrated in %1.2f secs. %3.1f Mbytes/second\n",
mbytes,
duration,
mbytes / duration);
return 0;
}
