int main(int argc, char *argv[]) {
int i;
int ret;
int nr = 2;
char c;
char *p;
int mapflag = MAP_ANONYMOUS;
int protflag = PROT_READ|PROT_WRITE;
unsigned long nr_nodes = numa_max_node() + 1;
struct bitmask *new_nodes;
unsigned long nodemask;
int do_unpoison = 0;
int loop = 3;
while ((c = getopt(argc, argv, "vp:m:n:ul:h:")) != -1) {
switch(c) {
case 'v':
verbose = 1;
break;
case 'p':
testpipe = optarg;
{
struct stat stat;
lstat(testpipe, &stat);
if (!S_ISFIFO(stat.st_mode))
errmsg("Given file is not fifo.\n");
}
break;
case 'm':
if (!strcmp(optarg, "private"))
mapflag |= MAP_PRIVATE;
else if (!strcmp(optarg, "shared"))
mapflag |= MAP_SHARED;
else
errmsg("invalid optarg for -m\n");
break;
case 'n':
nr = strtoul(optarg, NULL, 10);
break;
case 'u':
do_unpoison = 1;
break;
case 'l':
loop = strtoul(optarg, NULL, 10);
break;
case 'h':
HPS = strtoul(optarg, NULL, 10) * 1024;
mapflag |= MAP_HUGETLB;
/* todo: arch independent */
if (HPS != 2097152 && HPS != 1073741824)
errmsg("Invalid hugepage size\n");
break;
default:
errmsg("invalid option\n");
break;
}
}
if (nr_nodes < 2)
errmsg("A minimum of 2 nodes is required for this test.\n");
new_nodes = numa_bitmask_alloc(nr_nodes);
numa_bitmask_setbit(new_nodes, 1);
nodemask = 1; /* only node 0 allowed */
if (set_mempolicy(MPOL_BIND, &nodemask, nr_nodes) == -1)
err("set_mempolicy");
signal(SIGUSR2, sig_handle);
pprintf("start background migration\n");
pause();
signal(SIGUSR1, sig_handle_flag);
pprintf("hugepages prepared\n");
while (flag) {
p = checked_mmap((void *)ADDR_INPUT, nr * HPS, protflag, mapflag, -1, 0);
/* fault in */
memset(p, 'a', nr * HPS);
for (i = 0; i < nr; i++) {
ret = madvise(p + i * HPS, 4096, MADV_HWPOISON);
if (ret) {
perror("madvise");
pprintf("madvise returned %d\n", ret);
}
}
if (do_unpoison) {
pprintf("need unpoison\n");
pause();
}
checked_munmap(p, nr * HPS);
if (loop-- <= 0)
break;
}
pprintf("exit\n");
pause();
return 0;
}
int linuxNodeInfoCPUPopulate(FILE *cpuinfo,
virNodeInfoPtr nodeinfo,
bool need_hyperthreads)
{
char line[1024];
DIR *cpudir = NULL;
struct dirent *cpudirent = NULL;
unsigned int cpu;
unsigned long cur_threads;
int socket;
unsigned long long socket_mask = 0;
unsigned int remaining;
int online;
nodeinfo->cpus = 0;
nodeinfo->mhz = 0;
nodeinfo->cores = 1;
nodeinfo->nodes = 1;
# if HAVE_NUMACTL
if (numa_available() >= 0)
nodeinfo->nodes = numa_max_node() + 1;
# endif
/* NB: It is impossible to fill our nodes, since cpuinfo
* has no knowledge of NUMA nodes */
/* NOTE: hyperthreads are ignored here; they are parsed out of /sys */
while (fgets(line, sizeof(line), cpuinfo) != NULL) {
char *buf = line;
if (STRPREFIX(buf, "processor")) { /* aka a single logical CPU */
buf += 9;
while (*buf && c_isspace(*buf))
buf++;
if (*buf != ':') {
nodeReportError(VIR_ERR_INTERNAL_ERROR,
"%s", _("parsing cpuinfo processor"));
return -1;
}
nodeinfo->cpus++;
# if defined(__x86_64__) || \
defined(__amd64__) || \
defined(__i386__)
} else if (STRPREFIX(buf, "cpu MHz")) {
char *p;
unsigned int ui;
buf += 9;
while (*buf && c_isspace(*buf))
buf++;
if (*buf != ':' || !buf[1]) {
nodeReportError(VIR_ERR_INTERNAL_ERROR,
"%s", _("parsing cpuinfo cpu MHz"));
return -1;
}
if (virStrToLong_ui(buf+1, &p, 10, &ui) == 0
/* Accept trailing fractional part. */
&& (*p == '\0' || *p == '.' || c_isspace(*p)))
nodeinfo->mhz = ui;
} else if (STRPREFIX(buf, "cpu cores")) { /* aka cores */
char *p;
unsigned int id;
buf += 9;
while (*buf && c_isspace(*buf))
buf++;
if (*buf != ':' || !buf[1]) {
nodeReportError(VIR_ERR_INTERNAL_ERROR,
_("parsing cpuinfo cpu cores %c"), *buf);
return -1;
}
if (virStrToLong_ui(buf+1, &p, 10, &id) == 0
&& (*p == '\0' || c_isspace(*p))
&& id > nodeinfo->cores)
nodeinfo->cores = id;
# elif defined(__powerpc__) || \
defined(__powerpc64__)
} else if (STRPREFIX(buf, "clock")) {
char *p;
unsigned int ui;
buf += 5;
while (*buf && c_isspace(*buf))
buf++;
if (*buf != ':' || !buf[1]) {
nodeReportError(VIR_ERR_INTERNAL_ERROR,
"%s", _("parsing cpuinfo cpu MHz"));
return -1;
}
if (virStrToLong_ui(buf+1, &p, 10, &ui) == 0
/* Accept trailing fractional part. */
&& (*p == '\0' || *p == '.' || c_isspace(*p)))
nodeinfo->mhz = ui;
# elif defined(__s390__) || \
defined(__s390x__)
} else if (STRPREFIX(buf, "# processors")) {
char *p;
unsigned int ui;
buf += 12;
while (*buf && c_isspace(*buf))
buf++;
if (*buf != ':' || !buf[1]) {
nodeReportError(VIR_ERR_INTERNAL_ERROR,
_("parsing number of processors %c"), *buf);
return -1;
}
if (virStrToLong_ui(buf+1, &p, 10, &ui) == 0
&& (*p == '\0' || c_isspace(*p)))
nodeinfo->cpus = ui;
/* No other interesting infos are available in /proc/cpuinfo.
* However, there is a line identifying processor's version,
* identification and machine, but we don't want it to be caught
* and parsed in next iteration, because it is not in expected
* format and thus lead to error. */
break;
# else
# warning Parser for /proc/cpuinfo needs to be adapted for your architecture
# endif
}
}
if (!nodeinfo->cpus) {
nodeReportError(VIR_ERR_INTERNAL_ERROR,
"%s", _("no cpus found"));
return -1;
}
if (!need_hyperthreads)
return 0;
/* OK, we've parsed what we can out of /proc/cpuinfo. Get the socket
* and thread information from /sys
*/
remaining = nodeinfo->cpus;
cpudir = opendir(CPU_SYS_PATH);
if (cpudir == NULL) {
virReportSystemError(errno, _("cannot opendir %s"), CPU_SYS_PATH);
return -1;
}
while ((errno = 0), remaining && (cpudirent = readdir(cpudir))) {
if (sscanf(cpudirent->d_name, "cpu%u", &cpu) != 1)
continue;
online = cpu_online(cpu);
if (online < 0) {
closedir(cpudir);
return -1;
}
if (!online)
continue;
remaining--;
socket = parse_socket(cpu);
if (socket < 0) {
closedir(cpudir);
return -1;
}
if (!(socket_mask & (1 << socket))) {
socket_mask |= (1 << socket);
nodeinfo->sockets++;
}
cur_threads = count_thread_siblings(cpu);
if (cur_threads == 0) {
closedir(cpudir);
return -1;
}
if (cur_threads > nodeinfo->threads)
nodeinfo->threads = cur_threads;
}
if (errno) {
virReportSystemError(errno,
_("problem reading %s"), CPU_SYS_PATH);
closedir(cpudir);
return -1;
}
closedir(cpudir);
/* there should always be at least one socket and one thread */
if (nodeinfo->sockets == 0) {
nodeReportError(VIR_ERR_INTERNAL_ERROR,
"%s", _("no sockets found"));
return -1;
}
if (nodeinfo->threads == 0) {
nodeReportError(VIR_ERR_INTERNAL_ERROR,
"%s", _("no threads found"));
return -1;
}
/* nodeinfo->sockets is supposed to be a number of sockets per NUMA node,
* however if NUMA nodes are not composed of whole sockets, we just lie
* about the number of NUMA nodes and force apps to check capabilities XML
* for the actual NUMA topology.
*/
if (nodeinfo->sockets % nodeinfo->nodes == 0)
nodeinfo->sockets /= nodeinfo->nodes;
else
nodeinfo->nodes = 1;
return 0;
}
int
virNumaSetupMemoryPolicy(virDomainNumatuneMemMode mode,
virBitmapPtr nodeset)
{
nodemask_t mask;
int node = -1;
int ret = -1;
int bit = 0;
size_t i;
int maxnode = 0;
if (!nodeset)
return 0;
if (!virNumaNodesetIsAvailable(nodeset))
return -1;
maxnode = numa_max_node();
maxnode = maxnode < NUMA_NUM_NODES ? maxnode : NUMA_NUM_NODES;
/* Convert nodemask to NUMA bitmask. */
nodemask_zero(&mask);
bit = -1;
while ((bit = virBitmapNextSetBit(nodeset, bit)) >= 0) {
if (bit > maxnode) {
virReportError(VIR_ERR_INTERNAL_ERROR,
_("NUMA node %d is out of range"), bit);
return -1;
}
nodemask_set(&mask, bit);
}
switch (mode) {
case VIR_DOMAIN_NUMATUNE_MEM_STRICT:
numa_set_bind_policy(1);
numa_set_membind(&mask);
numa_set_bind_policy(0);
break;
case VIR_DOMAIN_NUMATUNE_MEM_PREFERRED:
{
int nnodes = 0;
for (i = 0; i < NUMA_NUM_NODES; i++) {
if (nodemask_isset(&mask, i)) {
node = i;
nnodes++;
}
}
if (nnodes != 1) {
virReportError(VIR_ERR_INTERNAL_ERROR,
"%s", _("NUMA memory tuning in 'preferred' mode "
"only supports single node"));
goto cleanup;
}
numa_set_bind_policy(0);
numa_set_preferred(node);
}
break;
case VIR_DOMAIN_NUMATUNE_MEM_INTERLEAVE:
numa_set_interleave_mask(&mask);
break;
case VIR_DOMAIN_NUMATUNE_MEM_LAST:
break;
}
ret = 0;
cleanup:
return ret;
}
/**
* @brief Returns an array of cores of size req_cores choosen
* round-robin from NUMA nodes in batches of req_step.
*
* @param req_step The step with - how many cores should be picked
* from each NUMA node in each iteration. Use a negative value
* for a "fill"-strategy, where NUMA nodes are completely filled
* before moving on to the next one.
*/
void placement(size_t req_cores, size_t req_step, coreid_t *cores)
{
// For convenience, allows to lookup 2*n for n in 0..n/2
if (req_step==0)
req_step=1;
size_t max_node = numa_max_node();
size_t num_cores = numa_num_configured_cpus();
size_t cores_per_node = num_cores/(max_node+1);
printf("req_cores: %zu\n", req_cores);
printf("req_step: %zu\n", req_step);
printf("cores / NUMA node: %zu\n", cores_per_node);
printf("max_node: %zu\n", max_node);
size_t num_selected = 0;
size_t curr_numa_idx = 0;
// How many nodes to choose from each NUMA node
size_t choose_per_node[max_node+1];
memset(choose_per_node, 0, sizeof(size_t)*(max_node+1));
// Step 1:
// Figure out how many cores to choose from each node
while (num_selected<req_cores) {
// Determine number of cores of that node
// How many cores should be choosen in this step?
// At max req_step
size_t num_choose = min(min(req_step, req_cores-num_selected),
cores_per_node-choose_per_node[curr_numa_idx]);
// Increment counter indicating how many to choose from this node
choose_per_node[curr_numa_idx] += num_choose;
num_selected += num_choose;
// Move on to the next NUMA node
curr_numa_idx = (curr_numa_idx + 1) % (max_node+1);
}
// Step 2:
// Get the cores from each NUMA node
//
// hyperthreads? -> should have higher core IDs, and hence picked in
// the end.
struct bitmask *mask = numa_allocate_cpumask();
size_t idx = 0;
for (size_t i=0; i<=max_node; i++) {
dbg_printf("node %2zu choosing %2zu\n", i, choose_per_node[i]);
// Determine which cores are on node i
numa_node_to_cpus(i, mask);
size_t choosen = 0;
for (coreid_t p=0; p<num_cores && choosen<choose_per_node[i]; p++) {
// Is processor p on node i
if (numa_bitmask_isbitset(mask, p)) {
cores[idx++] = p;
choosen++;
dbg_printf("Choosing %" PRIuCOREID " on node %zu\n", p, i);
}
}
}
assert (idx == req_cores);
}
int main(int argc, char **argv)
{
int core_max = -1;
if (argc>1) {
core_max = atoi(argv[1]);
fprintf(stderr, "Limiting number of cores to %d\n", core_max);
}
size_t num_cores = sysconf(_SC_NPROCESSORS_ONLN);
size_t total = num_cores;
if (core_max>0 && core_max<total) {
total = core_max;
}
fprintf(stderr, "Running %d threads\n", core_max);
gl_total = total;
// Allocate memory for Smelt messages
gl_msg = (struct smlt_msg**) malloc(sizeof(struct smlt_msg*)*num_cores);
COND_PANIC(gl_msg!=NULL, "Failed to allocate gl_msg");
for (size_t i=0; i<num_cores; i++) {
gl_msg[i] = smlt_message_alloc(56);
}
// Determine NUMA properties
size_t max_node = numa_max_node();
size_t cores_per_node = num_cores/(max_node+1);
if (total<num_cores) {
size_t tmp = cores_per_node;
cores_per_node = tmp/(num_cores/total);
fprintf(stderr, "Scaling down cores_per_node from %zu to %zu\n",
tmp, cores_per_node);
}
chan = (struct smlt_channel**) malloc(sizeof(struct smlt_channel*)*num_cores);
COND_PANIC(chan!=NULL, "Failed to allocate chan");
for (size_t i = 0; i < num_cores; i++) {
chan[i] = (struct smlt_channel*) malloc(sizeof(struct smlt_channel)*num_cores);
}
typedef void* (worker_func_t)(void*);
worker_func_t * workers[NUM_EXP] = {
&ab,
&reduction,
&agreement,
&barrier,
};
const char *labels[NUM_EXP] = {
"Atomic Broadcast",
"Reduction",
"Agreement",
"Barrier",
};
const char *topo_names[NUM_TOPO] = {
"mst",
"bintree",
"cluster",
"badtree",
"fibonacci",
"sequential",
"adaptivetree-shuffle-sort",
"adaptivetree",
};
errval_t err;
err = smlt_init(num_cores, true);
if (smlt_err_is_fail(err)) {
printf("FAILED TO INITIALIZE !\n");
return 1;
}
// Full mesh of channels
for (unsigned int i = 0; i < num_cores; i++) {
for (unsigned int j = 0; j < num_cores; j++) {
struct smlt_channel* ch = &chan[i][j];
err = smlt_channel_create(&ch, (uint32_t *)&i, (uint32_t*) &j, 1, 1);
if (smlt_err_is_fail(err)) {
printf("FAILED TO INITIALIZE CHANNELS !\n");
return 1;
}
}
}
// Foreach topology
for (int top = 0; top < NUM_TOPO; top++) {
// For an increasing number of threads
for (size_t num_threads = 2; num_threads<total+1;
num_threads+=INC_CORES) {
// For an increasing round-robin batch size
for (size_t step=0; step<=cores_per_node; step+=INC_STEPS) {
coreid_t cores[num_threads];
// Make available globally
gl_num_threads = num_threads;
gl_cores = cores;
gl_step = step;
placement(num_threads, step, cores);
printf("Cores (%zu threads): ", num_threads);
for (size_t dbg=0; dbg<num_threads; dbg++) {
printf(" %" PRIuCOREID, cores[dbg]);
}
printf("\n");
pthread_barrier_init(&bar, NULL, num_threads);
struct smlt_generated_model* model = NULL;
fprintf(stderr, "%s nthreads %zu \n", topo_names[top], num_threads);
err = smlt_generate_model(cores, num_threads, topo_names[top], &model);
if (smlt_err_is_fail(err)) {
printf("Failed to generated model, aborting\n");
return 1;
}
struct smlt_topology *topo = NULL;
smlt_topology_create(model, topo_names[top], &topo);
active_topo = topo;
err = smlt_context_create(topo, &context);
if (smlt_err_is_fail(err)) {
printf("FAILED TO INITIALIZE CONTEXT !\n");
return 1;
}
for (int i = 0; i < NUM_EXP; i++){
printf("----------------------------------------\n");
printf("Executing experiment %s\n", labels[i]);
printf("----------------------------------------\n");
struct smlt_node *node;
for (uint64_t j = 0; j < num_threads; j++) {
node = smlt_get_node_by_id(cores[j]);
assert(node!=NULL);
err = smlt_node_start(node, workers[i], (void*)(uint64_t) cores[j]);
if (smlt_err_is_fail(err)) {
printf("Staring node failed \n");
}
}
for (unsigned int j=0; j < num_threads; j++) {
node = smlt_get_node_by_id(cores[j]);
smlt_node_join(node);
}
}
}
}
}
// Free space for messages
for (size_t i=0; i<num_cores; i++) {
smlt_message_free(gl_msg[i]);
}
return 0;
}
/**
* @brief Do memory binding.
*
* This is handling the binding types map_mem, mask_mem and rank.
* The types local (default) and none are handled directly by the deamon.
*
* When using libnuma with API v1, this is a noop, just giving a warning.
*
* @param step Step structure
* @param task Task structure
*
* @return No return value.
*/
void doMemBind(Step_t *step, PStask_t *task)
{
# ifndef HAVE_NUMA_ALLOCATE_NODEMASK
mlog("%s: psslurm does not support memory binding types map_mem, mask_mem"
" and rank with libnuma v1\n", __func__);
fprintf(stderr, "Memory binding type not supported with used libnuma"
" version");
return;
# else
const char delimiters[] = ",";
uint32_t lTID;
char *next, *saveptr, *ents, *myent, *endptr;
char **entarray;
unsigned int numents;
uint16_t mynode;
struct bitmask *nodemask = NULL;
if (!(step->memBindType & MEM_BIND_MAP)
&& !(step->memBindType & MEM_BIND_MASK)
&& !(step->memBindType & MEM_BIND_RANK)) {
/* things are handled elsewhere */
return;
}
if (!PSIDnodes_bindMem(PSC_getMyID()) || getenv("__PSI_NO_MEMBIND")) {
// info messages already printed in doClamps()
return;
}
if (numa_available()==-1) {
fprintf(stderr, "NUMA not available:");
return;
}
nodemask = numa_allocate_nodemask();
if (!nodemask) {
fprintf(stderr, "Allocation of nodemask failed:");
return;
}
lTID = getLocalRankID(task->rank, step, step->localNodeId);
if (step->memBindType & MEM_BIND_RANK) {
if (lTID > (unsigned int)numa_max_node()) {
mlog("%s: memory binding to ranks not possible for rank %d."
" (local rank %d > #numa_nodes %d)\n", __func__,
task->rank, lTID, numa_max_node());
fprintf(stderr, "Memory binding to ranks not possible for rank %d,"
" local rank %u larger than max numa node %d.",
task->rank, lTID, numa_max_node());
if (nodemask) numa_free_nodemask(nodemask);
return;
}
if (numa_bitmask_isbitset(numa_get_mems_allowed(), lTID)) {
numa_bitmask_setbit(nodemask, lTID);
} else {
mlog("%s: setting bit %d in memory mask not allowed in rank"
" %d\n", __func__, lTID, task->rank);
fprintf(stderr, "Not allowed to set bit %u in memory mask"
" of rank %d\n", lTID, task->rank);
}
numa_set_membind(nodemask);
if (nodemask) numa_free_nodemask(nodemask);
return;
}
ents = ustrdup(step->memBind);
entarray = umalloc(step->tasksToLaunch[step->localNodeId] * sizeof(char*));
numents = 0;
myent = NULL;
entarray[0] = NULL;
next = strtok_r(ents, delimiters, &saveptr);
while (next && (numents < step->tasksToLaunch[step->localNodeId])) {
entarray[numents++] = next;
if (numents == lTID+1) {
myent = next;
break;
}
next = strtok_r(NULL, delimiters, &saveptr);
}
if (!myent && numents) {
myent = entarray[lTID % numents];
}
if (!myent) {
numa_set_membind(numa_all_nodes_ptr);
if (step->memBindType & MEM_BIND_MASK) {
mlog("%s: invalid mem mask string '%s'\n", __func__, ents);
}
else if (step->memBindType & MEM_BIND_MAP) {
mlog("%s: invalid mem map string '%s'\n", __func__, ents);
}
goto cleanup;
}
if (step->memBindType & MEM_BIND_MAP) {
if (strncmp(myent, "0x", 2) == 0) {
mynode = strtoul (myent+2, &endptr, 16);
} else {
mynode = strtoul (myent, &endptr, 10);
}
if (*endptr == '\0' && mynode <= numa_max_node()) {
if (numa_bitmask_isbitset(numa_get_mems_allowed(), mynode)) {
numa_bitmask_setbit(nodemask, mynode);
} else {
mlog("%s: setting bit %d in memory mask not allowed in rank"
" %d\n", __func__, mynode, task->rank);
fprintf(stderr, "Not allowed to set bit %d in memory mask"
" of rank %d\n", mynode, task->rank);
}
} else {
mlog("%s: invalid memory map entry '%s' (%d) for rank %d\n",
__func__, myent, mynode, task->rank);
fprintf(stderr, "Invalid memory map entry '%s' for rank %d\n",
myent, task->rank);
numa_set_membind(numa_all_nodes_ptr);
goto cleanup;
}
mdbg(PSSLURM_LOG_PART, "%s: (bind_map) node %i local task %i"
" memstr '%s'\n", __func__, step->localNodeId, lTID, myent);
} else if (step->memBindType & MEM_BIND_MASK) {
parseNUMAmask(nodemask, myent, task->rank);
}
numa_set_membind(nodemask);
cleanup:
ufree(ents);
ufree(entarray);
if (nodemask) numa_free_nodemask(nodemask);
# endif
return;
}
static int virLXCControllerSetupNUMAPolicy(virLXCControllerPtr ctrl)
{
nodemask_t mask;
int mode = -1;
int node = -1;
int ret = -1;
int i = 0;
int maxnode = 0;
bool warned = false;
if (!ctrl->def->numatune.memory.nodemask)
return 0;
VIR_DEBUG("Setting NUMA memory policy");
if (numa_available() < 0) {
virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
"%s", _("Host kernel is not aware of NUMA."));
return -1;
}
maxnode = numa_max_node() + 1;
/* Convert nodemask to NUMA bitmask. */
nodemask_zero(&mask);
i = -1;
while ((i = virBitmapNextSetBit(ctrl->def->numatune.memory.nodemask, i)) >= 0) {
if (i > NUMA_NUM_NODES) {
virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
_("Host cannot support NUMA node %d"), i);
return -1;
}
if (i > maxnode && !warned) {
VIR_WARN("nodeset is out of range, there is only %d NUMA "
"nodes on host", maxnode);
warned = true;
}
nodemask_set(&mask, i);
}
mode = ctrl->def->numatune.memory.mode;
if (mode == VIR_DOMAIN_NUMATUNE_MEM_STRICT) {
numa_set_bind_policy(1);
numa_set_membind(&mask);
numa_set_bind_policy(0);
} else if (mode == VIR_DOMAIN_NUMATUNE_MEM_PREFERRED) {
int nnodes = 0;
for (i = 0; i < NUMA_NUM_NODES; i++) {
if (nodemask_isset(&mask, i)) {
node = i;
nnodes++;
}
}
if (nnodes != 1) {
virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
"%s", _("NUMA memory tuning in 'preferred' mode "
"only supports single node"));
goto cleanup;
}
numa_set_bind_policy(0);
numa_set_preferred(node);
} else if (mode == VIR_DOMAIN_NUMATUNE_MEM_INTERLEAVE) {
numa_set_interleave_mask(&mask);
} else {
virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
_("Unable to set NUMA policy %s"),
virDomainNumatuneMemModeTypeToString(mode));
goto cleanup;
}
ret = 0;
cleanup:
return ret;
}
int main()
{
int available = numa_available();
if( available == -1 )
{
printf( "NUMA is not available on this system!\n" );
return -1;
}
printf( "numa_available() returned %d\n", available );
mai_init(BIND);
int nodes = numa_max_node();
printf( "There are %d NUMA nodes in this system:\n\n", nodes + 1 );
DumpMemoryStats();
// initialize the random number list
int x;
for(x = 0; x < WORKERS_COUNT * MAJOR_COUNT; x++ )
{
rand_nums[x] = 6 + rand() % MAX_BLOCK_FACTOR;
}
time_t time_start = time(NULL);
// start worker threads
printf( "\nStarting workers...\n" );
int thread_no = 0,node,m;
for(node = 0; node < CORES_COUNT; node++ )
{
for(m = 0; m < WORKERS_PER_CORE; m++ )
{
CreateWorkerThread( thread_no, node );
thread_no++;
}
}
// wait until all threads complete
printf( "Waiting for workers to complete...\n" );
int n;
for( n = 0; n < WORKERS_COUNT; n++ )
{
if( 0 != pthread_join( targs[n].tid, NULL ) )
{
printf( "pthread_join failed!\n" );
}
}
time_t time_end = time(NULL);
printf( "All worker threads completed!\n\n" );
DumpMemoryStats();
double seconds = difftime( time_end, time_start );
unsigned long long min = 0xffffffffffffffffll;
unsigned long long max = 0;
unsigned long long avg = 0;
for( n = 0; n < WORKERS_COUNT; n++ )
{
if( targs[n].tdiff < min )
min = targs[n].tdiff;
if( targs[n].tdiff > max )
max = targs[n].tdiff;
avg = avg + targs[n].tdiff;
}
printf( "\n%f seconds\t( avg: %llu,\tmin: %llu,\tmax: %llu ticks)\n", seconds, avg, min, max );
return 0;
}
