/*
* Set the memory policy for data allocation for a process
*/
int hw_set_mempol(int mempol, int node)
{
int error;
switch(mempol)
{
case OS:
if (local_topo->nnodes != 0 ){
hwloc_obj_t obj;
obj = hwloc_get_obj_by_type(topology, HWLOC_OBJ_MACHINE,0);
error = hwloc_set_membind_nodeset(topology,obj->nodeset,HWLOC_MEMBIND_FIRSTTOUCH,0);
}
break;
case LOCAL:
if (local_topo->nnodes != 0 ){
node = hw_get_proc_node();
hwloc_obj_t obj;
obj = hwloc_get_obj_by_type(topology, HWLOC_OBJ_NODE, node);
error = hwloc_set_membind_nodeset(topology,obj->nodeset,HWLOC_MEMBIND_BIND,0);
}
break;
case INTERLEAVE:
if (local_topo->nnodes != 0 ){
node = hw_get_proc_node();
hwloc_obj_t obj;
obj = hwloc_get_obj_by_type(topology, HWLOC_OBJ_NODE, node);
obj = obj->parent;
error = hwloc_set_membind_nodeset(topology,obj->nodeset,HWLOC_MEMBIND_INTERLEAVE,0);
}
break;
case MANUAL:
if (local_topo->nnodes != 0 ){
if (node == -1)
node = hw_get_proc_node();
hwloc_obj_t obj;
obj = hwloc_get_obj_by_type(topology, HWLOC_OBJ_NODE, node);
error = hwloc_set_membind_nodeset(topology,obj->nodeset,HWLOC_MEMBIND_BIND,0);
}
break;
default:
error = 1;
break;
}
return error;
}
int
hwloc_set_membind(hwloc_topology_t topology, hwloc_const_cpuset_t set, hwloc_membind_policy_t policy, int flags)
{
hwloc_nodeset_t nodeset = hwloc_bitmap_alloc();
int ret;
if (hwloc_fix_membind_cpuset(topology, nodeset, set))
ret = -1;
else
ret = hwloc_set_membind_nodeset(topology, nodeset, policy, flags);
hwloc_bitmap_free(nodeset);
return ret;
}
void bindCurrentThreadToNumaNode(int node) {
hwloc_topology_t topology = getHWTopology();
hwloc_cpuset_t cpuset;
hwloc_obj_t obj;
// The actual node
obj = hwloc_get_obj_by_type(topology, HWLOC_OBJ_NODE, node);
// obj is nullptr on non NUMA machines
if (obj == nullptr) {
fprintf(stderr, "Couldn't get hwloc object, bindCurrentThreadToNumaNode failed!\n");
return;
}
cpuset = hwloc_bitmap_dup(obj->cpuset);
// hwloc_bitmap_singlify(cpuset);
// bind
if (hwloc_set_cpubind(topology, cpuset, HWLOC_CPUBIND_STRICT | HWLOC_CPUBIND_NOMEMBIND | HWLOC_CPUBIND_THREAD)) {
char* str;
int error = errno;
hwloc_bitmap_asprintf(&str, obj->cpuset);
printf("Couldn't bind to cpuset %s: %s\n", str, strerror(error));
free(str);
throw std::runtime_error(strerror(error));
}
// free duplicated cpuset
hwloc_bitmap_free(cpuset);
// assuming single machine system
obj = hwloc_get_obj_by_type(topology, HWLOC_OBJ_MACHINE, 0);
// set membind policy interleave for this thread
if (hwloc_set_membind_nodeset(
topology, obj->nodeset, HWLOC_MEMBIND_INTERLEAVE, HWLOC_MEMBIND_STRICT | HWLOC_MEMBIND_THREAD) && errno != ENOSYS) {
char* str;
int error = errno;
hwloc_bitmap_asprintf(&str, obj->nodeset);
fprintf(stderr, "Couldn't membind to nodeset %s: %s\n", str, strerror(error));
fprintf(stderr, "Continuing as normal, however, no guarantees\n");
free(str);
}
}
void THardwareLocalityHelper::BindThreadForDevice(int deviceId) {
if (!HasContext) {
return;
}
THwlocSet deviceCpu;
THwlocSet numaNode;
int errCode = hwloc_cudart_get_device_cpuset(Context, deviceId, deviceCpu.Set);
hwloc_cpuset_to_nodeset(Context, deviceCpu.Set, numaNode.Set);
errCode = hwloc_set_cpubind(Context, deviceCpu.Set, HWLOC_CPUBIND_THREAD | HWLOC_CPUBIND_STRICT);
if (errCode == -1) {
MATRIXNET_ERROR_LOG << "Can't bind thread for " << deviceId << " with err " << errno << Endl;
}
errCode = hwloc_set_membind_nodeset(Context, numaNode.Set, HWLOC_MEMBIND_BIND, HWLOC_CPUBIND_THREAD | HWLOC_CPUBIND_STRICT);
if (errCode == -1) {
MATRIXNET_ERROR_LOG << "Can't bind memory for " << deviceId << " with err " << errno << Endl;
}
}