static void testmem(hwloc_const_bitmap_t nodeset, hwloc_membind_policy_t policy, int flags, int expected)
{
hwloc_bitmap_t new_nodeset = hwloc_bitmap_alloc();
hwloc_membind_policy_t newpolicy;
void *area;
size_t area_size = 1024;
result_set("Bind this singlethreaded process memory", hwloc_set_membind(topology, nodeset, policy, flags), (support->membind->set_thisproc_membind || support->membind->set_thisthread_membind) && expected);
result_get("Get this singlethreaded process memory", nodeset, new_nodeset, hwloc_get_membind(topology, new_nodeset, &newpolicy, flags), (support->membind->get_thisproc_membind || support->membind->get_thisthread_membind) && expected);
result_set("Bind this thread memory", hwloc_set_membind(topology, nodeset, policy, flags | HWLOC_MEMBIND_THREAD), support->membind->set_thisproc_membind && expected);
result_get("Get this thread memory", nodeset, new_nodeset, hwloc_get_membind(topology, new_nodeset, &newpolicy, flags | HWLOC_MEMBIND_THREAD), support->membind->get_thisproc_membind && expected);
result_set("Bind this whole process memory", hwloc_set_membind(topology, nodeset, policy, flags | HWLOC_MEMBIND_PROCESS), support->membind->set_thisproc_membind && expected);
result_get("Get this whole process memory", nodeset, new_nodeset, hwloc_get_membind(topology, new_nodeset, &newpolicy, flags | HWLOC_MEMBIND_PROCESS), support->membind->get_thisproc_membind && expected);
#ifdef HWLOC_WIN_SYS
result_set("Bind process memory", hwloc_set_proc_membind(topology, GetCurrentProcess(), nodeset, policy, flags), support->membind->set_proc_membind && expected);
result_get("Get process memory", nodeset, new_nodeset, hwloc_get_proc_membind(topology, GetCurrentProcess(), new_nodeset, &newpolicy, flags), support->membind->get_proc_membind && expected);
#else /* !HWLOC_WIN_SYS */
result_set("Bind process memory", hwloc_set_proc_membind(topology, getpid(), nodeset, policy, flags), support->membind->set_proc_membind && expected);
result_get("Get process memory", nodeset, new_nodeset, hwloc_get_proc_membind(topology, getpid(), new_nodeset, &newpolicy, flags), support->membind->get_proc_membind && expected);
#endif /* !HWLOC_WIN_SYS */
result_set("Bind area", hwloc_set_area_membind(topology, &new_nodeset, sizeof(new_nodeset), nodeset, policy, flags), support->membind->set_area_membind && expected);
result_get("Get area", nodeset, new_nodeset, hwloc_get_area_membind(topology, &new_nodeset, sizeof(new_nodeset), new_nodeset, &newpolicy, flags), support->membind->get_area_membind && expected);
if (!(flags & HWLOC_MEMBIND_MIGRATE)) {
result_set("Alloc bound area", (area = hwloc_alloc_membind(topology, area_size, nodeset, policy, flags)) == NULL, (support->membind->alloc_membind && expected) || !(flags & HWLOC_MEMBIND_STRICT));
if (area) {
memset(area, 0, area_size);
result_get("Get bound area", nodeset, new_nodeset, hwloc_get_area_membind(topology, area, area_size, new_nodeset, &newpolicy, flags), support->membind->get_area_membind && expected);
result_get("Free bound area", NULL, NULL, hwloc_free(topology, area, area_size), support->membind->alloc_membind && expected);
}
}
printf("\n");
hwloc_bitmap_free(new_nodeset);
}
inline void __pact_reuse_add(void *ary, long long start, long long end, long long mem_ac) {
hwloc_bitmap_t set = hwloc_bitmap_alloc();
hwloc_get_cpubind(__pact_topo, set, HWLOC_CPUBIND_THREAD);
hwloc_get_last_cpu_location(__pact_topo, set, HWLOC_CPUBIND_THREAD);
hwloc_bitmap_singlify(set);
hwloc_set_area_membind ( __pact_topo, (const void*)ary, abs(end-start), (hwloc_const_cpuset_t)set, HWLOC_MEMBIND_BIND, HWLOC_MEMBIND_MIGRATE );
hwloc_bitmap_free(set);
}
int opal_hwloc_base_memory_set(opal_hwloc_base_memory_segment_t *segments,
size_t num_segments)
{
int rc = OPAL_SUCCESS;
char *msg = NULL;
size_t i;
hwloc_cpuset_t cpuset = NULL;
/* bozo check */
if (OPAL_SUCCESS != opal_hwloc_base_get_topology()) {
msg = "hwloc_set_area_membind() failure - topology not available";
return opal_hwloc_base_report_bind_failure(__FILE__, __LINE__,
msg, rc);
}
/* This module won't be used unless the process is already
processor-bound. So find out where we're processor bound, and
bind our memory there, too. */
cpuset = hwloc_bitmap_alloc();
if (NULL == cpuset) {
rc = OPAL_ERR_OUT_OF_RESOURCE;
msg = "hwloc_bitmap_alloc() failure";
goto out;
}
hwloc_get_cpubind(opal_hwloc_topology, cpuset, 0);
for (i = 0; i < num_segments; ++i) {
if (0 != hwloc_set_area_membind(opal_hwloc_topology,
segments[i].mbs_start_addr,
segments[i].mbs_len, cpuset,
HWLOC_MEMBIND_BIND,
HWLOC_MEMBIND_STRICT)) {
rc = OPAL_ERROR;
msg = "hwloc_set_area_membind() failure";
goto out;
}
}
out:
if (NULL != cpuset) {
hwloc_bitmap_free(cpuset);
}
if (OPAL_SUCCESS != rc) {
return opal_hwloc_base_report_bind_failure(__FILE__, __LINE__, msg, rc);
}
return OPAL_SUCCESS;
}
int opal_hwloc_base_membind(opal_hwloc_base_memory_segment_t *segs,
size_t count, int node_id)
{
size_t i;
int rc = OPAL_SUCCESS;
char *msg = NULL;
hwloc_cpuset_t cpuset = NULL;
/* bozo check */
if (OPAL_SUCCESS != opal_hwloc_base_get_topology()) {
msg = "hwloc_set_area_membind() failure - topology not available";
return opal_hwloc_base_report_bind_failure(__FILE__, __LINE__,
msg, rc);
}
cpuset = hwloc_bitmap_alloc();
if (NULL == cpuset) {
rc = OPAL_ERR_OUT_OF_RESOURCE;
msg = "hwloc_bitmap_alloc() failure";
goto out;
}
hwloc_bitmap_set(cpuset, node_id);
for(i = 0; i < count; i++) {
if (0 != hwloc_set_area_membind(opal_hwloc_topology,
segs[i].mbs_start_addr,
segs[i].mbs_len, cpuset,
HWLOC_MEMBIND_BIND,
HWLOC_MEMBIND_STRICT)) {
rc = OPAL_ERROR;
msg = "hwloc_set_area_membind() failure";
goto out;
}
}
out:
if (NULL != cpuset) {
hwloc_bitmap_free(cpuset);
}
if (OPAL_SUCCESS != rc) {
return opal_hwloc_base_report_bind_failure(__FILE__, __LINE__, msg, rc);
}
return OPAL_SUCCESS;
}
void migrate(long PageStart, long PageEnd) {
SPMR_DEBUG(std::cout << "Runtime: migrate pages: " << PageStart << " to "
<< PageEnd << "\n");
SPMR_DEBUG(std::cout << "Runtime: hwloc call: " << (PageStart << PAGE_EXP)
<< ", " << ((PageEnd - PageStart) << PAGE_EXP) << "\n");
hwloc_bitmap_t set = hwloc_bitmap_alloc();
hwloc_get_cpubind(__spm_topo, set, HWLOC_CPUBIND_THREAD);
hwloc_get_last_cpu_location(__spm_topo, set, HWLOC_CPUBIND_THREAD);
hwloc_bitmap_singlify(set);
assert(
hwloc_set_area_membind(__spm_topo, (const void*)(PageStart << PAGE_EXP),
(PageEnd - PageStart) << PAGE_EXP,
(hwloc_const_cpuset_t)set, HWLOC_MEMBIND_BIND,
HWLOC_MEMBIND_MIGRATE)
!= -1 && "Unable to migrate requested pages");
hwloc_bitmap_free(set);
}
int main(void)
{
const struct hwloc_topology_support *support;
char *buffer;
hwloc_topology_t topology;
hwloc_bitmap_t set = hwloc_bitmap_alloc();
hwloc_bitmap_t total = hwloc_bitmap_alloc();
hwloc_obj_t node;
char *s;
int err;
err = hwloc_topology_init(&topology);
assert(!err);
err = hwloc_topology_load(topology);
assert(!err);
support = hwloc_topology_get_support(topology);
if (!support->membind->get_area_memlocation)
goto out;
buffer = hwloc_alloc(topology, LEN);
assert(buffer);
printf("buffer %p length %u\n", buffer, LEN);
err = hwloc_get_area_memlocation(topology, buffer, LEN, set, HWLOC_MEMBIND_BYNODESET);
if (err < 0 && errno == ENOSYS) {
fprintf(stderr, "hwloc_get_area_memlocation() failed with ENOSYS, aborting\n");
goto out_with_buffer;
}
assert(!err);
hwloc_bitmap_asprintf(&s, set);
printf("address %p length %u allocated in nodeset %s\n", buffer, LEN, s);
free(s);
hwloc_bitmap_copy(total, set);
node = NULL;
next1:
node = hwloc_get_next_obj_by_type(topology, HWLOC_OBJ_NUMANODE, node);
if (!node)
goto out_with_buffer;
if (!node->memory.local_memory)
goto next1;
printf("binding to 1st node and touching 1st quarter\n");
err = hwloc_set_area_membind(topology, buffer, LEN, node->nodeset, HWLOC_MEMBIND_BIND, HWLOC_MEMBIND_BYNODESET);
if (err < 0 && errno == ENOSYS) {
fprintf(stderr, "hwloc_set_area_membind() failed with ENOSYS, aborting\n");
goto out_with_buffer;
}
assert(!err);
memset(buffer, 0, LEN/4);
err = hwloc_get_area_memlocation(topology, buffer, 1, set, HWLOC_MEMBIND_BYNODESET);
assert(!err);
hwloc_bitmap_asprintf(&s, set);
printf("address %p length %u allocated in nodeset %s\n", buffer, LEN/4, s);
free(s);
hwloc_bitmap_or(total, total, set);
next2:
node = hwloc_get_next_obj_by_type(topology, HWLOC_OBJ_NUMANODE, node);
if (!node)
goto out_with_nomorenodes;
if (!node->memory.local_memory)
goto next2;
printf("binding to 2nd node and touching 2nd quarter\n");
err = hwloc_set_area_membind(topology, buffer, LEN, node->nodeset, HWLOC_MEMBIND_BIND, HWLOC_MEMBIND_BYNODESET);
assert(!err);
memset(buffer+LEN/4, 0, LEN/4);
err = hwloc_get_area_memlocation(topology, buffer+LEN/4, LEN/4, set, HWLOC_MEMBIND_BYNODESET);
assert(!err);
hwloc_bitmap_asprintf(&s, set);
printf("address %p length %u allocated in nodeset %s\n", buffer+LEN/4, LEN/4, s);
free(s);
hwloc_bitmap_or(total, total, set);
next3:
node = hwloc_get_next_obj_by_type(topology, HWLOC_OBJ_NUMANODE, node);
if (!node)
goto out_with_nomorenodes;
if (!node->memory.local_memory)
goto next3;
printf("binding to 3rd node and touching 3rd quarter\n");
err = hwloc_set_area_membind(topology, buffer, LEN, node->nodeset, HWLOC_MEMBIND_BIND, HWLOC_MEMBIND_BYNODESET);
assert(!err);
memset(buffer+LEN/2, 0, LEN/4);
err = hwloc_get_area_memlocation(topology, buffer+LEN/2, LEN/4, set, HWLOC_MEMBIND_BYNODESET);
assert(!err);
hwloc_bitmap_asprintf(&s, set);
printf("address %p length %u allocated in nodeset %s\n", buffer+LEN/2, LEN/4, s);
free(s);
hwloc_bitmap_or(total, total, set);
next4:
node = hwloc_get_next_obj_by_type(topology, HWLOC_OBJ_NUMANODE, node);
if (!node)
goto out_with_nomorenodes;
if (!node->memory.local_memory)
goto next4;
printf("binding to 4th node and touching 4th quarter\n");
err = hwloc_set_area_membind(topology, buffer, LEN, node->nodeset, HWLOC_MEMBIND_BIND, HWLOC_MEMBIND_BYNODESET);
assert(!err);
memset(buffer+3*LEN/4, 0, LEN/4);
err = hwloc_get_area_memlocation(topology, buffer+3*LEN/4, LEN/4, set, HWLOC_MEMBIND_BYNODESET);
assert(!err);
hwloc_bitmap_asprintf(&s, set);
printf("address %p length %u allocated in nodeset %s\n", buffer+3*LEN/4, LEN/4, s);
free(s);
hwloc_bitmap_or(total, total, set);
out_with_nomorenodes:
err = hwloc_get_area_memlocation(topology, buffer, LEN, set, HWLOC_MEMBIND_BYNODESET);
assert(!err);
hwloc_bitmap_asprintf(&s, set);
printf("address %p length %u located on %s\n", buffer, LEN, s);
free(s);
assert(hwloc_bitmap_isincluded(total, set));
out_with_buffer:
hwloc_free(topology, buffer, LEN);
out:
hwloc_topology_destroy(topology);
hwloc_bitmap_free(set);
hwloc_bitmap_free(total);
return 0;
}
