MEMKIND_EXPORT int memkind_default_mbind(struct memkind *kind, void *ptr,
size_t size)
{
nodemask_t nodemask;
int err = 0;
int mode;
if (MEMKIND_UNLIKELY(kind->ops->get_mbind_nodemask == NULL ||
kind->ops->get_mbind_mode == NULL)) {
log_err("memkind_ops->mbind_mode or memkind_ops->bind_nodemask is NULL.");
return MEMKIND_ERROR_BADOPS;
}
err = kind->ops->get_mbind_nodemask(kind, nodemask.n, NUMA_NUM_NODES);
if (MEMKIND_UNLIKELY(err)) {
return err;
}
err = kind->ops->get_mbind_mode(kind, &mode);
if (MEMKIND_UNLIKELY(err)) {
return err;
}
err = mbind(ptr, size, mode, nodemask.n, NUMA_NUM_NODES, 0);
if (MEMKIND_UNLIKELY(err)) {
log_err("syscall mbind() returned: %d", err);
return MEMKIND_ERROR_MBIND;
}
return err;
}
int main()
{
int ret;
int len;
int policy = -1;
unsigned char *p;
unsigned long mask[MAXNODE] = { 0 };
unsigned long retmask[MAXNODE] = { 0 };
len = getpagesize();
p = mmap(NULL, len, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS,
0, 0);
if (p == MAP_FAILED)
printf("mbind err: %d\n", errno);
mask[0] = 1;
ret = mbind(p, len, MPOL_BIND, mask, MAXNODE, 0);
if (ret < 0)
printf("mbind err: %d %d\n", ret, errno);
ret = get_mempolicy(&policy, retmask, MAXNODE, p, MPOL_F_ADDR);
if (ret < 0)
printf("get_mempolicy err: %d %d\n", ret, errno);
if (policy == MPOL_BIND)
printf("OK\n");
else
printf("ERROR: policy is %d\n", policy);
return 0;
}
static void arena_weighted_mbind(void *arena, size_t arena_size,
uint16_t *weights, size_t nr_weights) {
/* compute cumulative sum for weights
* cumulative sum starts at -1
* the method for determining a hit on a weight i is when the generated
* random number (modulo sum of weights) <= weights_cumsum[i]
*/
int64_t weights_cumsum[nr_weights];
weights_cumsum[0] = weights[0] - 1;
for (unsigned int i = 1; i < nr_weights; i++) {
weights_cumsum[i] = weights_cumsum[i-1] + weights[i];
}
const int32_t weight_sum = weights_cumsum[nr_weights-1]+1;
const int pagesize = getpagesize();
uint64_t mask = 0;
char *q = (char *)arena + arena_size;
rng_init(1);
for (char *p = arena; p < q; p += pagesize) {
uint32_t r = rng_int(1<<31) % weight_sum;
unsigned int node;
for (node = 0; node < nr_weights; node++) {
if (weights_cumsum[node] >= r) {
break;
}
}
mask = 1 << node;
if (mbind(p, pagesize, MPOL_BIND, &mask, nr_weights, MPOL_MF_STRICT)) {
perror("mbind");
exit(1);
}
*p = 0;
}
}
/*
* segment_mbind() - set memory policy for a range of specified segment
*
* NOTE: offset is relative to start of mapping, not start of file
*/
int
segment_mbind(char *name, range_t * range, int policy,
nodemask_t * nodemask, int flags)
{
glctx_t *gcp = &glctx;
segment_t *segp;
char *start;
off_t offset;
size_t length, maxlength;
int ret;
segp = segment_get(name);
if (segp == NULL) {
fprintf(stderr, "%s: no such segment: %s\n",
gcp->program_name, name);
return SEG_ERR;
}
if (segp->seg_start == MAP_FAILED) {
fprintf(stderr, "%s: segment %s not mapped\n",
gcp->program_name, name);
return SEG_ERR;
}
offset = round_down_to_pagesize(range->offset);
if (offset >= segp->seg_length) {
fprintf(stderr, "%s: offset %ld is past end of segment %s\n",
gcp->program_name, offset, name);
return SEG_ERR;
}
start = segp->seg_start + offset;
maxlength = segp->seg_length - offset;
length = range->length;
if (length)
length = round_up_to_pagesize(length);
/*
* note: we silently truncate to max length [end of segment]
*/
if (length == 0 || length > maxlength)
length = maxlength;
ret = mbind(segp->seg_start + offset, length, policy, nodemask->n,
NUMA_NUM_NODES, flags);
if (ret == -1) {
int err = errno;
fprintf(stderr, "%s: mbind() of segment %s failed - %s\n",
gcp->program_name, name, strerror(err));
return SEG_ERR;
}
return SEG_OK;
}
int main(void) {
size_t page_size = sysconf(_SC_PAGESIZE);
void* p = mmap(NULL, 16 * page_size, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
int ret;
test_assert(p != MAP_FAILED);
ret = mbind(p, 16 * page_size, MPOL_PREFERRED, NULL, 0, MPOL_MF_MOVE);
test_assert(ret == 0 || (ret == -1 && errno == ENOSYS));
atomic_puts("EXIT-SUCCESS");
return 0;
}
int main(void)
{
int max = numa_max_node();
int maxmask = numa_num_possible_nodes();
struct bitmask *nodes, *mask;
int pagesize = getpagesize();
int i;
int pol;
int node;
int err = 0;
nodes = numa_bitmask_alloc(maxmask);
mask = numa_bitmask_alloc(maxmask);
for (i = max; i >= 0; --i) {
char *mem = mmap(NULL, pagesize*(max+1), PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_ANONYMOUS, 0, 0);
char *adr = mem;
if (mem == (char *)-1)
err("mmap");
printf("%d offset %lx\n", i, (long)(adr - mem));
numa_bitmask_clearall(nodes);
numa_bitmask_clearall(mask);
numa_bitmask_setbit(nodes, i);
if (mbind(adr, pagesize, MPOL_PREFERRED, nodes->maskp,
nodes->size, 0) < 0)
err("mbind");
++*adr;
if (get_mempolicy(&pol, mask->maskp, mask->size, adr, MPOL_F_ADDR) < 0)
err("get_mempolicy");
assert(pol == MPOL_PREFERRED);
assert(numa_bitmask_isbitset(mask, i));
node = 0x123;
if (get_mempolicy(&node, NULL, 0, adr, MPOL_F_ADDR|MPOL_F_NODE) < 0)
err("get_mempolicy2");
printf("got node %d expected %d\n", node, i);
if (node != i)
err = 1;
}
return err;
}
int main(void)
{
void* ptr;
struct bitmask *nmask;
int err;
nmask = numa_allocate_nodemask();
numa_bitmask_setbit(nmask, 0);
ptr = shmem_open();
err = mbind(ptr, 4096 * 3, MPOL_INTERLEAVE,
nmask->maskp, nmask->size, 0);
if (err < 0)
perror("mbind1"), exit(1);
err = mbind(ptr + 4096, 4096, MPOL_BIND,
nmask->maskp, nmask->size, 0);
if (err < 0)
perror("mbind1"), exit(1);
return 0;
}
void *__mem_alloc_pages(void *base, int nr, int size,
struct bitmask *mask, int numa_policy)
{
void *vaddr;
int flags = MAP_PRIVATE | MAP_ANONYMOUS;
size_t len = nr * size;
switch (size) {
case PGSIZE_4KB:
break;
case PGSIZE_2MB:
flags |= MAP_HUGETLB | MAP_FIXED;
#ifdef MAP_HUGE_2MB
flags |= MAP_HUGE_2MB;
#endif
break;
case PGSIZE_1GB:
#ifdef MAP_HUGE_1GB
flags |= MAP_HUGETLB | MAP_HUGE_1GB | MAP_FIXED;
#else
return MAP_FAILED;
#endif
break;
default: /* fail on other sizes */
return MAP_FAILED;
}
vaddr = mmap(base, len, PROT_READ | PROT_WRITE, flags, -1, 0);
if (vaddr == MAP_FAILED)
return MAP_FAILED;
if (mbind(vaddr, len, numa_policy, mask ? mask->maskp : NULL,
mask ? mask->size : 0, MPOL_MF_STRICT))
goto fail;
if (vm_map_phys((physaddr_t) vaddr, (virtaddr_t) vaddr, nr, size,
VM_PERM_R | VM_PERM_W))
goto fail;
sighandler_t s = signal(SIGBUS, sigbus_error);
*(uint64_t *)vaddr = 0;
signal(SIGBUS, s);
return vaddr;
fail:
munmap(vaddr, len);
return MAP_FAILED;
}
void
proc_numa_membind(void* ptr, size_t size, int domainId)
{
int ret=0;
unsigned long mask = 0UL;
unsigned int flags = 0U;
flags |= MPOL_MF_STRICT;
mask |= (1UL<<domainId);
ret = mbind(ptr, size, &mask, numa_info.numberOfNodes+1, flags);
if (ret < 0)
{
ERROR;
}
}
void setpol(unsigned long offset, unsigned long length, int policy, unsigned long nodes)
{
long i, end;
printf("off:%lx length:%lx policy:%d nodes:%lx\n",
offset, length, policy, nodes);
if (mbind(map + offset*pagesize, length*pagesize, policy,
&nodes, 8, 0) < 0) {
printf("mbind: %s offset %lx length %lx policy %d nodes %lx\n",
strerror(errno),
offset*pagesize, length*pagesize,
policy, nodes);
return;
}
for (i = offset; i < offset+length; i++) {
pages[i].mask = nodes;
pages[i].policy = policy;
}
i = offset - 20;
if (i < 0)
i = 0;
end = offset+length+20;
if (end > PAGES)
end = PAGES;
for (; i < end; i++) {
int pol2;
unsigned long nodes2;
if (get_mempolicy(&pol2, &nodes2, sizeof(long)*8, map+i*pagesize,
MPOL_F_ADDR) < 0)
err("get_mempolicy");
if (pol2 != pages[i].policy) {
printf("%lx: got policy %d expected %d, nodes got %lx expected %lx\n",
i, pol2, pages[i].policy, nodes2, pages[i].mask);
}
if (policy != MPOL_DEFAULT && nodes2 != pages[i].mask) {
printf("%lx: nodes %lx, expected %lx, policy %d\n",
i, nodes2, pages[i].mask, policy);
}
}
}
int memkind_default_mbind(struct memkind *kind, void *ptr, size_t size)
{
nodemask_t nodemask;
int err = 0;
int mode;
if (kind->ops->get_mbind_nodemask == NULL ||
kind->ops->get_mbind_mode == NULL) {
err = MEMKIND_ERROR_BADOPS;
}
if (!err) {
err = kind->ops->get_mbind_nodemask(kind, nodemask.n, NUMA_NUM_NODES);
}
if (!err) {
err = kind->ops->get_mbind_mode(kind, &mode);
}
if (!err) {
err = mbind(ptr, size, mode, nodemask.n, NUMA_NUM_NODES, 0);
err = err ? MEMKIND_ERROR_MBIND : 0;
}
return err;
}
int main(int argc, char **argv)
{
int i;
int shmid = shmget(SHM_KEY, BUF_SIZE, IPC_CREAT | 0666 | SHM_HUGETLB);
char *buf = shmat(shmid, 0, 0);
assert(argc == 2);
int preferred_node = atoi(argv[1]);
const unsigned long nodemask = (1 << preferred_node);
printf("\tNodemask = %ld\n", nodemask);
mbind(buf, BUF_SIZE,
MPOL_BIND, /** < A strict assignment to nodes in nodemask */
&nodemask, /** < A bitmask of valid NUMA nodes */
32, /** < Num. of bits to consider. XXX: 2 does not work */
0); /** < Mode ?? */
for(i = 0; i < BUF_SIZE; i ++) {
buf[i] = (char) rand();
}
printf("\tRandom pick = %c\n", buf[rand() % BUF_SIZE]);
}
int
main(int argc, char *argv[])
{
int i;
if (argc < 2) {
printf("Invalid number of arguments!\n");
printf("Usage: %s [config file]\n", argv[0]);
exit(EXIT_FAILURE);
}
read_config_file(argv[1]);
// Initialize blocks
int nblocks;
get_int_config_value("nblocks", &nblocks);
nthreads = nblocks;
get_int_config_value("blocksize", &blocksize);
get_int_config_value("batchsize", &batchsize);
for (i = 0; i < nblocks; i++) {
// Allocate block in correct node
blocks[i] = valloc(blocksize * sizeof(long));
if (!blocks[i]) {
perror("valloc");
exit(EXIT_FAILURE);
}
#if defined(AFFINITY) && defined(linux)
char k[80];
sprintf(k, "block%d_node", i);
int node;
get_int_config_value(k, &node);
unsigned long nodemask = 1 << node;
if (mbind(blocks[i], blocksize * sizeof(long),
MPOL_BIND, &nodemask, sizeof(nodemask) + 1,
MPOL_MF_MOVE | MPOL_MF_STRICT) != 0) {
perror("mbind (1)");
exit(EXIT_FAILURE);
}
#endif
// Set block initial values
int j;
for (j = 0; j < blocksize; j++) {
blocks[i][j] = j;
}
// Allocate lock in correct node
locks[i] = valloc(sizeof(pthread_mutex_t));
if (!locks[i]) {
perror("valloc");
exit(EXIT_FAILURE);
}
#if defined(AFFINITY) && defined(linux)
if (mbind(locks[i], sizeof(pthread_mutex_t),
MPOL_BIND, &nodemask, sizeof(nodemask) + 1,
MPOL_MF_MOVE | MPOL_MF_STRICT) != 0) {
perror("mbind (2)");
exit(EXIT_FAILURE);
}
#endif
// Initialize lock
pthread_mutex_init(locks[i], NULL);
//printf("Initialized lock %d (address %p) in NUMA node %ld\n", i, locks[i], nodemask);
}
//long blockbytes = (blocksize * sizeof(long) / MB);
//printf("Total memory allocated: %ld MB (block size %ld MB)\n", nblocks * blockbytes, blockbytes);
// Launch measure thread
pthread_t th;
pthread_create(&th, NULL, measure_thread, NULL);
// Apply (crappy version of) Fisher-Yates to sort list of threads to launch
int shuffle[nthreads];
for (i = 0; i < nthreads; i++) {
shuffle[i] = 0;
}
int sorted[nthreads];
int j = 0;
while (j < nthreads) {
int n = rand() % nthreads;
i = 0;
while (1) {
if (shuffle[i] == 0) {
--n;
if (n < 0) break;
}
i = (i + 1) % nthreads;
}
shuffle[i] = 1;
sorted[j++] = i;
}
// Launch worker threads
struct worker_args args[nthreads];
pthread_t threads[nthreads];
#if defined(AFFINITY)
pthread_attr_t a;
pthread_attr_init(&a);
#endif
#if defined(AFFINITY) && defined(__sun)
// set system-level contention (instead of process-level contention)
pthread_attr_setscope(&a, PTHREAD_SCOPE_SYSTEM);
#endif
for (i = 0; i < nthreads; i++) {
int id = sorted[i];
char k[80];
int block;
sprintf(k, "thread%d_block", id);
get_int_config_value(k, &block);
#if defined(AFFINITY)
int cpu;
sprintf(k, "thread%d_core", id);
get_int_config_value(k, &cpu);
#endif
#if defined(AFFINITY) && defined(linux)
cpu_set_t c;
CPU_ZERO(&c);
CPU_SET(cpu, &c);
#endif
args[id].id = id;
args[id].block = block;
args[id].seed1 = rand();
args[id].seed2 = rand();
#if defined(AFFINITY)
args[id].cpu = cpu;
pthread_create(&threads[id], &a, worker_thread, &args[id]);
#else
pthread_create(&threads[id], NULL, worker_thread, &args[id]);
#endif
}
for (i = 0; i < nthreads; i++) {
pthread_join(threads[i], NULL);
}
return 0;
}
int
main(int argc, char *argv[])
{
int i;
if (argc < 2) {
printf("Invalid number of arguments!\n");
printf("Usage: %s [config file]\n", argv[0]);
exit(EXIT_FAILURE);
}
if (read_config_file(argv[1]) != 0) {
printf("Error reading configuration file.\n");
exit(EXIT_FAILURE);
}
/*
* Launch throughput measurement thread.
*/
pthread_t th;
pthread_create(&th, NULL, measure_thread, NULL);
/*
* Initialize counters.
*/
if (get_int_config_value("ncounters", &ncounters) != 0) {
printf("Error reading 'ncounters'.\n");
exit(EXIT_FAILURE);
}
if (ncounters > MAXCOUNTERS) {
printf("MAXCOUNTERS exceeded!\n");
exit(EXIT_FAILURE);
}
for (i = 0; i < ncounters; i++) {
counters[i] = valloc(sizeof(struct counter_aligned));
if (!counters[i]) {
perror("valloc");
exit(EXIT_FAILURE);
}
#if defined(AFFINITY)
/*
* Allocate counter in the desired memory node.
*/
char k[80];
int node;
sprintf(k, "counter%d_node", i);
if (get_int_config_value(k, &node) != 0) {
printf("Error reading '%s'.\n", k);
exit(EXIT_FAILURE);
}
unsigned long nodemask = 1 << node;
if (mbind(counters[i], sizeof(struct counter_aligned),
MPOL_BIND, &nodemask, sizeof(nodemask) + 1,
MPOL_MF_MOVE | MPOL_MF_STRICT) != 0) {
perror("mbind");
exit(EXIT_FAILURE);
}
#endif
counters[i]->c = 0;
#if defined(PTHREAD_SPIN)
pthread_spin_init(&counters[i]->spin, 0);
#elif defined(PTHREAD_MUTEX)
pthread_mutex_init(&counters[i]->mutex, NULL);
#endif
}
/*
* Launch worker threads.
*/
int nthreads;
if (get_int_config_value("nthreads", &nthreads) != 0) {
printf("Error reading 'nthreads'.\n");
exit(EXIT_FAILURE);
}
if (nthreads > MAXTHREADS) {
printf("MAXTHREADS exceeded!\n");
exit(EXIT_FAILURE);
}
pthread_t threads[nthreads];
pthread_attr_t a;
pthread_attr_init(&a);
/*
* Apply (crappy version of) Fisher-Yates to create a randomized list
* with the threads to launch. This prevents a deterministic behavior
* when affinity is OFF: the OS will be launching threads in different
* orders across runs, making the execution vary more.
*/
int shuffle[nthreads];
for (i = 0; i < nthreads; i++)
shuffle[i] = 0;
int random[nthreads]; /* holds randomized output list */
int j = 0;
while (j < nthreads) {
int n = rand() % nthreads;
i = 0;
while (1) {
if (shuffle[i] == 0) {
--n;
if (n < 0) break;
}
i = (i + 1) % nthreads;
}
shuffle[i] = 1;
random[j++] = i;
}
/*
* Launch worker threads.
*/
for (i = 0; i < nthreads; i++) {
char k[80];
int counter;
sprintf(k, "thread%d_counter", random[i]);
if (get_int_config_value(k, &counter) != 0) {
printf("Error reading '%s'.\n", k);
exit(EXIT_FAILURE);
}
#if defined(AFFINITY)
/*
* Allocate worker thread in the desired CPU core.
*/
int cpu;
sprintf(k, "thread%d_core", random[i]);
if (get_int_config_value(k, &cpu) != 0) {
printf("Error reading '%s'.\n", k);
exit(EXIT_FAILURE);
}
cpu_set_t c;
CPU_ZERO(&c);
CPU_SET(cpu, &c);
pthread_attr_setaffinity_np(&a, sizeof(c), &c);
#endif
pthread_create(&threads[random[i]], &a, worker_thread,
counters[counter]);
}
/*
* Wait for threads to finish.
*/
for (i = 0; i < nthreads; i++)
pthread_join(threads[i], NULL);
return 0;
}
void* StorageManager::allocateSlots(const std::size_t num_slots,
const int numa_node) {
#if defined(QUICKSTEP_HAVE_MMAP_LINUX_HUGETLB)
static constexpr int kLargePageMmapFlags
= MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB;
#elif defined(QUICKSTEP_HAVE_MMAP_BSD_SUPERPAGE)
static constexpr int kLargePageMmapFlags
= MAP_PRIVATE | MAP_ANONYMOUS | MAP_ALIGNED_SUPER;
#endif
makeRoomForBlockOrBlob(num_slots);
void *slots = nullptr;
#if defined(QUICKSTEP_HAVE_MMAP_LINUX_HUGETLB) || defined(QUICKSTEP_HAVE_MMAP_BSD_SUPERPAGE)
slots = mmap(nullptr,
num_slots * kSlotSizeBytes,
PROT_READ | PROT_WRITE,
kLargePageMmapFlags,
-1, 0);
// Fallback to regular mmap() if large page allocation failed. Even on
// systems with large page support, large page allocation may fail if the
// user running the executable is not a member of hugetlb_shm_group on Linux,
// or if all the reserved hugepages are already in use.
if (slots == MAP_FAILED) {
slots = mmap(nullptr,
num_slots * kSlotSizeBytes,
PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS,
-1, 0);
}
if (slots == MAP_FAILED) {
slots = nullptr;
}
#elif defined(QUICKSTEP_HAVE_MMAP_PLAIN)
slots = mmap(nullptr,
num_slots * kSlotSizeBytes,
PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS,
-1, 0);
if (slots == MAP_FAILED) {
slots = nullptr;
}
#else
slots = malloc_with_alignment(num_slots * kSlotSizeBytes,
kCacheLineBytes);
if (slots != nullptr) {
memset(slots, 0x0, num_slots * kSlotSizeBytes);
}
#endif
if (slots == nullptr) {
throw OutOfMemory();
}
#if defined(QUICKSTEP_HAVE_LIBNUMA)
if (numa_node != -1) {
DEBUG_ASSERT(numa_node < numa_num_configured_nodes());
struct bitmask *numa_node_bitmask = numa_allocate_nodemask();
// numa_node can be 0 through n-1, where n is the num of NUMA nodes.
numa_bitmask_setbit(numa_node_bitmask, numa_node);
long mbind_status = mbind(slots, // NOLINT(runtime/int)
num_slots * kSlotSizeBytes,
MPOL_PREFERRED,
numa_node_bitmask->maskp,
numa_node_bitmask->size,
0);
numa_free_nodemask(numa_node_bitmask);
if (mbind_status == -1) {
LOG(WARNING) << "mbind() failed with errno " << errno << " ("
<< std::strerror(errno) << ")";
}
}
#endif // QUICKSTEP_HAVE_LIBNUMA
total_memory_usage_ += num_slots;
return slots;
}
void test_ksm_merge_across_nodes(unsigned long nr_pages)
{
char **memory;
int i, ret;
int num_nodes, *nodes;
unsigned long length;
unsigned long pagesize;
#if HAVE_NUMA_H && HAVE_LINUX_MEMPOLICY_H && HAVE_NUMAIF_H \
&& HAVE_MPOL_CONSTANTS
unsigned long nmask[MAXNODES / BITS_PER_LONG] = { 0 };
#endif
ret = get_allowed_nodes_arr(NH_MEMS|NH_CPUS, &num_nodes, &nodes);
if (ret != 0)
tst_brkm(TBROK|TERRNO, cleanup, "get_allowed_nodes_arr");
if (num_nodes < 2) {
tst_resm(TINFO, "need NUMA system support");
free(nodes);
return;
}
pagesize = sysconf(_SC_PAGE_SIZE);
length = nr_pages * pagesize;
memory = malloc(num_nodes * sizeof(char *));
for (i = 0; i < num_nodes; i++) {
memory[i] = mmap(NULL, length, PROT_READ|PROT_WRITE,
MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);
if (memory[i] == MAP_FAILED)
tst_brkm(TBROK|TERRNO, tst_exit, "mmap");
#ifdef HAVE_MADV_MERGEABLE
if (madvise(memory[i], length, MADV_MERGEABLE) == -1)
tst_brkm(TBROK|TERRNO, tst_exit, "madvise");
#endif
#if HAVE_NUMA_H && HAVE_LINUX_MEMPOLICY_H && HAVE_NUMAIF_H \
&& HAVE_MPOL_CONSTANTS
clean_node(nmask);
set_node(nmask, nodes[i]);
/*
* Use mbind() to make sure each node contains
* length size memory.
*/
ret = mbind(memory[i], length, MPOL_BIND, nmask, MAXNODES, 0);
if (ret == -1)
tst_brkm(TBROK|TERRNO, tst_exit, "mbind");
#endif
memset(memory[i], 10, length);
}
SAFE_FILE_PRINTF(cleanup, PATH_KSM "sleep_millisecs", "0");
SAFE_FILE_PRINTF(cleanup, PATH_KSM "pages_to_scan", "%ld",
nr_pages * num_nodes);
/*
* merge_across_nodes setting can be changed only when there
* are no ksm shared pages in system, so set run 2 to unmerge
* pages first, then to 1 after changing merge_across_nodes,
* to remerge according to the new setting.
*/
SAFE_FILE_PRINTF(cleanup, PATH_KSM "run", "2");
wait_ksmd_done();
tst_resm(TINFO, "Start to test KSM with merge_across_nodes=1");
SAFE_FILE_PRINTF(cleanup, PATH_KSM "merge_across_nodes", "1");
SAFE_FILE_PRINTF(cleanup, PATH_KSM "run", "1");
group_check(1, 1, nr_pages * num_nodes - 1, 0, 0, 0,
nr_pages * num_nodes);
SAFE_FILE_PRINTF(cleanup, PATH_KSM "run", "2");
wait_ksmd_done();
tst_resm(TINFO, "Start to test KSM with merge_across_nodes=0");
SAFE_FILE_PRINTF(cleanup, PATH_KSM "merge_across_nodes", "0");
SAFE_FILE_PRINTF(cleanup, PATH_KSM "run", "1");
group_check(1, num_nodes, nr_pages * num_nodes - num_nodes,
0, 0, 0, nr_pages * num_nodes);
SAFE_FILE_PRINTF(cleanup, PATH_KSM "run", "2");
wait_ksmd_done();
}
int main(int argc, char *argv[])
{
char *p;
int option;
struct timespec result;
unsigned long bytes;
double duration, mbytes;
struct bitmask *from;
struct bitmask *to;
pagesize = getpagesize();
/* Command line processing */
opterr = 1;
cmd = argv[0];
while ((option = getopt(argc, argv, optstr)) != EOF)
switch (option) {
case 'h' :
case '?' :
usage();
break;
case 'v' :
verbose++;
break;
case 'p' :
pages = strtoul(optarg, &p, 0);
if (p == optarg || *p)
usage();
break;
}
if (!argv[optind])
usage();
if (verbose > 1)
printf("numa_max_node = %d\n", numa_max_node());
numa_exit_on_error = 1;
from = numa_parse_nodestring(argv[optind]);
if (!from) {
printf ("<%s> is invalid\n", argv[optind]);
exit(1);
}
if (errno) {
perror("from mask");
exit(1);
}
if (verbose)
printmask("From", from);
if (!argv[optind+1])
usage();
to = numa_parse_nodestring(argv[optind+1]);
if (!to) {
printf ("<%s> is invalid\n", argv[optind+1]);
exit(1);
}
if (errno) {
perror("to mask");
exit(1);
}
if (verbose)
printmask("To", to);
bytes = pages * pagesize;
if (verbose)
printf("Allocating %lu pages of %lu bytes of memory\n",
pages, pagesize);
memory = memalign(pagesize, bytes);
if (!memory) {
printf("Out of Memory\n");
exit(2);
}
if (mbind(memory, bytes, MPOL_BIND, from->maskp, from->size, 0) < 0)
numa_error("mbind");
if (verbose)
printf("Dirtying memory....\n");
for (p = memory; p <= memory + bytes; p += pagesize)
*p = 1;
if (verbose)
printf("Starting test\n");
displaymap();
clock_gettime(CLOCK_REALTIME, &start);
if (mbind(memory, bytes, MPOL_BIND, to->maskp, to->size, MPOL_MF_MOVE) <0)
numa_error("memory move");
clock_gettime(CLOCK_REALTIME, &end);
displaymap();
result.tv_sec = end.tv_sec - start.tv_sec;
result.tv_nsec = end.tv_nsec - start.tv_nsec;
if (result.tv_nsec < 0) {
result.tv_sec--;
result.tv_nsec += 1000000000;
}
if (result.tv_nsec >= 1000000000) {
result.tv_sec++;
result.tv_nsec -= 1000000000;
}
duration = result.tv_sec + result.tv_nsec / 1000000000.0;
mbytes = bytes / (1024*1024.0);
printf("%1.1f Mbyte migrated in %1.2f secs. %3.1f Mbytes/second\n",
mbytes,
duration,
mbytes / duration);
return 0;
}
int main(int argc, char **argv)
{
FILE *fp;
void *addr, *start, *end, *lastend;
int node, err, lc;
char buf[BUFSIZ];
struct bitmask *nmask = numa_allocate_nodemask();
pagesize = getpagesize();
tst_parse_opts(argc, argv, options, usage);
if (opt_node) {
node = SAFE_STRTOL(NULL, opt_nodestr, 1, LONG_MAX);
} else {
err = get_allowed_nodes(NH_MEMS | NH_MEMS, 1, &node);
if (err == -3)
tst_brkm(TCONF, NULL, "requires at least one node.");
else if (err < 0)
tst_brkm(TBROK | TERRNO, NULL, "get_allowed_nodes");
}
numa_bitmask_setbit(nmask, node);
for (lc = 0; TEST_LOOPING(lc); lc++) {
tst_count = 0;
addr = mmap(NULL, pagesize * 3, PROT_WRITE,
MAP_ANON | MAP_PRIVATE, 0, 0);
if (addr == MAP_FAILED)
tst_brkm(TBROK | TERRNO, NULL, "mmap");
tst_resm(TINFO, "pid = %d addr = %p", getpid(), addr);
/* make page populate */
memset(addr, 0, pagesize * 3);
/* first mbind */
err = mbind(addr + pagesize, pagesize, MPOL_BIND, nmask->maskp,
nmask->size, MPOL_MF_MOVE_ALL);
if (err != 0) {
if (errno != ENOSYS)
tst_brkm(TBROK | TERRNO, NULL, "mbind1");
else
tst_brkm(TCONF, NULL,
"mbind syscall not implemented on this system.");
}
/* second mbind */
err = mbind(addr, pagesize * 3, MPOL_DEFAULT, NULL, 0, 0);
if (err != 0)
tst_brkm(TBROK | TERRNO, NULL, "mbind2");
/* /proc/self/maps in the form of
"00400000-00406000 r-xp 00000000". */
fp = fopen("/proc/self/maps", "r");
if (fp == NULL)
tst_brkm(TBROK | TERRNO, NULL, "fopen");
while (fgets(buf, BUFSIZ, fp) != NULL) {
if (sscanf(buf, "%p-%p ", &start, &end) != 2)
continue;
if (start == addr) {
tst_resm(TINFO, "start = %p, end = %p",
start, end);
if (end == addr + pagesize * 3) {
tst_resm(TPASS, "only 1 VMA.");
break;
}
lastend = end;
while (fgets(buf, BUFSIZ, fp) != NULL) {
/* No more VMAs, break */
if (sscanf(buf, "%p-%p ", &start,
&end) != 2)
break;
tst_resm(TINFO, "start = %p, end = %p",
start, end);
/* more VMAs found */
if (start == lastend)
lastend = end;
if (end == addr + pagesize * 3) {
tst_resm(TFAIL,
">1 unmerged VMAs.");
break;
}
}
if (end != addr + pagesize * 3)
tst_resm(TFAIL, "no matched VMAs.");
break;
}
}
fclose(fp);
if (munmap(addr, pagesize * 3) == -1)
tst_brkm(TWARN | TERRNO, NULL, "munmap");
}
tst_exit();
}
static void
host_memory_backend_memory_complete(UserCreatable *uc, Error **errp)
{
HostMemoryBackend *backend = MEMORY_BACKEND(uc);
HostMemoryBackendClass *bc = MEMORY_BACKEND_GET_CLASS(uc);
Error *local_err = NULL;
void *ptr;
uint64_t sz;
if (bc->alloc) {
bc->alloc(backend, &local_err);
if (local_err) {
error_propagate(errp, local_err);
return;
}
ptr = memory_region_get_ram_ptr(&backend->mr);
sz = memory_region_size(&backend->mr);
if (backend->merge) {
qemu_madvise(ptr, sz, QEMU_MADV_MERGEABLE);
}
if (!backend->dump) {
qemu_madvise(ptr, sz, QEMU_MADV_DONTDUMP);
}
#ifdef CONFIG_NUMA
unsigned long lastbit = find_last_bit(backend->host_nodes, MAX_NODES);
/* lastbit == MAX_NODES means maxnode = 0 */
unsigned long maxnode = (lastbit + 1) % (MAX_NODES + 1);
/* ensure policy won't be ignored in case memory is preallocated
* before mbind(). note: MPOL_MF_STRICT is ignored on hugepages so
* this doesn't catch hugepage case. */
unsigned flags = MPOL_MF_STRICT | MPOL_MF_MOVE;
/* check for invalid host-nodes and policies and give more verbose
* error messages than mbind(). */
if (maxnode && backend->policy == MPOL_DEFAULT) {
error_setg(errp, "host-nodes must be empty for policy default,"
" or you should explicitly specify a policy other"
" than default");
return;
} else if (maxnode == 0 && backend->policy != MPOL_DEFAULT) {
error_setg(errp, "host-nodes must be set for policy %s",
HostMemPolicy_lookup[backend->policy]);
return;
}
/* We can have up to MAX_NODES nodes, but we need to pass maxnode+1
* as argument to mbind() due to an old Linux bug (feature?) which
* cuts off the last specified node. This means backend->host_nodes
* must have MAX_NODES+1 bits available.
*/
assert(sizeof(backend->host_nodes) >=
BITS_TO_LONGS(MAX_NODES + 1) * sizeof(unsigned long));
assert(maxnode <= MAX_NODES);
if (mbind(ptr, sz, backend->policy,
maxnode ? backend->host_nodes : NULL, maxnode + 1, flags)) {
error_setg_errno(errp, errno,
"cannot bind memory to host NUMA nodes");
return;
}
#endif
/* Preallocate memory after the NUMA policy has been instantiated.
* This is necessary to guarantee memory is allocated with
* specified NUMA policy in place.
*/
if (backend->prealloc) {
os_mem_prealloc(memory_region_get_fd(&backend->mr), ptr, sz);
}
}
}
int main(int argc, char *argv[]) {
int i;
int nr = 2;
int ret;
char c;
char *p;
int mapflag = MAP_ANONYMOUS;
int protflag = PROT_READ|PROT_WRITE;
unsigned long nr_nodes = numa_max_node() + 1;
struct bitmask *all_nodes;
struct bitmask *old_nodes;
struct bitmask *new_nodes;
while ((c = getopt(argc, argv, "vp:m:n:h:")) != -1) {
switch(c) {
case 'v':
verbose = 1;
break;
case 'p':
testpipe = optarg;
{
struct stat stat;
lstat(testpipe, &stat);
if (!S_ISFIFO(stat.st_mode))
errmsg("Given file is not fifo.\n");
}
break;
case 'm':
if (!strcmp(optarg, "private"))
mapflag |= MAP_PRIVATE;
else if (!strcmp(optarg, "shared"))
mapflag |= MAP_SHARED;
else
errmsg("invalid optarg for -m\n");
break;
case 'n':
nr = strtoul(optarg, NULL, 10);
break;
case 'h':
mapflag |= MAP_HUGETLB;
HPS = strtoul(optarg, NULL, 10) * 1024;
/* todo: arch independent */
if (HPS != 2097152 && HPS != 1073741824)
errmsg("Invalid hugepage size\n");
break;
default:
errmsg("invalid option\n");
break;
}
}
if (nr_nodes < 2)
errmsg("A minimum of 2 nodes is required for this test.\n");
all_nodes = numa_bitmask_alloc(nr_nodes);
old_nodes = numa_bitmask_alloc(nr_nodes);
new_nodes = numa_bitmask_alloc(nr_nodes);
numa_bitmask_setbit(all_nodes, 0);
numa_bitmask_setbit(all_nodes, 1);
numa_bitmask_setbit(old_nodes, 0);
numa_bitmask_setbit(new_nodes, 1);
numa_sched_setaffinity(0, old_nodes);
signal(SIGUSR1, sig_handle);
p = mmap((void *)ADDR_INPUT, nr * HPS, protflag, mapflag, -1, 0);
if (p == MAP_FAILED)
err("mmap");
/* fault in */
memset(p, 'a', nr * HPS);
pprintf("before mbind\n");
pause();
numa_sched_setaffinity(0, all_nodes);
printf("call mbind\n");
ret = mbind(p, nr * HPS, MPOL_BIND, new_nodes->maskp,
new_nodes->size + 1, MPOL_MF_MOVE|MPOL_MF_STRICT);
if (ret == -1)
err("mbind");
signal(SIGUSR1, sig_handle_flag);
pprintf("entering busy loop\n");
while (flag)
memset(p, 'a', nr * HPS);
pprintf("exited busy loop\n");
pause();
return 0;
}
int testMonad()
{
Susp<int> x = mjoin(fmap(ints(1, 4), sum));
Susp<int> y = mbind(ints(1, 4), sum);
return y.get();
}
int main (int argc, char** argv) {
int ret, c;
int i, repeat = 5;
int cpu = 2;
static int errortype = 1;
static int verbose = 1;
static int disableHuge = 0;
static int madvisePoison = 0;
static int poll_exit=0;
static long length;
struct bitmask *nodes, *gnodes;
int gpolicy;
unsigned long error_opt;
void *vaddrmin = (void *)-1UL, *vaddrmax = NULL;
static size_t pdcount=0;
unsigned long mattr, addrend, pages, count, nodeid, paddr = 0;
unsigned long addr_start=0, nodeid_start=-1, mattr_start=-1;
unsigned int pagesize = getpagesize();
char pte_str[20];
struct dlook_get_map_info req;
static page_desc_t *pdbegin=NULL;
page_desc_t *pd, *pdend;
length = memsize("100k");
nodes = numa_allocate_nodemask();
gnodes = numa_allocate_nodemask();
progname = argv[0];
while (1)
{
static struct option long_options[] =
{
{"verbose", no_argument, &verbose, 1},
{"delay", no_argument, &delay, 1},
{"disableHuge", no_argument, &disableHuge, 1},
{"poll", no_argument, &poll_exit, 1},
{"madvisePoison", no_argument, &madvisePoison, 1},
{"manual", no_argument, &manual, 1},
{"cpu", required_argument, 0, 'c'},
{"errortype", required_argument, 0, 'e'},
{"help", no_argument, 0, 'h'},
{"length", required_argument, 0, 'l'}
};
/* getopt_long stores the option index here. */
int option_index = 0;
c = getopt_long (argc, argv, "hc:e:l:",
long_options, &option_index);
/* Detect the end of the options. */
if (c == -1)
break;
switch (c)
{
case 'c':
cpu = atoi(optarg);
break;
case 'e':
errortype = atoi(optarg);
break;
case 'h':
help();
case 'l':
/* Not exposed */
printf ("option -l with value `%s'\n", optarg);
length = memsize("optarg");
break;
case '?':
/* getopt_long already printed an error message. */
exit(-1);
}
}
cpu_process_setaffinity(getpid(), cpu);
error_opt = get_etype(errortype);
buf = mmap(NULL, length, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, 0, 0);
if (mbind((void *)buf, length, MPOL_DEFAULT, nodes->maskp, nodes->size, 0) < 0){
perror("mbind error\n");
}
/* Disable Hugepages */
if (disableHuge)
madvise((void *)buf, length, MADV_NOHUGEPAGE);
if (madvisePoison)
madvise((void *)buf, length,MADV_HWPOISON );
gpolicy = -1;
if (get_mempolicy(&gpolicy, gnodes->maskp, gnodes->size, (void *)buf, MPOL_F_ADDR) < 0)
perror("get_mempolicy");
if (!numa_bitmask_equal(gnodes, nodes)) {
printf("nodes differ %lx, %lx!\n", gnodes->maskp[0], nodes->maskp[0]);
}
strcpy(pte_str, "");
addrend = ((unsigned long)buf)+length;
pages = (addrend-((unsigned long)buf))/pagesize;
if (pages > pdcount) {
pdbegin = realloc(pdbegin, sizeof(page_desc_t)*pages);
pdcount = pages;
}
req.pid = getpid();
req.start_vaddr = (unsigned long)buf;
req.end_vaddr = addrend;
req.pd = pdbegin;
sigaction(SIGBUS, &recover_act, NULL);
/*Fault in Pages */
if(!poll_exit)
hog((void *)buf, length);
/* Get mmap phys_addrs */
if ((fd = open(UVMCE_DEVICE, O_RDWR)) < 0) {
printf("Failed to open: %s\n", UVMCE_DEVICE);
exit (1);
}
if (ioctl(fd, UVMCE_DLOOK, &req ) < 0){
printf("Failed to INJECT_UCE\n");
exit(1);
}
process_map(pd,pdbegin, pdend, pages, buf, addrend, pagesize, mattr,
nodeid, paddr, pte_str, nodeid_start, mattr_start, addr_start);
printf("\n\tstart_vaddr\t 0x%016lx length\t 0x%x\n\tend_vaddr\t 0x%016lx pages\t %ld\n",
buf , length, addrend, pages);
uv_inject(pd,pdbegin, pdend, pages, (unsigned long)buf, addrend, pagesize, mattr,
nodeid, paddr, pte_str, nodeid_start,
mattr_start, addr_start, error_opt);
if (delay){
printf("Enter char to consume bad memory..");
getchar();
}
if (error_opt != UVMCE_PATROL_SCRUB_UCE){
consume_it((void *)buf, length);
}
out:
close(fd);
return 0;
}
