int
main(int argc, char *argv[])
{
START(argc, argv, "pmem_movnt_align");
if (argc != 2)
UT_FATAL("usage: %s type", argv[0]);
char type = argv[1][0];
char *src, *dst;
size_t s;
switch (type) {
case 'C': /* memcpy */
/* mmap with guard pages */
src = MMAP_ANON_ALIGNED(N_BYTES, 0);
dst = MMAP_ANON_ALIGNED(N_BYTES, 0);
if (src == NULL || dst == NULL)
UT_FATAL("!mmap");
/* check memcpy with 0 size */
check_memcpy(dst, src, 0);
/* check memcpy with unaligned size */
for (s = 0; s < CACHELINE; s++)
check_memcpy(dst, src, N_BYTES - s);
/* check memcpy with unaligned begin */
for (s = 0; s < CACHELINE; s++)
check_memcpy(dst + s, src, N_BYTES - s);
/* check memcpy with unaligned begin and end */
for (s = 0; s < CACHELINE; s++)
check_memcpy(dst + s, src + s, N_BYTES - 2 * s);
MUNMAP_ANON_ALIGNED(src, N_BYTES);
MUNMAP_ANON_ALIGNED(dst, N_BYTES);
break;
case 'B': /* memmove backward */
/* mmap with guard pages */
src = MMAP_ANON_ALIGNED(2 * N_BYTES - 4096, 0);
dst = src + N_BYTES - 4096;
if (src == NULL)
UT_FATAL("!mmap");
/* check memmove in backward direction with 0 size */
check_memmove(dst, src, 0);
/* check memmove in backward direction with unaligned size */
for (s = 0; s < CACHELINE; s++)
check_memmove(dst, src, N_BYTES - s);
/* check memmove in backward direction with unaligned begin */
for (s = 0; s < CACHELINE; s++)
check_memmove(dst + s, src, N_BYTES - s);
/*
* check memmove in backward direction with unaligned begin
* and end
*/
for (s = 0; s < CACHELINE; s++)
check_memmove(dst + s, src + s, N_BYTES - 2 * s);
MUNMAP_ANON_ALIGNED(src, 2 * N_BYTES - 4096);
break;
case 'F': /* memmove forward */
/* mmap with guard pages */
dst = MMAP_ANON_ALIGNED(2 * N_BYTES - 4096, 0);
src = dst + N_BYTES - 4096;
if (src == NULL)
UT_FATAL("!mmap");
/* check memmove in forward direction with 0 size */
check_memmove(dst, src, 0);
/* check memmove in forward direction with unaligned size */
for (s = 0; s < CACHELINE; s++)
check_memmove(dst, src, N_BYTES - s);
/* check memmove in forward direction with unaligned begin */
for (s = 0; s < CACHELINE; s++)
check_memmove(dst + s, src, N_BYTES - s);
/*
* check memmove in forward direction with unaligned begin
* and end
*/
for (s = 0; s < CACHELINE; s++)
check_memmove(dst + s, src + s, N_BYTES - 2 * s);
MUNMAP_ANON_ALIGNED(dst, 2 * N_BYTES - 4096);
break;
case 'S': /* memset */
/* mmap with guard pages */
dst = MMAP_ANON_ALIGNED(N_BYTES, 0);
if (dst == NULL)
UT_FATAL("!mmap");
/* check memset with 0 size */
check_memset(dst, 0);
/* check memset with unaligned size */
for (s = 0; s < CACHELINE; s++)
check_memset(dst, N_BYTES - s);
/* check memset with unaligned begin */
for (s = 0; s < CACHELINE; s++)
check_memset(dst + s, N_BYTES - s);
/* check memset with unaligned begin and end */
for (s = 0; s < CACHELINE; s++)
check_memset(dst + s, N_BYTES - 2 * s);
MUNMAP_ANON_ALIGNED(dst, N_BYTES);
break;
default:
UT_FATAL("!wrong type of test");
break;
}
DONE(NULL);
}
static void
test_recycler(void)
{
struct mock_pop *mpop = MMAP_ANON_ALIGNED(MOCK_POOL_SIZE,
Ut_mmap_align);
PMEMobjpool *pop = &mpop->p;
memset(pop, 0, MOCK_POOL_SIZE);
pop->heap_offset = (uint64_t)((uint64_t)&mpop->heap - (uint64_t)mpop);
pop->p_ops.persist = obj_heap_persist;
pop->p_ops.memset_persist = obj_heap_memset_persist;
pop->p_ops.base = pop;
pop->set = MALLOC(sizeof(*(pop->set)));
pop->set->options = 0;
pop->set->directory_based = 0;
void *heap_start = (char *)pop + pop->heap_offset;
uint64_t heap_size = MOCK_POOL_SIZE - sizeof(PMEMobjpool);
struct palloc_heap *heap = &pop->heap;
struct pmem_ops *p_ops = &pop->p_ops;
struct stats *s = stats_new(pop);
UT_ASSERTne(s, NULL);
UT_ASSERT(heap_check(heap_start, heap_size) != 0);
UT_ASSERT(heap_init(heap_start, heap_size,
&pop->heap_size, p_ops) == 0);
UT_ASSERT(heap_boot(heap, heap_start, heap_size,
&pop->heap_size,
pop, p_ops, s, pop->set) == 0);
UT_ASSERT(heap_buckets_init(heap) == 0);
UT_ASSERT(pop->heap.rt != NULL);
/* trigger heap bucket populate */
struct memory_block m = MEMORY_BLOCK_NONE;
m.size_idx = 1;
struct bucket *b = heap_bucket_acquire_by_id(heap,
DEFAULT_ALLOC_CLASS_ID);
UT_ASSERT(heap_get_bestfit_block(heap, b, &m) == 0);
heap_bucket_release(heap, b);
int ret;
struct recycler *r = recycler_new(&pop->heap, 10000 /* never recalc */);
UT_ASSERTne(r, NULL);
init_run_with_score(pop->heap.layout, 0, 64);
init_run_with_score(pop->heap.layout, 1, 128);
init_run_with_score(pop->heap.layout, 15, 0);
struct memory_block mrun = {0, 0, 1, 0};
struct memory_block mrun2 = {1, 0, 1, 0};
memblock_rebuild_state(&pop->heap, &mrun);
memblock_rebuild_state(&pop->heap, &mrun2);
ret = recycler_put(r, &mrun,
recycler_calc_score(&pop->heap, &mrun, NULL));
UT_ASSERTeq(ret, 0);
ret = recycler_put(r, &mrun2,
recycler_calc_score(&pop->heap, &mrun2, NULL));
UT_ASSERTeq(ret, 0);
struct memory_block mrun_ret = MEMORY_BLOCK_NONE;
mrun_ret.size_idx = 1;
struct memory_block mrun2_ret = MEMORY_BLOCK_NONE;
mrun2_ret.size_idx = 1;
ret = recycler_get(r, &mrun_ret);
UT_ASSERTeq(ret, 0);
ret = recycler_get(r, &mrun2_ret);
UT_ASSERTeq(ret, 0);
UT_ASSERTeq(mrun2.chunk_id, mrun2_ret.chunk_id);
UT_ASSERTeq(mrun.chunk_id, mrun_ret.chunk_id);
init_run_with_score(pop->heap.layout, 7, 256);
init_run_with_score(pop->heap.layout, 2, 64);
init_run_with_score(pop->heap.layout, 5, 512);
init_run_with_score(pop->heap.layout, 10, 128);
mrun.chunk_id = 7;
mrun2.chunk_id = 2;
struct memory_block mrun3 = {5, 0, 1, 0};
struct memory_block mrun4 = {10, 0, 1, 0};
memblock_rebuild_state(&pop->heap, &mrun3);
memblock_rebuild_state(&pop->heap, &mrun4);
mrun_ret.size_idx = 1;
mrun2_ret.size_idx = 1;
struct memory_block mrun3_ret = MEMORY_BLOCK_NONE;
mrun3_ret.size_idx = 1;
struct memory_block mrun4_ret = MEMORY_BLOCK_NONE;
mrun4_ret.size_idx = 1;
ret = recycler_put(r, &mrun,
recycler_calc_score(&pop->heap, &mrun, NULL));
UT_ASSERTeq(ret, 0);
ret = recycler_put(r, &mrun2,
recycler_calc_score(&pop->heap, &mrun2, NULL));
UT_ASSERTeq(ret, 0);
ret = recycler_put(r, &mrun3,
recycler_calc_score(&pop->heap, &mrun3, NULL));
UT_ASSERTeq(ret, 0);
ret = recycler_put(r, &mrun4,
recycler_calc_score(&pop->heap, &mrun4, NULL));
UT_ASSERTeq(ret, 0);
ret = recycler_get(r, &mrun2_ret);
UT_ASSERTeq(ret, 0);
ret = recycler_get(r, &mrun4_ret);
UT_ASSERTeq(ret, 0);
ret = recycler_get(r, &mrun_ret);
UT_ASSERTeq(ret, 0);
ret = recycler_get(r, &mrun3_ret);
UT_ASSERTeq(ret, 0);
UT_ASSERTeq(mrun.chunk_id, mrun_ret.chunk_id);
UT_ASSERTeq(mrun2.chunk_id, mrun2_ret.chunk_id);
UT_ASSERTeq(mrun3.chunk_id, mrun3_ret.chunk_id);
UT_ASSERTeq(mrun4.chunk_id, mrun4_ret.chunk_id);
init_run_with_max_block(pop->heap.layout, 1);
struct memory_block mrun5 = {1, 0, 1, 0};
memblock_rebuild_state(&pop->heap, &mrun5);
ret = recycler_put(r, &mrun5,
recycler_calc_score(&pop->heap, &mrun5, NULL));
UT_ASSERTeq(ret, 0);
struct memory_block mrun5_ret = MEMORY_BLOCK_NONE;
mrun5_ret.size_idx = 11;
ret = recycler_get(r, &mrun5_ret);
UT_ASSERTeq(ret, ENOMEM);
mrun5_ret = MEMORY_BLOCK_NONE;
mrun5_ret.size_idx = 10;
ret = recycler_get(r, &mrun5_ret);
UT_ASSERTeq(ret, 0);
recycler_delete(r);
stats_delete(pop, s);
heap_cleanup(heap);
UT_ASSERT(heap->rt == NULL);
FREE(pop->set);
MUNMAP_ANON_ALIGNED(mpop, MOCK_POOL_SIZE);
}
static void
test_mock_pool_allocs()
{
addr = MMAP_ANON_ALIGNED(MOCK_POOL_SIZE, Ut_pagesize);
mock_pop = &addr->p;
mock_pop->addr = addr;
mock_pop->size = MOCK_POOL_SIZE;
mock_pop->rdonly = 0;
mock_pop->is_pmem = 0;
mock_pop->heap_offset = offsetof(struct mock_pop, ptr);
UT_ASSERTeq(mock_pop->heap_offset % Ut_pagesize, 0);
mock_pop->heap_size = MOCK_POOL_SIZE - mock_pop->heap_offset;
mock_pop->nlanes = 1;
mock_pop->lanes_offset = sizeof(PMEMobjpool);
mock_pop->is_master_replica = 1;
mock_pop->persist_local = (persist_local_fn)pmem_msync;
mock_pop->flush_local = (flush_local_fn)pmem_msync;
mock_pop->drain_local = drain_empty;
mock_pop->p_ops.persist = obj_persist;
mock_pop->p_ops.flush = obj_flush;
mock_pop->p_ops.drain = obj_drain;
mock_pop->p_ops.memcpy_persist = obj_memcpy;
mock_pop->p_ops.memset_persist = obj_memset;
mock_pop->p_ops.base = mock_pop;
mock_pop->p_ops.pool_size = mock_pop->size;
mock_pop->redo = redo_log_config_new(addr, &mock_pop->p_ops,
redo_log_check_offset, mock_pop, REDO_NUM_ENTRIES);
void *heap_start = (char *)mock_pop + mock_pop->heap_offset;
uint64_t heap_size = mock_pop->heap_size;
heap_init(heap_start, heap_size, &mock_pop->p_ops);
heap_boot(&mock_pop->heap, heap_start, heap_size, mock_pop,
&mock_pop->p_ops);
heap_buckets_init(&mock_pop->heap);
/* initialize runtime lanes structure */
mock_pop->lanes_desc.runtime_nlanes = (unsigned)mock_pop->nlanes;
lane_boot(mock_pop);
UT_ASSERTne(mock_pop->heap.rt, NULL);
test_malloc_free_loop(MALLOC_FREE_SIZE);
/*
* Allocating till OOM and freeing the objects in a loop for different
* buckets covers basically all code paths except error cases.
*/
test_oom_allocs(TEST_HUGE_ALLOC_SIZE);
test_oom_allocs(TEST_TINY_ALLOC_SIZE);
test_oom_allocs(TEST_HUGE_ALLOC_SIZE);
test_oom_allocs(TEST_SMALL_ALLOC_SIZE);
test_oom_allocs(TEST_MEGA_ALLOC_SIZE);
test_realloc(TEST_SMALL_ALLOC_SIZE, TEST_MEDIUM_ALLOC_SIZE);
test_realloc(TEST_HUGE_ALLOC_SIZE, TEST_MEGA_ALLOC_SIZE);
lane_cleanup(mock_pop);
heap_cleanup(&mock_pop->heap);
MUNMAP_ANON_ALIGNED(addr, MOCK_POOL_SIZE);
}
static void
test_heap(void)
{
struct mock_pop *mpop = MMAP_ANON_ALIGNED(MOCK_POOL_SIZE,
Ut_mmap_align);
PMEMobjpool *pop = &mpop->p;
memset(pop, 0, MOCK_POOL_SIZE);
pop->heap_offset = (uint64_t)((uint64_t)&mpop->heap - (uint64_t)mpop);
pop->p_ops.persist = obj_heap_persist;
pop->p_ops.memset_persist = obj_heap_memset_persist;
pop->p_ops.base = pop;
pop->set = MALLOC(sizeof(*(pop->set)));
pop->set->options = 0;
pop->set->directory_based = 0;
struct stats *s = stats_new(pop);
UT_ASSERTne(s, NULL);
void *heap_start = (char *)pop + pop->heap_offset;
uint64_t heap_size = MOCK_POOL_SIZE - sizeof(PMEMobjpool);
struct palloc_heap *heap = &pop->heap;
struct pmem_ops *p_ops = &pop->p_ops;
UT_ASSERT(heap_check(heap_start, heap_size) != 0);
UT_ASSERT(heap_init(heap_start, heap_size,
&pop->heap_size, p_ops) == 0);
UT_ASSERT(heap_boot(heap, heap_start, heap_size,
&pop->heap_size,
pop, p_ops, s, pop->set) == 0);
UT_ASSERT(heap_buckets_init(heap) == 0);
UT_ASSERT(pop->heap.rt != NULL);
test_alloc_class_bitmap_correctness();
test_container((struct block_container *)container_new_ravl(heap),
heap);
test_container((struct block_container *)container_new_seglists(heap),
heap);
struct alloc_class *c_small = heap_get_best_class(heap, 1);
struct alloc_class *c_big = heap_get_best_class(heap, 2048);
UT_ASSERT(c_small->unit_size < c_big->unit_size);
/* new small buckets should be empty */
UT_ASSERT(c_big->type == CLASS_RUN);
struct memory_block blocks[MAX_BLOCKS] = {
{0, 0, 1, 0},
{0, 0, 1, 0},
{0, 0, 1, 0}
};
struct bucket *b_def = heap_bucket_acquire_by_id(heap,
DEFAULT_ALLOC_CLASS_ID);
for (int i = 0; i < MAX_BLOCKS; ++i) {
heap_get_bestfit_block(heap, b_def, &blocks[i]);
UT_ASSERT(blocks[i].block_off == 0);
}
heap_bucket_release(heap, b_def);
struct memory_block old_run = {0, 0, 1, 0};
struct memory_block new_run = {0, 0, 0, 0};
struct alloc_class *c_run = heap_get_best_class(heap, 1024);
struct bucket *b_run = heap_bucket_acquire(heap, c_run);
/*
* Allocate blocks from a run until one run is exhausted.
*/
UT_ASSERTne(heap_get_bestfit_block(heap, b_run, &old_run), ENOMEM);
int *nresv = bucket_current_resvp(b_run);
do {
new_run.chunk_id = 0;
new_run.block_off = 0;
new_run.size_idx = 1;
UT_ASSERTne(heap_get_bestfit_block(heap, b_run, &new_run),
ENOMEM);
UT_ASSERTne(new_run.size_idx, 0);
*nresv = 0;
} while (old_run.block_off != new_run.block_off);
*nresv = 0;
heap_bucket_release(heap, b_run);
stats_delete(pop, s);
UT_ASSERT(heap_check(heap_start, heap_size) == 0);
heap_cleanup(heap);
UT_ASSERT(heap->rt == NULL);
FREE(pop->set);
MUNMAP_ANON_ALIGNED(mpop, MOCK_POOL_SIZE);
}
int
main(int argc, char *argv[])
{
if (argc != 3)
UT_FATAL("usage: %s type heavy=[0|1]", argv[0]);
char type = argv[1][0];
Heavy = argv[2][0] == '1';
const char *thr = getenv("PMEM_MOVNT_THRESHOLD");
const char *avx = getenv("PMEM_AVX");
const char *avx512f = getenv("PMEM_AVX512F");
START(argc, argv, "pmem_movnt_align %c %s %savx %savx512f", type,
thr ? thr : "default",
avx ? "" : "!",
avx512f ? "" : "!");
size_t s;
switch (type) {
case 'C': /* memcpy */
/* mmap with guard pages */
Src = MMAP_ANON_ALIGNED(N_BYTES, 0);
Dst = MMAP_ANON_ALIGNED(N_BYTES, 0);
if (Src == NULL || Dst == NULL)
UT_FATAL("!mmap");
Scratch = MALLOC(N_BYTES);
/* check memcpy with 0 size */
check_memcpy_variants(0, 0, 0);
/* check memcpy with unaligned size */
for (s = 0; s < CACHELINE; s++)
check_memcpy_variants(0, 0, N_BYTES - s);
/* check memcpy with unaligned begin */
for (s = 0; s < CACHELINE; s++)
check_memcpy_variants(s, 0, N_BYTES - s);
/* check memcpy with unaligned begin and end */
for (s = 0; s < CACHELINE; s++)
check_memcpy_variants(s, s, N_BYTES - 2 * s);
MUNMAP_ANON_ALIGNED(Src, N_BYTES);
MUNMAP_ANON_ALIGNED(Dst, N_BYTES);
FREE(Scratch);
break;
case 'B': /* memmove backward */
/* mmap with guard pages */
Src = MMAP_ANON_ALIGNED(2 * N_BYTES - 4096, 0);
Dst = Src + N_BYTES - 4096;
if (Src == NULL)
UT_FATAL("!mmap");
/* check memmove in backward direction with 0 size */
check_memmove_variants(0, 0, 0);
/* check memmove in backward direction with unaligned size */
for (s = 0; s < CACHELINE; s++)
check_memmove_variants(0, 0, N_BYTES - s);
/* check memmove in backward direction with unaligned begin */
for (s = 0; s < CACHELINE; s++)
check_memmove_variants(s, 0, N_BYTES - s);
/*
* check memmove in backward direction with unaligned begin
* and end
*/
for (s = 0; s < CACHELINE; s++)
check_memmove_variants(s, s, N_BYTES - 2 * s);
MUNMAP_ANON_ALIGNED(Src, 2 * N_BYTES - 4096);
break;
case 'F': /* memmove forward */
/* mmap with guard pages */
Dst = MMAP_ANON_ALIGNED(2 * N_BYTES - 4096, 0);
Src = Dst + N_BYTES - 4096;
if (Src == NULL)
UT_FATAL("!mmap");
/* check memmove in forward direction with 0 size */
check_memmove_variants(0, 0, 0);
/* check memmove in forward direction with unaligned size */
for (s = 0; s < CACHELINE; s++)
check_memmove_variants(0, 0, N_BYTES - s);
/* check memmove in forward direction with unaligned begin */
for (s = 0; s < CACHELINE; s++)
check_memmove_variants(s, 0, N_BYTES - s);
/*
* check memmove in forward direction with unaligned begin
* and end
*/
for (s = 0; s < CACHELINE; s++)
check_memmove_variants(s, s, N_BYTES - 2 * s);
MUNMAP_ANON_ALIGNED(Dst, 2 * N_BYTES - 4096);
break;
case 'S': /* memset */
/* mmap with guard pages */
Dst = MMAP_ANON_ALIGNED(N_BYTES, 0);
if (Dst == NULL)
UT_FATAL("!mmap");
Scratch = MALLOC(N_BYTES);
/* check memset with 0 size */
check_memset_variants(0, 0);
/* check memset with unaligned size */
for (s = 0; s < CACHELINE; s++)
check_memset_variants(0, N_BYTES - s);
/* check memset with unaligned begin */
for (s = 0; s < CACHELINE; s++)
check_memset_variants(s, N_BYTES - s);
/* check memset with unaligned begin and end */
for (s = 0; s < CACHELINE; s++)
check_memset_variants(s, N_BYTES - 2 * s);
MUNMAP_ANON_ALIGNED(Dst, N_BYTES);
FREE(Scratch);
break;
default:
UT_FATAL("!wrong type of test");
break;
}
DONE(NULL);
}
