TEST_END
TEST_BEGIN(huge_mallocx) {
unsigned arena1, arena2;
size_t sz = sizeof(unsigned);
assert_d_eq(mallctl("arenas.create", &arena1, &sz, NULL, 0), 0,
"Failed to create arena");
void *huge = mallocx(HUGE_SZ, MALLOCX_ARENA(arena1));
assert_ptr_not_null(huge, "Fail to allocate huge size");
assert_d_eq(mallctl("arenas.lookup", &arena2, &sz, &huge,
sizeof(huge)), 0, "Unexpected mallctl() failure");
assert_u_eq(arena1, arena2, "Wrong arena used for mallocx");
dallocx(huge, MALLOCX_ARENA(arena1));
void *huge2 = mallocx(HUGE_SZ, 0);
assert_ptr_not_null(huge, "Fail to allocate huge size");
assert_d_eq(mallctl("arenas.lookup", &arena2, &sz, &huge2,
sizeof(huge2)), 0, "Unexpected mallctl() failure");
assert_u_ne(arena1, arena2,
"Huge allocation should not come from the manual arena.");
assert_u_ne(arena2, 0,
"Huge allocation should not come from the arena 0.");
dallocx(huge2, 0);
}
void *memkind_arena_calloc(struct memkind *kind, size_t num, size_t size)
{
void *result = NULL;
int err = 0;
unsigned int arena;
err = kind->ops->get_arena(kind, &arena, size);
if (!err) {
result = jemk_mallocx_check(num * size, MALLOCX_ARENA(arena) | MALLOCX_ZERO);
}
return result;
}
void *memkind_arena_realloc(struct memkind *kind, void *ptr, size_t size)
{
int err = 0;
unsigned int arena;
if (size == 0 && ptr != NULL) {
memkind_free(kind, ptr);
ptr = NULL;
}
else {
err = kind->ops->get_arena(kind, &arena, size);
if (!err) {
if (ptr == NULL) {
ptr = jemk_mallocx_check(size, MALLOCX_ARENA(arena));
}
else {
ptr = jemk_rallocx_check(ptr, size, MALLOCX_ARENA(arena));
}
}
}
return ptr;
}
TEST_END
TEST_BEGIN(test_arenas_lookup) {
unsigned arena, arena1;
void *ptr;
size_t sz = sizeof(unsigned);
assert_d_eq(mallctl("arenas.create", (void *)&arena, &sz, NULL, 0), 0,
"Unexpected mallctl() failure");
ptr = mallocx(42, MALLOCX_ARENA(arena) | MALLOCX_TCACHE_NONE);
assert_ptr_not_null(ptr, "Unexpected mallocx() failure");
assert_d_eq(mallctl("arenas.lookup", &arena1, &sz, &ptr, sizeof(ptr)),
0, "Unexpected mallctl() failure");
assert_u_eq(arena, arena1, "Unexpected arena index");
dallocx(ptr, 0);
}
static void
test_zero(size_t szmin, size_t szmax)
{
int flags = MALLOCX_ARENA(arena_ind()) | MALLOCX_ZERO;
size_t sz, nsz;
void *p;
#define FILL_BYTE 0x7aU
sz = szmax;
p = mallocx(sz, flags);
assert_ptr_not_null(p, "Unexpected mallocx() error");
assert_false(validate_fill(p, 0x00, 0, sz), "Memory not filled: sz=%zu",
sz);
/*
* Fill with non-zero so that non-debug builds are more likely to detect
* errors.
*/
memset(p, FILL_BYTE, sz);
assert_false(validate_fill(p, FILL_BYTE, 0, sz),
"Memory not filled: sz=%zu", sz);
/* Shrink in place so that we can expect growing in place to succeed. */
sz = szmin;
assert_zu_eq(xallocx(p, sz, 0, flags), sz,
"Unexpected xallocx() error");
assert_false(validate_fill(p, FILL_BYTE, 0, sz),
"Memory not filled: sz=%zu", sz);
for (sz = szmin; sz < szmax; sz = nsz) {
nsz = nallocx(sz+1, flags);
assert_zu_eq(xallocx(p, sz+1, 0, flags), nsz,
"Unexpected xallocx() failure");
assert_false(validate_fill(p, FILL_BYTE, 0, sz),
"Memory not filled: sz=%zu", sz);
assert_false(validate_fill(p, 0x00, sz, nsz-sz),
"Memory not filled: sz=%zu, nsz-sz=%zu", sz, nsz-sz);
memset((void *)((uintptr_t)p + sz), FILL_BYTE, nsz-sz);
assert_false(validate_fill(p, FILL_BYTE, 0, nsz),
"Memory not filled: nsz=%zu", nsz);
}
dallocx(p, flags);
}
bool JemallocNodumpAllocator::extend_and_setup_arena() {
#ifdef FOLLY_JEMALLOC_NODUMP_ALLOCATOR_SUPPORTED
if (mallctl == nullptr) {
// Not linked with jemalloc.
return false;
}
size_t len = sizeof(arena_index_);
if (auto ret = mallctl("arenas.extend", &arena_index_, &len, nullptr, 0)) {
LOG(FATAL) << "Unable to extend arena: " << errnoStr(ret);
}
flags_ = MALLOCX_ARENA(arena_index_) | MALLOCX_TCACHE_NONE;
// Set the custom alloc hook
const auto key =
folly::to<std::string>("arena.", arena_index_, ".chunk_hooks");
chunk_hooks_t hooks;
len = sizeof(hooks);
// Read the existing hooks
if (auto ret = mallctl(key.c_str(), &hooks, &len, nullptr, 0)) {
LOG(FATAL) << "Unable to get the hooks: " << errnoStr(ret);
}
if (original_chunk_alloc_ == nullptr) {
original_chunk_alloc_ = hooks.alloc;
} else {
DCHECK_EQ(original_chunk_alloc_, hooks.alloc);
}
// Set the custom hook
hooks.alloc = &JemallocNodumpAllocator::chunk_alloc;
if (auto ret =
mallctl(key.c_str(), nullptr, nullptr, &hooks, sizeof(hooks))) {
LOG(FATAL) << "Unable to set the hooks: " << errnoStr(ret);
}
return true;
#else // FOLLY_JEMALLOC_NODUMP_ALLOCATOR_SUPPORTED
return false;
#endif // FOLLY_JEMALLOC_NODUMP_ALLOCATOR_SUPPORTED
}
int memkind_arena_posix_memalign(struct memkind *kind, void **memptr, size_t alignment,
size_t size)
{
int err = 0;
unsigned int arena;
int errno_before;
*memptr = NULL;
err = kind->ops->get_arena(kind, &arena, size);
if (!err) {
err = memkind_posix_check_alignment(kind, alignment);
}
if (!err) {
/* posix_memalign should not change errno.
Set it to its previous value after calling jemalloc */
errno_before = errno;
*memptr = jemk_mallocx_check(size, MALLOCX_ALIGN(alignment) | MALLOCX_ARENA(arena));
errno = errno_before;
err = *memptr ? 0 : ENOMEM;
}
return err;
}
void as_join(int id) {
if (as_flags[id] != 0) {
log_gas("address space %d already joined\n", id);
return;
}
const chunk_hooks_t *hooks = _hooks[id];
// If there aren't any custom hooks set for this space, then the basic local
// allocator is fine, which means that we don't need any special flags for
// this address space.
if (!hooks) {
log_gas("no custom allocator for %d, using local\n", id);
return;
}
// Create an arena that uses the right hooks.
unsigned arena;
size_t sz = sizeof(arena);
dbg_check( je_mallctl("arenas.extend", &arena, &sz, NULL, 0) );
char path[128];
snprintf(path, 128, "arena.%u.chunk_hooks", arena);
dbg_check( je_mallctl(path, NULL, NULL, (void*)hooks, sizeof(*hooks)) );
// // Disable dirty page purging for this arena
// snprintf(path, 124, "arena.%u.lg_dirty_mult", arena);
// ssize_t i = -1;
// dbg_check( je_mallctl(path, NULL, NULL, (void*)&i, sizeof(i)) );
// Create a cache.
unsigned cache;
sz = sizeof(cache);
dbg_check( je_mallctl("tcache.create", &cache, &sz, NULL, 0) );
// And set the flags.
as_flags[id] = MALLOCX_ARENA(arena) | MALLOCX_TCACHE(cache);
}
TEST_END
TEST_BEGIN(test_extra_huge)
{
int flags = MALLOCX_ARENA(arena_ind());
size_t largemax, huge1, huge2, huge3, hugemax;
void *p;
/* Get size classes. */
largemax = get_large_size(get_nlarge()-1);
huge1 = get_huge_size(1);
huge2 = get_huge_size(2);
huge3 = get_huge_size(3);
hugemax = get_huge_size(get_nhuge()-1);
p = mallocx(huge3, flags);
assert_ptr_not_null(p, "Unexpected mallocx() error");
assert_zu_eq(xallocx(p, huge3, 0, flags), huge3,
"Unexpected xallocx() behavior");
/* Test size decrease with zero extra. */
assert_zu_ge(xallocx(p, huge1, 0, flags), huge1,
"Unexpected xallocx() behavior");
assert_zu_ge(xallocx(p, largemax, 0, flags), huge1,
"Unexpected xallocx() behavior");
assert_zu_eq(xallocx(p, huge3, 0, flags), huge3,
"Unexpected xallocx() behavior");
/* Test size decrease with non-zero extra. */
assert_zu_eq(xallocx(p, huge1, huge3 - huge1, flags), huge3,
"Unexpected xallocx() behavior");
assert_zu_eq(xallocx(p, huge2, huge3 - huge2, flags), huge3,
"Unexpected xallocx() behavior");
assert_zu_eq(xallocx(p, huge1, huge2 - huge1, flags), huge2,
"Unexpected xallocx() behavior");
assert_zu_ge(xallocx(p, largemax, huge1 - largemax, flags), huge1,
"Unexpected xallocx() behavior");
assert_zu_ge(xallocx(p, huge1, 0, flags), huge1,
"Unexpected xallocx() behavior");
/* Test size increase with zero extra. */
assert_zu_le(xallocx(p, huge3, 0, flags), huge3,
"Unexpected xallocx() behavior");
assert_zu_le(xallocx(p, hugemax+1, 0, flags), huge3,
"Unexpected xallocx() behavior");
assert_zu_ge(xallocx(p, huge1, 0, flags), huge1,
"Unexpected xallocx() behavior");
/* Test size increase with non-zero extra. */
assert_zu_le(xallocx(p, huge1, SIZE_T_MAX - huge1, flags), hugemax,
"Unexpected xallocx() behavior");
assert_zu_ge(xallocx(p, huge1, 0, flags), huge1,
"Unexpected xallocx() behavior");
/* Test size increase with non-zero extra. */
assert_zu_le(xallocx(p, huge1, huge3 - huge1, flags), huge3,
"Unexpected xallocx() behavior");
assert_zu_eq(xallocx(p, huge3, 0, flags), huge3,
"Unexpected xallocx() behavior");
/* Test size+extra overflow. */
assert_zu_le(xallocx(p, huge3, hugemax - huge3 + 1, flags), hugemax,
"Unexpected xallocx() behavior");
dallocx(p, flags);
}
TEST_END
TEST_BEGIN(test_extra_large)
{
int flags = MALLOCX_ARENA(arena_ind());
size_t smallmax, large0, large1, large2, huge0, hugemax;
void *p;
/* Get size classes. */
smallmax = get_small_size(get_nsmall()-1);
large0 = get_large_size(0);
large1 = get_large_size(1);
large2 = get_large_size(2);
huge0 = get_huge_size(0);
hugemax = get_huge_size(get_nhuge()-1);
p = mallocx(large2, flags);
assert_ptr_not_null(p, "Unexpected mallocx() error");
assert_zu_eq(xallocx(p, large2, 0, flags), large2,
"Unexpected xallocx() behavior");
/* Test size decrease with zero extra. */
assert_zu_eq(xallocx(p, large0, 0, flags), large0,
"Unexpected xallocx() behavior");
assert_zu_eq(xallocx(p, smallmax, 0, flags), large0,
"Unexpected xallocx() behavior");
assert_zu_eq(xallocx(p, large2, 0, flags), large2,
"Unexpected xallocx() behavior");
/* Test size decrease with non-zero extra. */
assert_zu_eq(xallocx(p, large0, large2 - large0, flags), large2,
"Unexpected xallocx() behavior");
assert_zu_eq(xallocx(p, large1, large2 - large1, flags), large2,
"Unexpected xallocx() behavior");
assert_zu_eq(xallocx(p, large0, large1 - large0, flags), large1,
"Unexpected xallocx() behavior");
assert_zu_eq(xallocx(p, smallmax, large0 - smallmax, flags), large0,
"Unexpected xallocx() behavior");
assert_zu_eq(xallocx(p, large0, 0, flags), large0,
"Unexpected xallocx() behavior");
/* Test size increase with zero extra. */
assert_zu_eq(xallocx(p, large2, 0, flags), large2,
"Unexpected xallocx() behavior");
assert_zu_eq(xallocx(p, huge0, 0, flags), large2,
"Unexpected xallocx() behavior");
assert_zu_eq(xallocx(p, large0, 0, flags), large0,
"Unexpected xallocx() behavior");
/* Test size increase with non-zero extra. */
assert_zu_lt(xallocx(p, large0, huge0 - large0, flags), huge0,
"Unexpected xallocx() behavior");
assert_zu_eq(xallocx(p, large0, 0, flags), large0,
"Unexpected xallocx() behavior");
/* Test size increase with non-zero extra. */
assert_zu_eq(xallocx(p, large0, large2 - large0, flags), large2,
"Unexpected xallocx() behavior");
assert_zu_eq(xallocx(p, large2, 0, flags), large2,
"Unexpected xallocx() behavior");
/* Test size+extra overflow. */
assert_zu_lt(xallocx(p, large2, hugemax - large2 + 1, flags), huge0,
"Unexpected xallocx() behavior");
dallocx(p, flags);
}