static void weak_clear_entry_no_lock(azone_t *azone, weak_entry_t *entry, uintptr_t *weak_refs_count, auto_weak_callback_block_t **head)
{
// clear referrers, update counters, update lists
unsigned count = entry->referrers.num_allocated;
unsigned index = 0;
for (; index < count; ++index) {
weak_referrer_t *ref = &entry->referrers.refs[index];
if (ref->referrer) {
if (azone->control.log & AUTO_LOG_WEAK) malloc_printf("%s: WEAK: clearing ref to %p at %p (value was %p)\n", auto_prelude(), entry->referent, ref->referrer, *ref->referrer);
if (*ref->referrer != entry->referent) {
malloc_printf("__weak value %p at location %p not equal to %p and so will not be cleared\n", *ref->referrer, ref->referrer, entry->referent);
void **base = (void **)auto_zone_base_pointer((auto_zone_t*)azone, ref->referrer);
if (base) {
auto_memory_type_t type = auto_zone_get_layout_type((auto_zone_t*)azone, base);
malloc_printf("...location is %s starting at %p with first slot value %p\n",
(type & AUTO_OBJECT) ? "an object" : "a data block",
base,
*base);
}
continue;
}
*ref->referrer = NULL;
++*weak_refs_count;
if (ref->block && ref->block->callback_function && !ref->block->next) {
// chain it if isn't already chained & there is a callout to call
ref->block->next = *head;
*head = ref->block;
}
}
}
weak_entry_remove_no_lock(azone, entry);
}
static test_status_t
p_test_impl(bool do_malloc_init, test_t *t, va_list ap)
{
test_status_t ret;
if (do_malloc_init) {
/*
* Make sure initialization occurs prior to running tests.
* Tests are special because they may use internal facilities
* prior to triggering initialization as a side effect of
* calling into the public API.
*/
if (nallocx(1, 0) == 0) {
malloc_printf("Initialization error");
return (test_status_fail);
}
}
ret = test_status_pass;
for (; t != NULL; t = va_arg(ap, test_t *)) {
t();
if (test_status > ret)
ret = test_status;
}
malloc_printf("--- %s: %u/%u, %s: %u/%u, %s: %u/%u ---\n",
test_status_string(test_status_pass),
test_counts[test_status_pass], test_count,
test_status_string(test_status_skip),
test_counts[test_status_skip], test_count,
test_status_string(test_status_fail),
test_counts[test_status_fail], test_count);
return (ret);
}
void *
je_thread_start(void *arg)
{
unsigned main_arena_ind = *(unsigned *)arg;
void *p;
unsigned arena_ind;
size_t size;
int err;
p = malloc(1);
if (p == NULL) {
malloc_printf("%s(): Error in malloc()\n", __func__);
return (void *)1;
}
free(p);
size = sizeof(arena_ind);
if ((err = mallctl("thread.arena", &arena_ind, &size, &main_arena_ind,
sizeof(main_arena_ind)))) {
malloc_printf("%s(): Error in mallctl(): %s\n", __func__,
strerror(err));
return (void *)1;
}
size = sizeof(arena_ind);
if ((err = mallctl("thread.arena", &arena_ind, &size, NULL,
0))) {
malloc_printf("%s(): Error in mallctl(): %s\n", __func__,
strerror(err));
return (void *)1;
}
assert(arena_ind == main_arena_ind);
return (NULL);
}
void* realloc(void* oldmem, size_t bytes)
{
void *retval;
int oldsize;
jmalloc_header_t *jhead;
void *ra = __builtin_return_address(0);
if (oldmem)
{
jhead = oldmem;
jhead--;
if (jhead->magic == JMALLOC_MAGIC)
oldsize = jhead->len;
else
{
if (jhead->magic == JFREE_MAGIC)
{
malloc_printf("MEMORY: %d about to realloc from %p memory %p "
"(%d->%d bytes), allocated from %p and already released "
"from %p\n", getpid(), ra, oldmem, jhead->len, bytes,
jhead->alloc_ra, jhead->free_ra);
__display_chunk(mem2chunk(jhead));
}
else
{
malloc_printf("MEMORY: %d about to realloc garbage %p from %p: "
"alloc_ra %p, free_ra %p, len %d, magic %#x\n",
getpid(), oldmem, ra, jhead->alloc_ra, jhead->free_ra,
jhead->len, jhead->magic);
}
#ifdef __CONFIG_RG_DBG_ULIBC_MALLOC_CRASH__
/* Let's give to complete the printing */
sleep(2);
*((int *)0) = 0xfaceface;
#endif
return NULL;
}
}
else
oldsize = 0;
retval = malloc(bytes);
if (retval && oldsize && bytes)
memcpy(retval, oldmem, bytes < oldsize? bytes : oldsize);
if (retval)
{
jhead = retval;
jhead--;
jhead->alloc_ra = ra;
}
free(oldmem);
return retval;
}
int
main(void)
{
malloc_printf("Test begin\n");
test_bitmap_size();
test_bitmap_init();
test_bitmap_set();
test_bitmap_unset();
test_bitmap_sfu();
malloc_printf("Test end\n");
return (0);
}
static void *
pages_map(void *addr, size_t size)
{
void *ret;
/*
* We don't use MAP_FIXED here, because it can cause the *replacement*
* of existing mappings, and we only want to create new mappings.
*/
ret = mmap(addr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON,
-1, 0);
assert(ret != NULL);
if (ret == MAP_FAILED)
ret = NULL;
else if (addr != NULL && ret != addr) {
/*
* We succeeded in mapping memory, but not in the right place.
*/
if (munmap(ret, size) == -1) {
char buf[BUFERROR_BUF];
buferror(errno, buf, sizeof(buf));
malloc_printf("<jemalloc: Error in munmap(): %s\n",
buf);
if (opt_abort)
abort();
}
ret = NULL;
}
assert(ret == NULL || (addr == NULL && ret != addr)
|| (addr != NULL && ret == addr));
return (ret);
}
// Register a new weak reference.
// referent is the object being weakly pointed to
// referrer is the memory location that will be zeroed when referent dies
// Does not check whether referent is currently live (fixme probably should)
// Does not check whether referrer is already a weak reference location for
// the given referent or any other referent (fixme maybe should)
// Does not change the scannability of referrer; referrer should be non-scanned
void weak_register(azone_t *azone, const void *referent, void **referrer, auto_weak_callback_block_t *block)
{
weak_entry_t *entry;
if (azone->control.log & AUTO_LOG_WEAK) malloc_printf("%s: WEAK: registering weak reference to %p at %p\n", auto_prelude(), referent, referrer);
spin_lock(&azone->weak_refs_table_lock);
if (*referrer) weak_unregister_no_lock(azone, *referrer, referrer);
if (referent) {
if ((entry = weak_entry_for_referent(azone, referent))) {
append_referrer_no_lock(&entry->referrers, referrer, block);
}
else {
weak_entry_t new_entry;
new_entry.referent = referent;
new_entry.referrers.refs = NULL;
new_entry.referrers.num_refs = 0;
new_entry.referrers.num_allocated = 0;
append_referrer_no_lock(&new_entry.referrers, referrer, block);
weak_grow_maybe_no_lock(azone);
azone->num_weak_refs++;
weak_entry_insert_no_lock(azone, &new_entry);
}
}
// make sure that anyone accessing this via a read gets the value
*referrer = (void *)referent;
spin_unlock(&azone->weak_refs_table_lock);
}
void
p_test_fini(void)
{
test_counts[test_status]++;
malloc_printf("%s: %s\n", test_name, test_status_string(test_status));
}
usword_t SubzoneBlockRef::dec_refcount_no_lock() const {
Admin *admin = subzone()->admin();
if (has_refcount()) {
// non-zero reference count, check the overflow table.
PtrIntHashMap &retains = admin->retains();
PtrIntHashMap::iterator retain_iter = retains.find(address());
if (retain_iter != retains.end() && retain_iter->first == address()) {
if (--retain_iter->second == 1) {
// transition from 2 -> 1
retains.erase(retain_iter);
return 1;
} else {
return retain_iter->second;
}
} else {
// transition from 1 -> 0
subzone()->clear_has_refcount(q());
return 0;
}
}
// underflow.
malloc_printf("reference count underflow for %p, break on auto_refcount_underflow_error to debug.\n", address());
auto_refcount_underflow_error(address());
return -1;
}
static void
os_pages_unmap(void *addr, size_t size) {
assert(ALIGNMENT_ADDR2BASE(addr, os_page) == (vaddr_t)addr);
assert(ALIGNMENT_CEILING(size, os_page) == size);
#ifdef _WIN32
if (VirtualFree(addr, 0, MEM_RELEASE) == 0)
#else
if (munmap(addr, size) == -1)
#endif
{
char buf[BUFERROR_BUF];
buferror(get_errno(), buf, sizeof(buf));
malloc_printf("<jemalloc>: Error in "
#ifdef _WIN32
"VirtualFree"
#else
"munmap"
#endif
"(): %s\n", buf);
if (opt_abort) {
abort();
}
}
}
static void *
pages_map(void *addr, size_t size
#ifdef JEMALLOC_ENABLE_MEMKIND
, unsigned partition
#endif
)
{
void *ret;
assert(size != 0);
#ifdef _WIN32
/*
* If VirtualAlloc can't allocate at the given address when one is
* given, it fails and returns NULL.
*/
ret = VirtualAlloc(addr, size, MEM_COMMIT | MEM_RESERVE,
PAGE_READWRITE);
#else
#ifdef JEMALLOC_ENABLE_MEMKIND
if (partition && memkind_partition_mmap) {
ret = memkind_partition_mmap(partition, addr, size);
}
else {
#endif /* JEMALLOC_ENABLE_MEMKIND */
/*
* We don't use MAP_FIXED here, because it can cause the *replacement*
* of existing mappings, and we only want to create new mappings.
*/
ret = mmap(addr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
#ifdef JEMALLOC_ENABLE_MEMKIND
}
#endif /* JEMALLOC_ENABLE_MEMKIND */
assert(ret != NULL);
if (ret == MAP_FAILED)
ret = NULL;
else if (addr != NULL && ret != addr) {
/*
* We succeeded in mapping memory, but not in the right place.
*/
if (munmap(ret, size) == -1) {
char buf[BUFERROR_BUF];
buferror(get_errno(), buf, sizeof(buf));
malloc_printf("<jemalloc>: Error in munmap(): %s\n",
buf);
if (opt_abort)
abort();
}
ret = NULL;
}
#endif
assert(ret == NULL || (addr == NULL && ret != addr)
|| (addr != NULL && ret == addr));
return (ret);
}
usword_t LargeBlockRef::dec_refcount_no_lock() const {
usword_t rc = refcount();
if (rc <= 0) {
malloc_printf("reference count underflow for %p, break on auto_refcount_underflow_error to debug\n", address());
auto_refcount_underflow_error(address());
} else {
rc = rc - 1;
_large->set_refcount(rc);
}
return rc;
}
// Add new_entry to the zone's table of weak references.
// Does not check whether the referent is already in the table.
// Does not update num_weak_refs.
static void weak_entry_insert_no_lock(azone_t *azone, weak_entry_t *new_entry)
{
weak_entry_t *table = azone->weak_refs_table;
if (!table) { malloc_printf("no auto weak ref table!\n"); return; }
unsigned table_size = azone->max_weak_refs;
unsigned hash_index = hash(new_entry->referent) % table_size;
unsigned index = hash_index;
do {
weak_entry_t *entry = table + index;
if (entry->referent == NULL) {
*entry = *new_entry;
return;
}
index++; if (index == table_size) index = 0;
} while (index != hash_index);
malloc_printf("no room for new entry in auto weak ref table!\n");
}
void
test_skip(const char *format, ...)
{
va_list ap;
va_start(ap, format);
malloc_vcprintf(NULL, NULL, format, ap);
va_end(ap);
malloc_printf("\n");
test_status = test_status_skip;
}
TEST_END
static void
print_filled_extents(const void *p, uint8_t c, size_t len)
{
const uint8_t *pc = (const uint8_t *)p;
size_t i, range0;
uint8_t c0;
malloc_printf(" p=%p, c=%#x, len=%zu:", p, c, len);
range0 = 0;
c0 = pc[0];
for (i = 0; i < len; i++) {
if (pc[i] != c0) {
malloc_printf(" %#x[%zu..%zu)", c0, range0, i);
range0 = i;
c0 = pc[i];
}
}
malloc_printf(" %#x[%zu..%zu)\n", c0, range0, i);
}
int
main(void)
{
int ret = 0;
void *p;
unsigned arena_ind;
size_t size;
int err;
je_thread_t threads[NTHREADS];
unsigned i;
malloc_printf("Test begin\n");
p = malloc(1);
if (p == NULL) {
malloc_printf("%s(): Error in malloc()\n", __func__);
ret = 1;
goto label_return;
}
size = sizeof(arena_ind);
if ((err = mallctl("thread.arena", &arena_ind, &size, NULL, 0))) {
malloc_printf("%s(): Error in mallctl(): %s\n", __func__,
strerror(err));
ret = 1;
goto label_return;
}
for (i = 0; i < NTHREADS; i++) {
je_thread_create(&threads[i], je_thread_start,
(void *)&arena_ind);
}
for (i = 0; i < NTHREADS; i++)
je_thread_join(threads[i], (void *)&ret);
label_return:
malloc_printf("Test end\n");
return (ret);
}
static void *
pages_map(void *addr, size_t size)
{
void *ret;
assert(size != 0);
#ifdef _WIN32
/*
* If VirtualAlloc can't allocate at the given address when one is
* given, it fails and returns NULL.
*/
ret = VirtualAlloc(addr, size, MEM_COMMIT | MEM_RESERVE,
PAGE_READWRITE);
#else
/*
* We don't use MAP_FIXED here, because it can cause the *replacement*
* of existing mappings, and we only want to create new mappings.
*/
ret = mmap(addr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON,
-1, 0);
assert(ret != NULL);
if (ret == MAP_FAILED)
ret = NULL;
else if (addr != NULL && ret != addr) {
/*
* We succeeded in mapping memory, but not in the right place.
*/
if (munmap(ret, size) == -1) {
char buf[BUFERROR_BUF];
buferror(get_errno(), buf, sizeof(buf));
malloc_printf("<jemalloc: Error in munmap(): %s\n",
buf);
if (opt_abort)
abort();
}
ret = NULL;
}
#endif
#if defined(__ANDROID__)
if (ret != NULL) {
/* Name this memory as being used by libc */
prctl(ANDROID_PR_SET_VMA, ANDROID_PR_SET_VMA_ANON_NAME, ret,
size, "libc_malloc");
}
#endif
assert(ret == NULL || (addr == NULL && ret != addr)
|| (addr != NULL && ret == addr));
return (ret);
}
void
p_test_fail(const char *format, ...)
{
va_list ap;
va_start(ap, format);
malloc_vcprintf(NULL, NULL, format, ap);
format = va_arg(ap, const char *);
malloc_vcprintf(NULL, NULL, format, ap);
va_end(ap);
malloc_printf("\n");
test_status = test_status_fail;
}
static void
pages_unmap(void *addr, size_t size)
{
if (munmap(addr, size) == -1) {
char buf[BUFERROR_BUF];
buferror(errno, buf, sizeof(buf));
malloc_printf("<jemalloc>: Error in munmap(): %s\n", buf);
if (opt_abort)
abort();
}
}
int
main(void)
{
int ret = 0;
je_thread_t thread;
malloc_printf("Test begin\n");
je_thread_start(NULL);
je_thread_create(&thread, je_thread_start, NULL);
je_thread_join(thread, (void *)&ret);
je_thread_start(NULL);
je_thread_create(&thread, je_thread_start, NULL);
je_thread_join(thread, (void *)&ret);
je_thread_start(NULL);
malloc_printf("Test end\n");
return (ret);
}
// Unregister an already-registered weak reference.
// This is used when referrer's storage is about to go away, but referent
// isn't dead yet. (Otherwise, zeroing referrer later would be a
// bad memory access.)
// Does nothing if referent/referrer is not a currently active weak reference.
// fixme currently requires old referent value to be passed in (lame)
// fixme unregistration should be automatic if referrer is collected
static void weak_unregister_no_lock(azone_t *azone, const void *referent, void **referrer)
{
weak_entry_t *entry;
if (azone->control.log & AUTO_LOG_WEAK) malloc_printf("%s: WEAK: unregistering weak reference to %p at %p\n", auto_prelude(), referent, referrer);
if ((entry = weak_entry_for_referent(azone, referent))) {
remove_referrer_no_lock(&entry->referrers, referrer);
if (entry->referrers.num_refs == 0) {
weak_entry_remove_no_lock(azone, entry);
azone->num_weak_refs--;
}
}
}
test_status_t
p_test(test_t *t, ...)
{
test_status_t ret;
va_list ap;
/*
* Make sure initialization occurs prior to running tests. Tests are
* special because they may use internal facilities prior to triggering
* initialization as a side effect of calling into the public API. This
* is a final safety that works even if jemalloc_constructor() doesn't
* run, as for MSVC builds.
*/
if (nallocx(1, 0) == 0) {
malloc_printf("Initialization error");
return (test_status_fail);
}
ret = test_status_pass;
va_start(ap, t);
for (; t != NULL; t = va_arg(ap, test_t *)) {
t();
if (test_status > ret)
ret = test_status;
}
va_end(ap);
malloc_printf("--- %s: %u/%u, %s: %u/%u, %s: %u/%u ---\n",
test_status_string(test_status_pass),
test_counts[test_status_pass], test_count,
test_status_string(test_status_skip),
test_counts[test_status_skip], test_count,
test_status_string(test_status_fail),
test_counts[test_status_fail], test_count);
return (ret);
}
TEST_END
int
main(void) {
/* Ensure tsd bootstrapped. */
if (nallocx(1, 0) == 0) {
malloc_printf("Initialization error");
return test_status_fail;
}
return test_no_reentrancy(
test_tsd_main_thread,
test_tsd_sub_thread,
test_tsd_reincarnation);
}
void
compare_funcs(uint64_t nwarmup, uint64_t niter, const char *name_a,
void (*func_a), const char *name_b, void (*func_b))
{
timedelta_t timer_a, timer_b;
char ratio_buf[6];
time_func(&timer_a, nwarmup, niter, func_a);
time_func(&timer_b, nwarmup, niter, func_b);
timer_ratio(&timer_a, &timer_b, ratio_buf, sizeof(ratio_buf));
malloc_printf("%"PRIu64" iterations, %s=%"PRIu64"us, "
"%s=%"PRIu64"us, ratio=1:%s\n",
niter, name_a, timer_usec(&timer_a), name_b, timer_usec(&timer_b),
ratio_buf);
}
TEST_END
int
main(void)
{
/* Core tsd bootstrapping must happen prior to data_tsd_boot(). */
if (nallocx(1, 0) == 0) {
malloc_printf("Initialization error");
return (test_status_fail);
}
data_tsd_boot();
return (test(
test_tsd_main_thread,
test_tsd_sub_thread));
}
// Remove old_referrer from list, if it's present.
// Does not remove duplicates.
// fixme this is slow if old_referrer is not present.
static void remove_referrer_no_lock(weak_referrer_array_t *list, void **old_referrer)
{
unsigned index = hash(old_referrer) % list->num_allocated;
unsigned start_index = index, hash_displacement = 0;
while (list->refs[index].referrer != old_referrer) {
index++;
hash_displacement++;
if (index == list->num_allocated)
index = 0;
if (index == start_index || hash_displacement > list->max_hash_displacement) {
malloc_printf("%s: attempted to remove unregistered weak referrer %p\n", auto_prelude(), old_referrer);
return;
}
}
list->refs[index].referrer = NULL;
list->num_refs--;
}
void
ckh_delete(tsd_t *tsd, ckh_t *ckh) {
assert(ckh != NULL);
#ifdef CKH_VERBOSE
malloc_printf(
"%s(%p): ngrows: %"FMTu64", nshrinks: %"FMTu64","
" nshrinkfails: %"FMTu64", ninserts: %"FMTu64","
" nrelocs: %"FMTu64"\n", __func__, ckh,
(unsigned long long)ckh->ngrows,
(unsigned long long)ckh->nshrinks,
(unsigned long long)ckh->nshrinkfails,
(unsigned long long)ckh->ninserts,
(unsigned long long)ckh->nrelocs);
#endif
idalloctm(tsd_tsdn(tsd), ckh->tab, NULL, NULL, true, true);
if (config_debug) {
memset(ckh, JEMALLOC_FREE_JUNK, sizeof(ckh_t));
}
}
void
ckh_delete(ckh_t *ckh)
{
assert(ckh != NULL);
#ifdef CKH_VERBOSE
malloc_printf(
"%s(%p): ngrows: %"PRIu64", nshrinks: %"PRIu64","
" nshrinkfails: %"PRIu64", ninserts: %"PRIu64","
" nrelocs: %"PRIu64"\n", __func__, ckh,
(unsigned long long)ckh->ngrows,
(unsigned long long)ckh->nshrinks,
(unsigned long long)ckh->nshrinkfails,
(unsigned long long)ckh->ninserts,
(unsigned long long)ckh->nrelocs);
#endif
idalloc(ckh->tab);
if (config_debug)
memset(ckh, 0x5a, sizeof(ckh_t));
}
void
ckh_delete(ckh_t *ckh)
{
assert(ckh != NULL);
dassert(ckh->magic == CKH_MAGIC);
#ifdef CKH_VERBOSE
malloc_printf(
"%s(%p): ngrows: %"PRIu64", nshrinks: %"PRIu64","
" nshrinkfails: %"PRIu64", ninserts: %"PRIu64","
" nrelocs: %"PRIu64"\n", __func__, ckh,
(unsigned long long)ckh->ngrows,
(unsigned long long)ckh->nshrinks,
(unsigned long long)ckh->nshrinkfails,
(unsigned long long)ckh->ninserts,
(unsigned long long)ckh->nrelocs);
#endif
idalloc(ckh->tab);
#ifdef JEMALLOC_DEBUG
memset(ckh, 0x5a, sizeof(ckh_t));
#endif
}
static void
pages_unmap(void *addr, size_t size)
{
#ifdef _WIN32
if (VirtualFree(addr, 0, MEM_RELEASE) == 0)
#else
if (munmap(addr, size) == -1)
#endif
{
char buf[BUFERROR_BUF];
buferror(get_errno(), buf, sizeof(buf));
malloc_printf("<jemalloc>: Error in "
#ifdef _WIN32
"VirtualFree"
#else
"munmap"
#endif
"(): %s\n", buf);
if (opt_abort)
abort();
}
}
