static void base_extent_bump_alloc_post(base_t *base, extent_t *extent, size_t gap_size, void *addr, size_t size) { if (extent_bsize_get(extent) > 0) { /* * Compute the index for the largest size class that does not * exceed extent's size. */ szind_t index_floor = sz_size2index(extent_bsize_get(extent) + 1) - 1; extent_heap_insert(&base->avail[index_floor], extent); } if (config_stats) { base->allocated += size; /* * Add one PAGE to base_resident for every page boundary that is * crossed by the new allocation. Adjust n_thp similarly when * metadata_thp is enabled. */ base->resident += PAGE_CEILING((vaddr_t)addr + size) - PAGE_CEILING((vaddr_t)addr - gap_size); assert(base->allocated <= base->resident); assert(base->resident <= base->mapped); if (metadata_thp_madvise() && (opt_metadata_thp == metadata_thp_always || base->auto_thp_switched)) { base->n_thp += (HUGEPAGE_CEILING((vaddr_t)addr + size) - HUGEPAGE_CEILING((vaddr_t)addr - gap_size)) >> LG_HUGEPAGE; assert(base->mapped >= base->n_thp << LG_HUGEPAGE); } }
/* * base_alloc() guarantees demand-zeroed memory, in order to make multi-page * sparse data structures such as radix tree nodes efficient with respect to * physical memory usage. */ void * base_alloc(tsdn_t *tsdn, size_t size) { void *ret; size_t csize, usize; extent_node_t *node; extent_node_t key; /* * Round size up to nearest multiple of the cacheline size, so that * there is no chance of false cache line sharing. */ csize = CACHELINE_CEILING(size); usize = s2u(csize); extent_node_init(&key, NULL, NULL, usize, 0, false, false); malloc_mutex_lock(tsdn, &base_mtx); node = extent_tree_szsnad_nsearch(&base_avail_szsnad, &key); if (node != NULL) { /* Use existing space. */ extent_tree_szsnad_remove(&base_avail_szsnad, node); } else { /* Try to allocate more space. */ node = base_chunk_alloc(tsdn, csize); } if (node == NULL) { ret = NULL; goto label_return; } ret = extent_node_addr_get(node); if (extent_node_size_get(node) > csize) { extent_node_addr_set(node, (void *)((uintptr_t)ret + csize)); extent_node_size_set(node, extent_node_size_get(node) - csize); extent_tree_szsnad_insert(&base_avail_szsnad, node); } else base_node_dalloc(tsdn, node); if (config_stats) { base_allocated += csize; /* * Add one PAGE to base_resident for every page boundary that is * crossed by the new allocation. */ base_resident += PAGE_CEILING((vaddr_t)ret + csize) - PAGE_CEILING((vaddr_t)ret); } JEMALLOC_VALGRIND_MAKE_MEM_DEFINED(ret, csize); label_return: malloc_mutex_unlock(tsdn, &base_mtx); return (ret); }
static extent_node_t * base_chunk_alloc(tsdn_t *tsdn, size_t minsize) { extent_node_t *node; size_t csize, nsize; void *addr; malloc_mutex_assert_owner(tsdn, &base_mtx); assert(minsize != 0); node = base_node_try_alloc(tsdn); /* Allocate enough space to also carve a node out if necessary. */ nsize = (node == NULL) ? CACHELINE_CEILING(sizeof(extent_node_t)) : 0; csize = CHUNK_CEILING(minsize + nsize); addr = chunk_alloc_base(csize); if (addr == NULL) { if (node != NULL) base_node_dalloc(tsdn, node); return (NULL); } base_mapped += csize; if (node == NULL) { node = (extent_node_t *)addr; addr = (void *)((uintptr_t)addr + nsize); csize -= nsize; if (config_stats) { base_allocated += nsize; base_resident += PAGE_CEILING(nsize); } } base_extent_node_init(node, addr, csize); return (node); }
bool pages_purge_lazy(void *addr, size_t size) { assert(PAGE_ADDR2BASE(addr) == addr); assert(PAGE_CEILING(size) == size); if (!pages_can_purge_lazy) { return true; } if (!pages_can_purge_lazy_runtime) { /* * Built with lazy purge enabled, but detected it was not * supported on the current system. */ return true; } #ifdef _WIN32 VirtualAlloc(addr, size, MEM_RESET, PAGE_READWRITE); return false; #elif defined(JEMALLOC_PURGE_MADVISE_FREE) return (madvise(addr, size, # ifdef MADV_FREE MADV_FREE # else JEMALLOC_MADV_FREE # endif ) != 0); #elif defined(JEMALLOC_PURGE_MADVISE_DONTNEED) && \ !defined(JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS) return (madvise(addr, size, MADV_DONTNEED) != 0); #else not_reached(); #endif }
static bool pages_commit_impl(void *addr, size_t size, bool commit) { assert(PAGE_ADDR2BASE(addr) == addr); assert(PAGE_CEILING(size) == size); if (os_overcommits) { return true; } #ifdef _WIN32 return (commit ? (addr != VirtualAlloc(addr, size, MEM_COMMIT, PAGE_READWRITE)) : (!VirtualFree(addr, size, MEM_DECOMMIT))); #else { int prot = commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT; void *result = mmap(addr, size, prot, mmap_flags | MAP_FIXED, -1, 0); if (result == MAP_FAILED) { return true; } if (result != addr) { /* * We succeeded in mapping memory, but not in the right * place. */ os_pages_unmap(result, size); return true; } return false; } #endif }
void pages_unmap(void *addr, size_t size) { assert(PAGE_ADDR2BASE(addr) == addr); assert(PAGE_CEILING(size) == size); os_pages_unmap(addr, size); }
bool pages_dodump(void *addr, size_t size) { assert(PAGE_ADDR2BASE(addr) == addr); assert(PAGE_CEILING(size) == size); #ifdef JEMALLOC_MADVISE_DONTDUMP return madvise(addr, size, MADV_DODUMP) != 0; #else return false; #endif }
bool pages_purge_forced(void *addr, size_t size) { assert(PAGE_ADDR2BASE(addr) == addr); assert(PAGE_CEILING(size) == size); if (!pages_can_purge_forced) { return true; } #if defined(JEMALLOC_PURGE_MADVISE_DONTNEED) && \ defined(JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS) return (madvise(addr, size, MADV_DONTNEED) != 0); #elif defined(JEMALLOC_MAPS_COALESCE) /* Try to overlay a new demand-zeroed mapping. */ return pages_commit(addr, size); #else not_reached(); #endif }
/* Iterate over a valid jemalloc chunk, calling callback for each large * allocation run, and calling je_iterate_small for each small allocation run */ static void je_iterate_chunk(arena_chunk_t *chunk, void (*callback)(uintptr_t ptr, size_t size, void* arg), void* arg) { size_t pageind; pageind = map_bias; while (pageind < chunk_npages) { size_t mapbits; size_t size; mapbits = arena_mapbits_get(chunk, pageind); if (!arena_mapbits_allocated_get(chunk, pageind)) { /* Unallocated run */ size = arena_mapbits_unallocated_size_get(chunk, pageind); } else if (arena_mapbits_large_get(chunk, pageind)) { /* Large allocation run */ void *rpages; size = arena_mapbits_large_size_get(chunk, pageind); rpages = arena_miscelm_to_rpages(arena_miscelm_get(chunk, pageind)); callback((uintptr_t)rpages, size, arg); } else { /* Run of small allocations */ szind_t binind; arena_run_t *run; assert(arena_mapbits_small_runind_get(chunk, pageind) == pageind); binind = arena_mapbits_binind_get(chunk, pageind); run = &arena_miscelm_get(chunk, pageind)->run; assert(run->binind == binind); size = arena_bin_info[binind].run_size; je_iterate_small(run, callback, arg); } assert(size == PAGE_CEILING(size)); assert(size > 0); pageind += size >> LG_PAGE; } }
bool chunk_swap_enable(const int *fds, unsigned nfds, bool prezeroed) { bool ret; unsigned i; off_t off; void *vaddr; size_t cumsize, voff; size_t sizes[nfds]; malloc_mutex_lock(&swap_mtx); /* Get file sizes. */ for (i = 0, cumsize = 0; i < nfds; i++) { off = lseek(fds[i], 0, SEEK_END); if (off == ((off_t)-1)) { ret = true; goto RETURN; } if (PAGE_CEILING(off) != off) { /* Truncate to a multiple of the page size. */ off &= ~PAGE_MASK; if (ftruncate(fds[i], off) != 0) { ret = true; goto RETURN; } } sizes[i] = off; if (cumsize + off < cumsize) { /* * Cumulative file size is greater than the total * address space. Bail out while it's still obvious * what the problem is. */ ret = true; goto RETURN; } cumsize += off; } /* Round down to a multiple of the chunk size. */ cumsize &= ~chunksize_mask; if (cumsize == 0) { ret = true; goto RETURN; } /* * Allocate a chunk-aligned region of anonymous memory, which will * be the final location for the memory-mapped files. */ vaddr = chunk_alloc_mmap_noreserve(cumsize); if (vaddr == NULL) { ret = true; goto RETURN; } /* Overlay the files onto the anonymous mapping. */ for (i = 0, voff = 0; i < nfds; i++) { void *addr = mmap((void *)((uintptr_t)vaddr + voff), sizes[i], PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, fds[i], 0); if (addr == MAP_FAILED) { char buf[BUFERROR_BUF]; buferror(errno, buf, sizeof(buf)); malloc_write( "<jemalloc>: Error in mmap(..., MAP_FIXED, ...): "); malloc_write(buf); malloc_write("\n"); if (opt_abort) abort(); if (munmap(vaddr, voff) == -1) { buferror(errno, buf, sizeof(buf)); malloc_write("<jemalloc>: Error in munmap(): "); malloc_write(buf); malloc_write("\n"); } ret = true; goto RETURN; } assert(addr == (void *)((uintptr_t)vaddr + voff)); /* * Tell the kernel that the mapping will be accessed randomly, * and that it should not gratuitously sync pages to the * filesystem. */ #ifdef MADV_RANDOM madvise(addr, sizes[i], MADV_RANDOM); #endif #ifdef MADV_NOSYNC madvise(addr, sizes[i], MADV_NOSYNC); #endif voff += sizes[i]; } swap_prezeroed = prezeroed; swap_base = vaddr; swap_end = swap_base; swap_max = (void *)((uintptr_t)vaddr + cumsize); /* Copy the fds array for mallctl purposes. */ swap_fds = (int *)base_alloc(nfds * sizeof(int)); if (swap_fds == NULL) { ret = true; goto RETURN; } memcpy(swap_fds, fds, nfds * sizeof(int)); swap_nfds = nfds; #ifdef JEMALLOC_STATS swap_avail = cumsize; #endif swap_enabled = true; ret = false; RETURN: malloc_mutex_unlock(&swap_mtx); return (ret); }