void * chunk_alloc_mmap(size_t size, size_t alignment, bool *zero) { void *ret; size_t offset; /* * Ideally, there would be a way to specify alignment to mmap() (like * NetBSD has), but in the absence of such a feature, we have to work * hard to efficiently create aligned mappings. The reliable, but * slow method is to create a mapping that is over-sized, then trim the * excess. However, that always results in one or two calls to * pages_unmap(). * * Optimistically try mapping precisely the right amount before falling * back to the slow method, with the expectation that the optimistic * approach works most of the time. */ assert(alignment != 0); assert((alignment & chunksize_mask) == 0); ret = pages_map(NULL, size); if (ret == NULL) return (NULL); offset = ALIGNMENT_ADDR2OFFSET(ret, alignment); if (offset != 0) { pages_unmap(ret, size); return (chunk_alloc_mmap_slow(size, alignment, zero)); } assert(ret != NULL); *zero = true; return (ret); }
void * chunk_alloc_mmap(size_t size, size_t alignment, bool *zero) { void *ret; size_t offset; /* * Ideally, there would be a way to specify alignment to mmap() (like * NetBSD has), but in the absence of such a feature, we have to work * hard to efficiently create aligned mappings. The reliable, but * slow method is to create a mapping that is over-sized, then trim the * excess. However, that always results in at least one call to * pages_unmap(). * * A more optimistic approach is to try mapping precisely the right * amount, then try to append another mapping if alignment is off. In * practice, this works out well as long as the application is not * interleaving mappings via direct mmap() calls. If we do run into a * situation where there is an interleaved mapping and we are unable to * extend an unaligned mapping, our best option is to switch to the * slow method until mmap() returns another aligned mapping. This will * tend to leave a gap in the memory map that is too small to cause * later problems for the optimistic method. * * Another possible confounding factor is address space layout * randomization (ASLR), which causes mmap(2) to disregard the * requested address. As such, repeatedly trying to extend unaligned * mappings could result in an infinite loop, so if extension fails, * immediately fall back to the reliable method of over-allocation * followed by trimming. */ ret = pages_map(NULL, size); if (ret == NULL) return (NULL); offset = ALIGNMENT_ADDR2OFFSET(ret, alignment); if (offset != 0) { /* Try to extend chunk boundary. */ if (pages_map((void *)((uintptr_t)ret + size), chunksize - offset) == NULL) { /* * Extension failed. Clean up, then fall back to the * reliable-but-expensive method. */ pages_unmap(ret, size); return (chunk_alloc_mmap_slow(size, alignment, true, zero)); } else { /* Clean up unneeded leading space. */ pages_unmap(ret, chunksize - offset); ret = (void *)((uintptr_t)ret + (chunksize - offset)); } } assert(ret != NULL); *zero = true; return (ret); }
static void * chunk_alloc_mmap_internal(size_t size, bool noreserve) { void *ret; /* * Ideally, there would be a way to specify alignment to mmap() (like * NetBSD has), but in the absence of such a feature, we have to work * hard to efficiently create aligned mappings. The reliable, but * slow method is to create a mapping that is over-sized, then trim the * excess. However, that always results in at least one call to * pages_unmap(). * * A more optimistic approach is to try mapping precisely the right * amount, then try to append another mapping if alignment is off. In * practice, this works out well as long as the application is not * interleaving mappings via direct mmap() calls. If we do run into a * situation where there is an interleaved mapping and we are unable to * extend an unaligned mapping, our best option is to switch to the * slow method until mmap() returns another aligned mapping. This will * tend to leave a gap in the memory map that is too small to cause * later problems for the optimistic method. * * Another possible confounding factor is address space layout * randomization (ASLR), which causes mmap(2) to disregard the * requested address. mmap_unaligned tracks whether the previous * chunk_alloc_mmap() execution received any unaligned or relocated * mappings, and if so, the current execution will immediately fall * back to the slow method. However, we keep track of whether the fast * method would have succeeded, and if so, we make a note to try the * fast method next time. */ if (MMAP_UNALIGNED_GET() == false) { size_t offset; ret = pages_map(NULL, size, noreserve); if (ret == NULL) return (NULL); offset = CHUNK_ADDR2OFFSET(ret); if (offset != 0) { MMAP_UNALIGNED_SET(true); /* Try to extend chunk boundary. */ if (pages_map((void *)((uintptr_t)ret + size), chunksize - offset, noreserve) == NULL) { /* * Extension failed. Clean up, then revert to * the reliable-but-expensive method. */ pages_unmap(ret, size); ret = chunk_alloc_mmap_slow(size, true, noreserve); } else { /* Clean up unneeded leading space. */ pages_unmap(ret, chunksize - offset); ret = (void *)((uintptr_t)ret + (chunksize - offset)); } } } else ret = chunk_alloc_mmap_slow(size, false, noreserve); return (ret); }