static bool do_slabs_adjust_mem_limit(size_t new_mem_limit) { /* Cannot adjust memory limit at runtime if prealloc'ed */ if (mem_base != NULL) return false; settings.maxbytes = new_mem_limit; mem_limit = new_mem_limit; mem_limit_reached = false; /* Will reset on next alloc */ memory_release(); /* free what might already be in the global pool */ return true; }
void memory_finalize() { if (!use_allocator) return; if (memory_table != NULL) { for (int memid = 0; memid < memory_table_size; memid++) memory_release(memid); FREE_ALIGN(memory_table); memory_table = NULL; } use_allocator = 0; }
static void slab_rebalance_finish(void) { slabclass_t *s_cls; slabclass_t *d_cls; int x; uint32_t rescues; uint32_t evictions_nomem; uint32_t inline_reclaim; pthread_mutex_lock(&slabs_lock); s_cls = &slabclass[slab_rebal.s_clsid]; d_cls = &slabclass[slab_rebal.d_clsid]; #ifdef DEBUG_SLAB_MOVER /* If the algorithm is broken, live items can sneak in. */ slab_rebal.slab_pos = slab_rebal.slab_start; while (1) { item *it = slab_rebal.slab_pos; assert(it->it_flags == (ITEM_SLABBED|ITEM_FETCHED)); assert(memcmp(ITEM_key(it), "deadbeef", 8) == 0); it->it_flags = ITEM_SLABBED|ITEM_FETCHED; slab_rebal.slab_pos = (char *)slab_rebal.slab_pos + s_cls->size; if (slab_rebal.slab_pos >= slab_rebal.slab_end) break; } #endif /* At this point the stolen slab is completely clear. * We always kill the "first"/"oldest" slab page in the slab_list, so * shuffle the page list backwards and decrement. */ s_cls->slabs--; for (x = 0; x < s_cls->slabs; x++) { s_cls->slab_list[x] = s_cls->slab_list[x+1]; } d_cls->slab_list[d_cls->slabs++] = slab_rebal.slab_start; /* Don't need to split the page into chunks if we're just storing it */ if (slab_rebal.d_clsid > SLAB_GLOBAL_PAGE_POOL) { memset(slab_rebal.slab_start, 0, (size_t)settings.item_size_max); split_slab_page_into_freelist(slab_rebal.slab_start, slab_rebal.d_clsid); } else if (slab_rebal.d_clsid == SLAB_GLOBAL_PAGE_POOL) { /* mem_malloc'ed might be higher than mem_limit. */ memory_release(); } slab_rebal.done = 0; slab_rebal.s_clsid = 0; slab_rebal.d_clsid = 0; slab_rebal.slab_start = NULL; slab_rebal.slab_end = NULL; slab_rebal.slab_pos = NULL; evictions_nomem = slab_rebal.evictions_nomem; inline_reclaim = slab_rebal.inline_reclaim; rescues = slab_rebal.rescues; slab_rebal.evictions_nomem = 0; slab_rebal.inline_reclaim = 0; slab_rebal.rescues = 0; slab_rebalance_signal = 0; pthread_mutex_unlock(&slabs_lock); STATS_LOCK(); stats.slabs_moved++; stats.slab_reassign_rescues += rescues; stats.slab_reassign_evictions_nomem += evictions_nomem; stats.slab_reassign_inline_reclaim += inline_reclaim; stats_state.slab_reassign_running = false; STATS_UNLOCK(); if (settings.verbose > 1) { fprintf(stderr, "finished a slab move\n"); } }
int memory_register(void* baseaddr, size_t mem_size, const char* uuid, int do_create) { if (!use_allocator) return 0; int memid; size_t offset = 0; size_t max_quant_params_count = 1; for (memid = 0; memid < memory_table_size; memid++) if (!memory_table[memid].in_use) break; if (memid == memory_table_size) memory_table_grow(EP_MEMORY_TABLE_INC); memory_t* memptr = &memory_table[memid]; memptr->in_use = 1; memptr->do_create = do_create; memptr->shm_base = NULL; memptr->client_shm_base = baseaddr; memptr->shm_mspace = NULL; memptr->shm_id = -1; memptr->mem_size = mem_size; if (baseaddr == NULL) offset = (((sizeof(intptr_t*) * (max_ep + 1)) / PAGE_SIZE) + 1) * PAGE_SIZE; if (do_create) { /* Client */ /* Create a shared memory segment */ snprintf(memptr->shm_filename, SHM_FILENAME_LEN, SHM_FILENAME_PREFIX"%s%d", uuid, memid); if ((memptr->shm_id = shm_open(memptr->shm_filename, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IROTH)) < 0) { PRINT("CLIENT: shm_open failed (%s)\n", strerror(errno)); memory_release(memid); return -1; } /* Adjust the size of the shared memory segment */ if (ftruncate(memptr->shm_id, mem_size) < 0) { PRINT("CLIENT: ftruncate failed (%s)\n", strerror(errno)); memory_release(memid); return -1; } /* Create mmap region */ if ((memptr->shm_base = (void*)mmap(baseaddr, mem_size, (PROT_WRITE|PROT_READ), (baseaddr) ? (MAP_FIXED|MAP_SHARED) : MAP_SHARED, memptr->shm_id, 0)) == MAP_FAILED) { PRINT("CLIENT: mmap failed (%s)\n", strerror(errno)); memory_release(memid); return -1; } } else { /* Server */ /* Open shared memory region */ snprintf(memptr->shm_filename, SHM_FILENAME_LEN, SHM_FILENAME_PREFIX"%s%d", uuid, memid); if ((memptr->shm_id = shm_open(memptr->shm_filename, O_RDWR, S_IRWXU|S_IRWXG|S_IROTH)) < 0) { DEBUG_PRINT("SERVER: shm_open failed (%s)\n", strerror(errno)); memory_release(memid); return -1; } /* Create mmap region */ if ((memptr->shm_base = (void*)mmap(NULL, mem_size, PROT_WRITE|PROT_READ, MAP_SHARED, memptr->shm_id, 0)) == MAP_FAILED) { DEBUG_PRINT("SERVER: mmap failed (%s)\n", strerror(errno)); memory_release(memid); return -1; } } ASSERT(memptr->shm_base != NULL); if (do_create) { memptr->client_shm_base = memptr->shm_base; if (baseaddr == NULL) { /* Track client shmem address and server cqueue address */ for (int i = 0; i < max_ep + 1 + max_quant_params_count; i++) { void* cqueue_base = (intptr_t*)memptr->shm_base + i; if (i == 0) /* Index 0: client shared memory region */ *(intptr_t*)cqueue_base = (intptr_t)memptr->shm_base; else /* Index 1 to max_ep: server cqueue address */ *(intptr_t*)cqueue_base = (intptr_t)-1; } /* Create dlmalloc mspace only for allocated regions */ intptr_t mspace_start = (intptr_t)memptr->shm_base + (intptr_t)offset; memptr->shm_mspace = create_mspace_with_base((void*)mspace_start, mem_size-offset, 1); if (memptr->shm_mspace == NULL) { memory_release(memid); return -1; } } DEBUG_PRINT("CLIENT: %p %ld %ld %p %ld %p\n", memptr->shm_base, *(intptr_t*)memptr->shm_base, offset, memptr->shm_base + (intptr_t)offset, mem_size, memptr->shm_mspace); } else DEBUG_PRINT("SERVER: %p %ld %ld %p %ld\n", memptr->shm_base, *(intptr_t*)memptr->shm_base, offset, memptr->shm_base + (intptr_t)offset, mem_size); return memid; }