void* __hlt_realloc(void* p, uint64_t size, uint64_t old_size, const char* type, const char* location) { #ifdef DEBUG if ( p ) { ++__hlt_globals()->num_deallocs; _dbg_mem_raw("free", p, size, type, location, "realloc", 0); } #endif if ( size > old_size ) { p = realloc(p, size); memset((char*)p + old_size, 0, size - old_size); if ( ! p ) { fputs("out of memory in hlt_malloc, aborting", stderr); exit(1); } } #ifdef DEBUG ++__hlt_globals()->num_allocs; _dbg_mem_raw("malloc", p, size, type, location, "realloc", 0); #endif return p; }
void __hlt_fiber_done() { if ( ! hlt_is_multi_threaded() ) return; __hlt_fiber_pool_delete(__hlt_globals()->synced_fiber_pool); if ( hlt_is_multi_threaded() && pthread_mutex_destroy(&__hlt_globals()->synced_fiber_pool_lock) != 0 ) fatal_error("cannot destroy mutex"); }
void __hlt_fiber_init() { if ( ! hlt_is_multi_threaded() ) { __hlt_globals()->synced_fiber_pool = 0; return; } if ( pthread_mutex_init(&__hlt_globals()->synced_fiber_pool_lock, 0) != 0 ) fatal_error("cannot init mutex"); __hlt_globals()->synced_fiber_pool = __hlt_fiber_pool_new(); }
void hlt_stack_free(void* stack, size_t size) { if ( munmap(stack, size) < 0 ) { fprintf(stderr, "munmap failed: %s\n", strerror(errno)); exit(1); } #ifdef DEBUG --__hlt_globals()->num_stacks; __hlt_globals()->size_stacks -= size; #endif }
void __hlt_object_ref(const hlt_type_info* ti, void* obj, hlt_execution_context* ctx) { __hlt_gchdr* hdr = (__hlt_gchdr*)obj; ; #ifdef DEBUG if ( ! ti->gc ) { _dbg_mem_gc("! ref", ti, obj, 0, 0, ctx); _internal_memory_error(obj, "__hlt_object_ref", "object not garbage collected", ti); } #endif #ifdef HLT_ATOMIC_REF_COUNTING __atomic_add_fetch(&hdr->ref_cnt, 1, __ATOMIC_SEQ_CST); #else ++hdr->ref_cnt; #endif #if 0 // This is ok now! if ( new_ref_cnt <= 0 ) { _dbg_mem_gc("! ref", ti, obj, 0, 0, ctx); _internal_memory_error(obj, "__hlt_object_ref", "bad reference count", ti); } #endif #ifdef DEBUG ++__hlt_globals()->num_refs; _dbg_mem_gc("ref", ti, obj, 0, 0, ctx); #endif }
static inline void release_lock(int i) { if ( pthread_mutex_unlock(&__hlt_globals()->synced_fiber_pool_lock) != 0 ) fatal_error("cannot unlock mutex"); hlt_pthread_setcancelstate(i, NULL); }
hlt_execution_context* __hlt_execution_context_new_ref(hlt_vthread_id vid, int8_t run_module_init) { hlt_execution_context* ctx = (hlt_execution_context*)hlt_malloc(sizeof(hlt_execution_context) + __hlt_globals()->globals_size); ctx->vid = vid; ctx->nullbuffer = __hlt_memory_nullbuffer_new(); // init first ctx->excpt = 0; ctx->fiber = 0; ctx->fiber_pool = __hlt_fiber_pool_new(); ctx->worker = 0; ctx->tcontext = 0; ctx->tcontext_type = 0; ctx->pstate = 0; ctx->blockable = 0; ctx->tmgr = hlt_timer_mgr_new(&ctx->excpt, ctx); GC_CCTOR(ctx->tmgr, hlt_timer_mgr, ctx); __hlt_globals_init(ctx); if ( run_module_init ) __hlt_modules_init(ctx); if ( ctx->excpt ) hlt_exception_print_uncaught_abort(ctx->excpt, ctx); return ctx; }
static inline void acqire_lock(int* i) { hlt_pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, i); if ( pthread_mutex_lock(&__hlt_globals()->synced_fiber_pool_lock) != 0 ) fatal_error("cannot lock mutex"); }
hlt_fiber* hlt_fiber_create(hlt_fiber_func func, hlt_execution_context* fctx, void* p, hlt_execution_context* ctx) { assert(ctx); // If there's a fiber available in the local pool, use that. Otherwise // check the global. Otherwise, create one. __hlt_fiber_pool* fiber_pool = ctx->worker ? ctx->worker->fiber_pool : ctx->fiber_pool; assert(fiber_pool); hlt_fiber* fiber = 0; if ( ! fiber_pool->head && hlt_is_multi_threaded() ) { __hlt_fiber_pool* global_pool = __hlt_globals()->synced_fiber_pool; // We do this without locking first, should be fine to encounter a // race. if ( global_pool->size ) { int s = 0; acqire_lock(&s); int n = hlt_config_get()->fiber_max_pool_size / 5; // 20% while ( global_pool->head && n-- ) { fiber = global_pool->head; global_pool->head = fiber->next; --global_pool->size; fiber->next = fiber_pool->head; fiber_pool->head = fiber; ++fiber_pool->size; } // fprintf(stderr, "vid %lu took %lu from global, that now at %lu\n", ctx->vid, hlt_config_get()->fiber_max_pool_size / 10, global_pool->size); release_lock(s); } } if ( fiber_pool->head ) { fiber = fiber_pool->head; fiber_pool->head = fiber_pool->head->next; --fiber_pool->size; fiber->next = 0; assert(fiber->state == IDLE); } else fiber = __hlt_fiber_create(fctx); fiber->run = func; fiber->context = fctx; fiber->cookie = p; return fiber; }
void __hlt_free(void* memory, const char* type, const char* location) { if ( ! memory ) return; #ifdef DEBUG ++__hlt_globals()->num_deallocs; _dbg_mem_raw("free", memory, 0, type, location, 0, 0); #endif free(memory); }
void* hlt_stack_alloc(size_t size) { #ifdef DARWIN void* stack = mmap(0, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); #else void* stack = mmap(0, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK | MAP_GROWSDOWN | MAP_NORESERVE, -1, 0); #endif if ( stack == MAP_FAILED ) { fprintf(stderr, "mmap failed: %s\n", strerror(errno)); exit(1); } #ifdef DEBUG ++__hlt_globals()->num_stacks; __hlt_globals()->size_stacks += size; #endif return stack; }
void __hlt_memory_nullbuffer_remove(__hlt_memory_nullbuffer* nbuf, void* obj) { // TODO: Unclear if it's worth keeping the buffer sorted to speed this up? for ( size_t i = 0; i < nbuf->used; i++ ) { struct __obj_with_rtti x = nbuf->objs[i]; if ( x.obj == obj ) { // Mark as done. x.obj = 0; --__hlt_globals()->num_nullbuffer; break; } } }
void __hlt_memory_nullbuffer_flush(__hlt_memory_nullbuffer* nbuf, hlt_execution_context* ctx) { if ( nbuf->flush_pos >= 0 ) return; #ifdef DEBUG _dbg_mem_raw("nullbuffer_flush", nbuf, nbuf->used, 0, "start", 0, ctx); #endif // Note, flush_pos is examined during flushing by nullbuffer_add(). for ( nbuf->flush_pos = 0; nbuf->flush_pos < nbuf->used; ++nbuf->flush_pos ) { struct __obj_with_rtti x = nbuf->objs[nbuf->flush_pos]; if ( ! x.obj ) // May have been removed. continue; #ifdef DEBUG --__hlt_globals()->num_nullbuffer; #endif __hlt_gchdr* hdr = (__hlt_gchdr*)x.obj; if ( hdr->ref_cnt > 0 ) // Still alive actually. continue; if ( x.ti->obj_dtor ) (*(x.ti->obj_dtor))(x.ti, x.obj, ctx); __hlt_free(x.obj, x.ti->tag, "nullbuffer_flush"); } nbuf->used = 0; if ( nbuf->allocated > __INITIAL_NULLBUFFER_SIZE ) { hlt_free(nbuf->objs); nbuf->allocated = __INITIAL_NULLBUFFER_SIZE; nbuf->objs = (struct __obj_with_rtti*)hlt_malloc(sizeof(struct __obj_with_rtti) * nbuf->allocated); } #ifdef DEBUG _dbg_mem_raw("nullbuffer_flush", nbuf, nbuf->used, 0, "end", 0, ctx); #endif nbuf->flush_pos = -1; }
void* __hlt_calloc(uint64_t count, uint64_t size, const char* type, const char* location) { void* p = calloc(count, size); if ( ! p ) { fputs("out of memory in hlt_calloc, aborting", stderr); exit(1); } #ifdef DEBUG ++__hlt_globals()->num_allocs; _dbg_mem_raw("calloc", p, count * size, type, location, 0, 0); #endif return p; }
void* __hlt_malloc_no_init(uint64_t size, const char* type, const char* location) { void* p = malloc(size); if ( ! p ) { fputs("out of memory in hlt_malloc_no_init, aborting", stderr); exit(1); } #ifdef DEBUG ++__hlt_globals()->num_allocs; _dbg_mem_raw("malloc_no_init", p, size, type, location, 0, 0); #endif return p; }
hlt_memory_stats hlt_memory_statistics() { hlt_memory_stats stats; hlt_memory_usage(&stats.size_heap, &stats.size_alloced); __hlt_global_state* globals = __hlt_globals(); stats.num_allocs = globals->num_allocs; stats.num_deallocs = globals->num_deallocs; stats.num_refs = globals->num_refs; stats.num_unrefs = globals->num_unrefs; stats.size_stacks = globals->size_stacks; stats.num_stacks = globals->num_stacks; stats.num_nullbuffer = globals->num_nullbuffer; stats.max_nullbuffer = globals->max_nullbuffer; return stats; }
void __hlt_object_unref(const hlt_type_info* ti, void* obj, hlt_execution_context* ctx) { if ( ! obj ) return; __hlt_gchdr* hdr = (__hlt_gchdr*)obj; #ifdef DEBUG if ( ! ti->gc ) { _dbg_mem_gc("! unref", ti, obj, 0, "", ctx); _internal_memory_error(obj, "__hlt_object_unref", "object not garbage collected", ti); } #endif #ifdef HLT_ATOMIC_REF_COUNTING int64_t new_ref_cnt = __atomic_sub_fetch(&hdr->ref_cnt, 1, __ATOMIC_SEQ_CST); #else int64_t new_ref_cnt = --hdr->ref_cnt; #endif #ifdef DEBUG const char* aux = 0; if ( new_ref_cnt == 0 ) aux = "dtor"; #if 0 // This is now ok ! if ( new_ref_cnt < 0 ) { _dbg_mem_gc("! unref", ti, obj, 0, aux, ctx); _internal_memory_error(obj, "__hlt_object_unref", "bad reference count", ti); } #endif #endif #ifdef DEBUG ++__hlt_globals()->num_unrefs; _dbg_mem_gc("unref", ti, obj, 0, aux, ctx); #endif if ( new_ref_cnt == 0 ) __hlt_memory_nullbuffer_add(ctx->nullbuffer, ti, hdr, ctx); }
void __hlt_memory_nullbuffer_add(__hlt_memory_nullbuffer* nbuf, const hlt_type_info* ti, void* obj, hlt_execution_context* ctx) { int64_t nbpos = _nullbuffer_index(nbuf, obj); if ( nbuf->flush_pos >= 0 ) { // We're flushing. if ( nbpos == nbuf->flush_pos ) // Deleting this right now. return; if ( nbpos > nbuf->flush_pos ) // In the buffer, and still getting there. return; // Either not yet in the buffer, or we passed it already. Delete // directly, we must not further modify the nullbuffer while flushing. #ifdef DEBUG __hlt_gchdr* hdr = (__hlt_gchdr*)obj; assert(hdr->ref_cnt <= 0); _UNUSED(hdr); #endif if ( ti->obj_dtor ) (*(ti->obj_dtor))(ti, obj, ctx); if ( nbpos >= 0 ) // Just to be safe. nbuf->objs[nbpos].obj = 0; __hlt_free(obj, ti->tag, "nullbuffer_add (during flush)"); return; } if ( nbpos >= 0 ) // Already in the buffer. return; #ifdef DEBUG __hlt_gchdr* hdr = (__hlt_gchdr*)obj; _dbg_mem_gc("nullbuffer_add", ti, hdr, "", 0, ctx); #endif if ( nbuf->used >= nbuf->allocated ) { size_t nsize = (nbuf->allocated * 2); nbuf->objs = (struct __obj_with_rtti*)hlt_realloc(nbuf->objs, sizeof(struct __obj_with_rtti) * nsize, sizeof(struct __obj_with_rtti) * nbuf->allocated); nbuf->allocated = nsize; } struct __obj_with_rtti x; x.ti = ti; x.obj = obj; nbuf->objs[nbuf->used++] = x; #ifdef DEBUG ++__hlt_globals()->num_nullbuffer; if ( nbuf->used > __hlt_globals()->max_nullbuffer ) // Not thread-safe, but doesn't matter. __hlt_globals()->max_nullbuffer = nbuf->used; #endif }
void hlt_fiber_delete(hlt_fiber* fiber, hlt_execution_context* ctx) { assert(! fiber->next); if ( ! ctx ) { __hlt_fiber_delete(fiber); return; } __hlt_fiber_pool* fiber_pool = ctx->worker ? ctx->worker->fiber_pool : ctx->fiber_pool; // Return the fiber to the local pool as long as we haven't reached us // maximum pool size yet. If we have, return the pool to the global pool first. if ( fiber_pool->size >= hlt_config_get()->fiber_max_pool_size ) { if ( hlt_is_multi_threaded() ) { __hlt_fiber_pool* global_pool = __hlt_globals()->synced_fiber_pool; // We do this without locking, should be fine to encounter a // race. if ( global_pool->size <= 10 * hlt_config_get()->fiber_max_pool_size ) { int s = 0; acqire_lock(&s); // Check again. if ( global_pool->size <= 10 * hlt_config_get()->fiber_max_pool_size ) { // fprintf(stderr, "vid %lu gives %lu to global, that then at %lu\n", ctx->vid, fiber_pool->size, global_pool->size + fiber_pool->size); hlt_fiber* tail; for ( tail = global_pool->head; tail && tail->next; tail = tail->next ) ; if ( tail ) tail->next = fiber_pool->head; else global_pool->head = fiber_pool->head; global_pool->size += fiber_pool->size; fiber_pool->head = 0; fiber_pool->size = 0; release_lock(s); goto return_to_local; } release_lock(s); } } // Local and global have reached their size, just return. __hlt_fiber_delete(fiber); return; } return_to_local: hlt_stack_invalidate(fiber->uctx.uc_stack.ss_sp, fiber->uctx.uc_stack.ss_size); fiber->next = fiber_pool->head; fiber_pool->head = fiber; ++fiber_pool->size; }