void* __hlt_realloc(void* p, uint64_t size, uint64_t old_size, const char* type,
const char* location)
{
#ifdef DEBUG
if ( p ) {
++__hlt_globals()->num_deallocs;
_dbg_mem_raw("free", p, size, type, location, "realloc", 0);
}
#endif
if ( size > old_size ) {
p = realloc(p, size);
memset((char*)p + old_size, 0, size - old_size);
if ( ! p ) {
fputs("out of memory in hlt_malloc, aborting", stderr);
exit(1);
}
}
#ifdef DEBUG
++__hlt_globals()->num_allocs;
_dbg_mem_raw("malloc", p, size, type, location, "realloc", 0);
#endif
return p;
}
void __hlt_fiber_done()
{
if ( ! hlt_is_multi_threaded() )
return;
__hlt_fiber_pool_delete(__hlt_globals()->synced_fiber_pool);
if ( hlt_is_multi_threaded() && pthread_mutex_destroy(&__hlt_globals()->synced_fiber_pool_lock) != 0 )
fatal_error("cannot destroy mutex");
}
void __hlt_fiber_init()
{
if ( ! hlt_is_multi_threaded() ) {
__hlt_globals()->synced_fiber_pool = 0;
return;
}
if ( pthread_mutex_init(&__hlt_globals()->synced_fiber_pool_lock, 0) != 0 )
fatal_error("cannot init mutex");
__hlt_globals()->synced_fiber_pool = __hlt_fiber_pool_new();
}
void hlt_stack_free(void* stack, size_t size)
{
if ( munmap(stack, size) < 0 ) {
fprintf(stderr, "munmap failed: %s\n", strerror(errno));
exit(1);
}
#ifdef DEBUG
--__hlt_globals()->num_stacks;
__hlt_globals()->size_stacks -= size;
#endif
}
void __hlt_object_ref(const hlt_type_info* ti, void* obj, hlt_execution_context* ctx)
{
__hlt_gchdr* hdr = (__hlt_gchdr*)obj;
;
#ifdef DEBUG
if ( ! ti->gc ) {
_dbg_mem_gc("! ref", ti, obj, 0, 0, ctx);
_internal_memory_error(obj, "__hlt_object_ref", "object not garbage collected", ti);
}
#endif
#ifdef HLT_ATOMIC_REF_COUNTING
__atomic_add_fetch(&hdr->ref_cnt, 1, __ATOMIC_SEQ_CST);
#else
++hdr->ref_cnt;
#endif
#if 0
// This is ok now!
if ( new_ref_cnt <= 0 ) {
_dbg_mem_gc("! ref", ti, obj, 0, 0, ctx);
_internal_memory_error(obj, "__hlt_object_ref", "bad reference count", ti);
}
#endif
#ifdef DEBUG
++__hlt_globals()->num_refs;
_dbg_mem_gc("ref", ti, obj, 0, 0, ctx);
#endif
}
static inline void release_lock(int i)
{
if ( pthread_mutex_unlock(&__hlt_globals()->synced_fiber_pool_lock) != 0 )
fatal_error("cannot unlock mutex");
hlt_pthread_setcancelstate(i, NULL);
}
hlt_execution_context* __hlt_execution_context_new_ref(hlt_vthread_id vid, int8_t run_module_init)
{
hlt_execution_context* ctx = (hlt_execution_context*)hlt_malloc(sizeof(hlt_execution_context) +
__hlt_globals()->globals_size);
ctx->vid = vid;
ctx->nullbuffer = __hlt_memory_nullbuffer_new(); // init first
ctx->excpt = 0;
ctx->fiber = 0;
ctx->fiber_pool = __hlt_fiber_pool_new();
ctx->worker = 0;
ctx->tcontext = 0;
ctx->tcontext_type = 0;
ctx->pstate = 0;
ctx->blockable = 0;
ctx->tmgr = hlt_timer_mgr_new(&ctx->excpt, ctx);
GC_CCTOR(ctx->tmgr, hlt_timer_mgr, ctx);
__hlt_globals_init(ctx);
if ( run_module_init )
__hlt_modules_init(ctx);
if ( ctx->excpt )
hlt_exception_print_uncaught_abort(ctx->excpt, ctx);
return ctx;
}
static inline void acqire_lock(int* i)
{
hlt_pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, i);
if ( pthread_mutex_lock(&__hlt_globals()->synced_fiber_pool_lock) != 0 )
fatal_error("cannot lock mutex");
}
hlt_fiber* hlt_fiber_create(hlt_fiber_func func, hlt_execution_context* fctx, void* p, hlt_execution_context* ctx)
{
assert(ctx);
// If there's a fiber available in the local pool, use that. Otherwise
// check the global. Otherwise, create one.
__hlt_fiber_pool* fiber_pool = ctx->worker ? ctx->worker->fiber_pool : ctx->fiber_pool;
assert(fiber_pool);
hlt_fiber* fiber = 0;
if ( ! fiber_pool->head && hlt_is_multi_threaded() ) {
__hlt_fiber_pool* global_pool = __hlt_globals()->synced_fiber_pool;
// We do this without locking first, should be fine to encounter a
// race.
if ( global_pool->size ) {
int s = 0;
acqire_lock(&s);
int n = hlt_config_get()->fiber_max_pool_size / 5; // 20%
while ( global_pool->head && n-- ) {
fiber = global_pool->head;
global_pool->head = fiber->next;
--global_pool->size;
fiber->next = fiber_pool->head;
fiber_pool->head = fiber;
++fiber_pool->size;
}
// fprintf(stderr, "vid %lu took %lu from global, that now at %lu\n", ctx->vid, hlt_config_get()->fiber_max_pool_size / 10, global_pool->size);
release_lock(s);
}
}
if ( fiber_pool->head ) {
fiber = fiber_pool->head;
fiber_pool->head = fiber_pool->head->next;
--fiber_pool->size;
fiber->next = 0;
assert(fiber->state == IDLE);
}
else
fiber = __hlt_fiber_create(fctx);
fiber->run = func;
fiber->context = fctx;
fiber->cookie = p;
return fiber;
}
void __hlt_free(void* memory, const char* type, const char* location)
{
if ( ! memory )
return;
#ifdef DEBUG
++__hlt_globals()->num_deallocs;
_dbg_mem_raw("free", memory, 0, type, location, 0, 0);
#endif
free(memory);
}
void* hlt_stack_alloc(size_t size)
{
#ifdef DARWIN
void* stack = mmap(0, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
#else
void* stack =
mmap(0, size, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK | MAP_GROWSDOWN | MAP_NORESERVE, -1, 0);
#endif
if ( stack == MAP_FAILED ) {
fprintf(stderr, "mmap failed: %s\n", strerror(errno));
exit(1);
}
#ifdef DEBUG
++__hlt_globals()->num_stacks;
__hlt_globals()->size_stacks += size;
#endif
return stack;
}
void __hlt_memory_nullbuffer_remove(__hlt_memory_nullbuffer* nbuf, void* obj)
{
// TODO: Unclear if it's worth keeping the buffer sorted to speed this up?
for ( size_t i = 0; i < nbuf->used; i++ ) {
struct __obj_with_rtti x = nbuf->objs[i];
if ( x.obj == obj ) {
// Mark as done.
x.obj = 0;
--__hlt_globals()->num_nullbuffer;
break;
}
}
}
void __hlt_memory_nullbuffer_flush(__hlt_memory_nullbuffer* nbuf, hlt_execution_context* ctx)
{
if ( nbuf->flush_pos >= 0 )
return;
#ifdef DEBUG
_dbg_mem_raw("nullbuffer_flush", nbuf, nbuf->used, 0, "start", 0, ctx);
#endif
// Note, flush_pos is examined during flushing by nullbuffer_add().
for ( nbuf->flush_pos = 0; nbuf->flush_pos < nbuf->used; ++nbuf->flush_pos ) {
struct __obj_with_rtti x = nbuf->objs[nbuf->flush_pos];
if ( ! x.obj )
// May have been removed.
continue;
#ifdef DEBUG
--__hlt_globals()->num_nullbuffer;
#endif
__hlt_gchdr* hdr = (__hlt_gchdr*)x.obj;
if ( hdr->ref_cnt > 0 )
// Still alive actually.
continue;
if ( x.ti->obj_dtor )
(*(x.ti->obj_dtor))(x.ti, x.obj, ctx);
__hlt_free(x.obj, x.ti->tag, "nullbuffer_flush");
}
nbuf->used = 0;
if ( nbuf->allocated > __INITIAL_NULLBUFFER_SIZE ) {
hlt_free(nbuf->objs);
nbuf->allocated = __INITIAL_NULLBUFFER_SIZE;
nbuf->objs =
(struct __obj_with_rtti*)hlt_malloc(sizeof(struct __obj_with_rtti) * nbuf->allocated);
}
#ifdef DEBUG
_dbg_mem_raw("nullbuffer_flush", nbuf, nbuf->used, 0, "end", 0, ctx);
#endif
nbuf->flush_pos = -1;
}
void* __hlt_calloc(uint64_t count, uint64_t size, const char* type, const char* location)
{
void* p = calloc(count, size);
if ( ! p ) {
fputs("out of memory in hlt_calloc, aborting", stderr);
exit(1);
}
#ifdef DEBUG
++__hlt_globals()->num_allocs;
_dbg_mem_raw("calloc", p, count * size, type, location, 0, 0);
#endif
return p;
}
void* __hlt_malloc_no_init(uint64_t size, const char* type, const char* location)
{
void* p = malloc(size);
if ( ! p ) {
fputs("out of memory in hlt_malloc_no_init, aborting", stderr);
exit(1);
}
#ifdef DEBUG
++__hlt_globals()->num_allocs;
_dbg_mem_raw("malloc_no_init", p, size, type, location, 0, 0);
#endif
return p;
}
hlt_memory_stats hlt_memory_statistics()
{
hlt_memory_stats stats;
hlt_memory_usage(&stats.size_heap, &stats.size_alloced);
__hlt_global_state* globals = __hlt_globals();
stats.num_allocs = globals->num_allocs;
stats.num_deallocs = globals->num_deallocs;
stats.num_refs = globals->num_refs;
stats.num_unrefs = globals->num_unrefs;
stats.size_stacks = globals->size_stacks;
stats.num_stacks = globals->num_stacks;
stats.num_nullbuffer = globals->num_nullbuffer;
stats.max_nullbuffer = globals->max_nullbuffer;
return stats;
}
void __hlt_object_unref(const hlt_type_info* ti, void* obj, hlt_execution_context* ctx)
{
if ( ! obj )
return;
__hlt_gchdr* hdr = (__hlt_gchdr*)obj;
#ifdef DEBUG
if ( ! ti->gc ) {
_dbg_mem_gc("! unref", ti, obj, 0, "", ctx);
_internal_memory_error(obj, "__hlt_object_unref", "object not garbage collected", ti);
}
#endif
#ifdef HLT_ATOMIC_REF_COUNTING
int64_t new_ref_cnt = __atomic_sub_fetch(&hdr->ref_cnt, 1, __ATOMIC_SEQ_CST);
#else
int64_t new_ref_cnt = --hdr->ref_cnt;
#endif
#ifdef DEBUG
const char* aux = 0;
if ( new_ref_cnt == 0 )
aux = "dtor";
#if 0
// This is now ok !
if ( new_ref_cnt < 0 ) {
_dbg_mem_gc("! unref", ti, obj, 0, aux, ctx);
_internal_memory_error(obj, "__hlt_object_unref", "bad reference count", ti);
}
#endif
#endif
#ifdef DEBUG
++__hlt_globals()->num_unrefs;
_dbg_mem_gc("unref", ti, obj, 0, aux, ctx);
#endif
if ( new_ref_cnt == 0 )
__hlt_memory_nullbuffer_add(ctx->nullbuffer, ti, hdr, ctx);
}
void __hlt_memory_nullbuffer_add(__hlt_memory_nullbuffer* nbuf, const hlt_type_info* ti, void* obj,
hlt_execution_context* ctx)
{
int64_t nbpos = _nullbuffer_index(nbuf, obj);
if ( nbuf->flush_pos >= 0 ) {
// We're flushing.
if ( nbpos == nbuf->flush_pos )
// Deleting this right now.
return;
if ( nbpos > nbuf->flush_pos )
// In the buffer, and still getting there.
return;
// Either not yet in the buffer, or we passed it already. Delete
// directly, we must not further modify the nullbuffer while flushing.
#ifdef DEBUG
__hlt_gchdr* hdr = (__hlt_gchdr*)obj;
assert(hdr->ref_cnt <= 0);
_UNUSED(hdr);
#endif
if ( ti->obj_dtor )
(*(ti->obj_dtor))(ti, obj, ctx);
if ( nbpos >= 0 )
// Just to be safe.
nbuf->objs[nbpos].obj = 0;
__hlt_free(obj, ti->tag, "nullbuffer_add (during flush)");
return;
}
if ( nbpos >= 0 )
// Already in the buffer.
return;
#ifdef DEBUG
__hlt_gchdr* hdr = (__hlt_gchdr*)obj;
_dbg_mem_gc("nullbuffer_add", ti, hdr, "", 0, ctx);
#endif
if ( nbuf->used >= nbuf->allocated ) {
size_t nsize = (nbuf->allocated * 2);
nbuf->objs =
(struct __obj_with_rtti*)hlt_realloc(nbuf->objs, sizeof(struct __obj_with_rtti) * nsize,
sizeof(struct __obj_with_rtti) * nbuf->allocated);
nbuf->allocated = nsize;
}
struct __obj_with_rtti x;
x.ti = ti;
x.obj = obj;
nbuf->objs[nbuf->used++] = x;
#ifdef DEBUG
++__hlt_globals()->num_nullbuffer;
if ( nbuf->used > __hlt_globals()->max_nullbuffer )
// Not thread-safe, but doesn't matter.
__hlt_globals()->max_nullbuffer = nbuf->used;
#endif
}
void hlt_fiber_delete(hlt_fiber* fiber, hlt_execution_context* ctx)
{
assert(! fiber->next);
if ( ! ctx ) {
__hlt_fiber_delete(fiber);
return;
}
__hlt_fiber_pool* fiber_pool = ctx->worker ? ctx->worker->fiber_pool : ctx->fiber_pool;
// Return the fiber to the local pool as long as we haven't reached us
// maximum pool size yet. If we have, return the pool to the global pool first.
if ( fiber_pool->size >= hlt_config_get()->fiber_max_pool_size ) {
if ( hlt_is_multi_threaded() ) {
__hlt_fiber_pool* global_pool = __hlt_globals()->synced_fiber_pool;
// We do this without locking, should be fine to encounter a
// race.
if ( global_pool->size <= 10 * hlt_config_get()->fiber_max_pool_size ) {
int s = 0;
acqire_lock(&s);
// Check again.
if ( global_pool->size <= 10 * hlt_config_get()->fiber_max_pool_size ) {
// fprintf(stderr, "vid %lu gives %lu to global, that then at %lu\n", ctx->vid, fiber_pool->size, global_pool->size + fiber_pool->size);
hlt_fiber* tail;
for ( tail = global_pool->head; tail && tail->next; tail = tail->next )
;
if ( tail )
tail->next = fiber_pool->head;
else
global_pool->head = fiber_pool->head;
global_pool->size += fiber_pool->size;
fiber_pool->head = 0;
fiber_pool->size = 0;
release_lock(s);
goto return_to_local;
}
release_lock(s);
}
}
// Local and global have reached their size, just return.
__hlt_fiber_delete(fiber);
return;
}
return_to_local:
hlt_stack_invalidate(fiber->uctx.uc_stack.ss_sp, fiber->uctx.uc_stack.ss_size);
fiber->next = fiber_pool->head;
fiber_pool->head = fiber;
++fiber_pool->size;
}