DUK_LOCAL void duk__run_voluntary_gc(duk_heap *heap) {
if (DUK_HEAP_HAS_MARKANDSWEEP_RUNNING(heap)) {
DUK_DD(DUK_DDPRINT("mark-and-sweep in progress -> skip voluntary mark-and-sweep now"));
} else {
duk_small_uint_t flags;
duk_bool_t rc;
DUK_D(DUK_DPRINT("triggering voluntary mark-and-sweep"));
flags = 0;
rc = duk_heap_mark_and_sweep(heap, flags);
DUK_UNREF(rc);
}
}
DUK_INTERNAL void *duk_heap_mem_alloc(duk_heap *heap, duk_size_t size) {
void *res;
duk_bool_t rc;
duk_small_int_t i;
DUK_ASSERT(heap != NULL);
DUK_ASSERT_DISABLE(size >= 0);
/*
* Voluntary periodic GC (if enabled)
*/
DUK__VOLUNTARY_PERIODIC_GC(heap);
/*
* First attempt
*/
#if defined(DUK_USE_GC_TORTURE)
/* simulate alloc failure on every alloc (except when mark-and-sweep is running) */
if (!DUK_HEAP_HAS_MARKANDSWEEP_RUNNING(heap)) {
DUK_DDD(DUK_DDDPRINT("gc torture enabled, pretend that first alloc attempt fails"));
res = NULL;
DUK_UNREF(res);
goto skip_attempt;
}
#endif
res = heap->alloc_func(heap->heap_udata, size);
if (res || size == 0) {
/* for zero size allocations NULL is allowed */
return res;
}
#if defined(DUK_USE_GC_TORTURE)
skip_attempt:
#endif
DUK_D(DUK_DPRINT("first alloc attempt failed, attempt to gc and retry"));
/*
* Avoid a GC if GC is already running. This can happen at a late
* stage in a GC when we try to e.g. resize the stringtable
* or compact objects.
*/
if (DUK_HEAP_HAS_MARKANDSWEEP_RUNNING(heap)) {
DUK_D(DUK_DPRINT("duk_heap_mem_alloc() failed, gc in progress (gc skipped), alloc size %ld", (long) size));
return NULL;
}
/*
* Retry with several GC attempts. Initial attempts are made without
* emergency mode; later attempts use emergency mode which minimizes
* memory allocations forcibly.
*/
for (i = 0; i < DUK_HEAP_ALLOC_FAIL_MARKANDSWEEP_LIMIT; i++) {
duk_small_uint_t flags;
flags = 0;
if (i >= DUK_HEAP_ALLOC_FAIL_MARKANDSWEEP_EMERGENCY_LIMIT - 1) {
flags |= DUK_MS_FLAG_EMERGENCY;
}
rc = duk_heap_mark_and_sweep(heap, flags);
DUK_UNREF(rc);
res = heap->alloc_func(heap->heap_udata, size);
if (res) {
DUK_D(DUK_DPRINT("duk_heap_mem_alloc() succeeded after gc (pass %ld), alloc size %ld",
(long) (i + 1), (long) size));
return res;
}
}
DUK_D(DUK_DPRINT("duk_heap_mem_alloc() failed even after gc, alloc size %ld", (long) size));
return NULL;
}
DUK_INTERNAL void *duk_heap_mem_realloc_indirect(duk_heap *heap, duk_mem_getptr cb, void *ud, duk_size_t newsize) {
void *res;
duk_bool_t rc;
duk_small_int_t i;
DUK_ASSERT(heap != NULL);
DUK_ASSERT_DISABLE(newsize >= 0);
/*
* Voluntary periodic GC (if enabled)
*/
DUK__VOLUNTARY_PERIODIC_GC(heap);
/*
* First attempt
*/
#if defined(DUK_USE_GC_TORTURE)
/* simulate alloc failure on every realloc (except when mark-and-sweep is running) */
if (!DUK_HEAP_HAS_MARKANDSWEEP_RUNNING(heap)) {
DUK_DDD(DUK_DDDPRINT("gc torture enabled, pretend that first indirect realloc attempt fails"));
res = NULL;
DUK_UNREF(res);
goto skip_attempt;
}
#endif
res = heap->realloc_func(heap->heap_udata, cb(heap, ud), newsize);
if (res || newsize == 0) {
/* for zero size allocations NULL is allowed */
return res;
}
#if defined(DUK_USE_GC_TORTURE)
skip_attempt:
#endif
DUK_D(DUK_DPRINT("first indirect realloc attempt failed, attempt to gc and retry"));
/*
* Avoid a GC if GC is already running. See duk_heap_mem_alloc().
*/
if (DUK_HEAP_HAS_MARKANDSWEEP_RUNNING(heap)) {
DUK_D(DUK_DPRINT("duk_heap_mem_realloc_indirect() failed, gc in progress (gc skipped), alloc size %ld", (long) newsize));
return NULL;
}
/*
* Retry with several GC attempts. Initial attempts are made without
* emergency mode; later attempts use emergency mode which minimizes
* memory allocations forcibly.
*/
for (i = 0; i < DUK_HEAP_ALLOC_FAIL_MARKANDSWEEP_LIMIT; i++) {
duk_small_uint_t flags;
#if defined(DUK_USE_ASSERTIONS)
void *ptr_pre; /* ptr before mark-and-sweep */
void *ptr_post;
#endif
#if defined(DUK_USE_ASSERTIONS)
ptr_pre = cb(heap, ud);
#endif
flags = 0;
if (i >= DUK_HEAP_ALLOC_FAIL_MARKANDSWEEP_EMERGENCY_LIMIT - 1) {
flags |= DUK_MS_FLAG_EMERGENCY;
}
rc = duk_heap_mark_and_sweep(heap, flags);
DUK_UNREF(rc);
#if defined(DUK_USE_ASSERTIONS)
ptr_post = cb(heap, ud);
if (ptr_pre != ptr_post) {
/* useful for debugging */
DUK_DD(DUK_DDPRINT("note: base pointer changed by mark-and-sweep: %p -> %p",
(void *) ptr_pre, (void *) ptr_post));
}
#endif
/* Note: key issue here is to re-lookup the base pointer on every attempt.
* The pointer being reallocated may change after every mark-and-sweep.
*/
res = heap->realloc_func(heap->heap_udata, cb(heap, ud), newsize);
if (res || newsize == 0) {
DUK_D(DUK_DPRINT("duk_heap_mem_realloc_indirect() succeeded after gc (pass %ld), alloc size %ld",
(long) (i + 1), (long) newsize));
return res;
}
}
DUK_D(DUK_DPRINT("duk_heap_mem_realloc_indirect() failed even after gc, alloc size %ld", (long) newsize));
return NULL;
}
void duk_heap_heaphdr_decref(duk_hthread *thr, duk_heaphdr *h) {
duk_heap *heap;
#if 0
DUK_DDD(DUK_DDDPRINT("heaphdr decref %p (%ld->%ld): %!O",
(void *) h,
(h != NULL ? (long) h->h_refcount : (long) 0),
(h != NULL ? (long) (h->h_refcount - 1) : (long) 0),
(duk_heaphdr *) h));
#endif
DUK_ASSERT(thr != NULL);
DUK_ASSERT(thr->heap != NULL);
if (!h) {
return;
}
DUK_ASSERT(DUK_HEAPHDR_HTYPE_VALID(h));
DUK_ASSERT(h->h_refcount >= 1);
if (--h->h_refcount != 0) {
return;
}
heap = thr->heap;
DUK_DDD(DUK_DDDPRINT("refzero %p: %!O", (void *) h, (duk_heaphdr *) h));
#ifdef DUK_USE_MARK_AND_SWEEP
/*
* If mark-and-sweep is running, don't process 'refzero' situations at all.
* They may happen because mark-and-sweep needs to finalize refcounts for
* each object it sweeps. Otherwise the target objects of swept objects
* would have incorrect refcounts.
*
* Note: mark-and-sweep could use a separate decref handler to avoid coming
* here at all. However, mark-and-sweep may also call finalizers, which
* can do arbitrary operations and would use this decref variant anyway.
*/
if (DUK_HEAP_HAS_MARKANDSWEEP_RUNNING(heap)) {
DUK_DDD(DUK_DDDPRINT("refzero handling suppressed when mark-and-sweep running, object: %p", (void *) h));
return;
}
#endif
switch ((duk_small_int_t) DUK_HEAPHDR_GET_TYPE(h)) {
case DUK_HTYPE_STRING:
/*
* Strings have no internal references but do have "weak"
* references in the string cache. Also note that strings
* are not on the heap_allocated list like other heap
* elements.
*/
duk_heap_strcache_string_remove(heap, (duk_hstring *) h);
duk_heap_string_remove(heap, (duk_hstring *) h);
duk_heap_free_heaphdr_raw(heap, h);
break;
case DUK_HTYPE_OBJECT:
/*
* Objects have internal references. Must finalize through
* the "refzero" work list.
*/
duk_heap_remove_any_from_heap_allocated(heap, h);
duk__queue_refzero(heap, h);
duk__refzero_free_pending(thr);
break;
case DUK_HTYPE_BUFFER:
/*
* Buffers have no internal references. However, a dynamic
* buffer has a separate allocation for the buffer. This is
* freed by duk_heap_free_heaphdr_raw().
*/
duk_heap_remove_any_from_heap_allocated(heap, h);
duk_heap_free_heaphdr_raw(heap, h);
break;
default:
DUK_D(DUK_DPRINT("invalid heap type in decref: %ld", (long) DUK_HEAPHDR_GET_TYPE(h)));
DUK_UNREACHABLE();
}
}
DUK_INTERNAL duk_bool_t duk_heap_mark_and_sweep(duk_heap *heap, duk_small_uint_t flags) {
duk_hthread *thr;
duk_size_t count_keep_obj;
duk_size_t count_keep_str;
#if defined(DUK_USE_VOLUNTARY_GC)
duk_size_t tmp;
#endif
/* XXX: thread selection for mark-and-sweep is currently a hack.
* If we don't have a thread, the entire mark-and-sweep is now
* skipped (although we could just skip finalizations).
*/
/* If thr != NULL, the thr may still be in the middle of
* initialization.
* XXX: Improve the thread viability test.
*/
thr = duk__get_temp_hthread(heap);
if (thr == NULL) {
DUK_D(DUK_DPRINT("gc skipped because we don't have a temp thread"));
/* reset voluntary gc trigger count */
#if defined(DUK_USE_VOLUNTARY_GC)
heap->mark_and_sweep_trigger_counter = DUK_HEAP_MARK_AND_SWEEP_TRIGGER_SKIP;
#endif
return 0; /* OK */
}
/* If debugger is paused, garbage collection is disabled by default. */
/* XXX: will need a force flag if garbage collection is triggered
* explicitly during paused state.
*/
#if defined(DUK_USE_DEBUGGER_SUPPORT)
if (DUK_HEAP_IS_PAUSED(heap)) {
/* Checking this here rather that in memory alloc primitives
* reduces checking code there but means a failed allocation
* will go through a few retries before giving up. That's
* fine because this only happens during debugging.
*/
DUK_D(DUK_DPRINT("gc skipped because debugger is paused"));
return 0;
}
#endif
DUK_D(DUK_DPRINT("garbage collect (mark-and-sweep) starting, requested flags: 0x%08lx, effective flags: 0x%08lx",
(unsigned long) flags, (unsigned long) (flags | heap->mark_and_sweep_base_flags)));
flags |= heap->mark_and_sweep_base_flags;
/*
* Assertions before
*/
#if defined(DUK_USE_ASSERTIONS)
DUK_ASSERT(!DUK_HEAP_HAS_MARKANDSWEEP_RUNNING(heap));
DUK_ASSERT(!DUK_HEAP_HAS_MARKANDSWEEP_RECLIMIT_REACHED(heap));
DUK_ASSERT(heap->mark_and_sweep_recursion_depth == 0);
duk__assert_heaphdr_flags(heap);
#if defined(DUK_USE_REFERENCE_COUNTING)
/* Note: DUK_HEAP_HAS_REFZERO_FREE_RUNNING(heap) may be true; a refcount
* finalizer may trigger a mark-and-sweep.
*/
duk__assert_valid_refcounts(heap);
#endif /* DUK_USE_REFERENCE_COUNTING */
#endif /* DUK_USE_ASSERTIONS */
/*
* Begin
*/
DUK_HEAP_SET_MARKANDSWEEP_RUNNING(heap);
/*
* Mark roots, hoping that recursion limit is not normally hit.
* If recursion limit is hit, run additional reachability rounds
* starting from "temproots" until marking is complete.
*
* Marking happens in two phases: first we mark actual reachability
* roots (and run "temproots" to complete the process). Then we
* check which objects are unreachable and are finalizable; such
* objects are marked as FINALIZABLE and marked as reachability
* (and "temproots" is run again to complete the process).
*
* The heap finalize_list must also be marked as a reachability root.
* There may be objects on the list from a previous round if the
* previous run had finalizer skip flag.
*/
duk__mark_roots_heap(heap); /* main reachability roots */
#if defined(DUK_USE_REFERENCE_COUNTING)
duk__mark_refzero_list(heap); /* refzero_list treated as reachability roots */
#endif
duk__mark_temproots_by_heap_scan(heap); /* temproots */
#if defined(DUK_USE_FINALIZER_SUPPORT)
duk__mark_finalizable(heap); /* mark finalizable as reachability roots */
duk__mark_finalize_list(heap); /* mark finalizer work list as reachability roots */
#endif
duk__mark_temproots_by_heap_scan(heap); /* temproots */
/*
* Sweep garbage and remove marking flags, and move objects with
* finalizers to the finalizer work list.
*
* Objects to be swept need to get their refcounts finalized before
* they are swept. In other words, their target object refcounts
* need to be decreased. This has to be done before freeing any
* objects to avoid decref'ing dangling pointers (which may happen
* even without bugs, e.g. with reference loops)
*
* Because strings don't point to other heap objects, similar
* finalization is not necessary for strings.
*/
/* XXX: more emergency behavior, e.g. find smaller hash sizes etc */
#if defined(DUK_USE_REFERENCE_COUNTING)
duk__finalize_refcounts(heap);
#endif
duk__sweep_heap(heap, flags, &count_keep_obj);
duk__sweep_stringtable(heap, &count_keep_str);
#if defined(DUK_USE_REFERENCE_COUNTING)
duk__clear_refzero_list_flags(heap);
#endif
#if defined(DUK_USE_FINALIZER_SUPPORT)
duk__clear_finalize_list_flags(heap);
#endif
/*
* Object compaction (emergency only).
*
* Object compaction is a separate step after sweeping, as there is
* more free memory for it to work with. Also, currently compaction
* may insert new objects into the heap allocated list and the string
* table which we don't want to do during a sweep (the reachability
* flags of such objects would be incorrect). The objects inserted
* are currently:
*
* - a temporary duk_hbuffer for a new properties allocation
* - if array part is abandoned, string keys are interned
*
* The object insertions go to the front of the list, so they do not
* cause an infinite loop (they are not compacted).
*/
if ((flags & DUK_MS_FLAG_EMERGENCY) &&
!(flags & DUK_MS_FLAG_NO_OBJECT_COMPACTION)) {
duk__compact_objects(heap);
}
/*
* String table resize check.
*
* This is mainly useful in emergency GC: if the string table load
* factor is really low for some reason, we can shrink the string
* table to a smaller size and free some memory in the process.
* Only execute in emergency GC. String table has internal flags
* to protect against recursive resizing if this mark-and-sweep pass
* was triggered by a string table resize.
*/
if (flags & DUK_MS_FLAG_EMERGENCY) {
DUK_D(DUK_DPRINT("stringtable resize check in emergency gc"));
duk_heap_strtable_force_resize(heap);
}
/*
* Finalize objects in the finalization work list. Finalized
* objects are queued back to heap_allocated with FINALIZED set.
*
* Since finalizers may cause arbitrary side effects, they are
* prevented during string table and object property allocation
* resizing using the DUK_MS_FLAG_NO_FINALIZERS flag in
* heap->mark_and_sweep_base_flags. In this case the objects
* remain in the finalization work list after mark-and-sweep
* exits and they may be finalized on the next pass.
*
* Finalization currently happens inside "MARKANDSWEEP_RUNNING"
* protection (no mark-and-sweep may be triggered by the
* finalizers). As a side effect:
*
* 1) an out-of-memory error inside a finalizer will not
* cause a mark-and-sweep and may cause the finalizer
* to fail unnecessarily
*
* 2) any temporary objects whose refcount decreases to zero
* during finalization will not be put into refzero_list;
* they can only be collected by another mark-and-sweep
*
* This is not optimal, but since the sweep for this phase has
* already happened, this is probably good enough for now.
*/
#if defined(DUK_USE_FINALIZER_SUPPORT)
#if defined(DUK_USE_MARKANDSWEEP_FINALIZER_TORTURE)
/* Cannot simulate individual finalizers because finalize_list only
* contains objects with actual finalizers. But simulate side effects
* from finalization by doing a bogus function call and resizing the
* stacks.
*/
if (flags & DUK_MS_FLAG_NO_FINALIZERS) {
DUK_D(DUK_DPRINT("skip mark-and-sweep torture finalizer, DUK_MS_FLAG_NO_FINALIZERS is set"));
} else if (!(thr->valstack != NULL && thr->callstack != NULL && thr->catchstack != NULL)) {
DUK_D(DUK_DPRINT("skip mark-and-sweep torture finalizer, thread not yet viable"));
} else {
DUK_D(DUK_DPRINT("run mark-and-sweep torture finalizer"));
duk__markandsweep_torture_finalizer(thr);
}
#endif /* DUK_USE_MARKANDSWEEP_FINALIZER_TORTURE */
if (flags & DUK_MS_FLAG_NO_FINALIZERS) {
DUK_D(DUK_DPRINT("finalizer run skipped because DUK_MS_FLAG_NO_FINALIZERS is set"));
} else {
duk__run_object_finalizers(heap, flags);
}
#endif /* DUK_USE_FINALIZER_SUPPORT */
/*
* Finish
*/
DUK_HEAP_CLEAR_MARKANDSWEEP_RUNNING(heap);
/*
* Assertions after
*/
#if defined(DUK_USE_ASSERTIONS)
DUK_ASSERT(!DUK_HEAP_HAS_MARKANDSWEEP_RUNNING(heap));
DUK_ASSERT(!DUK_HEAP_HAS_MARKANDSWEEP_RECLIMIT_REACHED(heap));
DUK_ASSERT(heap->mark_and_sweep_recursion_depth == 0);
duk__assert_heaphdr_flags(heap);
#if defined(DUK_USE_REFERENCE_COUNTING)
/* Note: DUK_HEAP_HAS_REFZERO_FREE_RUNNING(heap) may be true; a refcount
* finalizer may trigger a mark-and-sweep.
*/
duk__assert_valid_refcounts(heap);
#endif /* DUK_USE_REFERENCE_COUNTING */
#endif /* DUK_USE_ASSERTIONS */
/*
* Reset trigger counter
*/
#if defined(DUK_USE_VOLUNTARY_GC)
tmp = (count_keep_obj + count_keep_str) / 256;
heap->mark_and_sweep_trigger_counter = (duk_int_t) (
(tmp * DUK_HEAP_MARK_AND_SWEEP_TRIGGER_MULT) +
DUK_HEAP_MARK_AND_SWEEP_TRIGGER_ADD);
DUK_D(DUK_DPRINT("garbage collect (mark-and-sweep) finished: %ld objects kept, %ld strings kept, trigger reset to %ld",
(long) count_keep_obj, (long) count_keep_str, (long) heap->mark_and_sweep_trigger_counter));
#else
DUK_D(DUK_DPRINT("garbage collect (mark-and-sweep) finished: %ld objects kept, %ld strings kept, no voluntary trigger",
(long) count_keep_obj, (long) count_keep_str));
#endif
return 0; /* OK */
}
int duk_heap_mark_and_sweep(duk_heap *heap, int flags) {
duk_size_t count_keep_obj;
duk_size_t count_keep_str;
duk_size_t tmp;
/* FIXME: thread selection for mark-and-sweep is currently a hack.
* If we don't have a thread, the entire mark-and-sweep is now
* skipped (although we could just skip finalizations).
*/
if (duk__get_temp_hthread(heap) == NULL) {
DUK_D(DUK_DPRINT("temporary hack: gc skipped because we don't have a temp thread"));
/* reset voluntary gc trigger count */
#ifdef DUK_USE_VOLUNTARY_GC
heap->mark_and_sweep_trigger_counter = DUK_HEAP_MARK_AND_SWEEP_TRIGGER_SKIP;
#endif
return 0; /* OK */
}
DUK_D(DUK_DPRINT("garbage collect (mark-and-sweep) starting, requested flags: 0x%08x, effective flags: 0x%08x",
flags, flags | heap->mark_and_sweep_base_flags));
flags |= heap->mark_and_sweep_base_flags;
/*
* Assertions before
*/
#ifdef DUK_USE_ASSERTIONS
DUK_ASSERT(!DUK_HEAP_HAS_MARKANDSWEEP_RUNNING(heap));
DUK_ASSERT(!DUK_HEAP_HAS_MARKANDSWEEP_RECLIMIT_REACHED(heap));
DUK_ASSERT(heap->mark_and_sweep_recursion_depth == 0);
duk__assert_heaphdr_flags(heap);
#ifdef DUK_USE_REFERENCE_COUNTING
/* Note: DUK_HEAP_HAS_REFZERO_FREE_RUNNING(heap) may be true; a refcount
* finalizer may trigger a mark-and-sweep.
*/
duk__assert_valid_refcounts(heap);
#endif /* DUK_USE_REFERENCE_COUNTING */
#endif /* DUK_USE_ASSERTIONS */
/*
* Begin
*/
DUK_HEAP_SET_MARKANDSWEEP_RUNNING(heap);
/*
* Mark roots, hoping that recursion limit is not normally hit.
* If recursion limit is hit, run additional reachability rounds
* starting from "temproots" until marking is complete.
*
* Marking happens in two phases: first we mark actual reachability
* roots (and run "temproots" to complete the process). Then we
* check which objects are unreachable and are finalizable; such
* objects are marked as FINALIZABLE and marked as reachability
* (and "temproots" is run again to complete the process).
*/
duk__mark_roots_heap(heap); /* main reachability roots */
#ifdef DUK_USE_REFERENCE_COUNTING
duk__mark_refzero_list(heap); /* refzero_list treated as reachability roots */
#endif
duk__mark_temproots_by_heap_scan(heap); /* temproots */
duk__mark_finalizable(heap); /* mark finalizable as reachability roots */
duk__mark_temproots_by_heap_scan(heap); /* temproots */
/*
* Sweep garbage and remove marking flags, and move objects with
* finalizers to the finalizer work list.
*
* Objects to be swept need to get their refcounts finalized before
* they are swept. In other words, their target object refcounts
* need to be decreased. This has to be done before freeing any
* objects to avoid decref'ing dangling pointers (which may happen
* even without bugs, e.g. with reference loops)
*
* Because strings don't point to other heap objects, similar
* finalization is not necessary for strings.
*/
/* XXX: more emergency behavior, e.g. find smaller hash sizes etc */
#ifdef DUK_USE_REFERENCE_COUNTING
duk__finalize_refcounts(heap);
#endif
duk__sweep_heap(heap, flags, &count_keep_obj);
duk__sweep_stringtable(heap, &count_keep_str);
#ifdef DUK_USE_REFERENCE_COUNTING
duk__clear_refzero_list_flags(heap);
#endif
/*
* Object compaction (emergency only).
*
* Object compaction is a separate step after sweeping, as there is
* more free memory for it to work with. Also, currently compaction
* may insert new objects into the heap allocated list and the string
* table which we don't want to do during a sweep (the reachability
* flags of such objects would be incorrect). The objects inserted
* are currently:
*
* - a temporary duk_hbuffer for a new properties allocation
* - if array part is abandoned, string keys are interned
*
* The object insertions go to the front of the list, so they do not
* cause an infinite loop (they are not compacted).
*/
if ((flags & DUK_MS_FLAG_EMERGENCY) &&
!(flags & DUK_MS_FLAG_NO_OBJECT_COMPACTION)) {
duk__compact_objects(heap);
}
/*
* String table resize check.
*
* Note: this may silently (and safely) fail if GC is caused by an
* allocation call in stringtable resize_hash(). Resize_hash()
* will prevent a recursive call to itself by setting the
* DUK_MS_FLAG_NO_STRINGTABLE_RESIZE in heap->mark_and_sweep_base_flags.
*/
/* XXX: stringtable emergency compaction? */
#if defined(DUK_USE_MS_STRINGTABLE_RESIZE)
if (!(flags & DUK_MS_FLAG_NO_STRINGTABLE_RESIZE)) {
DUK_DD(DUK_DDPRINT("resize stringtable: %p", (void *) heap));
duk_heap_force_stringtable_resize(heap);
} else {
DUK_D(DUK_DPRINT("stringtable resize skipped because DUK_MS_FLAG_NO_STRINGTABLE_RESIZE is set"));
}
#endif
/*
* Finalize objects in the finalization work list. Finalized
* objects are queued back to heap_allocated with FINALIZED set.
*
* Since finalizers may cause arbitrary side effects, they are
* prevented during string table and object property allocation
* resizing using the DUK_MS_FLAG_NO_FINALIZERS flag in
* heap->mark_and_sweep_base_flags. In this case the objects
* remain in the finalization work list after mark-and-sweep
* exits and they may be finalized on the next pass.
*
* Finalization currently happens inside "MARKANDSWEEP_RUNNING"
* protection (no mark-and-sweep may be triggered by the
* finalizers). As a side effect:
*
* 1) an out-of-memory error inside a finalizer will not
* cause a mark-and-sweep and may cause the finalizer
* to fail unnecessarily
*
* 2) any temporary objects whose refcount decreases to zero
* during finalization will not be put into refzero_list;
* they can only be collected by another mark-and-sweep
*
* This is not optimal, but since the sweep for this phase has
* already happened, this is probably good enough for now.
*/
if (!(flags & DUK_MS_FLAG_NO_FINALIZERS)) {
duk__run_object_finalizers(heap);
} else {
DUK_D(DUK_DPRINT("finalizer run skipped because DUK_MS_FLAG_NO_FINALIZERS is set"));
}
/*
* Finish
*/
DUK_HEAP_CLEAR_MARKANDSWEEP_RUNNING(heap);
/*
* Assertions after
*/
#ifdef DUK_USE_ASSERTIONS
DUK_ASSERT(!DUK_HEAP_HAS_MARKANDSWEEP_RUNNING(heap));
DUK_ASSERT(!DUK_HEAP_HAS_MARKANDSWEEP_RECLIMIT_REACHED(heap));
DUK_ASSERT(heap->mark_and_sweep_recursion_depth == 0);
duk__assert_heaphdr_flags(heap);
#ifdef DUK_USE_REFERENCE_COUNTING
/* Note: DUK_HEAP_HAS_REFZERO_FREE_RUNNING(heap) may be true; a refcount
* finalizer may trigger a mark-and-sweep.
*/
duk__assert_valid_refcounts(heap);
#endif /* DUK_USE_REFERENCE_COUNTING */
#endif /* DUK_USE_ASSERTIONS */
/*
* Reset trigger counter
*/
#ifdef DUK_USE_VOLUNTARY_GC
tmp = (count_keep_obj + count_keep_str) / 256;
heap->mark_and_sweep_trigger_counter =
(tmp * DUK_HEAP_MARK_AND_SWEEP_TRIGGER_MULT) +
DUK_HEAP_MARK_AND_SWEEP_TRIGGER_ADD;
DUK_D(DUK_DPRINT("garbage collect (mark-and-sweep) finished: %d objects kept, %d strings kept, trigger reset to %d",
(int) count_keep_obj, (int) count_keep_str, (int) heap->mark_and_sweep_trigger_counter));
#else
DUK_D(DUK_DPRINT("garbage collect (mark-and-sweep) finished: %d objects kept, %d strings kept, no voluntary trigger",
(int) count_keep_obj, (int) count_keep_str));
#endif
return 0; /* OK */
}
void *duk_heap_mem_realloc(duk_heap *heap, void *ptr, size_t newsize) {
void *res;
int rc;
int i;
DUK_ASSERT(heap != NULL);
/* ptr may be NULL */
DUK_ASSERT_DISABLE(newsize >= 0);
/*
* Voluntary periodic GC (if enabled)
*/
DUK__VOLUNTARY_PERIODIC_GC(heap);
/*
* First attempt
*/
#ifdef DUK_USE_GC_TORTURE
/* simulate alloc failure on every realloc (except when mark-and-sweep is running) */
if (!DUK_HEAP_HAS_MARKANDSWEEP_RUNNING(heap)) {
DUK_DDD(DUK_DDDPRINT("gc torture enabled, pretend that first realloc attempt fails"));
res = NULL;
DUK_UNREF(res);
goto skip_attempt;
}
#endif
res = heap->realloc_func(heap->alloc_udata, ptr, newsize);
if (res || newsize == 0) {
/* for zero size allocations NULL is allowed */
return res;
}
#ifdef DUK_USE_GC_TORTURE
skip_attempt:
#endif
DUK_D(DUK_DPRINT("first realloc attempt failed, attempt to gc and retry"));
/*
* Avoid a GC if GC is already running. See duk_heap_mem_alloc().
*/
if (DUK_HEAP_HAS_MARKANDSWEEP_RUNNING(heap)) {
DUK_D(DUK_DPRINT("duk_heap_mem_realloc() failed, gc in progress (gc skipped), alloc size %d", newsize));
return NULL;
}
/*
* Retry with several GC attempts. Initial attempts are made without
* emergency mode; later attempts use emergency mode which minimizes
* memory allocations forcibly.
*/
for (i = 0; i < DUK_HEAP_ALLOC_FAIL_MARKANDSWEEP_LIMIT; i++) {
int flags;
flags = 0;
if (i >= DUK_HEAP_ALLOC_FAIL_MARKANDSWEEP_EMERGENCY_LIMIT - 1) {
flags |= DUK_MS_FLAG_EMERGENCY;
}
rc = duk_heap_mark_and_sweep(heap, flags);
DUK_UNREF(rc);
res = heap->realloc_func(heap->alloc_udata, ptr, newsize);
if (res) {
DUK_D(DUK_DPRINT("duk_heap_mem_realloc() succeeded after gc (pass %d), alloc size %d",
i + 1, newsize));
return res;
}
}
DUK_D(DUK_DPRINT("duk_heap_mem_realloc() failed even after gc, alloc size %d", newsize));
return NULL;
}
DUK_LOCAL void duk__free_run_finalizers(duk_heap *heap) {
duk_hthread *thr;
duk_heaphdr *curr;
duk_uint_t round_no;
duk_size_t count_all;
duk_size_t count_finalized;
duk_size_t curr_limit;
DUK_ASSERT(heap != NULL);
DUK_ASSERT(heap->heap_thread != NULL);
#if defined(DUK_USE_REFERENCE_COUNTING)
DUK_ASSERT(heap->refzero_list == NULL); /* refzero not running -> must be empty */
#endif
#if defined(DUK_USE_MARK_AND_SWEEP)
DUK_ASSERT(heap->finalize_list == NULL); /* mark-and-sweep not running -> must be empty */
#endif
/* XXX: here again finalizer thread is the heap_thread which needs
* to be coordinated with finalizer thread fixes.
*/
thr = heap->heap_thread;
DUK_ASSERT(thr != NULL);
/* Prevent mark-and-sweep for the pending finalizers, also prevents
* refzero handling from moving objects away from the heap_allocated
* list. (The flag meaning is slightly abused here.)
*/
DUK_ASSERT(!DUK_HEAP_HAS_MARKANDSWEEP_RUNNING(heap));
DUK_HEAP_SET_MARKANDSWEEP_RUNNING(heap);
curr_limit = 0; /* suppress warning, not used */
for (round_no = 0; ; round_no++) {
curr = heap->heap_allocated;
count_all = 0;
count_finalized = 0;
while (curr) {
count_all++;
if (DUK_HEAPHDR_GET_TYPE(curr) == DUK_HTYPE_OBJECT) {
/* Only objects in heap_allocated may have finalizers. Check that
* the object itself has a _Finalizer property (own or inherited)
* so that we don't execute finalizers for e.g. Proxy objects.
*/
DUK_ASSERT(thr != NULL);
DUK_ASSERT(curr != NULL);
if (duk_hobject_hasprop_raw(thr, (duk_hobject *) curr, DUK_HTHREAD_STRING_INT_FINALIZER(thr))) {
if (!DUK_HEAPHDR_HAS_FINALIZED((duk_heaphdr *) curr)) {
DUK_ASSERT(DUK_HEAP_HAS_FINALIZER_NORESCUE(heap)); /* maps to finalizer 2nd argument */
duk_hobject_run_finalizer(thr, (duk_hobject *) curr);
count_finalized++;
}
}
}
curr = DUK_HEAPHDR_GET_NEXT(heap, curr);
}
/* Each round of finalizer execution may spawn new finalizable objects
* which is normal behavior for some applications. Allow multiple
* rounds of finalization, but use a shrinking limit based on the
* first round to detect the case where a runaway finalizer creates
* an unbounded amount of new finalizable objects. Finalizer rescue
* is not supported: the semantics are unclear because most of the
* objects being finalized here are already reachable. The finalizer
* is given a boolean to indicate that rescue is not possible.
*
* See discussion in: https://github.com/svaarala/duktape/pull/473
*/
if (round_no == 0) {
/* Cannot wrap: each object is at least 8 bytes so count is
* at most 1/8 of that.
*/
curr_limit = count_all * 2;
} else {
curr_limit = (curr_limit * 3) / 4; /* Decrease by 25% every round */
}
DUK_D(DUK_DPRINT("finalizer round %ld complete, %ld objects, tried to execute %ld finalizers, current limit is %ld",
(long) round_no, (long) count_all, (long) count_finalized, (long) curr_limit));
if (count_finalized == 0) {
DUK_D(DUK_DPRINT("no more finalizable objects, forced finalization finished"));
break;
}
if (count_finalized >= curr_limit) {
DUK_D(DUK_DPRINT("finalizer count above limit, potentially runaway finalizer; skip remaining finalizers"));
break;
}
}
DUK_ASSERT(DUK_HEAP_HAS_MARKANDSWEEP_RUNNING(heap));
DUK_HEAP_CLEAR_MARKANDSWEEP_RUNNING(heap);
}
DUK_INTERNAL void duk_heaphdr_refzero(duk_hthread *thr, duk_heaphdr *h) {
duk_heap *heap;
DUK_ASSERT(thr != NULL);
DUK_ASSERT(h != NULL);
heap = thr->heap;
DUK_DDD(DUK_DDDPRINT("refzero %p: %!O", (void *) h, (duk_heaphdr *) h));
/*
* If mark-and-sweep is running, don't process 'refzero' situations at all.
* They may happen because mark-and-sweep needs to finalize refcounts for
* each object it sweeps. Otherwise the target objects of swept objects
* would have incorrect refcounts.
*
* This check must be enabled also when mark-and-sweep support has been
* disabled: the flag is also used in heap destruction when running
* finalizers for remaining objects, and the flag prevents objects from
* being moved around in heap linked lists.
*
* Note: mark-and-sweep could use a separate decref handler to avoid coming
* here at all. However, mark-and-sweep may also call finalizers, which
* can do arbitrary operations and would use this decref variant anyway.
*/
if (DUK_UNLIKELY(DUK_HEAP_HAS_MARKANDSWEEP_RUNNING(heap))) {
DUK_DDD(DUK_DDDPRINT("refzero handling suppressed when mark-and-sweep running, object: %p", (void *) h));
return;
}
switch ((duk_small_int_t) DUK_HEAPHDR_GET_TYPE(h)) {
case DUK_HTYPE_STRING:
/*
* Strings have no internal references but do have "weak"
* references in the string cache. Also note that strings
* are not on the heap_allocated list like other heap
* elements.
*/
duk_heap_strcache_string_remove(heap, (duk_hstring *) h);
duk_heap_string_remove(heap, (duk_hstring *) h);
duk_heap_free_heaphdr_raw(heap, h);
break;
case DUK_HTYPE_OBJECT:
/*
* Objects have internal references. Must finalize through
* the "refzero" work list.
*/
duk_heap_remove_any_from_heap_allocated(heap, h);
duk__queue_refzero(heap, h);
duk__refzero_free_pending(thr);
break;
case DUK_HTYPE_BUFFER:
/*
* Buffers have no internal references. However, a dynamic
* buffer has a separate allocation for the buffer. This is
* freed by duk_heap_free_heaphdr_raw().
*/
duk_heap_remove_any_from_heap_allocated(heap, h);
duk_heap_free_heaphdr_raw(heap, h);
break;
default:
DUK_D(DUK_DPRINT("invalid heap type in decref: %ld", (long) DUK_HEAPHDR_GET_TYPE(h)));
DUK_UNREACHABLE();
}
}
DUK_INTERNAL void duk_heaphdr_refzero(duk_hthread *thr, duk_heaphdr *h) {
duk_heap *heap;
DUK_ASSERT(thr != NULL);
DUK_ASSERT(h != NULL);
heap = thr->heap;
DUK_DDD(DUK_DDDPRINT("refzero %p: %!O", (void *) h, (duk_heaphdr *) h));
/*
* Refzero handling is skipped entirely if (1) mark-and-sweep is
* running or (2) execution is paused in the debugger. The objects
* are left in the heap, and will be freed by mark-and-sweep or
* eventual heap destruction.
*
* This is necessary during mark-and-sweep because refcounts are also
* updated during the sweep phase (otherwise objects referenced by a
* swept object would have incorrect refcounts) which then calls here.
* This could be avoided by using separate decref macros in
* mark-and-sweep; however, mark-and-sweep also calls finalizers which
* would use the ordinary decref macros anyway and still call this
* function.
*
* This check must be enabled also when mark-and-sweep support has been
* disabled: the flag is also used in heap destruction when running
* finalizers for remaining objects, and the flag prevents objects from
* being moved around in heap linked lists.
*/
/* XXX: ideally this would be just one flag (maybe a derived one) so
* that a single bit test is sufficient to check the condition.
*/
#if defined(DUK_USE_DEBUGGER_SUPPORT)
if (DUK_UNLIKELY(DUK_HEAP_HAS_MARKANDSWEEP_RUNNING(heap) || DUK_HEAP_IS_PAUSED(heap))) {
#else
if (DUK_UNLIKELY(DUK_HEAP_HAS_MARKANDSWEEP_RUNNING(heap))) {
#endif
DUK_DDD(DUK_DDDPRINT("refzero handling suppressed when mark-and-sweep running, object: %p", (void *) h));
return;
}
switch ((duk_small_int_t) DUK_HEAPHDR_GET_TYPE(h)) {
case DUK_HTYPE_STRING:
/*
* Strings have no internal references but do have "weak"
* references in the string cache. Also note that strings
* are not on the heap_allocated list like other heap
* elements.
*/
duk_heap_strcache_string_remove(heap, (duk_hstring *) h);
duk_heap_string_remove(heap, (duk_hstring *) h);
duk_heap_free_heaphdr_raw(heap, h);
break;
case DUK_HTYPE_OBJECT:
/*
* Objects have internal references. Must finalize through
* the "refzero" work list.
*/
duk_heap_remove_any_from_heap_allocated(heap, h);
duk__queue_refzero(heap, h);
duk__refzero_free_pending(thr);
break;
case DUK_HTYPE_BUFFER:
/*
* Buffers have no internal references. However, a dynamic
* buffer has a separate allocation for the buffer. This is
* freed by duk_heap_free_heaphdr_raw().
*/
duk_heap_remove_any_from_heap_allocated(heap, h);
duk_heap_free_heaphdr_raw(heap, h);
break;
default:
DUK_D(DUK_DPRINT("invalid heap type in decref: %ld", (long) DUK_HEAPHDR_GET_TYPE(h)));
DUK_UNREACHABLE();
}
}
#if !defined(DUK_USE_FAST_REFCOUNT_DEFAULT)
DUK_INTERNAL void duk_tval_incref(duk_tval *tv) {
DUK_ASSERT(tv != NULL);
if (DUK_TVAL_NEEDS_REFCOUNT_UPDATE(tv)) {
duk_heaphdr *h = DUK_TVAL_GET_HEAPHDR(tv);
DUK_ASSERT(h != NULL);
DUK_ASSERT(DUK_HEAPHDR_HTYPE_VALID(h));
DUK_ASSERT_DISABLE(h->h_refcount >= 0);
DUK_HEAPHDR_PREINC_REFCOUNT(h);
}
}
