ZEND_API void ZEND_FASTCALL gc_possible_root(zend_refcounted *ref)
{
uint32_t idx;
gc_root_buffer *newRoot;
if (UNEXPECTED(GC_G(gc_protected))) {
return;
}
GC_BENCH_INC(zval_possible_root);
if (EXPECTED(GC_HAS_UNUSED())) {
idx = GC_FETCH_UNUSED();
} else if (EXPECTED(GC_HAS_NEXT_UNUSED_UNDER_THRESHOLD())) {
idx = GC_FETCH_NEXT_UNUSED();
} else {
gc_possible_root_when_full(ref);
return;
}
ZEND_ASSERT(GC_TYPE(ref) == IS_ARRAY || GC_TYPE(ref) == IS_OBJECT);
ZEND_ASSERT(GC_INFO(ref) == 0);
newRoot = GC_IDX2PTR(idx);
newRoot->ref = ref; /* GC_ROOT tag is 0 */
GC_TRACE_SET_COLOR(ref, GC_PURPLE);
idx = gc_compress(idx);
GC_REF_SET_INFO(ref, idx | GC_PURPLE);
GC_G(num_roots)++;
GC_BENCH_INC(zval_buffered);
GC_BENCH_INC(root_buf_length);
GC_BENCH_PEAK(root_buf_peak, root_buf_length);
}
ZEND_API void zend_gc_get_status(zend_gc_status *status)
{
status->runs = GC_G(gc_runs);
status->collected = GC_G(collected);
status->threshold = GC_G(gc_threshold);
status->num_roots = GC_G(num_roots);
}
static zend_always_inline uint32_t gc_fetch_next_unused(void)
{
uint32_t idx;
ZEND_ASSERT(GC_HAS_NEXT_UNUSED());
idx = GC_G(first_unused);
GC_G(first_unused) = GC_G(first_unused) + 1;
return idx;
}
static zend_always_inline uint32_t gc_fetch_unused(void)
{
uint32_t idx;
gc_root_buffer *root;
ZEND_ASSERT(GC_HAS_UNUSED());
idx = GC_G(unused);
root = GC_IDX2PTR(idx);
ZEND_ASSERT(GC_IS_UNUSED(root->ref));
GC_G(unused) = GC_LIST2IDX(root->ref);
return idx;
}
static void gc_adjust_threshold(int count)
{
uint32_t new_threshold;
/* TODO Very simple heuristic for dynamic GC buffer resizing:
* If there are "too few" collections, increase the collection threshold
* by a fixed step */
if (count < GC_THRESHOLD_TRIGGER) {
/* increase */
if (GC_G(gc_threshold) < GC_THRESHOLD_MAX) {
new_threshold = GC_G(gc_threshold) + GC_THRESHOLD_STEP;
if (new_threshold > GC_THRESHOLD_MAX) {
new_threshold = GC_THRESHOLD_MAX;
}
if (new_threshold > GC_G(buf_size)) {
gc_grow_root_buffer();
}
if (new_threshold <= GC_G(buf_size)) {
GC_G(gc_threshold) = new_threshold;
}
}
} else if (GC_G(gc_threshold) > GC_THRESHOLD_DEFAULT) {
new_threshold = GC_G(gc_threshold) - GC_THRESHOLD_STEP;
if (new_threshold < GC_THRESHOLD_DEFAULT) {
new_threshold = GC_THRESHOLD_DEFAULT;
}
GC_G(gc_threshold) = new_threshold;
}
}
/* Two-Finger compaction algorithm */
static void gc_compact(void)
{
if (GC_G(num_roots) + GC_FIRST_ROOT != GC_G(first_unused)) {
if (GC_G(num_roots)) {
gc_root_buffer *free = GC_IDX2PTR(GC_FIRST_ROOT);
gc_root_buffer *scan = GC_IDX2PTR(GC_G(first_unused) - 1);
gc_root_buffer *end = GC_IDX2PTR(GC_G(num_roots));
uint32_t idx;
zend_refcounted *p;
while (free < scan) {
while (!GC_IS_UNUSED(free->ref)) {
free++;
}
while (GC_IS_UNUSED(scan->ref)) {
scan--;
}
if (scan > free) {
p = scan->ref;
free->ref = p;
p = GC_GET_PTR(p);
idx = gc_compress(GC_PTR2IDX(free));
GC_REF_SET_INFO(p, idx | GC_REF_COLOR(p));
free++;
scan--;
if (scan <= end) {
break;
}
}
}
}
GC_G(unused) = GC_INVALID;
GC_G(first_unused) = GC_G(num_roots) + GC_FIRST_ROOT;
}
}
static zend_never_inline void ZEND_FASTCALL gc_possible_root_when_full(zend_refcounted *ref)
{
uint32_t idx;
gc_root_buffer *newRoot;
ZEND_ASSERT(GC_TYPE(ref) == IS_ARRAY || GC_TYPE(ref) == IS_OBJECT);
ZEND_ASSERT(GC_INFO(ref) == 0);
if (GC_G(gc_enabled) && !GC_G(gc_active)) {
GC_ADDREF(ref);
gc_adjust_threshold(gc_collect_cycles());
if (UNEXPECTED(GC_DELREF(ref)) == 0) {
rc_dtor_func(ref);
return;
} else if (UNEXPECTED(GC_INFO(ref))) {
return;
}
}
if (GC_HAS_UNUSED()) {
idx = GC_FETCH_UNUSED();
} else if (EXPECTED(GC_HAS_NEXT_UNUSED())) {
idx = GC_FETCH_NEXT_UNUSED();
} else {
gc_grow_root_buffer();
if (UNEXPECTED(!GC_HAS_NEXT_UNUSED())) {
return;
}
idx = GC_FETCH_NEXT_UNUSED();
}
newRoot = GC_IDX2PTR(idx);
newRoot->ref = ref; /* GC_ROOT tag is 0 */
GC_TRACE_SET_COLOR(ref, GC_PURPLE);
idx = gc_compress(idx);
GC_REF_SET_INFO(ref, idx | GC_PURPLE);
GC_G(num_roots)++;
GC_BENCH_INC(zval_buffered);
GC_BENCH_INC(root_buf_length);
GC_BENCH_PEAK(root_buf_peak, root_buf_length);
}
static ZEND_INI_MH(OnUpdateGCEnabled) /* {{{ */
{
OnUpdateBool(entry, new_value, mh_arg1, mh_arg2, mh_arg3, stage);
if (GC_G(gc_enabled)) {
gc_init();
}
return SUCCESS;
}
static int gc_collect_roots(uint32_t *flags, gc_stack *stack)
{
uint32_t idx, end;
zend_refcounted *ref;
int count = 0;
gc_root_buffer *current = GC_IDX2PTR(GC_FIRST_ROOT);
gc_root_buffer *last = GC_IDX2PTR(GC_G(first_unused));
/* remove non-garbage from the list */
while (current != last) {
if (GC_IS_ROOT(current->ref)) {
if (GC_REF_CHECK_COLOR(current->ref, GC_BLACK)) {
GC_REF_SET_INFO(current->ref, 0); /* reset GC_ADDRESS() and keep GC_BLACK */
gc_remove_from_roots(current);
}
}
current++;
}
gc_compact();
/* Root buffer might be reallocated during gc_collect_white,
* make sure to reload pointers. */
idx = GC_FIRST_ROOT;
end = GC_G(first_unused);
while (idx != end) {
current = GC_IDX2PTR(idx);
ref = current->ref;
ZEND_ASSERT(GC_IS_ROOT(ref));
current->ref = GC_MAKE_GARBAGE(ref);
if (GC_REF_CHECK_COLOR(ref, GC_WHITE)) {
GC_REF_SET_BLACK(ref);
count += gc_collect_white(ref, flags, stack);
}
idx++;
}
return count;
}
ZEND_API zend_bool gc_enable(zend_bool enable)
{
zend_bool old_enabled = GC_G(gc_enabled);
GC_G(gc_enabled) = enable;
if (enable && !old_enabled && GC_G(buf) == NULL) {
GC_G(buf) = (gc_root_buffer*) pemalloc(sizeof(gc_root_buffer) * GC_DEFAULT_BUF_SIZE, 1);
GC_G(buf)[0].ref = NULL;
GC_G(buf_size) = GC_DEFAULT_BUF_SIZE;
GC_G(gc_threshold) = GC_THRESHOLD_DEFAULT + GC_FIRST_ROOT;
gc_reset();
}
return old_enabled;
}
static void gc_scan_roots(gc_stack *stack)
{
gc_root_buffer *current = GC_IDX2PTR(GC_FIRST_ROOT);
gc_root_buffer *last = GC_IDX2PTR(GC_G(first_unused));
while (current != last) {
if (GC_IS_ROOT(current->ref)) {
if (GC_REF_CHECK_COLOR(current->ref, GC_GREY)) {
GC_REF_SET_COLOR(current->ref, GC_WHITE);
gc_scan(current->ref, stack);
}
}
current++;
}
}
static zend_always_inline gc_root_buffer* gc_decompress(zend_refcounted *ref, uint32_t idx)
{
gc_root_buffer *root = GC_IDX2PTR(idx);
if (EXPECTED(GC_GET_PTR(root->ref) == ref)) {
return root;
}
while (1) {
idx += GC_MAX_UNCOMPRESSED;
ZEND_ASSERT(idx < GC_G(first_unused));
root = GC_IDX2PTR(idx);
if (GC_GET_PTR(root->ref) == ref) {
return root;
}
}
}
static void gc_mark_roots(gc_stack *stack)
{
gc_root_buffer *current, *last;
gc_compact();
current = GC_IDX2PTR(GC_FIRST_ROOT);
last = GC_IDX2PTR(GC_G(first_unused));
while (current != last) {
if (GC_IS_ROOT(current->ref)) {
if (GC_REF_CHECK_COLOR(current->ref, GC_PURPLE)) {
GC_REF_SET_COLOR(current->ref, GC_GREY);
gc_mark_grey(current->ref, stack);
}
}
current++;
}
}
ZEND_API void ZEND_FASTCALL gc_remove_from_buffer(zend_refcounted *ref)
{
gc_root_buffer *root;
uint32_t idx = GC_REF_ADDRESS(ref);
GC_BENCH_INC(zval_remove_from_buffer);
if (!GC_REF_CHECK_COLOR(ref, GC_BLACK)) {
GC_TRACE_SET_COLOR(ref, GC_BLACK);
}
GC_REF_SET_INFO(ref, 0);
/* Perform decompression only in case of large buffers */
if (UNEXPECTED(GC_G(first_unused) >= GC_MAX_UNCOMPRESSED)) {
gc_remove_compressed(ref, idx);
return;
}
ZEND_ASSERT(idx);
root = GC_IDX2PTR(idx);
gc_remove_from_roots(root);
}
static void gc_add_garbage(zend_refcounted *ref)
{
uint32_t idx;
gc_root_buffer *buf;
if (GC_HAS_UNUSED()) {
idx = GC_FETCH_UNUSED();
} else if (GC_HAS_NEXT_UNUSED()) {
idx = GC_FETCH_NEXT_UNUSED();
} else {
gc_grow_root_buffer();
if (UNEXPECTED(!GC_HAS_NEXT_UNUSED())) {
return;
}
idx = GC_FETCH_NEXT_UNUSED();
}
buf = GC_IDX2PTR(idx);
buf->ref = GC_MAKE_GARBAGE(ref);
idx = gc_compress(idx);
GC_REF_SET_INFO(ref, idx | GC_BLACK);
GC_G(num_roots)++;
}
ZEND_API zend_bool gc_enabled(void)
{
return GC_G(gc_enabled);
}
#define GC_ROOT_BUFFER_MAX_ENTRIES 10001
#ifdef ZTS
ZEND_API int gc_globals_id;
#else
ZEND_API zend_gc_globals gc_globals;
#endif
#define GC_REMOVE_FROM_ROOTS(current) \
gc_remove_from_roots((current) TSRMLS_CC)
static zend_always_inline void gc_remove_from_roots(gc_root_buffer *root TSRMLS_DC)
{
root->next->prev = root->prev;
root->prev->next = root->next;
root->prev = GC_G(unused);
GC_G(unused) = root;
GC_BENCH_DEC(root_buf_length);
}
static void root_buffer_dtor(zend_gc_globals *gc_globals TSRMLS_DC)
{
if (gc_globals->buf) {
free(gc_globals->buf);
gc_globals->buf = NULL;
}
}
static void gc_globals_ctor_ex(zend_gc_globals *gc_globals TSRMLS_DC)
{
gc_globals->gc_enabled = 0;
ZEND_API int zend_gc_collect_cycles(void)
{
int count = 0;
if (GC_G(num_roots)) {
gc_root_buffer *current, *last;
zend_refcounted *p;
uint32_t gc_flags = 0;
uint32_t idx, end;
gc_stack stack;
stack.prev = NULL;
stack.next = NULL;
if (GC_G(gc_active)) {
return 0;
}
GC_TRACE("Collecting cycles");
GC_G(gc_runs)++;
GC_G(gc_active) = 1;
GC_TRACE("Marking roots");
gc_mark_roots(&stack);
GC_TRACE("Scanning roots");
gc_scan_roots(&stack);
GC_TRACE("Collecting roots");
count = gc_collect_roots(&gc_flags, &stack);
gc_stack_free(&stack);
if (!GC_G(num_roots)) {
/* nothing to free */
GC_TRACE("Nothing to free");
GC_G(gc_active) = 0;
return 0;
}
end = GC_G(first_unused);
if (gc_flags & GC_HAS_DESTRUCTORS) {
uint32_t *refcounts;
GC_TRACE("Calling destructors");
// TODO: may be use emalloc() ???
refcounts = pemalloc(sizeof(uint32_t) * end, 1);
/* Remember reference counters before calling destructors */
idx = GC_FIRST_ROOT;
current = GC_IDX2PTR(GC_FIRST_ROOT);
while (idx != end) {
if (GC_IS_GARBAGE(current->ref)) {
p = GC_GET_PTR(current->ref);
refcounts[idx] = GC_REFCOUNT(p);
}
current++;
idx++;
}
/* Call destructors
*
* The root buffer might be reallocated during destructors calls,
* make sure to reload pointers as necessary. */
idx = GC_FIRST_ROOT;
while (idx != end) {
current = GC_IDX2PTR(idx);
if (GC_IS_GARBAGE(current->ref)) {
p = GC_GET_PTR(current->ref);
if (GC_TYPE(p) == IS_OBJECT
&& !(OBJ_FLAGS(p) & IS_OBJ_DESTRUCTOR_CALLED)) {
zend_object *obj = (zend_object*)p;
GC_TRACE_REF(obj, "calling destructor");
GC_ADD_FLAGS(obj, IS_OBJ_DESTRUCTOR_CALLED);
if (obj->handlers->dtor_obj != zend_objects_destroy_object
|| obj->ce->destructor) {
GC_ADDREF(obj);
obj->handlers->dtor_obj(obj);
GC_DELREF(obj);
}
}
}
idx++;
}
/* Remove values captured in destructors */
idx = GC_FIRST_ROOT;
current = GC_IDX2PTR(GC_FIRST_ROOT);
while (idx != end) {
if (GC_IS_GARBAGE(current->ref)) {
p = GC_GET_PTR(current->ref);
if (GC_REFCOUNT(p) > refcounts[idx]) {
gc_remove_nested_data_from_buffer(p, current);
}
}
current++;
idx++;
}
pefree(refcounts, 1);
if (GC_G(gc_protected)) {
/* something went wrong */
return 0;
}
}
/* Destroy zvals */
GC_TRACE("Destroying zvals");
GC_G(gc_protected) = 1;
current = GC_IDX2PTR(GC_FIRST_ROOT);
last = GC_IDX2PTR(GC_G(first_unused));
while (current != last) {
if (GC_IS_GARBAGE(current->ref)) {
p = GC_GET_PTR(current->ref);
GC_TRACE_REF(p, "destroying");
if (GC_TYPE(p) == IS_OBJECT) {
zend_object *obj = (zend_object*)p;
EG(objects_store).object_buckets[obj->handle] = SET_OBJ_INVALID(obj);
GC_TYPE_INFO(obj) = IS_NULL |
(GC_TYPE_INFO(obj) & ~GC_TYPE_MASK);
if (!(OBJ_FLAGS(obj) & IS_OBJ_FREE_CALLED)) {
GC_ADD_FLAGS(obj, IS_OBJ_FREE_CALLED);
GC_ADDREF(obj);
obj->handlers->free_obj(obj);
GC_DELREF(obj);
}
ZEND_OBJECTS_STORE_ADD_TO_FREE_LIST(obj->handle);
current->ref = GC_MAKE_GARBAGE(((char*)obj) - obj->handlers->offset);
} else if (GC_TYPE(p) == IS_ARRAY) {
zend_array *arr = (zend_array*)p;
GC_TYPE_INFO(arr) = IS_NULL |
(GC_TYPE_INFO(arr) & ~GC_TYPE_MASK);
/* GC may destroy arrays with rc>1. This is valid and safe. */
HT_ALLOW_COW_VIOLATION(arr);
zend_hash_destroy(arr);
}
}
current++;
}
/* Free objects */
current = GC_IDX2PTR(GC_FIRST_ROOT);
while (current != last) {
if (GC_IS_GARBAGE(current->ref)) {
p = GC_GET_PTR(current->ref);
GC_LINK_UNUSED(current);
GC_G(num_roots)--;
efree(p);
}
current++;
}
GC_TRACE("Collection finished");
GC_G(collected) += count;
GC_G(gc_protected) = 0;
GC_G(gc_active) = 0;
}
gc_compact();
return count;
}
static zend_always_inline void gc_link_unused(gc_root_buffer *root)
{
root->ref = GC_IDX2LIST(GC_G(unused));
GC_G(unused) = GC_PTR2IDX(root);
}
static zend_always_inline void gc_remove_from_roots(gc_root_buffer *root)
{
GC_LINK_UNUSED(root);
GC_G(num_roots)--;
GC_BENCH_DEC(root_buf_length);
}
static void gc_grow_root_buffer(void)
{
size_t new_size;
if (GC_G(buf_size) >= GC_MAX_BUF_SIZE) {
if (!GC_G(gc_full)) {
zend_error(E_WARNING, "GC buffer overflow (GC disabled)\n");
GC_G(gc_active) = 1;
GC_G(gc_protected) = 1;
GC_G(gc_full) = 1;
return;
}
}
if (GC_G(buf_size) < GC_BUF_GROW_STEP) {
new_size = GC_G(buf_size) * 2;
} else {
new_size = GC_G(buf_size) + GC_BUF_GROW_STEP;
}
if (new_size > GC_MAX_BUF_SIZE) {
new_size = GC_MAX_BUF_SIZE;
}
GC_G(buf) = perealloc(GC_G(buf), sizeof(gc_root_buffer) * new_size, 1);
GC_G(buf_size) = new_size;
}
ZEND_API zend_bool gc_protected(void)
{
return GC_G(gc_protected);
}
ZEND_API zend_bool gc_protect(zend_bool protect)
{
zend_bool old_protected = GC_G(gc_protected);
GC_G(gc_protected) = protect;
return old_protected;
}
void gc_reset(void)
{
if (GC_G(buf)) {
GC_G(gc_active) = 0;
GC_G(gc_protected) = 0;
GC_G(gc_full) = 0;
GC_G(unused) = GC_INVALID;
GC_G(first_unused) = GC_FIRST_ROOT;
GC_G(num_roots) = 0;
GC_G(gc_runs) = 0;
GC_G(collected) = 0;
#if GC_BENCH
GC_G(root_buf_length) = 0;
GC_G(root_buf_peak) = 0;
GC_G(zval_possible_root) = 0;
GC_G(zval_buffered) = 0;
GC_G(zval_remove_from_buffer) = 0;
GC_G(zval_marked_grey) = 0;
#endif
}
}
