static void gc_mark(void)
{
// mark all roots
// active tasks
gc_push_root(jl_root_task);
gc_push_root(jl_current_task);
// modules
gc_push_root(jl_main_module);
gc_push_root(jl_current_module);
// invisible builtin values
if (jl_an_empty_cell) gc_push_root(jl_an_empty_cell);
gc_push_root(jl_exception_in_transit);
gc_push_root(jl_task_arg_in_transit);
gc_push_root(jl_unprotect_stack_func);
gc_push_root(jl_bottom_func);
gc_push_root(jl_typetype_type);
gc_push_root(jl_tupletype_type);
// constants
gc_push_root(jl_null);
gc_push_root(jl_true);
gc_push_root(jl_false);
// libuv loops
gc_mark_uv_state(jl_global_event_loop());
jl_mark_box_caches();
size_t i;
// stuff randomly preserved
for(i=0; i < preserved_values.len; i++) {
gc_push_root((jl_value_t*)preserved_values.items[i]);
}
// objects currently being finalized
for(i=0; i < to_finalize.len; i++) {
gc_push_root(to_finalize.items[i]);
}
gc_mark_all();
// find unmarked objects that need to be finalized.
// this must happen last.
for(i=0; i < finalizer_table.size; i+=2) {
if (finalizer_table.table[i+1] != HT_NOTFOUND) {
jl_value_t *v = finalizer_table.table[i];
if (!gc_marked(v)) {
gc_push_root(v);
schedule_finalization(v);
}
gc_push_root(finalizer_table.table[i+1]);
}
}
gc_mark_all();
}
static void push_root(jl_value_t *v)
{
assert(v != NULL);
if (gc_marked(v)) return;
jl_value_t *vt = (jl_value_t*)jl_typeof(v);
#ifdef OBJPROFILE
void **bp = ptrhash_bp(&obj_counts, vt);
if (*bp == HT_NOTFOUND)
*bp = (void*)2;
else
(*((ptrint_t*)bp))++;
#endif
gc_setmark(v);
if (vt == (jl_value_t*)jl_weakref_type ||
(jl_is_datatype(vt) && ((jl_datatype_t*)vt)->pointerfree)) {
return;
}
if (mark_sp >= mark_stack_size) {
size_t newsz = mark_stack_size>0 ? mark_stack_size*2 : 32000;
mark_stack = (jl_value_t**)realloc(mark_stack,newsz*sizeof(void*));
if (mark_stack == NULL) exit(1);
mark_stack_size = newsz;
}
mark_stack[mark_sp++] = v;
}
static void sweep_weak_refs(void)
{
size_t n=0, ndel=0, l=weak_refs.len;
jl_weakref_t *wr;
void **lst = weak_refs.items;
void *tmp;
#define SWAP_wr(a,b) (tmp=a,a=b,b=tmp,1)
if (l == 0)
return;
do {
wr = (jl_weakref_t*)lst[n];
if (gc_marked(wr)) {
// weakref itself is alive
if (!gc_marked(wr->value))
wr->value = (jl_value_t*)jl_nothing;
n++;
}
else {
ndel++;
}
} while ((n < l-ndel) && SWAP_wr(lst[n],lst[n+ndel]));
weak_refs.len -= ndel;
}
static void sweep_malloced_arrays()
{
mallocarray_t *ma = mallocarrays;
mallocarray_t **pma = &mallocarrays;
while (ma != NULL) {
mallocarray_t *nxt = ma->next;
if (gc_marked(ma->a)) {
pma = &ma->next;
}
else {
*pma = nxt;
assert(ma->a->how == 2);
jl_gc_free_array(ma->a);
ma->next = mafreelist;
mafreelist = ma;
}
ma = nxt;
}
}
static void big_obj_stats(void)
{
bigval_t *v = big_objects;
size_t nused=0, nbytes=0;
while (v != NULL) {
if (v->marked) {
nused++;
nbytes += v->sz;
}
v = v->next;
}
mallocarray_t *ma = mallocarrays;
while (ma != NULL) {
if (gc_marked(ma->a)) {
nused++;
nbytes += array_nbytes(ma->a);
}
ma = ma->next;
}
JL_PRINTF(JL_STDOUT, "%d bytes in %d large objects\n", nbytes, nused);
}
static void gc_mark(void)
{
// mark all roots
// active tasks
gc_push_root(jl_root_task, 0);
gc_push_root(jl_current_task, 0);
// modules
gc_push_root(jl_main_module, 0);
gc_push_root(jl_internal_main_module, 0);
gc_push_root(jl_current_module, 0);
if (jl_old_base_module) gc_push_root(jl_old_base_module, 0);
// invisible builtin values
if (jl_an_empty_cell) gc_push_root(jl_an_empty_cell, 0);
gc_push_root(jl_exception_in_transit, 0);
gc_push_root(jl_task_arg_in_transit, 0);
gc_push_root(jl_unprotect_stack_func, 0);
gc_push_root(jl_bottom_func, 0);
gc_push_root(jl_typetype_type, 0);
gc_push_root(jl_tupletype_type, 0);
gc_push_root(typeToTypeId, 0);
if (jl_module_init_order != NULL)
gc_push_root(jl_module_init_order, 0);
// constants
gc_push_root(jl_null, 0);
gc_push_root(jl_true, 0);
gc_push_root(jl_false, 0);
jl_mark_box_caches();
size_t i;
// stuff randomly preserved
for(i=0; i < preserved_values.len; i++) {
gc_push_root((jl_value_t*)preserved_values.items[i], 0);
}
// objects currently being finalized
for(i=0; i < to_finalize.len; i++) {
gc_push_root(to_finalize.items[i], 0);
}
visit_mark_stack();
// find unmarked objects that need to be finalized.
// this must happen last.
for(i=0; i < finalizer_table.size; i+=2) {
if (finalizer_table.table[i+1] != HT_NOTFOUND) {
jl_value_t *v = (jl_value_t*)finalizer_table.table[i];
if (!gc_marked(v)) {
jl_value_t *fin = (jl_value_t*)finalizer_table.table[i+1];
if (gc_typeof(fin) == (jl_value_t*)jl_voidpointer_type) {
void *p = ((void**)fin)[1];
if (p)
((void (*)(void*))p)(jl_data_ptr(v));
finalizer_table.table[i+1] = HT_NOTFOUND;
continue;
}
gc_push_root(v, 0);
schedule_finalization(v);
}
gc_push_root(finalizer_table.table[i+1], 0);
}
}
visit_mark_stack();
}
static void push_root(jl_value_t *v, int d)
{
assert(v != NULL);
jl_value_t *vt = (jl_value_t*)gc_typeof(v);
#ifdef OBJPROFILE
if (!gc_marked(v)) {
void **bp = ptrhash_bp(&obj_counts, vt);
if (*bp == HT_NOTFOUND)
*bp = (void*)2;
else
(*((ptrint_t*)bp))++;
}
#endif
gc_setmark(v);
if (vt == (jl_value_t*)jl_weakref_type ||
(jl_is_datatype(vt) && ((jl_datatype_t*)vt)->pointerfree)) {
return;
}
if (d >= MAX_MARK_DEPTH)
goto queue_the_root;
d++;
// some values have special representations
if (vt == (jl_value_t*)jl_tuple_type) {
size_t l = jl_tuple_len(v);
jl_value_t **data = ((jl_tuple_t*)v)->data;
for(size_t i=0; i < l; i++) {
jl_value_t *elt = data[i];
if (elt != NULL)
gc_push_root(elt, d);
}
}
else if (((jl_datatype_t*)(vt))->name == jl_array_typename) {
jl_array_t *a = (jl_array_t*)v;
if (a->how == 3) {
jl_value_t *owner = jl_array_data_owner(a);
gc_push_root(owner, d);
return;
}
else if (a->how == 1) {
gc_setmark_buf((char*)a->data - a->offset*a->elsize);
}
if (a->ptrarray && a->data!=NULL) {
size_t l = jl_array_len(a);
if (l > 100000 && d > MAX_MARK_DEPTH-10) {
// don't mark long arrays at high depth, to try to avoid
// copying the whole array into the mark queue
goto queue_the_root;
}
else {
void *data = a->data;
for(size_t i=0; i < l; i++) {
jl_value_t *elt = ((jl_value_t**)data)[i];
if (elt != NULL) gc_push_root(elt, d);
}
}
}
}
else if (vt == (jl_value_t*)jl_module_type) {
gc_mark_module((jl_module_t*)v, d);
}
else if (vt == (jl_value_t*)jl_task_type) {
gc_mark_task((jl_task_t*)v, d);
}
else {
jl_datatype_t *dt = (jl_datatype_t*)vt;
int nf = (int)jl_tuple_len(dt->names);
for(int i=0; i < nf; i++) {
if (dt->fields[i].isptr) {
jl_value_t *fld = *(jl_value_t**)((char*)v + dt->fields[i].offset + sizeof(void*));
if (fld)
gc_push_root(fld, d);
}
}
}
return;
queue_the_root:
if (mark_sp >= mark_stack_size) {
size_t newsz = mark_stack_size>0 ? mark_stack_size*2 : 32000;
mark_stack = (jl_value_t**)realloc(mark_stack,newsz*sizeof(void*));
if (mark_stack == NULL) exit(1);
mark_stack_size = newsz;
}
mark_stack[mark_sp++] = v;
}
static void gc_mark_all()
{
while (mark_sp > 0) {
jl_value_t *v = mark_stack[--mark_sp];
jl_value_t *vt = (jl_value_t*)gc_typeof(v);
// some values have special representations
if (vt == (jl_value_t*)jl_tuple_type) {
size_t l = jl_tuple_len(v);
jl_value_t **data = ((jl_tuple_t*)v)->data;
for(size_t i=0; i < l; i++) {
jl_value_t *elt = data[i];
if (elt != NULL)
gc_push_root(elt);
}
}
else if (((jl_datatype_t*)(vt))->name == jl_array_typename) {
jl_array_t *a = (jl_array_t*)v;
char *data = a->data;
if (data == NULL) continue;
int ndims = jl_array_ndims(a);
char *data0 = data;
if (ndims == 1) data0 -= a->offset*a->elsize;
if (!a->isinline) {
jl_value_t *owner = jl_array_data_owner(a);
if (a->ismalloc) {
// jl_mallocptr_t
if (gc_marked(owner))
continue;
gc_setmark(owner);
}
else {
// an array
v = owner;
if (v != (jl_value_t*)a) {
gc_push_root(v);
continue;
}
}
}
if (a->ptrarray) {
size_t l = jl_array_len(a);
for(size_t i=0; i < l; i++) {
jl_value_t *elt = ((jl_value_t**)data)[i];
if (elt != NULL) gc_push_root(elt);
}
}
}
else if (vt == (jl_value_t*)jl_module_type) {
gc_mark_module((jl_module_t*)v);
}
else if (vt == (jl_value_t*)jl_task_type) {
jl_task_t *ta = (jl_task_t*)v;
if (ta->on_exit) gc_push_root(ta->on_exit);
gc_push_root(ta->last);
gc_push_root(ta->tls);
gc_push_root(ta->consumers);
if (ta->start) gc_push_root(ta->start);
if (ta->result) gc_push_root(ta->result);
if (ta->stkbuf != NULL || ta == jl_current_task) {
if (ta->stkbuf != NULL)
gc_setmark_buf(ta->stkbuf);
#ifdef COPY_STACKS
ptrint_t offset;
if (ta == jl_current_task) {
offset = 0;
gc_mark_stack(jl_pgcstack, offset);
}
else {
offset = (char *)ta->stkbuf - ((char *)ta->stackbase - ta->ssize);
gc_mark_stack(ta->gcstack, offset);
}
#else
gc_mark_stack(ta->gcstack, 0);
#endif
}
}
else {
jl_datatype_t *dt = (jl_datatype_t*)vt;
int nf = (int)jl_tuple_len(dt->names);
for(int i=0; i < nf; i++) {
if (dt->fields[i].isptr) {
jl_value_t *fld = *(jl_value_t**)((char*)v + dt->fields[i].offset + sizeof(void*));
if (fld)
gc_push_root(fld);
}
}
}
}
}