void
gc_collect(void)
{
int rc;
switch (gc_state_c->gs_mark_state) {
case GC_MS_MARK:
gc_resume_marking();
break;
case GC_MS_SWEEP:
gc_resume_sweeping();
break;
case GC_MS_NONE:
gc_debug("beginning a new collection");
#ifdef GC_COLLECT_STATS
gc_state_c->gs_nmark = 0;
gc_state_c->gs_nmarkbytes = 0;
gc_state_c->gs_nsweep = 0;
gc_state_c->gs_nsweepbytes = 0;
gc_state_c->gs_ntcollect++;
#endif
/* Update the VM info. */
if (gc_vm_tbl_update(&gc_state_c->gs_vt) != GC_SUCC) {
gc_error("gc_vm_tbl_update");
return;
}
gc_print_vm_tbl(&gc_state_c->gs_vt);
/* Get the trusted stack. */
rc = gc_cheri_get_ts(gc_state_c->gs_gts_c);
if (rc != 0) {
gc_error("gc_cheri_get_ts error: %d", rc);
return;
}
gc_start_marking();
/* Because we're not incremental yet: */
/*while (gc_state_c->mark_state != GC_MS_SWEEP)
gc_resume_marking();
while (gc_state_c->mark_state != GC_MS_NONE)
gc_resume_sweeping();*/
while (gc_state_c->gs_mark_state != GC_MS_NONE)
gc_resume_marking();
/* Restore the trusted stack. */
rc = gc_cheri_put_ts(gc_state_c->gs_gts_c);
if (rc != 0) {
gc_error("gc_cheri_put_ts error: %d", rc);
return;
}
break;
default:
/* NOTREACHABLE */
GC_NOTREACHABLE_ERROR();
break;
}
}
int
gc_vm_tbl_update(_gc_cap struct gc_vm_tbl *vt)
{
#ifdef GC_USE_LIBPROCSTAT
struct procstat *ps;
struct kinfo_vmentry *kv;
struct kinfo_proc *kp;
unsigned cnt, i;
ps = procstat_open_sysctl();
if (ps == NULL)
return (GC_ERROR);
cnt = 0;
kp = procstat_getprocs(ps, KERN_PROC_PID, getpid(), &cnt);
if (kp == NULL)
return (GC_ERROR);
gc_debug("getprocs retrieved %u procs", cnt);
kv = procstat_getvmmap(ps, kp, &cnt);
if (kv == NULL)
return (GC_ERROR);
gc_debug("getvmmap retrieved %u entries", cnt);
if (vt->vt_sz < cnt)
return (GC_TOO_SMALL);
vt->vt_nent = cnt;
for (i = 0; i < vt->vt_nent; i++) {
vt->vt_ent[i].ve_start = kv[i].kve_start;
vt->vt_ent[i].ve_end = kv[i].kve_end;
vt->vt_ent[i].ve_prot = kv[i].kve_protection;
vt->vt_ent[i].ve_type = kv[i].kve_type;
vt->vt_ent[i].ve_gctype = 0;
gc_vm_tbl_track(vt, &vt->vt_ent[i]);
}
procstat_freevmmap(ps, kv);
procstat_freeprocs(ps, kp);
procstat_close(ps);
return (GC_SUCC);
#else /* !GC_USE_LIBPROCSTAT */
return (GC_ERROR);
#endif /* GC_USE_LIBPROCSTAT */
}
/*
* GC collection.
*/
extern void GC_collect(void)
{
// Is collection enabled?
if (!gc_enabled)
return;
// Initialize marking
gc_debug("collect [stage=init_marks]");
gc_mark_init();
gc_debug("collect [stage=mark]");
struct gc_root_s root_0;
gc_root_t root = &root_0;
root->ptr = (void *)gc_stacktop();
root->size = gc_stackbottom - root->ptr;
root->ptrptr = &root->ptr;
root->sizeptr = &root->size;
root->elemsize = 1;
root->next = gc_roots;
gc_root_t roots = root;
gc_mark(roots);
gc_sweep();
}
/*
* GC handle error.
*/
extern void GC_handle_error(bool fatal, int err)
{
if (err != 0)
errno = err;
gc_debug("error occured [fatal=%d, errno=%s]\n", fatal,
strerror(errno));
gc_error_func_t func = gc_error_func;
if (func != NULL)
func();
if (fatal)
{
fprintf(stderr, "GC fatal error (%s)\n", strerror(errno));
abort();
}
}
_gc_cap struct gc_vm_ent *
gc_vm_tbl_find_btbl(_gc_cap struct gc_vm_tbl *vt, _gc_cap struct gc_btbl *bt)
{
size_t i;
_gc_cap struct gc_vm_ent *ve;
uint64_t start, end;
start = gc_cheri_getbase(bt->bt_base);
end = start + gc_cheri_getlen(bt->bt_base);
for (i = 0; i < vt->vt_nent; i++) {
ve = &vt->vt_ent[i];
gc_debug("search:[%llx,%llx] cur:[%llx,%llx]",
start,end,ve->ve_start,ve->ve_end);
if (ve->ve_start == start && ve->ve_end == end)
return (ve);
}
return (NULL);
}
/*
* GC marking.
*/
static void gc_mark(gc_root_t roots)
{
gc_markstack_t stack =
(gc_markstack_t)(gc_markstack + GC_MARK_STACK_SIZE);
stack--;
stack->startptr = NULL;
stack->endptr = NULL;
gc_used_size = 0;
while (true)
{
void **ptrptr = stack->startptr;
void **endptr = stack->endptr;
if (ptrptr == NULL)
{
// Attempt to find some work from the root list.
if (roots != NULL)
{
ptrptr = (void **)*roots->ptrptr;
size_t size = (*roots->sizeptr)*roots->elemsize;
endptr = ptrptr + size/sizeof(void *);
roots = roots->next;
goto gc_mark_loop_inner;
}
gc_debug("collect [stage=sweep]");
return;
}
else
stack++;
uint32_t pushed;
gc_mark_loop_inner:
pushed = 0;
while (ptrptr < endptr)
{
void *ptr = *ptrptr;
ptrptr++;
if (!gc_isptr(ptr))
{
// 'ptr' is not a value that points to anywhere in the GC's
// reserved virtual memory addresses; not a GC pointer.
continue;
}
gc_read_prefetch(ptrptr);
size_t idx = gc_index(ptr);
gc_region_t region = __gc_regions + idx;
if (ptr >= region->freeptr || ptr < region->startptr)
{
// 'ptr' points to memory that hasn't been allocated yet, or
// cannot be collected yet; not a GC pointer.
continue;
}
// 'ptr' has been deemed to be a GC pointer; check if it has been
// marked;
uint32_t size = region->size;
uint32_t ptridx = (uint32_t)(gc_objidx(ptr) - region->startidx);
if (!gc_mark_index(region->markptr, ptridx))
{
// 'ptr' is already marked; no need to follow it.
continue;
}
gc_used_size += size;
ptr = region->startptr + (size_t)ptridx*(size_t)size;
gc_read_prefetch(ptr);
// Push onto mark stack:
stack--;
stack->startptr = (void **)ptr;
stack->endptr = (void **)(ptr + size);
if (pushed > GC_MAX_MARK_PUSH)
{
void **tmp_ptrptr = stack[pushed].startptr;
void **tmp_endptr = stack[pushed].endptr;
stack[pushed].startptr = ptrptr;
stack[pushed].endptr = endptr;
ptrptr = tmp_ptrptr;
endptr = tmp_endptr;
pushed = 0;
}
pushed++;
}
}
}
/*
* GC memory allocation.
*/
extern void *GC_malloc_index(size_t idx)
{
gc_region_t region = __gc_regions + idx;
void *ptr;
// (0) Check if we need to collect.
gc_maybe_collect(region->size);
// (1) First, attempt to allocate from the freelist.
gc_freelist_t freelist = region->freelist, next;
if (freelist != NULL)
{
nonempty_freelist:
next = gc_unhide(freelist->next);
region->freelist = next;
ptr = (void *)freelist;
return ptr;
}
// (2) Next, attempt to allocated from marked memory.
if (region->markstartptr < region->markendptr)
{
ptr = region->markstartptr;
uint32_t ptridx = (uint32_t)(gc_objidx(ptr) - region->startidx);
uint8_t *markptr = region->markptr;
for (size_t i = 0; i < GC_FREELIST_LEN && ptr < region->markendptr; )
{
if (!gc_is_marked_index(markptr, ptridx))
{
gc_freelist_t freenode = (gc_freelist_t)ptr;
freenode->next = gc_hide(freelist);
freelist = freenode;
i++;
}
ptr += region->size;
ptridx++;
}
region->markstartptr = ptr;
region->freelist = freelist;
if (freelist != NULL)
goto nonempty_freelist;
}
// (3) Next, attempt to allocate from fresh space.
ptr = region->freeptr;
region->freeptr = ptr + region->size;
if (ptr >= region->endptr)
{
gc_handle_error(false, ENOMEM);
return NULL;
}
// Check if we can access the memory.
if (ptr + region->size >= region->protectptr)
{
void *protectptr = region->protectptr;
size_t protectlen = GC_PROTECT_LEN*GC_PAGESIZE;
protectlen = (protectlen < region->size? region->size:
protectlen);
if (gc_protect_memory(protectptr, protectlen) != 0)
{
gc_debug("protect failed");
gc_handle_error(false, 0);
return NULL;
}
region->protectptr = protectptr + protectlen;
}
return ptr;
}
/*
* GC initialization.
*/
extern bool GC_init(void)
{
if (gc_inited)
return true; // Already initialised.
gc_debug("initializing");
// Check that we are in a 64-bit environment.
if (sizeof(void *) != sizeof(uint64_t) ||
sizeof(double) != sizeof(uint64_t))
{
errno = ENOEXEC;
return false;
}
// Find the stack:
gc_stackbottom = gc_get_stackbottom();
// Reserve a large chunk of the virtual address space for the GC.
void *gc_memory = gc_get_memory();
if (gc_memory != GC_MEMORY)
goto init_error;
// Initialize all of the region information structures.
for (size_t i = 0; i < GC_NUM_REGIONS; i++)
{
void *startptr = GC_MEMORY + i*GC_REGION_SIZE;
size_t unit = gc_index_unit(i);
size_t size = (i - gc_unit_offset(unit))*unit + unit;
uintptr_t offset = (uintptr_t)startptr % size;
if (offset != 0)
startptr += size - offset;
gc_region_t region = __gc_regions + i;
region->size = size;
region->inv_size = (UINT64_MAX / size) + 1;
region->freelist = NULL;
region->startptr = startptr;
region->endptr = startptr + GC_REGION_SIZE;
region->freeptr = startptr;
region->protectptr = startptr;
region->markstartptr = startptr;
region->markendptr = startptr;
region->markptr = NULL;
region->startidx = gc_objidx(startptr);
}
// Reserve virtual space for the mark stack.
gc_markstack = gc_get_mark_memory(GC_MARK_STACK_SIZE);
if (gc_markstack == NULL)
goto init_error;
gc_inited = true;
return true;
int saved_errno;
init_error:
saved_errno = errno;
if (gc_markstack != NULL)
gc_free_memory(gc_markstack, GC_MARK_STACK_SIZE);
if (gc_memory != NULL)
gc_free_memory(GC_MEMORY, GC_NUM_REGIONS*GC_REGION_SIZE);
errno = saved_errno;
return false;
}