GC_INNER void GC_free_inner(void * p) { struct hblk *h; hdr *hhdr; size_t sz; /* bytes */ size_t ngranules; /* sz in granules */ void ** flh; int knd; struct obj_kind * ok; h = HBLKPTR(p); hhdr = HDR(h); knd = hhdr -> hb_obj_kind; sz = hhdr -> hb_sz; ngranules = BYTES_TO_GRANULES(sz); ok = &GC_obj_kinds[knd]; if (ngranules <= MAXOBJGRANULES) { GC_bytes_freed += sz; if (IS_UNCOLLECTABLE(knd)) GC_non_gc_bytes -= sz; if (ok -> ok_init) { BZERO((word *)p + 1, sz-sizeof(word)); } flh = &(ok -> ok_freelist[ngranules]); obj_link(p) = *flh; *flh = (ptr_t)p; } else { size_t nblocks = OBJ_SZ_TO_BLOCKS(sz); GC_bytes_freed += sz; if (IS_UNCOLLECTABLE(knd)) GC_non_gc_bytes -= sz; if (nblocks > 1) { GC_large_allocd_bytes -= nblocks * HBLKSIZE; } GC_freehblk(h); } }
/* Explicitly deallocate an object p. */ GC_API void GC_CALL GC_free(void * p) { struct hblk *h; hdr *hhdr; size_t sz; /* In bytes */ size_t ngranules; /* sz in granules */ void **flh; int knd; struct obj_kind * ok; DCL_LOCK_STATE; if (p == 0) return; /* Required by ANSI. It's not my fault ... */ # ifdef LOG_ALLOCS GC_log_printf("GC_free(%p) after GC #%lu\n", p, (unsigned long)GC_gc_no); # endif h = HBLKPTR(p); hhdr = HDR(h); # if defined(REDIRECT_MALLOC) && \ (defined(GC_SOLARIS_THREADS) || defined(GC_LINUX_THREADS) \ || defined(MSWIN32)) /* For Solaris, we have to redirect malloc calls during */ /* initialization. For the others, this seems to happen */ /* implicitly. */ /* Don't try to deallocate that memory. */ if (0 == hhdr) return; # endif GC_ASSERT(GC_base(p) == p); sz = hhdr -> hb_sz; ngranules = BYTES_TO_GRANULES(sz); knd = hhdr -> hb_obj_kind; ok = &GC_obj_kinds[knd]; if (EXPECT(ngranules <= MAXOBJGRANULES, TRUE)) { LOCK(); GC_bytes_freed += sz; if (IS_UNCOLLECTABLE(knd)) GC_non_gc_bytes -= sz; /* Its unnecessary to clear the mark bit. If the */ /* object is reallocated, it doesn't matter. O.w. the */ /* collector will do it, since it's on a free list. */ if (ok -> ok_init) { BZERO((word *)p + 1, sz-sizeof(word)); } flh = &(ok -> ok_freelist[ngranules]); obj_link(p) = *flh; *flh = (ptr_t)p; UNLOCK(); } else { size_t nblocks = OBJ_SZ_TO_BLOCKS(sz); LOCK(); GC_bytes_freed += sz; if (IS_UNCOLLECTABLE(knd)) GC_non_gc_bytes -= sz; if (nblocks > 1) { GC_large_allocd_bytes -= nblocks * HBLKSIZE; } GC_freehblk(h); UNLOCK(); } }
/* EXTRA_BYTES were already added to lb. */ STATIC ptr_t GC_alloc_large_and_clear(size_t lb, int k, unsigned flags) { ptr_t result = GC_alloc_large(lb, k, flags); word n_blocks = OBJ_SZ_TO_BLOCKS(lb); if (0 == result) return 0; if (GC_debugging_started || GC_obj_kinds[k].ok_init) { /* Clear the whole block, in case of GC_realloc call. */ BZERO(result, n_blocks * HBLKSIZE); } return result; }
GC_API GC_ATTR_MALLOC void * GC_CALL GC_generic_malloc(size_t lb, int k) { void * result; DCL_LOCK_STATE; if (EXPECT(GC_have_errors, FALSE)) GC_print_all_errors(); GC_INVOKE_FINALIZERS(); GC_DBG_COLLECT_AT_MALLOC(lb); if (SMALL_OBJ(lb)) { LOCK(); result = GC_generic_malloc_inner(lb, k); UNLOCK(); } else { size_t lg; size_t lb_rounded; word n_blocks; GC_bool init; lg = ROUNDED_UP_GRANULES(lb); lb_rounded = GRANULES_TO_BYTES(lg); if (lb_rounded < lb) return((*GC_get_oom_fn())(lb)); n_blocks = OBJ_SZ_TO_BLOCKS(lb_rounded); init = GC_obj_kinds[k].ok_init; LOCK(); result = (ptr_t)GC_alloc_large(lb_rounded, k, 0); if (0 != result) { if (GC_debugging_started) { BZERO(result, n_blocks * HBLKSIZE); } else { # ifdef THREADS /* Clear any memory that might be used for GC descriptors */ /* before we release the lock. */ ((word *)result)[0] = 0; ((word *)result)[1] = 0; ((word *)result)[GRANULES_TO_WORDS(lg)-1] = 0; ((word *)result)[GRANULES_TO_WORDS(lg)-2] = 0; # endif } } GC_bytes_allocd += lb_rounded; UNLOCK(); if (init && !GC_debugging_started && 0 != result) { BZERO(result, n_blocks * HBLKSIZE); } } if (0 == result) { return((*GC_get_oom_fn())(lb)); } else { return(result); } }
/* EXTRA_BYTES were already added to lb. */ GC_INNER ptr_t GC_alloc_large(size_t lb, int k, unsigned flags) { struct hblk * h; word n_blocks; ptr_t result; GC_bool retry = FALSE; GC_ASSERT(I_HOLD_LOCK()); lb = ROUNDUP_GRANULE_SIZE(lb); n_blocks = OBJ_SZ_TO_BLOCKS(lb); if (!EXPECT(GC_is_initialized, TRUE)) { DCL_LOCK_STATE; UNLOCK(); /* just to unset GC_lock_holder */ GC_init(); LOCK(); } /* Do our share of marking work */ if (GC_incremental && !GC_dont_gc) GC_collect_a_little_inner((int)n_blocks); h = GC_allochblk(lb, k, flags); # ifdef USE_MUNMAP if (0 == h) { GC_merge_unmapped(); h = GC_allochblk(lb, k, flags); } # endif while (0 == h && GC_collect_or_expand(n_blocks, flags != 0, retry)) { h = GC_allochblk(lb, k, flags); retry = TRUE; } if (h == 0) { result = 0; } else { size_t total_bytes = n_blocks * HBLKSIZE; if (n_blocks > 1) { GC_large_allocd_bytes += total_bytes; if (GC_large_allocd_bytes > GC_max_large_allocd_bytes) GC_max_large_allocd_bytes = GC_large_allocd_bytes; } /* FIXME: Do we need some way to reset GC_max_large_allocd_bytes? */ result = h -> hb_body; } return result; }
/* EXTRA_BYTES were already added to lb. */ GC_INNER ptr_t GC_alloc_large(size_t lb, int k, unsigned flags) { struct hblk * h; word n_blocks; ptr_t result; GC_bool retry = FALSE; /* Round up to a multiple of a granule. */ lb = (lb + GRANULE_BYTES - 1) & ~(GRANULE_BYTES - 1); n_blocks = OBJ_SZ_TO_BLOCKS(lb); if (!GC_is_initialized) GC_init(); /* Do our share of marking work */ if (GC_incremental && !GC_dont_gc) GC_collect_a_little_inner((int)n_blocks); h = GC_allochblk(lb, k, flags); # ifdef USE_MUNMAP if (0 == h) { GC_merge_unmapped(); h = GC_allochblk(lb, k, flags); } # endif while (0 == h && GC_collect_or_expand(n_blocks, flags != 0, retry)) { h = GC_allochblk(lb, k, flags); retry = TRUE; } if (h == 0) { result = 0; } else { size_t total_bytes = n_blocks * HBLKSIZE; if (n_blocks > 1) { GC_large_allocd_bytes += total_bytes; if (GC_large_allocd_bytes > GC_max_large_allocd_bytes) GC_max_large_allocd_bytes = GC_large_allocd_bytes; } result = h -> hb_body; } return result; }
/// The one difference between this one and the one above /// is the addition of the flag clear. This shouldn't really /// matter, but I was getting a bit of weird behavior in the baseline /// results (milc 100ms longer than previously measured) and wanted to /// eliminate it STATIC void HINTGC_reclaim_block(struct hblk *hbp, word report_if_found) { hdr * hhdr = HDR(hbp); size_t sz = hhdr -> hb_sz; /* size of objects in current block */ struct obj_kind * ok = &GC_obj_kinds[hhdr -> hb_obj_kind]; struct hblk ** rlh; // This line added for hinted collection since we have to // visit every block header anyways. Clear out the // pending free flags. hhdr->hb_flags &= ~HAS_PENDING_FREE; if( sz > MAXOBJBYTES ) { /* 1 big object */ if( !mark_bit_from_hdr(hhdr, 0) ) { if (report_if_found) { GC_add_leaked((ptr_t)hbp); } else { size_t blocks = OBJ_SZ_TO_BLOCKS(sz); if (blocks > 1) { GC_large_allocd_bytes -= blocks * HBLKSIZE; } GC_bytes_found += sz; GC_freehblk(hbp); } } else { if (hhdr -> hb_descr != 0) { GC_composite_in_use += sz; } else { GC_atomic_in_use += sz; } } } else { GC_bool empty = GC_block_empty(hhdr); # ifdef PARALLEL_MARK /* Count can be low or one too high because we sometimes */ /* have to ignore decrements. Objects can also potentially */ /* be repeatedly marked by each marker. */ /* Here we assume two markers, but this is extremely */ /* unlikely to fail spuriously with more. And if it does, it */ /* should be looked at. */ GC_ASSERT(hhdr -> hb_n_marks <= 2 * (HBLKSIZE/sz + 1) + 16); # else GC_ASSERT(sz * hhdr -> hb_n_marks <= HBLKSIZE); # endif if (hhdr -> hb_descr != 0) { GC_composite_in_use += sz * hhdr -> hb_n_marks; } else { GC_atomic_in_use += sz * hhdr -> hb_n_marks; } if (report_if_found) { GC_reclaim_small_nonempty_block(hbp, TRUE /* report_if_found */); } else if (empty) { GC_bytes_found += HBLKSIZE; GC_freehblk(hbp); } else if (GC_find_leak || !GC_block_nearly_full(hhdr)) { /* group of smaller objects, enqueue the real work */ rlh = &(ok -> ok_reclaim_list[BYTES_TO_GRANULES(sz)]); hhdr -> hb_next = *rlh; *rlh = hbp; } /* else not worth salvaging. */ /* We used to do the nearly_full check later, but we */ /* already have the right cache context here. Also */ /* doing it here avoids some silly lock contention in */ /* GC_malloc_many. */ } }