GC_API void * GC_generic_malloc_kind(size_t bytes, int kind)
{
size_t granules = ROUNDED_UP_GRANULES(bytes);
void *tsd;
void *result;
void **tiny_fl;
# if !defined(USE_PTHREAD_SPECIFIC) && !defined(USE_WIN32_SPECIFIC)
GC_key_t k = GC_thread_key;
if (EXPECT(0 == k, FALSE)) {
/* We haven't yet run GC_init_parallel. That means */
/* we also aren't locking, so this is fairly cheap. */
return GC_generic_malloc_kind_global(bytes, kind);
}
tsd = GC_getspecific(k);
# else
tsd = GC_getspecific(GC_thread_key);
# endif
# if !defined(USE_COMPILER_TLS) && !defined(USE_WIN32_COMPILER_TLS)
if (EXPECT(0 == tsd, FALSE)) {
return GC_generic_malloc_kind_global(bytes, kind);
}
# endif
GC_ASSERT(GC_is_initialized);
GC_ASSERT(GC_is_thread_tsd_valid(tsd));
tiny_fl = ((GC_tlfs)tsd)->freelists[kind];
GC_FAST_MALLOC_GRANS(result, granules, tiny_fl, DIRECT_GRANULES,
kind, GC_generic_malloc_kind_global(bytes, kind), obj_link(result) = 0);
return result;
}
GC_API GC_ATTR_MALLOC void * GC_CALL GC_malloc_atomic(size_t bytes)
{
size_t granules = ROUNDED_UP_GRANULES(bytes);
void *tsd;
void *result;
void **tiny_fl;
# if !defined(USE_PTHREAD_SPECIFIC) && !defined(USE_WIN32_SPECIFIC)
GC_key_t k = GC_thread_key;
if (EXPECT(0 == k, FALSE)) {
/* We haven't yet run GC_init_parallel. That means */
/* we also aren't locking, so this is fairly cheap. */
return GC_core_malloc_atomic(bytes);
}
tsd = GC_getspecific(k);
# else
tsd = GC_getspecific(GC_thread_key);
# endif
# if !defined(USE_COMPILER_TLS) && !defined(USE_WIN32_COMPILER_TLS)
if (EXPECT(0 == tsd, FALSE)) {
return GC_core_malloc_atomic(bytes);
}
# endif
GC_ASSERT(GC_is_initialized);
tiny_fl = ((GC_tlfs)tsd) -> ptrfree_freelists;
GC_FAST_MALLOC_GRANS(result, granules, tiny_fl, DIRECT_GRANULES, PTRFREE,
GC_core_malloc_atomic(bytes), (void)0 /* no init */);
return result;
}
GC_PTR GC_local_malloc_atomic(size_t bytes)
{
if (EXPECT(!SMALL_ENOUGH(bytes), 0)) {
return(GC_malloc_atomic(bytes));
} else {
int index = INDEX_FROM_BYTES(bytes);
ptr_t * my_fl = ((GC_thread)GC_getspecific(GC_thread_key))
-> ptrfree_freelists + index;
ptr_t my_entry = *my_fl;
if (EXPECT((word)my_entry >= HBLKSIZE, 1)) {
GC_PTR result = (GC_PTR)my_entry;
*my_fl = obj_link(my_entry);
return result;
} else if ((word)my_entry - 1 < DIRECT_GRANULES) {
*my_fl = my_entry + index + 1;
return GC_malloc_atomic(bytes);
} else {
GC_generic_malloc_many(BYTES_FROM_INDEX(index), PTRFREE, my_fl);
/* *my_fl is updated while the collector is excluded; */
/* the free list is always visible to the collector as */
/* such. */
if (*my_fl == 0) return GC_oom_fn(bytes);
return GC_local_malloc_atomic(bytes);
}
}
}
GC_API GC_ATTR_MALLOC void * GC_CALL GC_malloc_kind(size_t bytes, int knd)
{
size_t granules;
void *tsd;
void *result;
# if MAXOBJKINDS > THREAD_FREELISTS_KINDS
if (EXPECT(knd >= THREAD_FREELISTS_KINDS, FALSE)) {
return GC_malloc_kind_global(bytes, knd);
}
# endif
# if !defined(USE_PTHREAD_SPECIFIC) && !defined(USE_WIN32_SPECIFIC)
{
GC_key_t k = GC_thread_key;
if (EXPECT(0 == k, FALSE)) {
/* We haven't yet run GC_init_parallel. That means */
/* we also aren't locking, so this is fairly cheap. */
return GC_malloc_kind_global(bytes, knd);
}
tsd = GC_getspecific(k);
}
# else
tsd = GC_getspecific(GC_thread_key);
# endif
# if !defined(USE_COMPILER_TLS) && !defined(USE_WIN32_COMPILER_TLS)
if (EXPECT(0 == tsd, FALSE)) {
return GC_malloc_kind_global(bytes, knd);
}
# endif
GC_ASSERT(GC_is_initialized);
GC_ASSERT(GC_is_thread_tsd_valid(tsd));
granules = ROUNDED_UP_GRANULES(bytes);
GC_FAST_MALLOC_GRANS(result, granules,
((GC_tlfs)tsd) -> _freelists[knd], DIRECT_GRANULES,
knd, GC_malloc_kind_global(bytes, knd),
(void)(knd == PTRFREE ? NULL
: (obj_link(result) = 0)));
# ifdef LOG_ALLOCS
GC_log_printf("GC_malloc_kind(%lu, %d) returned %p, recent GC #%lu\n",
(unsigned long)bytes, knd, result,
(unsigned long)GC_gc_no);
# endif
return result;
}
/* We hold the allocator lock. */
void GC_destroy_thread_local(GC_thread p)
{
/* We currently only do this from the thread itself or from */
/* the fork handler for a child process. */
# ifndef HANDLE_FORK
GC_ASSERT(GC_getspecific(GC_thread_key) == (void *)p);
# endif
return_freelists(p -> ptrfree_freelists, GC_aobjfreelist);
return_freelists(p -> normal_freelists, GC_objfreelist);
# ifdef GC_GCJ_SUPPORT
return_freelists(p -> gcj_freelists, GC_gcjobjfreelist);
# endif
}
GC_API GC_ATTR_MALLOC void * GC_CALL GC_malloc(size_t bytes)
{
size_t granules = ROUNDED_UP_GRANULES(bytes);
void *tsd;
void *result;
void **tiny_fl;
# if !defined(USE_PTHREAD_SPECIFIC) && !defined(USE_WIN32_SPECIFIC)
GC_key_t k = GC_thread_key;
if (EXPECT(0 == k, FALSE)) {
/* We haven't yet run GC_init_parallel. That means */
/* we also aren't locking, so this is fairly cheap. */
return GC_core_malloc(bytes);
}
tsd = GC_getspecific(k);
# else
tsd = GC_getspecific(GC_thread_key);
# endif
# if !defined(USE_COMPILER_TLS) && !defined(USE_WIN32_COMPILER_TLS)
if (EXPECT(0 == tsd, FALSE)) {
return GC_core_malloc(bytes);
}
# endif
GC_ASSERT(GC_is_initialized);
GC_ASSERT(GC_is_thread_tsd_valid(tsd));
tiny_fl = ((GC_tlfs)tsd) -> normal_freelists;
GC_FAST_MALLOC_GRANS(result, granules, tiny_fl, DIRECT_GRANULES,
NORMAL, GC_core_malloc(bytes), obj_link(result)=0);
# ifdef LOG_ALLOCS
GC_log_printf("GC_malloc(%lu) returned %p, recent GC #%lu\n",
(unsigned long)bytes, result, (unsigned long)GC_gc_no);
# endif
return result;
}
GC_PTR GC_local_malloc(size_t bytes)
{
if (EXPECT(!SMALL_ENOUGH(bytes),0)) {
return(GC_malloc(bytes));
} else {
int index = INDEX_FROM_BYTES(bytes);
ptr_t * my_fl;
ptr_t my_entry;
# if defined(REDIRECT_MALLOC) && !defined(USE_PTHREAD_SPECIFIC)
GC_key_t k = GC_thread_key;
# endif
void * tsd;
# if defined(REDIRECT_MALLOC) && !defined(USE_PTHREAD_SPECIFIC)
if (EXPECT(0 == k, 0)) {
/* This can happen if we get called when the world is */
/* being initialized. Whether we can actually complete */
/* the initialization then is unclear. */
GC_init_parallel();
k = GC_thread_key;
}
# endif
tsd = GC_getspecific(GC_thread_key);
# ifdef GC_ASSERTIONS
LOCK();
GC_ASSERT(tsd == (void *)GC_lookup_thread(pthread_self()));
UNLOCK();
# endif
my_fl = ((GC_thread)tsd) -> normal_freelists + index;
my_entry = *my_fl;
if (EXPECT((word)my_entry >= HBLKSIZE, 1)) {
ptr_t next = obj_link(my_entry);
GC_PTR result = (GC_PTR)my_entry;
*my_fl = next;
obj_link(my_entry) = 0;
PREFETCH_FOR_WRITE(next);
return result;
} else if ((word)my_entry - 1 < DIRECT_GRANULES) {
*my_fl = my_entry + index + 1;
return GC_malloc(bytes);
} else {
GC_generic_malloc_many(BYTES_FROM_INDEX(index), NORMAL, my_fl);
if (*my_fl == 0) return GC_oom_fn(bytes);
return GC_local_malloc(bytes);
}
}
}
/* collector has been explicitly initialized. */
GC_API GC_ATTR_MALLOC void * GC_CALL GC_gcj_malloc(size_t bytes,
void * ptr_to_struct_containing_descr)
{
if (EXPECT(GC_incremental, FALSE)) {
return GC_core_gcj_malloc(bytes, ptr_to_struct_containing_descr);
} else {
size_t granules = ROUNDED_UP_GRANULES(bytes);
void *result;
void **tiny_fl = ((GC_tlfs)GC_getspecific(GC_thread_key))
-> gcj_freelists;
GC_ASSERT(GC_gcj_malloc_initialized);
GC_FAST_MALLOC_GRANS(result, granules, tiny_fl, DIRECT_GRANULES,
GC_gcj_kind,
GC_core_gcj_malloc(bytes,
ptr_to_struct_containing_descr),
{AO_compiler_barrier();
*(void **)result = ptr_to_struct_containing_descr;});
GC_PTR GC_local_gcj_malloc(size_t bytes,
void * ptr_to_struct_containing_descr)
{
GC_ASSERT(GC_gcj_malloc_initialized);
if (EXPECT(!SMALL_ENOUGH(bytes), 0)) {
return GC_gcj_malloc(bytes, ptr_to_struct_containing_descr);
} else {
int index = INDEX_FROM_BYTES(bytes);
ptr_t * my_fl = ((GC_thread)GC_getspecific(GC_thread_key))
-> gcj_freelists + index;
ptr_t my_entry = *my_fl;
if (EXPECT((word)my_entry >= HBLKSIZE, 1)) {
GC_PTR result = (GC_PTR)my_entry;
GC_ASSERT(!GC_incremental);
/* We assert that any concurrent marker will stop us. */
/* Thus it is impossible for a mark procedure to see the */
/* allocation of the next object, but to see this object */
/* still containing a free list pointer. Otherwise the */
/* marker might find a random "mark descriptor". */
*(volatile ptr_t *)my_fl = obj_link(my_entry);
/* We must update the freelist before we store the pointer. */
/* Otherwise a GC at this point would see a corrupted */
/* free list. */
/* A memory barrier is probably never needed, since the */
/* action of stopping this thread will cause prior writes */
/* to complete. */
GC_ASSERT(((void * volatile *)result)[1] == 0);
*(void * volatile *)result = ptr_to_struct_containing_descr;
return result;
} else if ((word)my_entry - 1 < DIRECT_GRANULES) {
if (!GC_incremental) *my_fl = my_entry + index + 1;
/* In the incremental case, we always have to take this */
/* path. Thus we leave the counter alone. */
return GC_gcj_malloc(bytes, ptr_to_struct_containing_descr);
} else {
GC_generic_malloc_many(BYTES_FROM_INDEX(index), GC_gcj_kind, my_fl);
if (*my_fl == 0) return GC_oom_fn(bytes);
return GC_local_gcj_malloc(bytes, ptr_to_struct_containing_descr);
}
}
}
GC_API void * GC_CALL GC_finalized_malloc(size_t client_lb,
const struct GC_finalizer_closure *fclos)
{
size_t lb = client_lb + sizeof(void *);
size_t lg = ROUNDED_UP_GRANULES(lb);
GC_tlfs tsd;
void *result;
void **tiny_fl, **my_fl, *my_entry;
void *next;
if (EXPECT(lg >= GC_TINY_FREELISTS, FALSE))
return GC_core_finalized_malloc(client_lb, fclos);
tsd = GC_getspecific(GC_thread_key);
tiny_fl = tsd->finalized_freelists;
my_fl = tiny_fl + lg;
my_entry = *my_fl;
while (EXPECT((word)my_entry
<= DIRECT_GRANULES + GC_TINY_FREELISTS + 1, FALSE)) {
if ((word)my_entry - 1 < DIRECT_GRANULES) {
*my_fl = (ptr_t)my_entry + lg + 1;
return GC_core_finalized_malloc(client_lb, fclos);
} else {
GC_generic_malloc_many(GC_RAW_BYTES_FROM_INDEX(lg),
GC_finalized_kind, my_fl);
my_entry = *my_fl;
if (my_entry == 0) {
return (*GC_get_oom_fn())(lb);
}
}
}
next = obj_link(my_entry);
result = (void *)my_entry;
*my_fl = next;
obj_link(result) = 0;
((const void **)result)[GRANULES_TO_WORDS(lg) - 1] = fclos;
PREFETCH_FOR_WRITE(next);
return result;
}
