MonoMethod*
mono_gc_get_managed_array_allocator (MonoVTable *vtable, int rank)
{
#ifdef MANAGED_ALLOCATION
MonoClass *klass = vtable->klass;
#ifdef HAVE_KW_THREAD
int tlab_next_offset = -1;
int tlab_temp_end_offset = -1;
MONO_THREAD_VAR_OFFSET (tlab_next, tlab_next_offset);
MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
if (tlab_next_offset == -1 || tlab_temp_end_offset == -1)
return NULL;
#endif
if (rank != 1)
return NULL;
if (!mono_runtime_has_tls_get ())
return NULL;
if (mono_profiler_get_events () & MONO_PROFILE_ALLOCATIONS)
return NULL;
if (has_per_allocation_action)
return NULL;
g_assert (!mono_class_has_finalizer (klass) && !klass->marshalbyref);
return mono_gc_get_managed_allocator_by_type (ATYPE_VECTOR);
#else
return NULL;
#endif
}
void
mono_tls_init_gc_keys (void)
{
#ifdef MONO_KEYWORD_THREAD
MONO_THREAD_VAR_OFFSET (mono_tls_sgen_thread_info, tls_offsets [TLS_KEY_SGEN_THREAD_INFO]);
#else
mono_native_tls_alloc (&mono_tls_key_sgen_thread_info, NULL);
MONO_THREAD_VAR_OFFSET (mono_tls_key_sgen_thread_info, tls_offsets [TLS_KEY_SGEN_THREAD_INFO]);
#endif
}
/**
* mono_gc_get_managed_allocator_by_type:
*
* Return a managed allocator method corresponding to allocator type ATYPE.
*/
MonoMethod*
mono_gc_get_managed_allocator_by_type (int atype)
{
int offset = -1;
MonoMethod *res;
MONO_THREAD_VAR_OFFSET (GC_thread_tls, offset);
mono_tls_key_set_offset (TLS_KEY_BOEHM_GC_THREAD, offset);
res = alloc_method_cache [atype];
if (res)
return res;
res = create_allocator (atype, TLS_KEY_BOEHM_GC_THREAD);
mono_mutex_lock (&mono_gc_lock);
if (alloc_method_cache [atype]) {
mono_free_method (res);
res = alloc_method_cache [atype];
} else {
mono_memory_barrier ();
alloc_method_cache [atype] = res;
}
mono_mutex_unlock (&mono_gc_lock);
return res;
}
void
sgen_init_allocator (void)
{
#if defined(HAVE_KW_THREAD) && !defined(SGEN_WITHOUT_MONO)
int tlab_next_addr_offset = -1;
int tlab_temp_end_offset = -1;
MONO_THREAD_VAR_OFFSET (tlab_next_addr, tlab_next_addr_offset);
MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
mono_tls_key_set_offset (TLS_KEY_SGEN_TLAB_NEXT_ADDR, tlab_next_addr_offset);
mono_tls_key_set_offset (TLS_KEY_SGEN_TLAB_TEMP_END, tlab_temp_end_offset);
#endif
#ifdef HEAVY_STATISTICS
mono_counters_register ("# objects allocated", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_objects_alloced);
mono_counters_register ("bytes allocated", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_bytes_alloced);
mono_counters_register ("bytes allocated in LOS", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_bytes_alloced_los);
#endif
}
/*
* Generate an allocator method implementing the fast path of mono_gc_alloc_obj ().
* The signature of the called method is:
* object allocate (MonoVTable *vtable)
*/
MonoMethod*
mono_gc_get_managed_allocator (MonoClass *klass, gboolean for_box, gboolean known_instance_size)
{
#ifdef MANAGED_ALLOCATION
#ifdef HAVE_KW_THREAD
int tlab_next_offset = -1;
int tlab_temp_end_offset = -1;
MONO_THREAD_VAR_OFFSET (tlab_next, tlab_next_offset);
MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
if (tlab_next_offset == -1 || tlab_temp_end_offset == -1)
return NULL;
#endif
if (collect_before_allocs)
return NULL;
if (!mono_runtime_has_tls_get ())
return NULL;
if (klass->instance_size > tlab_size)
return NULL;
if (known_instance_size && ALIGN_TO (klass->instance_size, ALLOC_ALIGN) >= MAX_SMALL_OBJ_SIZE)
return NULL;
if (klass->has_finalize || mono_class_is_marshalbyref (klass) || (mono_profiler_get_events () & MONO_PROFILE_ALLOCATIONS))
return NULL;
if (klass->rank)
return NULL;
if (klass->byval_arg.type == MONO_TYPE_STRING)
return mono_gc_get_managed_allocator_by_type (ATYPE_STRING);
/* Generic classes have dynamic field and can go above MAX_SMALL_OBJ_SIZE. */
if (known_instance_size)
return mono_gc_get_managed_allocator_by_type (ATYPE_SMALL);
else
return mono_gc_get_managed_allocator_by_type (ATYPE_NORMAL);
#else
return NULL;
#endif
}
/**
* mono_gc_get_managed_allocator_by_type:
*
* Return a managed allocator method corresponding to allocator type ATYPE.
*/
MonoMethod*
mono_gc_get_managed_allocator_by_type (int atype)
{
int offset = -1;
MonoMethod *res;
MONO_THREAD_VAR_OFFSET (GC_thread_tls, offset);
mono_loader_lock ();
res = alloc_method_cache [atype];
if (!res)
res = alloc_method_cache [atype] = create_allocator (atype, offset);
mono_loader_unlock ();
return res;
}
/*
* Generate an allocator method implementing the fast path of mono_gc_alloc_obj ().
* The signature of the called method is:
* object allocate (MonoVTable *vtable)
*/
MonoMethod*
mono_gc_get_managed_allocator (MonoVTable *vtable, gboolean for_box)
{
#ifdef MANAGED_ALLOCATION
MonoClass *klass = vtable->klass;
#ifdef HAVE_KW_THREAD
int tlab_next_offset = -1;
int tlab_temp_end_offset = -1;
MONO_THREAD_VAR_OFFSET (tlab_next, tlab_next_offset);
MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
if (tlab_next_offset == -1 || tlab_temp_end_offset == -1)
return NULL;
#endif
if (!mono_runtime_has_tls_get ())
return NULL;
if (klass->instance_size > tlab_size)
return NULL;
if (klass->has_finalize || klass->marshalbyref || (mono_profiler_get_events () & MONO_PROFILE_ALLOCATIONS))
return NULL;
if (klass->rank)
return NULL;
if (klass->byval_arg.type == MONO_TYPE_STRING)
return NULL;
if (collect_before_allocs)
return NULL;
if (ALIGN_TO (klass->instance_size, ALLOC_ALIGN) < MAX_SMALL_OBJ_SIZE)
return mono_gc_get_managed_allocator_by_type (ATYPE_SMALL);
else
return mono_gc_get_managed_allocator_by_type (ATYPE_NORMAL);
#else
return NULL;
#endif
}
void
mono_tls_init_runtime_keys (void)
{
#ifdef MONO_KEYWORD_THREAD
MONO_THREAD_VAR_OFFSET (mono_tls_thread, tls_offsets [TLS_KEY_THREAD]);
MONO_THREAD_VAR_OFFSET (mono_tls_jit_tls, tls_offsets [TLS_KEY_JIT_TLS]);
MONO_THREAD_VAR_OFFSET (mono_tls_domain, tls_offsets [TLS_KEY_DOMAIN]);
MONO_THREAD_VAR_OFFSET (mono_tls_lmf_addr, tls_offsets [TLS_KEY_LMF_ADDR]);
#else
mono_native_tls_alloc (&mono_tls_key_thread, NULL);
MONO_THREAD_VAR_OFFSET (mono_tls_key_thread, tls_offsets [TLS_KEY_THREAD]);
mono_native_tls_alloc (&mono_tls_key_jit_tls, NULL);
MONO_THREAD_VAR_OFFSET (mono_tls_key_jit_tls, tls_offsets [TLS_KEY_JIT_TLS]);
mono_native_tls_alloc (&mono_tls_key_domain, NULL);
MONO_THREAD_VAR_OFFSET (mono_tls_key_domain, tls_offsets [TLS_KEY_DOMAIN]);
mono_native_tls_alloc (&mono_tls_key_lmf_addr, NULL);
MONO_THREAD_VAR_OFFSET (mono_tls_key_lmf_addr, tls_offsets [TLS_KEY_LMF_ADDR]);
#endif
}
MonoMethod*
mono_gc_get_managed_allocator (MonoClass *klass, gboolean for_box, gboolean known_instance_size)
{
int offset = -1;
int atype;
MONO_THREAD_VAR_OFFSET (GC_thread_tls, offset);
/*g_print ("thread tls: %d\n", offset);*/
if (offset == -1)
return NULL;
if (!SMALL_ENOUGH (klass->instance_size))
return NULL;
if (mono_class_has_finalizer (klass) || mono_class_is_marshalbyref (klass) || (mono_profiler_get_events () & MONO_PROFILE_ALLOCATIONS))
return NULL;
if (klass->rank)
return NULL;
if (mono_class_is_open_constructed_type (&klass->byval_arg))
return NULL;
if (klass->byval_arg.type == MONO_TYPE_STRING) {
atype = ATYPE_STRING;
} else if (!known_instance_size) {
return NULL;
} else if (!klass->has_references) {
if (for_box)
atype = ATYPE_FREEPTR_FOR_BOX;
else
atype = ATYPE_FREEPTR;
} else {
return NULL;
/*
* disabled because we currently do a runtime choice anyway, to
* deal with multiple appdomains.
if (vtable->gc_descr != GC_NO_DESCRIPTOR)
atype = ATYPE_GCJ;
else
atype = ATYPE_NORMAL;
*/
}
return mono_gc_get_managed_allocator_by_type (atype, FALSE);
}
/* FIXME: Do this in the JIT, where specialized allocation sequences can be created
* for each class. This is currently not easy to do, as it is hard to generate basic
* blocks + branches, but it is easy with the linear IL codebase.
*
* For this to work we'd need to solve the TLAB race, first. Now we
* require the allocator to be in a few known methods to make sure
* that they are executed atomically via the restart mechanism.
*/
static MonoMethod*
create_allocator (int atype)
{
int p_var, size_var;
guint32 slowpath_branch, max_size_branch;
MonoMethodBuilder *mb;
MonoMethod *res;
MonoMethodSignature *csig;
static gboolean registered = FALSE;
int tlab_next_addr_var, new_next_var;
int num_params, i;
const char *name = NULL;
AllocatorWrapperInfo *info;
#ifdef HAVE_KW_THREAD
int tlab_next_addr_offset = -1;
int tlab_temp_end_offset = -1;
MONO_THREAD_VAR_OFFSET (tlab_next_addr, tlab_next_addr_offset);
MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
g_assert (tlab_next_addr_offset != -1);
g_assert (tlab_temp_end_offset != -1);
#endif
if (!registered) {
mono_register_jit_icall (mono_gc_alloc_obj, "mono_gc_alloc_obj", mono_create_icall_signature ("object ptr int"), FALSE);
mono_register_jit_icall (mono_gc_alloc_vector, "mono_gc_alloc_vector", mono_create_icall_signature ("object ptr int int"), FALSE);
registered = TRUE;
}
if (atype == ATYPE_SMALL) {
num_params = 1;
name = "AllocSmall";
} else if (atype == ATYPE_NORMAL) {
num_params = 1;
name = "Alloc";
} else if (atype == ATYPE_VECTOR) {
num_params = 2;
name = "AllocVector";
} else {
g_assert_not_reached ();
}
csig = mono_metadata_signature_alloc (mono_defaults.corlib, num_params);
csig->ret = &mono_defaults.object_class->byval_arg;
for (i = 0; i < num_params; ++i)
csig->params [i] = &mono_defaults.int_class->byval_arg;
mb = mono_mb_new (mono_defaults.object_class, name, MONO_WRAPPER_ALLOC);
size_var = mono_mb_add_local (mb, &mono_defaults.int32_class->byval_arg);
if (atype == ATYPE_NORMAL || atype == ATYPE_SMALL) {
/* size = vtable->klass->instance_size; */
mono_mb_emit_ldarg (mb, 0);
mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoVTable, klass));
mono_mb_emit_byte (mb, CEE_ADD);
mono_mb_emit_byte (mb, CEE_LDIND_I);
mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoClass, instance_size));
mono_mb_emit_byte (mb, CEE_ADD);
/* FIXME: assert instance_size stays a 4 byte integer */
mono_mb_emit_byte (mb, CEE_LDIND_U4);
mono_mb_emit_stloc (mb, size_var);
} else if (atype == ATYPE_VECTOR) {
MonoExceptionClause *clause;
int pos, pos_leave;
MonoClass *oom_exc_class;
MonoMethod *ctor;
/* n > MONO_ARRAY_MAX_INDEX -> OverflowException */
mono_mb_emit_ldarg (mb, 1);
mono_mb_emit_icon (mb, MONO_ARRAY_MAX_INDEX);
pos = mono_mb_emit_short_branch (mb, CEE_BLE_UN_S);
mono_mb_emit_exception (mb, "OverflowException", NULL);
mono_mb_patch_short_branch (mb, pos);
clause = mono_image_alloc0 (mono_defaults.corlib, sizeof (MonoExceptionClause));
clause->try_offset = mono_mb_get_label (mb);
/* vtable->klass->sizes.element_size */
mono_mb_emit_ldarg (mb, 0);
mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoVTable, klass));
mono_mb_emit_byte (mb, CEE_ADD);
mono_mb_emit_byte (mb, CEE_LDIND_I);
mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoClass, sizes.element_size));
mono_mb_emit_byte (mb, CEE_ADD);
mono_mb_emit_byte (mb, CEE_LDIND_U4);
/* * n */
mono_mb_emit_ldarg (mb, 1);
mono_mb_emit_byte (mb, CEE_MUL_OVF_UN);
/* + sizeof (MonoArray) */
mono_mb_emit_icon (mb, sizeof (MonoArray));
mono_mb_emit_byte (mb, CEE_ADD_OVF_UN);
mono_mb_emit_stloc (mb, size_var);
pos_leave = mono_mb_emit_branch (mb, CEE_LEAVE);
/* catch */
clause->flags = MONO_EXCEPTION_CLAUSE_NONE;
clause->try_len = mono_mb_get_pos (mb) - clause->try_offset;
clause->data.catch_class = mono_class_from_name (mono_defaults.corlib,
"System", "OverflowException");
g_assert (clause->data.catch_class);
clause->handler_offset = mono_mb_get_label (mb);
oom_exc_class = mono_class_from_name (mono_defaults.corlib,
"System", "OutOfMemoryException");
g_assert (oom_exc_class);
ctor = mono_class_get_method_from_name (oom_exc_class, ".ctor", 0);
g_assert (ctor);
mono_mb_emit_byte (mb, CEE_POP);
mono_mb_emit_op (mb, CEE_NEWOBJ, ctor);
mono_mb_emit_byte (mb, CEE_THROW);
clause->handler_len = mono_mb_get_pos (mb) - clause->handler_offset;
mono_mb_set_clauses (mb, 1, clause);
mono_mb_patch_branch (mb, pos_leave);
/* end catch */
} else {
g_assert_not_reached ();
}
/* size += ALLOC_ALIGN - 1; */
mono_mb_emit_ldloc (mb, size_var);
mono_mb_emit_icon (mb, ALLOC_ALIGN - 1);
mono_mb_emit_byte (mb, CEE_ADD);
/* size &= ~(ALLOC_ALIGN - 1); */
mono_mb_emit_icon (mb, ~(ALLOC_ALIGN - 1));
mono_mb_emit_byte (mb, CEE_AND);
mono_mb_emit_stloc (mb, size_var);
/* if (size > MAX_SMALL_OBJ_SIZE) goto slowpath */
if (atype != ATYPE_SMALL) {
mono_mb_emit_ldloc (mb, size_var);
mono_mb_emit_icon (mb, MAX_SMALL_OBJ_SIZE);
max_size_branch = mono_mb_emit_short_branch (mb, MONO_CEE_BGT_UN_S);
}
/*
* We need to modify tlab_next, but the JIT only supports reading, so we read
* another tls var holding its address instead.
*/
/* tlab_next_addr (local) = tlab_next_addr (TLS var) */
tlab_next_addr_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
EMIT_TLS_ACCESS (mb, tlab_next_addr, tlab_next_addr_offset);
mono_mb_emit_stloc (mb, tlab_next_addr_var);
/* p = (void**)tlab_next; */
p_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
mono_mb_emit_ldloc (mb, tlab_next_addr_var);
mono_mb_emit_byte (mb, CEE_LDIND_I);
mono_mb_emit_stloc (mb, p_var);
/* new_next = (char*)p + size; */
new_next_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
mono_mb_emit_ldloc (mb, p_var);
mono_mb_emit_ldloc (mb, size_var);
mono_mb_emit_byte (mb, CEE_CONV_I);
mono_mb_emit_byte (mb, CEE_ADD);
mono_mb_emit_stloc (mb, new_next_var);
/* if (G_LIKELY (new_next < tlab_temp_end)) */
mono_mb_emit_ldloc (mb, new_next_var);
EMIT_TLS_ACCESS (mb, tlab_temp_end, tlab_temp_end_offset);
slowpath_branch = mono_mb_emit_short_branch (mb, MONO_CEE_BLT_UN_S);
/* Slowpath */
if (atype != ATYPE_SMALL)
mono_mb_patch_short_branch (mb, max_size_branch);
mono_mb_emit_byte (mb, MONO_CUSTOM_PREFIX);
mono_mb_emit_byte (mb, CEE_MONO_NOT_TAKEN);
/* FIXME: mono_gc_alloc_obj takes a 'size_t' as an argument, not an int32 */
mono_mb_emit_ldarg (mb, 0);
mono_mb_emit_ldloc (mb, size_var);
if (atype == ATYPE_NORMAL || atype == ATYPE_SMALL) {
mono_mb_emit_icall (mb, mono_gc_alloc_obj);
} else if (atype == ATYPE_VECTOR) {
mono_mb_emit_ldarg (mb, 1);
mono_mb_emit_icall (mb, mono_gc_alloc_vector);
} else {
g_assert_not_reached ();
}
mono_mb_emit_byte (mb, CEE_RET);
/* Fastpath */
mono_mb_patch_short_branch (mb, slowpath_branch);
/* FIXME: Memory barrier */
/* tlab_next = new_next */
mono_mb_emit_ldloc (mb, tlab_next_addr_var);
mono_mb_emit_ldloc (mb, new_next_var);
mono_mb_emit_byte (mb, CEE_STIND_I);
/*The tlab store must be visible before the the vtable store. This could be replaced with a DDS but doing it with IL would be tricky. */
mono_mb_emit_byte ((mb), MONO_CUSTOM_PREFIX);
mono_mb_emit_op (mb, CEE_MONO_MEMORY_BARRIER, StoreStoreBarrier);
/* *p = vtable; */
mono_mb_emit_ldloc (mb, p_var);
mono_mb_emit_ldarg (mb, 0);
mono_mb_emit_byte (mb, CEE_STIND_I);
if (atype == ATYPE_VECTOR) {
/* arr->max_length = max_length; */
mono_mb_emit_ldloc (mb, p_var);
mono_mb_emit_ldflda (mb, G_STRUCT_OFFSET (MonoArray, max_length));
mono_mb_emit_ldarg (mb, 1);
#ifdef MONO_BIG_ARRAYS
mono_mb_emit_byte (mb, CEE_STIND_I);
#else
mono_mb_emit_byte (mb, CEE_STIND_I4);
#endif
}
/*
We must make sure both vtable and max_length are globaly visible before returning to managed land.
*/
mono_mb_emit_byte ((mb), MONO_CUSTOM_PREFIX);
mono_mb_emit_op (mb, CEE_MONO_MEMORY_BARRIER, StoreStoreBarrier);
/* return p */
mono_mb_emit_ldloc (mb, p_var);
mono_mb_emit_byte (mb, CEE_RET);
res = mono_mb_create_method (mb, csig, 8);
mono_mb_free (mb);
mono_method_get_header (res)->init_locals = FALSE;
info = mono_image_alloc0 (mono_defaults.corlib, sizeof (AllocatorWrapperInfo));
info->gc_name = "sgen";
info->alloc_type = atype;
mono_marshal_set_wrapper_info (res, info);
return res;
}
