/* FIXME: Do this in the JIT, where specialized allocation sequences can be created
* for each class. This is currently not easy to do, as it is hard to generate basic
* blocks + branches, but it is easy with the linear IL codebase.
*
* For this to work we'd need to solve the TLAB race, first. Now we
* require the allocator to be in a few known methods to make sure
* that they are executed atomically via the restart mechanism.
*/
static MonoMethod*
create_allocator (int atype)
{
int p_var, size_var;
guint32 slowpath_branch, max_size_branch;
MonoMethodBuilder *mb;
MonoMethod *res;
MonoMethodSignature *csig;
static gboolean registered = FALSE;
int tlab_next_addr_var, new_next_var;
int num_params, i;
const char *name = NULL;
AllocatorWrapperInfo *info;
#ifdef HAVE_KW_THREAD
int tlab_next_addr_offset = -1;
int tlab_temp_end_offset = -1;
MONO_THREAD_VAR_OFFSET (tlab_next_addr, tlab_next_addr_offset);
MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
g_assert (tlab_next_addr_offset != -1);
g_assert (tlab_temp_end_offset != -1);
#endif
if (!registered) {
mono_register_jit_icall (mono_gc_alloc_obj, "mono_gc_alloc_obj", mono_create_icall_signature ("object ptr int"), FALSE);
mono_register_jit_icall (mono_gc_alloc_vector, "mono_gc_alloc_vector", mono_create_icall_signature ("object ptr int int"), FALSE);
registered = TRUE;
}
if (atype == ATYPE_SMALL) {
num_params = 1;
name = "AllocSmall";
} else if (atype == ATYPE_NORMAL) {
num_params = 1;
name = "Alloc";
} else if (atype == ATYPE_VECTOR) {
num_params = 2;
name = "AllocVector";
} else {
g_assert_not_reached ();
}
csig = mono_metadata_signature_alloc (mono_defaults.corlib, num_params);
csig->ret = &mono_defaults.object_class->byval_arg;
for (i = 0; i < num_params; ++i)
csig->params [i] = &mono_defaults.int_class->byval_arg;
mb = mono_mb_new (mono_defaults.object_class, name, MONO_WRAPPER_ALLOC);
size_var = mono_mb_add_local (mb, &mono_defaults.int32_class->byval_arg);
if (atype == ATYPE_NORMAL || atype == ATYPE_SMALL) {
/* size = vtable->klass->instance_size; */
mono_mb_emit_ldarg (mb, 0);
mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoVTable, klass));
mono_mb_emit_byte (mb, CEE_ADD);
mono_mb_emit_byte (mb, CEE_LDIND_I);
mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoClass, instance_size));
mono_mb_emit_byte (mb, CEE_ADD);
/* FIXME: assert instance_size stays a 4 byte integer */
mono_mb_emit_byte (mb, CEE_LDIND_U4);
mono_mb_emit_stloc (mb, size_var);
} else if (atype == ATYPE_VECTOR) {
MonoExceptionClause *clause;
int pos, pos_leave;
MonoClass *oom_exc_class;
MonoMethod *ctor;
/* n > MONO_ARRAY_MAX_INDEX -> OverflowException */
mono_mb_emit_ldarg (mb, 1);
mono_mb_emit_icon (mb, MONO_ARRAY_MAX_INDEX);
pos = mono_mb_emit_short_branch (mb, CEE_BLE_UN_S);
mono_mb_emit_exception (mb, "OverflowException", NULL);
mono_mb_patch_short_branch (mb, pos);
clause = mono_image_alloc0 (mono_defaults.corlib, sizeof (MonoExceptionClause));
clause->try_offset = mono_mb_get_label (mb);
/* vtable->klass->sizes.element_size */
mono_mb_emit_ldarg (mb, 0);
mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoVTable, klass));
mono_mb_emit_byte (mb, CEE_ADD);
mono_mb_emit_byte (mb, CEE_LDIND_I);
mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoClass, sizes.element_size));
mono_mb_emit_byte (mb, CEE_ADD);
mono_mb_emit_byte (mb, CEE_LDIND_U4);
/* * n */
mono_mb_emit_ldarg (mb, 1);
mono_mb_emit_byte (mb, CEE_MUL_OVF_UN);
/* + sizeof (MonoArray) */
mono_mb_emit_icon (mb, sizeof (MonoArray));
mono_mb_emit_byte (mb, CEE_ADD_OVF_UN);
mono_mb_emit_stloc (mb, size_var);
pos_leave = mono_mb_emit_branch (mb, CEE_LEAVE);
/* catch */
clause->flags = MONO_EXCEPTION_CLAUSE_NONE;
clause->try_len = mono_mb_get_pos (mb) - clause->try_offset;
clause->data.catch_class = mono_class_from_name (mono_defaults.corlib,
"System", "OverflowException");
g_assert (clause->data.catch_class);
clause->handler_offset = mono_mb_get_label (mb);
oom_exc_class = mono_class_from_name (mono_defaults.corlib,
"System", "OutOfMemoryException");
g_assert (oom_exc_class);
ctor = mono_class_get_method_from_name (oom_exc_class, ".ctor", 0);
g_assert (ctor);
mono_mb_emit_byte (mb, CEE_POP);
mono_mb_emit_op (mb, CEE_NEWOBJ, ctor);
mono_mb_emit_byte (mb, CEE_THROW);
clause->handler_len = mono_mb_get_pos (mb) - clause->handler_offset;
mono_mb_set_clauses (mb, 1, clause);
mono_mb_patch_branch (mb, pos_leave);
/* end catch */
} else {
g_assert_not_reached ();
}
/* size += ALLOC_ALIGN - 1; */
mono_mb_emit_ldloc (mb, size_var);
mono_mb_emit_icon (mb, ALLOC_ALIGN - 1);
mono_mb_emit_byte (mb, CEE_ADD);
/* size &= ~(ALLOC_ALIGN - 1); */
mono_mb_emit_icon (mb, ~(ALLOC_ALIGN - 1));
mono_mb_emit_byte (mb, CEE_AND);
mono_mb_emit_stloc (mb, size_var);
/* if (size > MAX_SMALL_OBJ_SIZE) goto slowpath */
if (atype != ATYPE_SMALL) {
mono_mb_emit_ldloc (mb, size_var);
mono_mb_emit_icon (mb, MAX_SMALL_OBJ_SIZE);
max_size_branch = mono_mb_emit_short_branch (mb, MONO_CEE_BGT_UN_S);
}
/*
* We need to modify tlab_next, but the JIT only supports reading, so we read
* another tls var holding its address instead.
*/
/* tlab_next_addr (local) = tlab_next_addr (TLS var) */
tlab_next_addr_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
EMIT_TLS_ACCESS (mb, tlab_next_addr, tlab_next_addr_offset);
mono_mb_emit_stloc (mb, tlab_next_addr_var);
/* p = (void**)tlab_next; */
p_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
mono_mb_emit_ldloc (mb, tlab_next_addr_var);
mono_mb_emit_byte (mb, CEE_LDIND_I);
mono_mb_emit_stloc (mb, p_var);
/* new_next = (char*)p + size; */
new_next_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
mono_mb_emit_ldloc (mb, p_var);
mono_mb_emit_ldloc (mb, size_var);
mono_mb_emit_byte (mb, CEE_CONV_I);
mono_mb_emit_byte (mb, CEE_ADD);
mono_mb_emit_stloc (mb, new_next_var);
/* if (G_LIKELY (new_next < tlab_temp_end)) */
mono_mb_emit_ldloc (mb, new_next_var);
EMIT_TLS_ACCESS (mb, tlab_temp_end, tlab_temp_end_offset);
slowpath_branch = mono_mb_emit_short_branch (mb, MONO_CEE_BLT_UN_S);
/* Slowpath */
if (atype != ATYPE_SMALL)
mono_mb_patch_short_branch (mb, max_size_branch);
mono_mb_emit_byte (mb, MONO_CUSTOM_PREFIX);
mono_mb_emit_byte (mb, CEE_MONO_NOT_TAKEN);
/* FIXME: mono_gc_alloc_obj takes a 'size_t' as an argument, not an int32 */
mono_mb_emit_ldarg (mb, 0);
mono_mb_emit_ldloc (mb, size_var);
if (atype == ATYPE_NORMAL || atype == ATYPE_SMALL) {
mono_mb_emit_icall (mb, mono_gc_alloc_obj);
} else if (atype == ATYPE_VECTOR) {
mono_mb_emit_ldarg (mb, 1);
mono_mb_emit_icall (mb, mono_gc_alloc_vector);
} else {
g_assert_not_reached ();
}
mono_mb_emit_byte (mb, CEE_RET);
/* Fastpath */
mono_mb_patch_short_branch (mb, slowpath_branch);
/* FIXME: Memory barrier */
/* tlab_next = new_next */
mono_mb_emit_ldloc (mb, tlab_next_addr_var);
mono_mb_emit_ldloc (mb, new_next_var);
mono_mb_emit_byte (mb, CEE_STIND_I);
/*The tlab store must be visible before the the vtable store. This could be replaced with a DDS but doing it with IL would be tricky. */
mono_mb_emit_byte ((mb), MONO_CUSTOM_PREFIX);
mono_mb_emit_op (mb, CEE_MONO_MEMORY_BARRIER, StoreStoreBarrier);
/* *p = vtable; */
mono_mb_emit_ldloc (mb, p_var);
mono_mb_emit_ldarg (mb, 0);
mono_mb_emit_byte (mb, CEE_STIND_I);
if (atype == ATYPE_VECTOR) {
/* arr->max_length = max_length; */
mono_mb_emit_ldloc (mb, p_var);
mono_mb_emit_ldflda (mb, G_STRUCT_OFFSET (MonoArray, max_length));
mono_mb_emit_ldarg (mb, 1);
#ifdef MONO_BIG_ARRAYS
mono_mb_emit_byte (mb, CEE_STIND_I);
#else
mono_mb_emit_byte (mb, CEE_STIND_I4);
#endif
}
/*
We must make sure both vtable and max_length are globaly visible before returning to managed land.
*/
mono_mb_emit_byte ((mb), MONO_CUSTOM_PREFIX);
mono_mb_emit_op (mb, CEE_MONO_MEMORY_BARRIER, StoreStoreBarrier);
/* return p */
mono_mb_emit_ldloc (mb, p_var);
mono_mb_emit_byte (mb, CEE_RET);
res = mono_mb_create_method (mb, csig, 8);
mono_mb_free (mb);
mono_method_get_header (res)->init_locals = FALSE;
info = mono_image_alloc0 (mono_defaults.corlib, sizeof (AllocatorWrapperInfo));
info->gc_name = "sgen";
info->alloc_type = atype;
mono_marshal_set_wrapper_info (res, info);
return res;
}
static MonoMethod*
create_allocator (int atype, int offset)
{
int index_var, bytes_var, my_fl_var, my_entry_var;
guint32 no_freelist_branch, not_small_enough_branch = 0;
guint32 size_overflow_branch = 0;
MonoMethodBuilder *mb;
MonoMethod *res;
MonoMethodSignature *csig;
AllocatorWrapperInfo *info;
if (atype == ATYPE_STRING) {
csig = mono_metadata_signature_alloc (mono_defaults.corlib, 2);
csig->ret = &mono_defaults.string_class->byval_arg;
csig->params [0] = &mono_defaults.int_class->byval_arg;
csig->params [1] = &mono_defaults.int32_class->byval_arg;
} else {
csig = mono_metadata_signature_alloc (mono_defaults.corlib, 1);
csig->ret = &mono_defaults.object_class->byval_arg;
csig->params [0] = &mono_defaults.int_class->byval_arg;
}
mb = mono_mb_new (mono_defaults.object_class, "Alloc", MONO_WRAPPER_ALLOC);
bytes_var = mono_mb_add_local (mb, &mono_defaults.int32_class->byval_arg);
if (atype == ATYPE_STRING) {
/* a string alloator method takes the args: (vtable, len) */
/* bytes = (sizeof (MonoString) + ((len + 1) * 2)); */
mono_mb_emit_ldarg (mb, 1);
mono_mb_emit_icon (mb, 1);
mono_mb_emit_byte (mb, MONO_CEE_ADD);
mono_mb_emit_icon (mb, 1);
mono_mb_emit_byte (mb, MONO_CEE_SHL);
// sizeof (MonoString) might include padding
mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoString, chars));
mono_mb_emit_byte (mb, MONO_CEE_ADD);
mono_mb_emit_stloc (mb, bytes_var);
} else {
/* bytes = vtable->klass->instance_size */
mono_mb_emit_ldarg (mb, 0);
mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoVTable, klass));
mono_mb_emit_byte (mb, MONO_CEE_ADD);
mono_mb_emit_byte (mb, MONO_CEE_LDIND_I);
mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoClass, instance_size));
mono_mb_emit_byte (mb, MONO_CEE_ADD);
/* FIXME: assert instance_size stays a 4 byte integer */
mono_mb_emit_byte (mb, MONO_CEE_LDIND_U4);
mono_mb_emit_stloc (mb, bytes_var);
}
/* this is needed for strings/arrays only as the other big types are never allocated with this method */
if (atype == ATYPE_STRING) {
/* check for size */
/* if (!SMALL_ENOUGH (bytes)) jump slow_path;*/
mono_mb_emit_ldloc (mb, bytes_var);
mono_mb_emit_icon (mb, (NFREELISTS-1) * GRANULARITY);
not_small_enough_branch = mono_mb_emit_short_branch (mb, MONO_CEE_BGT_UN_S);
/* check for overflow */
mono_mb_emit_ldloc (mb, bytes_var);
mono_mb_emit_icon (mb, sizeof (MonoString));
size_overflow_branch = mono_mb_emit_short_branch (mb, MONO_CEE_BLE_UN_S);
}
/* int index = INDEX_FROM_BYTES(bytes); */
index_var = mono_mb_add_local (mb, &mono_defaults.int32_class->byval_arg);
mono_mb_emit_ldloc (mb, bytes_var);
mono_mb_emit_icon (mb, GRANULARITY - 1);
mono_mb_emit_byte (mb, MONO_CEE_ADD);
mono_mb_emit_icon (mb, shift_amount (GRANULARITY));
mono_mb_emit_byte (mb, MONO_CEE_SHR_UN);
mono_mb_emit_icon (mb, shift_amount (sizeof (gpointer)));
mono_mb_emit_byte (mb, MONO_CEE_SHL);
/* index var is already adjusted into bytes */
mono_mb_emit_stloc (mb, index_var);
my_fl_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
my_entry_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
/* my_fl = ((GC_thread)tsd) -> ptrfree_freelists + index; */
mono_mb_emit_byte (mb, MONO_CUSTOM_PREFIX);
mono_mb_emit_byte (mb, 0x0D); /* CEE_MONO_TLS */
mono_mb_emit_i4 (mb, offset);
if (atype == ATYPE_FREEPTR || atype == ATYPE_FREEPTR_FOR_BOX || atype == ATYPE_STRING)
mono_mb_emit_icon (mb, G_STRUCT_OFFSET (struct GC_Thread_Rep, ptrfree_freelists));
else if (atype == ATYPE_NORMAL)
mono_mb_emit_icon (mb, G_STRUCT_OFFSET (struct GC_Thread_Rep, normal_freelists));
else if (atype == ATYPE_GCJ)
mono_mb_emit_icon (mb, G_STRUCT_OFFSET (struct GC_Thread_Rep, gcj_freelists));
else
g_assert_not_reached ();
mono_mb_emit_byte (mb, MONO_CEE_ADD);
mono_mb_emit_ldloc (mb, index_var);
mono_mb_emit_byte (mb, MONO_CEE_ADD);
mono_mb_emit_stloc (mb, my_fl_var);
/* my_entry = *my_fl; */
mono_mb_emit_ldloc (mb, my_fl_var);
mono_mb_emit_byte (mb, MONO_CEE_LDIND_I);
mono_mb_emit_stloc (mb, my_entry_var);
/* if (EXPECT((word)my_entry >= HBLKSIZE, 1)) { */
mono_mb_emit_ldloc (mb, my_entry_var);
mono_mb_emit_icon (mb, HBLKSIZE);
no_freelist_branch = mono_mb_emit_short_branch (mb, MONO_CEE_BLT_UN_S);
/* ptr_t next = obj_link(my_entry); *my_fl = next; */
mono_mb_emit_ldloc (mb, my_fl_var);
mono_mb_emit_ldloc (mb, my_entry_var);
mono_mb_emit_byte (mb, MONO_CEE_LDIND_I);
mono_mb_emit_byte (mb, MONO_CEE_STIND_I);
/* set the vtable and clear the words in the object */
mono_mb_emit_ldloc (mb, my_entry_var);
mono_mb_emit_ldarg (mb, 0);
mono_mb_emit_byte (mb, MONO_CEE_STIND_I);
if (atype == ATYPE_FREEPTR) {
int start_var, end_var, start_loop;
/* end = my_entry + bytes; start = my_entry + sizeof (gpointer);
*/
start_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
end_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
mono_mb_emit_ldloc (mb, my_entry_var);
mono_mb_emit_ldloc (mb, bytes_var);
mono_mb_emit_byte (mb, MONO_CEE_ADD);
mono_mb_emit_stloc (mb, end_var);
mono_mb_emit_ldloc (mb, my_entry_var);
mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoObject, synchronisation));
mono_mb_emit_byte (mb, MONO_CEE_ADD);
mono_mb_emit_stloc (mb, start_var);
/*
* do {
* *start++ = NULL;
* } while (start < end);
*/
start_loop = mono_mb_get_label (mb);
mono_mb_emit_ldloc (mb, start_var);
mono_mb_emit_icon (mb, 0);
mono_mb_emit_byte (mb, MONO_CEE_STIND_I);
mono_mb_emit_ldloc (mb, start_var);
mono_mb_emit_icon (mb, sizeof (gpointer));
mono_mb_emit_byte (mb, MONO_CEE_ADD);
mono_mb_emit_stloc (mb, start_var);
mono_mb_emit_ldloc (mb, start_var);
mono_mb_emit_ldloc (mb, end_var);
mono_mb_emit_byte (mb, MONO_CEE_BLT_UN_S);
mono_mb_emit_byte (mb, start_loop - (mono_mb_get_label (mb) + 1));
} else if (atype == ATYPE_FREEPTR_FOR_BOX || atype == ATYPE_STRING) {
/* need to clear just the sync pointer */
mono_mb_emit_ldloc (mb, my_entry_var);
mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoObject, synchronisation));
mono_mb_emit_byte (mb, MONO_CEE_ADD);
mono_mb_emit_icon (mb, 0);
mono_mb_emit_byte (mb, MONO_CEE_STIND_I);
}
if (atype == ATYPE_STRING) {
/* need to set length and clear the last char */
/* s->length = len; */
mono_mb_emit_ldloc (mb, my_entry_var);
mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoString, length));
mono_mb_emit_byte (mb, MONO_CEE_ADD);
mono_mb_emit_ldarg (mb, 1);
mono_mb_emit_byte (mb, MONO_CEE_STIND_I4);
/* s->chars [len] = 0; */
mono_mb_emit_ldloc (mb, my_entry_var);
mono_mb_emit_ldloc (mb, bytes_var);
mono_mb_emit_icon (mb, 2);
mono_mb_emit_byte (mb, MONO_CEE_SUB);
mono_mb_emit_byte (mb, MONO_CEE_ADD);
mono_mb_emit_icon (mb, 0);
mono_mb_emit_byte (mb, MONO_CEE_STIND_I2);
}
/* return my_entry; */
mono_mb_emit_ldloc (mb, my_entry_var);
mono_mb_emit_byte (mb, MONO_CEE_RET);
mono_mb_patch_short_branch (mb, no_freelist_branch);
if (not_small_enough_branch > 0)
mono_mb_patch_short_branch (mb, not_small_enough_branch);
if (size_overflow_branch > 0)
mono_mb_patch_short_branch (mb, size_overflow_branch);
/* the slow path: we just call back into the runtime */
if (atype == ATYPE_STRING) {
mono_mb_emit_ldarg (mb, 1);
mono_mb_emit_icall (mb, mono_string_alloc);
} else {
mono_mb_emit_ldarg (mb, 0);
mono_mb_emit_icall (mb, mono_object_new_specific);
}
mono_mb_emit_byte (mb, MONO_CEE_RET);
res = mono_mb_create_method (mb, csig, 8);
mono_mb_free (mb);
mono_method_get_header (res)->init_locals = FALSE;
info = mono_image_alloc0 (mono_defaults.corlib, sizeof (AllocatorWrapperInfo));
info->gc_name = "boehm";
info->alloc_type = atype;
mono_marshal_set_wrapper_info (res, info);
return res;
}