/*
Search @list for element with key @key.
The nodes next, cur and prev are returned in @hp
Returns true if @value was removed by this call.
This function cannot be called from a signal nor with the world stopped.
*/
gboolean
mono_lls_remove (MonoLinkedListSet *list, MonoThreadHazardPointers *hp, MonoLinkedListSetNode *value)
{
MonoLinkedListSetNode *cur, **prev, *next;
while (1) {
if (!mono_lls_find (list, hp, value->key))
return FALSE;
next = mono_hazard_pointer_get_val (hp, 0);
cur = mono_hazard_pointer_get_val (hp, 1);
prev = mono_hazard_pointer_get_val (hp, 2);
g_assert (cur == value);
if (InterlockedCompareExchangePointer ((volatile gpointer*)&cur->next, mask (next, 1), next) != next)
continue;
/* The second CAS must happen before the first. */
mono_memory_write_barrier ();
if (InterlockedCompareExchangePointer ((volatile gpointer*)prev, mono_lls_pointer_unmask (next), cur) == cur) {
/* The CAS must happen before the hazard pointer clear. */
mono_memory_write_barrier ();
mono_hazard_pointer_clear (hp, 1);
if (list->free_node_func)
mono_thread_hazardous_free_or_queue (value, list->free_node_func, FALSE, TRUE);
} else
mono_lls_find (list, hp, value->key);
return TRUE;
}
}
void
mono_hazard_pointer_restore_for_signal_handler (int small_id)
{
MonoThreadHazardPointers *hp = mono_hazard_pointer_get ();
MonoThreadHazardPointers *hp_overflow;
int i;
if (small_id < 0)
return;
g_assert (small_id < HAZARD_TABLE_OVERFLOW);
g_assert (overflow_busy [small_id]);
for (i = 0; i < HAZARD_POINTER_COUNT; ++i)
g_assert (!hp->hazard_pointers [i]);
hp_overflow = &hazard_table [small_id];
*hp = *hp_overflow;
mono_memory_write_barrier ();
memset (hp_overflow, 0, sizeof (MonoThreadHazardPointers));
mono_memory_write_barrier ();
overflow_busy [small_id] = 0;
}
void
mono_lock_free_queue_enqueue (MonoLockFreeQueue *q, MonoLockFreeQueueNode *node)
{
MonoThreadHazardPointers *hp = mono_hazard_pointer_get ();
MonoLockFreeQueueNode *tail;
#ifdef QUEUE_DEBUG
g_assert (!node->in_queue);
node->in_queue = TRUE;
mono_memory_write_barrier ();
#endif
g_assert (node->next == FREE_NEXT);
node->next = END_MARKER;
for (;;) {
MonoLockFreeQueueNode *next;
tail = (MonoLockFreeQueueNode *) get_hazardous_pointer ((gpointer volatile*)&q->tail, hp, 0);
mono_memory_read_barrier ();
/*
* We never dereference next so we don't need a
* hazardous load.
*/
next = tail->next;
mono_memory_read_barrier ();
/* Are tail and next consistent? */
if (tail == q->tail) {
g_assert (next != INVALID_NEXT && next != FREE_NEXT);
g_assert (next != tail);
if (next == END_MARKER) {
/*
* Here we require that nodes that
* have been dequeued don't have
* next==END_MARKER. If they did, we
* might append to a node that isn't
* in the queue anymore here.
*/
if (InterlockedCompareExchangePointer ((gpointer volatile*)&tail->next, node, END_MARKER) == END_MARKER)
break;
} else {
/* Try to advance tail */
InterlockedCompareExchangePointer ((gpointer volatile*)&q->tail, next, tail);
}
}
mono_memory_write_barrier ();
mono_hazard_pointer_clear (hp, 0);
}
/* Try to advance tail */
InterlockedCompareExchangePointer ((gpointer volatile*)&q->tail, node, tail);
mono_memory_write_barrier ();
mono_hazard_pointer_clear (hp, 0);
}
mono_handle_new (MonoObject *obj, MonoThreadInfo *info, const char *owner)
#endif
{
info = info ? info : mono_thread_info_current ();
HandleStack *handles = info->handle_stack;
HandleChunk *top = handles->top;
#ifdef MONO_HANDLE_TRACK_SP
mono_handle_chunk_leak_check (handles);
#endif
retry:
if (G_LIKELY (top->size < OBJECTS_PER_HANDLES_CHUNK)) {
int idx = top->size;
gpointer* objslot = &top->elems [idx].o;
/* can be interrupted anywhere here, so:
* 1. make sure the new slot is null
* 2. make the new slot scannable (increment size)
* 3. put a valid object in there
*
* (have to do 1 then 3 so that if we're interrupted
* between 1 and 2, the object is still live)
*/
*objslot = NULL;
SET_OWNER (top,idx);
SET_SP (handles, top, idx);
mono_memory_write_barrier ();
top->size++;
mono_memory_write_barrier ();
*objslot = obj;
return objslot;
}
if (G_LIKELY (top->next)) {
top->next->size = 0;
/* make sure size == 0 is visible to a GC thread before it sees the new top */
mono_memory_write_barrier ();
top = top->next;
handles->top = top;
goto retry;
}
HandleChunk *new_chunk = new_handle_chunk ();
new_chunk->size = 0;
new_chunk->prev = top;
new_chunk->next = NULL;
/* make sure size == 0 before new chunk is visible */
mono_memory_write_barrier ();
top->next = new_chunk;
handles->top = new_chunk;
goto retry;
}
void
sgen_gray_object_enqueue_section (SgenGrayQueue *queue, GrayQueueSection *section, gboolean is_parallel)
{
STATE_TRANSITION (section, GRAY_QUEUE_SECTION_STATE_FLOATING, GRAY_QUEUE_SECTION_STATE_ENQUEUED);
if (queue->first)
queue->first->size = queue->cursor - queue->first->entries + 1;
section->next = queue->first;
section->prev = NULL;
if (queue->first)
queue->first->prev = section;
else
queue->last = section;
queue->first = section;
queue->cursor = queue->first->entries + queue->first->size - 1;
#ifdef SGEN_CHECK_GRAY_OBJECT_ENQUEUE
if (queue->enqueue_check_func) {
int i;
for (i = 0; i < section->size; ++i)
queue->enqueue_check_func (section->entries [i].obj);
}
#endif
if (is_parallel) {
mono_memory_write_barrier ();
mono_atomic_inc_i32 (&queue->num_sections);
} else {
queue->num_sections++;
}
}
static gboolean
add_stage_entry (int num_entries, volatile gint32 *next_entry, StageEntry *entries, MonoObject *obj, void *user_data)
{
gint32 index;
do {
do {
index = *next_entry;
if (index >= num_entries)
return FALSE;
} while (InterlockedCompareExchange (next_entry, index + 1, index) != index);
/*
* We don't need a write barrier here. *next_entry is just a
* help for finding an index, its value is irrelevant for
* correctness.
*/
} while (entries [index].state != STAGE_ENTRY_FREE ||
InterlockedCompareExchange (&entries [index].state, STAGE_ENTRY_BUSY, STAGE_ENTRY_FREE) != STAGE_ENTRY_FREE);
entries [index].obj = obj;
entries [index].user_data = user_data;
mono_memory_write_barrier ();
entries [index].state = STAGE_ENTRY_USED;
return TRUE;
}
/* LOCKING: requires that the GC lock is held */
static void
process_stage_entries (int num_entries, volatile gint32 *next_entry, StageEntry *entries, void (*process_func) (MonoObject*, void*))
{
int i;
int num_registered = 0;
int num_busy = 0;
for (i = 0; i < num_entries; ++i) {
gint32 state = entries [i].state;
if (state == STAGE_ENTRY_BUSY)
++num_busy;
if (state != STAGE_ENTRY_USED ||
InterlockedCompareExchange (&entries [i].state, STAGE_ENTRY_BUSY, STAGE_ENTRY_USED) != STAGE_ENTRY_USED) {
continue;
}
process_func (entries [i].obj, entries [i].user_data);
entries [i].obj = NULL;
entries [i].user_data = NULL;
mono_memory_write_barrier ();
entries [i].state = STAGE_ENTRY_FREE;
++num_registered;
}
*next_entry = 0;
/* g_print ("stage busy %d reg %d\n", num_busy, num_registered); */
}
void
sgen_workers_start_all_workers (SgenObjectOperations *object_ops)
{
idle_func_object_ops = object_ops;
mono_memory_write_barrier ();
sgen_workers_ensure_awake ();
}
/*
Search @list for element with key @key.
The nodes next, cur and prev are returned in @hp.
Returns true if a node with key @key was found.
This function cannot be called from a signal nor within interrupt context*.
XXX A variant that works within interrupted is possible if needed.
* interrupt context is when the current thread is reposible for another thread
been suspended at an arbritary point. This is a limitation of our SMR implementation.
*/
gboolean
mono_lls_find (MonoLinkedListSet *list, MonoThreadHazardPointers *hp, uintptr_t key)
{
MonoLinkedListSetNode *cur, *next;
MonoLinkedListSetNode **prev;
uintptr_t cur_key;
try_again:
prev = &list->head;
/*
* prev is not really a hazardous pointer, but we return prev
* in hazard pointer 2, so we set it here. Note also that
* prev is not a pointer to a node. We use here the fact that
* the first element in a node is the next pointer, so it
* works, but it's not pretty.
*/
mono_hazard_pointer_set (hp, 2, prev);
cur = get_hazardous_pointer_with_mask ((gpointer*)prev, hp, 1);
while (1) {
if (cur == NULL)
return FALSE;
next = get_hazardous_pointer_with_mask ((gpointer*)&cur->next, hp, 0);
cur_key = cur->key;
/*
* We need to make sure that we dereference prev below
* after reading cur->next above, so we need a read
* barrier.
*/
mono_memory_read_barrier ();
if (*prev != cur)
goto try_again;
if (!mono_lls_pointer_get_mark (next)) {
if (cur_key >= key)
return cur_key == key;
prev = &cur->next;
mono_hazard_pointer_set (hp, 2, cur);
} else {
next = mono_lls_pointer_unmask (next);
if (InterlockedCompareExchangePointer ((volatile gpointer*)prev, next, cur) == cur) {
/* The hazard pointer must be cleared after the CAS. */
mono_memory_write_barrier ();
mono_hazard_pointer_clear (hp, 1);
if (list->free_node_func)
mono_thread_hazardous_free_or_queue (cur, list->free_node_func);
} else
goto try_again;
}
cur = mono_lls_pointer_unmask (next);
mono_hazard_pointer_set (hp, 1, cur);
}
}
mono_handle_new (MonoObject *object, const char *owner)
#endif
{
MonoThreadInfo *info = mono_thread_info_current ();
HandleStack *handles = (HandleStack *)info->handle_stack;
HandleChunk *top = handles->top;
retry:
if (G_LIKELY (top->size < OBJECTS_PER_HANDLES_CHUNK)) {
int idx = top->size;
MonoObject** objslot = chunk_element_objslot (top, idx);
/* can be interrupted anywhere here, so:
* 1. make sure the new slot is null
* 2. make the new slot scannable (increment size)
* 3. put a valid object in there
*
* (have to do 1 then 3 so that if we're interrupted
* between 1 and 2, the object is still live)
*/
*objslot = NULL;
mono_memory_write_barrier ();
top->size++;
mono_memory_write_barrier ();
*objslot = object;
SET_OWNER (top,idx);
return objslot;
}
if (G_LIKELY (top->next)) {
top->next->size = 0;
/* make sure size == 0 is visible to a GC thread before it sees the new top */
mono_memory_write_barrier ();
top = top->next;
handles->top = top;
goto retry;
}
HandleChunk *new_chunk = g_new (HandleChunk, 1);
new_chunk->size = 0;
new_chunk->prev = top;
new_chunk->next = NULL;
/* make sure size == 0 before new chunk is visible */
mono_memory_write_barrier ();
top->next = new_chunk;
handles->top = new_chunk;
goto retry;
}
static void
free_dummy (gpointer _dummy)
{
MonoLockFreeQueueDummy *dummy = (MonoLockFreeQueueDummy *) _dummy;
mono_lock_free_queue_node_free (&dummy->node);
g_assert (dummy->in_use);
mono_memory_write_barrier ();
dummy->in_use = 0;
}
HandleStack*
mono_handle_stack_alloc (void)
{
HandleStack *stack = g_new (HandleStack, 1);
HandleChunk *chunk = g_new (HandleChunk, 1);
chunk->size = 0;
chunk->prev = chunk->next = NULL;
mono_memory_write_barrier ();
stack->top = stack->bottom = chunk;
return stack;
}
static Descriptor*
desc_alloc (MonoMemAccountType type)
{
MonoThreadHazardPointers *hp = mono_hazard_pointer_get ();
Descriptor *desc;
for (;;) {
gboolean success;
desc = (Descriptor *) mono_get_hazardous_pointer ((volatile gpointer *)&desc_avail, hp, 1);
if (desc) {
Descriptor *next = desc->next;
success = (mono_atomic_cas_ptr ((volatile gpointer *)&desc_avail, next, desc) == desc);
} else {
size_t desc_size = sizeof (Descriptor);
Descriptor *d;
int i;
desc = (Descriptor *) mono_valloc (NULL, desc_size * NUM_DESC_BATCH, prot_flags_for_activate (TRUE), type);
g_assertf (desc, "Failed to allocate memory for the lock free allocator");
/* Organize into linked list. */
d = desc;
for (i = 0; i < NUM_DESC_BATCH; ++i) {
Descriptor *next = (i == (NUM_DESC_BATCH - 1)) ? NULL : (Descriptor*)((char*)desc + ((i + 1) * desc_size));
d->next = next;
mono_lock_free_queue_node_init (&d->node, TRUE);
d = next;
}
mono_memory_write_barrier ();
success = (mono_atomic_cas_ptr ((volatile gpointer *)&desc_avail, desc->next, NULL) == NULL);
if (!success)
mono_vfree (desc, desc_size * NUM_DESC_BATCH, type);
}
mono_hazard_pointer_clear (hp, 1);
if (success)
break;
}
g_assert (!desc->in_use);
desc->in_use = TRUE;
return desc;
}
static void
desc_enqueue_avail (gpointer _desc)
{
Descriptor *desc = (Descriptor *) _desc;
Descriptor *old_head;
g_assert (desc->anchor.data.state == STATE_EMPTY);
g_assert (!desc->in_use);
do {
old_head = desc_avail;
desc->next = old_head;
mono_memory_write_barrier ();
} while (mono_atomic_cas_ptr ((volatile gpointer *)&desc_avail, desc, old_head) != old_head);
}
static void
desc_enqueue_avail (gpointer _desc)
{
Descriptor *desc = (Descriptor *) _desc;
Descriptor *old_head;
g_assert (desc->anchor.data.state == STATE_EMPTY);
g_assert (!desc->in_use);
do {
old_head = desc_avail;
desc->next = old_head;
mono_memory_write_barrier ();
} while (InterlockedCompareExchangePointer ((gpointer * volatile)&desc_avail, desc, old_head) != old_head);
}
HandleStack*
mono_handle_stack_alloc (void)
{
HandleStack *stack = new_handle_stack ();
HandleChunk *chunk = new_handle_chunk ();
chunk->prev = chunk->next = NULL;
chunk->size = 0;
mono_memory_write_barrier ();
stack->top = stack->bottom = chunk;
#ifdef MONO_HANDLE_TRACK_SP
stack->stackmark_sp = NULL;
#endif
return stack;
}
static Descriptor*
desc_alloc (void)
{
MonoThreadHazardPointers *hp = mono_hazard_pointer_get ();
Descriptor *desc;
for (;;) {
gboolean success;
desc = (Descriptor *) get_hazardous_pointer ((gpointer * volatile)&desc_avail, hp, 1);
if (desc) {
Descriptor *next = desc->next;
success = (InterlockedCompareExchangePointer ((gpointer * volatile)&desc_avail, next, desc) == desc);
} else {
size_t desc_size = sizeof (Descriptor);
Descriptor *d;
int i;
desc = (Descriptor *) mono_valloc (0, desc_size * NUM_DESC_BATCH, prot_flags_for_activate (TRUE));
/* Organize into linked list. */
d = desc;
for (i = 0; i < NUM_DESC_BATCH; ++i) {
Descriptor *next = (i == (NUM_DESC_BATCH - 1)) ? NULL : (Descriptor*)((char*)desc + ((i + 1) * desc_size));
d->next = next;
mono_lock_free_queue_node_init (&d->node, TRUE);
d = next;
}
mono_memory_write_barrier ();
success = (InterlockedCompareExchangePointer ((gpointer * volatile)&desc_avail, desc->next, NULL) == NULL);
if (!success)
mono_vfree (desc, desc_size * NUM_DESC_BATCH);
}
mono_hazard_pointer_clear (hp, 1);
if (success)
break;
}
g_assert (!desc->in_use);
desc->in_use = TRUE;
return desc;
}
void
mono_handle_stack_free (HandleStack *stack)
{
if (!stack)
return;
HandleChunk *c = stack->bottom;
stack->top = stack->bottom = NULL;
mono_memory_write_barrier ();
while (c) {
HandleChunk *next = c->next;
free_handle_chunk (c);
c = next;
}
free_handle_chunk (c);
free_handle_stack (stack);
}
void
sgen_gray_object_alloc_queue_section (SgenGrayQueue *queue, gboolean is_parallel)
{
GrayQueueSection *section;
if (queue->free_list) {
/* Use the previously allocated queue sections if possible */
section = queue->free_list;
queue->free_list = section->next;
STATE_TRANSITION (section, GRAY_QUEUE_SECTION_STATE_FREE_LIST, GRAY_QUEUE_SECTION_STATE_FLOATING);
} else {
HEAVY_STAT (stat_gray_queue_section_alloc ++);
/* Allocate a new section */
section = (GrayQueueSection *)sgen_alloc_internal (INTERNAL_MEM_GRAY_QUEUE);
STATE_SET (section, GRAY_QUEUE_SECTION_STATE_FLOATING);
}
/* Section is empty */
section->size = 0;
STATE_TRANSITION (section, GRAY_QUEUE_SECTION_STATE_FLOATING, GRAY_QUEUE_SECTION_STATE_ENQUEUED);
/* Link it with the others */
section->next = queue->first;
section->prev = NULL;
if (queue->first)
queue->first->prev = section;
else
queue->last = section;
queue->first = section;
queue->cursor = section->entries - 1;
if (is_parallel) {
mono_memory_write_barrier ();
/*
* FIXME
* we could probably optimize the code to only rely on the write barrier
* for synchronization with the stealer thread. Additionally we could also
* do a write barrier once every other gray queue change, and request
* to have a minimum of sections before stealing, to keep consistency.
*/
mono_atomic_inc_i32 (&queue->num_sections);
} else {
queue->num_sections++;
}
}
/*
* Clean up the threadpool of all domain jobs.
* Can only be called as part of the domain unloading process as
* it will wait for all jobs to be visible to the interruption code.
*/
gboolean
mono_thread_pool_remove_domain_jobs (MonoDomain *domain, int timeout)
{
HANDLE sem_handle;
int result = TRUE;
guint32 start_time = 0;
g_assert (domain->state == MONO_APPDOMAIN_UNLOADING);
threadpool_clear_queue (&async_tp, domain);
threadpool_clear_queue (&async_io_tp, domain);
EnterCriticalSection (&socket_io_data.io_lock);
if (socket_io_data.sock_to_state)
mono_g_hash_table_foreach_remove (socket_io_data.sock_to_state, remove_sockstate_for_domain, domain);
LeaveCriticalSection (&socket_io_data.io_lock);
/*
* There might be some threads out that could be about to execute stuff from the given domain.
* We avoid that by setting up a semaphore to be pulsed by the thread that reaches zero.
*/
sem_handle = CreateSemaphore (NULL, 0, 1, NULL);
domain->cleanup_semaphore = sem_handle;
/*
* The memory barrier here is required to have global ordering between assigning to cleanup_semaphone
* and reading threadpool_jobs.
* Otherwise this thread could read a stale version of threadpool_jobs and wait forever.
*/
mono_memory_write_barrier ();
if (domain->threadpool_jobs && timeout != -1)
start_time = mono_msec_ticks ();
while (domain->threadpool_jobs) {
WaitForSingleObject (sem_handle, timeout);
if (timeout != -1 && (mono_msec_ticks () - start_time) > timeout) {
result = FALSE;
break;
}
}
domain->cleanup_semaphore = NULL;
CloseHandle (sem_handle);
return result;
}
int
mono_hazard_pointer_save_for_signal_handler (void)
{
int small_id, i;
MonoThreadHazardPointers *hp = mono_hazard_pointer_get ();
MonoThreadHazardPointers *hp_overflow;
for (i = 0; i < HAZARD_POINTER_COUNT; ++i)
if (hp->hazard_pointers [i])
goto search;
return -1;
search:
for (small_id = 0; small_id < HAZARD_TABLE_OVERFLOW; ++small_id) {
if (!overflow_busy [small_id])
break;
}
/*
* If this assert fails we don't have enough overflow slots.
* We should contemplate adding them dynamically. If we can
* make mono_thread_small_id_alloc() lock-free we can just
* allocate them on-demand.
*/
g_assert (small_id < HAZARD_TABLE_OVERFLOW);
if (mono_atomic_cas_i32 (&overflow_busy [small_id], 1, 0) != 0)
goto search;
hp_overflow = &hazard_table [small_id];
for (i = 0; i < HAZARD_POINTER_COUNT; ++i)
g_assert (!hp_overflow->hazard_pointers [i]);
*hp_overflow = *hp;
mono_memory_write_barrier ();
memset (hp, 0, sizeof (MonoThreadHazardPointers));
return small_id;
}
static gpointer
alloc_from_new_sb (MonoLockFreeAllocator *heap)
{
unsigned int slot_size, block_size, count, i;
Descriptor *desc = desc_alloc (heap->account_type);
slot_size = desc->slot_size = heap->sc->slot_size;
block_size = desc->block_size = heap->sc->block_size;
count = LOCK_FREE_ALLOC_SB_USABLE_SIZE (block_size) / slot_size;
desc->heap = heap;
/*
* Setting avail to 1 because 0 is the block we're allocating
* right away.
*/
desc->anchor.data.avail = 1;
desc->slot_size = heap->sc->slot_size;
desc->max_count = count;
desc->anchor.data.count = desc->max_count - 1;
desc->anchor.data.state = STATE_PARTIAL;
desc->sb = alloc_sb (desc);
/* Organize blocks into linked list. */
for (i = 1; i < count - 1; ++i)
*(unsigned int*)((char*)desc->sb + i * slot_size) = i + 1;
*(unsigned int*)((char*)desc->sb + (count - 1) * slot_size) = 0;
mono_memory_write_barrier ();
/* Make it active or free it again. */
if (mono_atomic_cas_ptr ((volatile gpointer *)&heap->active, desc, NULL) == NULL) {
return desc->sb;
} else {
desc->anchor.data.state = STATE_EMPTY;
desc_retire (desc);
return NULL;
}
}
/*
Insert @value into @list.
The nodes value, cur and prev are returned in @hp.
Return true if @value was inserted by this call. If it returns FALSE, it's the caller
resposibility to release memory.
This function cannot be called from a signal nor with the world stopped.
*/
gboolean
mono_lls_insert (MonoLinkedListSet *list, MonoThreadHazardPointers *hp, MonoLinkedListSetNode *value)
{
MonoLinkedListSetNode *cur, **prev;
/*We must do a store barrier before inserting
to make sure all values in @node are globally visible.*/
mono_memory_barrier ();
while (1) {
if (mono_lls_find (list, hp, value->key))
return FALSE;
cur = mono_hazard_pointer_get_val (hp, 1);
prev = mono_hazard_pointer_get_val (hp, 2);
value->next = cur;
mono_hazard_pointer_set (hp, 0, value);
/* The CAS must happen after setting the hazard pointer. */
mono_memory_write_barrier ();
if (InterlockedCompareExchangePointer ((volatile gpointer*)prev, value, cur) == cur)
return TRUE;
}
}
static gpointer
alloc_from_new_sb (MonoLockFreeAllocator *heap)
{
unsigned int slot_size, count, i;
Descriptor *desc = desc_alloc ();
desc->sb = alloc_sb (desc);
slot_size = desc->slot_size = heap->sc->slot_size;
count = SB_USABLE_SIZE / slot_size;
/* Organize blocks into linked list. */
for (i = 1; i < count - 1; ++i)
*(unsigned int*)((char*)desc->sb + i * slot_size) = i + 1;
desc->heap = heap;
/*
* Setting avail to 1 because 0 is the block we're allocating
* right away.
*/
desc->anchor.data.avail = 1;
desc->slot_size = heap->sc->slot_size;
desc->max_count = count;
desc->anchor.data.count = desc->max_count - 1;
desc->anchor.data.state = STATE_PARTIAL;
mono_memory_write_barrier ();
/* Make it active or free it again. */
if (InterlockedCompareExchangePointer ((gpointer * volatile)&heap->active, desc, NULL) == NULL) {
return desc->sb;
} else {
desc->anchor.data.state = STATE_EMPTY;
desc_retire (desc);
return NULL;
}
}
static void*
alloc_from_fragment (Fragment *frag, size_t size)
{
char *p = frag->fragment_next;
char *end = p + size;
if (end > frag->fragment_end)
return NULL;
/* p = frag->fragment_next must happen before */
mono_memory_barrier ();
if (InterlockedCompareExchangePointer ((volatile gpointer*)&frag->fragment_next, end, p) != p)
return NULL;
if (frag->fragment_end - end < SGEN_MAX_NURSERY_WASTE) {
Fragment *next, **prev_ptr;
/*
* Before we clean the remaining nursery, we must claim the remaining space
* as it could end up been used by the range allocator since it can end up
* allocating from this dying fragment as it doesn't respect SGEN_MAX_NURSERY_WASTE
* when doing second chance allocation.
*/
if (mono_sgen_get_nursery_clear_policy () == CLEAR_AT_TLAB_CREATION && claim_remaining_size (frag, end)) {
/* Clear the remaining space, pinning depends on this. FIXME move this to use phony arrays */
memset (end, 0, frag->fragment_end - end);
HEAVY_STAT (InterlockedExchangeAdd (&stat_wasted_bytes_trailer, frag->fragment_end - end));
#ifdef NALLOC_DEBUG
add_alloc_record (end, frag->fragment_end - end, BLOCK_ZEROING);
#endif
}
prev_ptr = find_previous_pointer_fragment (frag);
/*Use Michaels linked list remove*/
/*prev_ptr will be null is the fragment was removed concurrently */
while (prev_ptr) {
next = frag->next;
/*already deleted*/
if (!get_mark (next)) {
/*frag->next read must happen before the first CAS*/
mono_memory_write_barrier ();
/*Fail if the next done is removed concurrently and its CAS wins */
if (InterlockedCompareExchangePointer ((volatile gpointer*)&frag->next, mask (next, 1), next) != next) {
continue;
}
}
/* The second CAS must happen after the first CAS or frag->next. */
mono_memory_write_barrier ();
/* Fail if the previous node was deleted and its CAS wins */
if (InterlockedCompareExchangePointer ((volatile gpointer*)prev_ptr, next, frag) != frag) {
prev_ptr = find_previous_pointer_fragment (frag);
continue;
}
/* No need to membar here since the worst that can happen is a CAS failure. */
do {
frag->next_free = fragment_freelist;
} while (InterlockedCompareExchangePointer ((volatile gpointer*)&fragment_freelist, frag, frag->next_free) != frag->next_free);
break;
}
}
return p;
}
/*
* Allocate a small thread id.
*
* FIXME: The biggest part of this function is very similar to
* domain_id_alloc() in domain.c and should be merged.
*/
int
mono_thread_small_id_alloc (void)
{
int i, id = -1;
mono_os_mutex_lock (&small_id_mutex);
if (!small_id_table)
small_id_table = mono_bitset_new (1, 0);
id = mono_bitset_find_first_unset (small_id_table, small_id_next - 1);
if (id == -1)
id = mono_bitset_find_first_unset (small_id_table, -1);
if (id == -1) {
MonoBitSet *new_table;
if (small_id_table->size * 2 >= (1 << 16))
g_assert_not_reached ();
new_table = mono_bitset_clone (small_id_table, small_id_table->size * 2);
id = mono_bitset_find_first_unset (new_table, small_id_table->size - 1);
mono_bitset_free (small_id_table);
small_id_table = new_table;
}
g_assert (!mono_bitset_test_fast (small_id_table, id));
mono_bitset_set_fast (small_id_table, id);
small_id_next++;
if (small_id_next >= small_id_table->size)
small_id_next = 0;
g_assert (id < HAZARD_TABLE_MAX_SIZE);
if (id >= hazard_table_size) {
#if MONO_SMALL_CONFIG
hazard_table = g_malloc0 (sizeof (MonoThreadHazardPointers) * HAZARD_TABLE_MAX_SIZE);
hazard_table_size = HAZARD_TABLE_MAX_SIZE;
#else
gpointer page_addr;
#if defined(__PASE__)
/*
* HACK: allocating the table with none prot will cause i 7.1
* to segfault when accessing or protecting it
*/
int table_prot = MONO_MMAP_READ | MONO_MMAP_WRITE;
#else
int table_prot = MONO_MMAP_NONE;
#endif
int pagesize = mono_pagesize ();
int num_pages = (hazard_table_size * sizeof (MonoThreadHazardPointers) + pagesize - 1) / pagesize;
if (hazard_table == NULL) {
hazard_table = (MonoThreadHazardPointers *volatile) mono_valloc (NULL,
sizeof (MonoThreadHazardPointers) * HAZARD_TABLE_MAX_SIZE,
table_prot, MONO_MEM_ACCOUNT_HAZARD_POINTERS);
}
g_assert (hazard_table != NULL);
page_addr = (guint8*)hazard_table + num_pages * pagesize;
mono_mprotect (page_addr, pagesize, MONO_MMAP_READ | MONO_MMAP_WRITE);
++num_pages;
hazard_table_size = num_pages * pagesize / sizeof (MonoThreadHazardPointers);
#endif
g_assert (id < hazard_table_size);
for (i = 0; i < HAZARD_POINTER_COUNT; ++i)
hazard_table [id].hazard_pointers [i] = NULL;
}
if (id > highest_small_id) {
highest_small_id = id;
mono_memory_write_barrier ();
}
mono_os_mutex_unlock (&small_id_mutex);
return id;
}
static void*
par_alloc_from_fragment (SgenFragmentAllocator *allocator, SgenFragment *frag, size_t size)
{
char *p = frag->fragment_next;
char *end = p + size;
if (end > frag->fragment_end)
return NULL;
/* p = frag->fragment_next must happen before */
mono_memory_barrier ();
if (InterlockedCompareExchangePointer ((volatile gpointer*)&frag->fragment_next, end, p) != p)
return NULL;
if (frag->fragment_end - end < SGEN_MAX_NURSERY_WASTE) {
SgenFragment *next, **prev_ptr;
/*
* Before we clean the remaining nursery, we must claim the remaining space
* as it could end up been used by the range allocator since it can end up
* allocating from this dying fragment as it doesn't respect SGEN_MAX_NURSERY_WASTE
* when doing second chance allocation.
*/
if ((sgen_get_nursery_clear_policy () == CLEAR_AT_TLAB_CREATION || sgen_get_nursery_clear_policy () == CLEAR_AT_TLAB_CREATION_DEBUG) && claim_remaining_size (frag, end)) {
sgen_clear_range (end, frag->fragment_end);
HEAVY_STAT (stat_wasted_bytes_trailer += frag->fragment_end - end);
#ifdef NALLOC_DEBUG
add_alloc_record (end, frag->fragment_end - end, BLOCK_ZEROING);
#endif
}
prev_ptr = find_previous_pointer_fragment (allocator, frag);
/*Use Michaels linked list remove*/
/*prev_ptr will be null if the fragment was removed concurrently */
while (prev_ptr) {
next = frag->next;
/*already deleted*/
if (!get_mark (next)) {
/*frag->next read must happen before the first CAS*/
mono_memory_write_barrier ();
/*Fail if the next node is removed concurrently and its CAS wins */
if (InterlockedCompareExchangePointer ((volatile gpointer*)&frag->next, mask (next, 1), next) != next) {
continue;
}
}
/* The second CAS must happen after the first CAS or frag->next. */
mono_memory_write_barrier ();
/* Fail if the previous node was deleted and its CAS wins */
if (InterlockedCompareExchangePointer ((volatile gpointer*)prev_ptr, unmask (next), frag) != frag) {
prev_ptr = find_previous_pointer_fragment (allocator, frag);
continue;
}
break;
}
}
return p;
}
MonoLockFreeQueueNode*
mono_lock_free_queue_dequeue (MonoLockFreeQueue *q)
{
MonoThreadHazardPointers *hp = mono_hazard_pointer_get ();
MonoLockFreeQueueNode *head;
retry:
for (;;) {
MonoLockFreeQueueNode *tail, *next;
head = (MonoLockFreeQueueNode *) get_hazardous_pointer ((gpointer volatile*)&q->head, hp, 0);
tail = (MonoLockFreeQueueNode*)q->tail;
mono_memory_read_barrier ();
next = head->next;
mono_memory_read_barrier ();
/* Are head, tail and next consistent? */
if (head == q->head) {
g_assert (next != INVALID_NEXT && next != FREE_NEXT);
g_assert (next != head);
/* Is queue empty or tail behind? */
if (head == tail) {
if (next == END_MARKER) {
/* Queue is empty */
mono_hazard_pointer_clear (hp, 0);
/*
* We only continue if we
* reenqueue the dummy
* ourselves, so as not to
* wait for threads that might
* not actually run.
*/
if (!is_dummy (q, head) && try_reenqueue_dummy (q))
continue;
return NULL;
}
/* Try to advance tail */
InterlockedCompareExchangePointer ((gpointer volatile*)&q->tail, next, tail);
} else {
g_assert (next != END_MARKER);
/* Try to dequeue head */
if (InterlockedCompareExchangePointer ((gpointer volatile*)&q->head, next, head) == head)
break;
}
}
mono_memory_write_barrier ();
mono_hazard_pointer_clear (hp, 0);
}
/*
* The head is dequeued now, so we know it's this thread's
* responsibility to free it - no other thread can.
*/
mono_memory_write_barrier ();
mono_hazard_pointer_clear (hp, 0);
g_assert (head->next);
/*
* Setting next here isn't necessary for correctness, but we
* do it to make sure that we catch dereferencing next in a
* node that's not in the queue anymore.
*/
head->next = INVALID_NEXT;
#if QUEUE_DEBUG
g_assert (head->in_queue);
head->in_queue = FALSE;
mono_memory_write_barrier ();
#endif
if (is_dummy (q, head)) {
g_assert (q->has_dummy);
q->has_dummy = 0;
mono_memory_write_barrier ();
mono_thread_hazardous_free_or_queue (head, free_dummy, FALSE, TRUE);
if (try_reenqueue_dummy (q))
goto retry;
return NULL;
}
/* The caller must hazardously free the node. */
return head;
}
/*
* Allocate a small thread id.
*
* FIXME: The biggest part of this function is very similar to
* domain_id_alloc() in domain.c and should be merged.
*/
int
mono_thread_small_id_alloc (void)
{
int i, id = -1;
EnterCriticalSection (&small_id_mutex);
if (!small_id_table)
small_id_table = mono_bitset_new (1, 0);
id = mono_bitset_find_first_unset (small_id_table, small_id_next);
if (id == -1)
id = mono_bitset_find_first_unset (small_id_table, -1);
if (id == -1) {
MonoBitSet *new_table;
if (small_id_table->size * 2 >= (1 << 16))
g_assert_not_reached ();
new_table = mono_bitset_clone (small_id_table, small_id_table->size * 2);
id = mono_bitset_find_first_unset (new_table, small_id_table->size - 1);
mono_bitset_free (small_id_table);
small_id_table = new_table;
}
g_assert (!mono_bitset_test_fast (small_id_table, id));
mono_bitset_set_fast (small_id_table, id);
small_id_next++;
if (small_id_next >= small_id_table->size)
small_id_next = 0;
g_assert (id < HAZARD_TABLE_MAX_SIZE);
if (id >= hazard_table_size) {
#if MONO_SMALL_CONFIG
hazard_table = g_malloc0 (sizeof (MonoThreadHazardPointers) * HAZARD_TABLE_MAX_SIZE);
hazard_table_size = HAZARD_TABLE_MAX_SIZE;
#else
gpointer page_addr;
int pagesize = mono_pagesize ();
int num_pages = (hazard_table_size * sizeof (MonoThreadHazardPointers) + pagesize - 1) / pagesize;
if (hazard_table == NULL) {
hazard_table = mono_valloc (NULL,
sizeof (MonoThreadHazardPointers) * HAZARD_TABLE_MAX_SIZE,
MONO_MMAP_NONE);
}
g_assert (hazard_table != NULL);
page_addr = (guint8*)hazard_table + num_pages * pagesize;
mono_mprotect (page_addr, pagesize, MONO_MMAP_READ | MONO_MMAP_WRITE);
++num_pages;
hazard_table_size = num_pages * pagesize / sizeof (MonoThreadHazardPointers);
#endif
g_assert (id < hazard_table_size);
for (i = 0; i < HAZARD_POINTER_COUNT; ++i)
hazard_table [id].hazard_pointers [i] = NULL;
}
if (id > highest_small_id) {
highest_small_id = id;
mono_memory_write_barrier ();
}
LeaveCriticalSection (&small_id_mutex);
return id;
}
/* We add elements to the table by first making space for them by
* shifting the elements at the back to the right, one at a time.
* This results in duplicate entries during the process, but during
* all the time the table is in a sorted state. Also, when an element
* is replaced by another one, the element that replaces it has an end
* address that is equal to or lower than that of the replaced
* element. That property is necessary to guarantee that when
* searching for an element we end up at a position not higher than
* the one we're looking for (i.e. we either find the element directly
* or we end up to the left of it).
*/
static void
jit_info_table_add (MonoDomain *domain, MonoJitInfoTable *volatile *table_ptr, MonoJitInfo *ji)
{
MonoJitInfoTable *table;
MonoJitInfoTableChunk *chunk;
int chunk_pos, pos;
int num_elements;
int i;
table = *table_ptr;
restart:
chunk_pos = jit_info_table_index (table, (gint8*)ji->code_start + ji->code_size);
g_assert (chunk_pos < table->num_chunks);
chunk = table->chunks [chunk_pos];
if (chunk->num_elements >= MONO_JIT_INFO_TABLE_CHUNK_SIZE) {
MonoJitInfoTable *new_table = jit_info_table_chunk_overflow (table, chunk);
/* Debugging code, should be removed. */
//jit_info_table_check (new_table);
*table_ptr = new_table;
mono_memory_barrier ();
domain->num_jit_info_tables++;
mono_thread_hazardous_free_or_queue (table, (MonoHazardousFreeFunc)mono_jit_info_table_free, TRUE, FALSE);
table = new_table;
goto restart;
}
/* Debugging code, should be removed. */
//jit_info_table_check (table);
num_elements = chunk->num_elements;
pos = jit_info_table_chunk_index (chunk, NULL, (gint8*)ji->code_start + ji->code_size);
/* First we need to size up the chunk by one, by copying the
last item, or inserting the first one, if the table is
empty. */
if (num_elements > 0)
chunk->data [num_elements] = chunk->data [num_elements - 1];
else
chunk->data [0] = ji;
mono_memory_write_barrier ();
chunk->num_elements = ++num_elements;
/* Shift the elements up one by one. */
for (i = num_elements - 2; i >= pos; --i) {
mono_memory_write_barrier ();
chunk->data [i + 1] = chunk->data [i];
}
/* Now we have room and can insert the new item. */
mono_memory_write_barrier ();
chunk->data [pos] = ji;
/* Set the high code end address chunk entry. */
chunk->last_code_end = (gint8*)chunk->data [chunk->num_elements - 1]->code_start
+ chunk->data [chunk->num_elements - 1]->code_size;
/* Debugging code, should be removed. */
//jit_info_table_check (table);
}