static void
unregister_thread (void *arg)
{
MonoThreadInfo *info = arg;
int small_id = info->small_id;
g_assert (info);
THREADS_DEBUG ("unregistering info %p\n", info);
/*
* TLS destruction order is not reliable so small_id might be cleaned up
* before us.
*/
mono_native_tls_set_value (small_id_key, GUINT_TO_POINTER (info->small_id + 1));
info->thread_state = STATE_SHUTTING_DOWN;
mono_thread_info_suspend_lock ();
if (threads_callbacks.thread_unregister)
threads_callbacks.thread_unregister (info);
mono_threads_unregister_current_thread (info);
info->thread_state = STATE_DEAD;
mono_thread_info_suspend_unlock ();
/*now it's safe to free the thread info.*/
mono_thread_hazardous_free_or_queue (info, free_thread_info, TRUE, FALSE);
mono_thread_small_id_free (small_id);
}
static void
unregister_thread (void *arg)
{
MonoThreadInfo *info = arg;
int small_id = info->small_id;
g_assert (info);
THREADS_DEBUG ("unregistering info %p\n", info);
/*
* TLS destruction order is not reliable so small_id might be cleaned up
* before us.
*/
mono_native_tls_set_value (small_id_key, GUINT_TO_POINTER (info->small_id + 1));
/*
The unregister callback is reposible for calling mono_threads_unregister_current_thread
since it usually needs to be done in sync with the GC does a stop-the-world.
*/
if (threads_callbacks.thread_unregister)
threads_callbacks.thread_unregister (info);
else
mono_threads_unregister_current_thread (info);
/*now it's safe to free the thread info.*/
mono_thread_hazardous_free_or_queue (info, free_thread_info);
mono_thread_small_id_free (small_id);
}
/*
Search @list for element with key @key.
The nodes next, cur and prev are returned in @hp
Returns true if @value was removed by this call.
This function cannot be called from a signal nor with the world stopped.
*/
gboolean
mono_lls_remove (MonoLinkedListSet *list, MonoThreadHazardPointers *hp, MonoLinkedListSetNode *value)
{
MonoLinkedListSetNode *cur, **prev, *next;
while (1) {
if (!mono_lls_find (list, hp, value->key))
return FALSE;
next = mono_hazard_pointer_get_val (hp, 0);
cur = mono_hazard_pointer_get_val (hp, 1);
prev = mono_hazard_pointer_get_val (hp, 2);
g_assert (cur == value);
if (InterlockedCompareExchangePointer ((volatile gpointer*)&cur->next, mask (next, 1), next) != next)
continue;
/* The second CAS must happen before the first. */
mono_memory_write_barrier ();
if (InterlockedCompareExchangePointer ((volatile gpointer*)prev, next, cur) == cur) {
/* The CAS must happen before the hazard pointer clear. */
mono_memory_write_barrier ();
mono_hazard_pointer_clear (hp, 1);
if (list->free_node_func)
mono_thread_hazardous_free_or_queue (value, list->free_node_func);
} else
mono_lls_find (list, hp, value->key);
return TRUE;
}
}
/*
Search @list for element with key @key.
The nodes next, cur and prev are returned in @hp.
Returns true if a node with key @key was found.
This function cannot be called from a signal nor within interrupt context*.
XXX A variant that works within interrupted is possible if needed.
* interrupt context is when the current thread is reposible for another thread
been suspended at an arbritary point. This is a limitation of our SMR implementation.
*/
gboolean
mono_lls_find (MonoLinkedListSet *list, MonoThreadHazardPointers *hp, uintptr_t key)
{
MonoLinkedListSetNode *cur, *next;
MonoLinkedListSetNode **prev;
uintptr_t cur_key;
try_again:
prev = &list->head;
/*
* prev is not really a hazardous pointer, but we return prev
* in hazard pointer 2, so we set it here. Note also that
* prev is not a pointer to a node. We use here the fact that
* the first element in a node is the next pointer, so it
* works, but it's not pretty.
*/
mono_hazard_pointer_set (hp, 2, prev);
cur = get_hazardous_pointer_with_mask ((gpointer*)prev, hp, 1);
while (1) {
if (cur == NULL)
return FALSE;
next = get_hazardous_pointer_with_mask ((gpointer*)&cur->next, hp, 0);
cur_key = cur->key;
/*
* We need to make sure that we dereference prev below
* after reading cur->next above, so we need a read
* barrier.
*/
mono_memory_read_barrier ();
if (*prev != cur)
goto try_again;
if (!mono_lls_pointer_get_mark (next)) {
if (cur_key >= key)
return cur_key == key;
prev = &cur->next;
mono_hazard_pointer_set (hp, 2, cur);
} else {
next = mono_lls_pointer_unmask (next);
if (InterlockedCompareExchangePointer ((volatile gpointer*)prev, next, cur) == cur) {
/* The hazard pointer must be cleared after the CAS. */
mono_memory_write_barrier ();
mono_hazard_pointer_clear (hp, 1);
if (list->free_node_func)
mono_thread_hazardous_free_or_queue (cur, list->free_node_func);
} else
goto try_again;
}
cur = mono_lls_pointer_unmask (next);
mono_hazard_pointer_set (hp, 1, cur);
}
}
static void
desc_retire (Descriptor *desc)
{
g_assert (desc->anchor.data.state == STATE_EMPTY);
g_assert (desc->in_use);
desc->in_use = FALSE;
free_sb (desc->sb, desc->block_size);
mono_thread_hazardous_free_or_queue (desc, desc_enqueue_avail, FALSE, TRUE);
}
static void
unregister_thread (void *arg)
{
MonoThreadInfo *info = (MonoThreadInfo *) arg;
int small_id = info->small_id;
g_assert (info);
THREADS_DEBUG ("unregistering info %p\n", info);
mono_native_tls_set_value (thread_exited_key, GUINT_TO_POINTER (1));
mono_threads_core_unregister (info);
/*
* TLS destruction order is not reliable so small_id might be cleaned up
* before us.
*/
#ifndef HAVE_KW_THREAD
mono_native_tls_set_value (small_id_key, GUINT_TO_POINTER (info->small_id + 1));
#endif
/*
First perform the callback that requires no locks.
This callback has the potential of taking other locks, so we do it before.
After it completes, the thread remains functional.
*/
if (threads_callbacks.thread_detach)
threads_callbacks.thread_detach (info);
/*
Since the thread info lock is taken from within blocking sections, we can't check from there, so it must be done here.
This ensures that we won't lose any suspend requests as a suspend initiator must hold the lock.
Once we're holding the suspend lock, no threads can suspend us and once we unregister, no thread can find us.
*/
MONO_PREPARE_BLOCKING
mono_thread_info_suspend_lock ();
MONO_FINISH_BLOCKING
/*
Now perform the callback that must be done under locks.
This will render the thread useless and non-suspendable, so it must
be done while holding the suspend lock to give no other thread chance
to suspend it.
*/
if (threads_callbacks.thread_unregister)
threads_callbacks.thread_unregister (info);
mono_threads_unregister_current_thread (info);
mono_threads_transition_detach (info);
mono_thread_info_suspend_unlock ();
/*now it's safe to free the thread info.*/
mono_thread_hazardous_free_or_queue (info, free_thread_info, TRUE, FALSE);
mono_thread_small_id_free (small_id);
}
/*
* LOCKING: domain lock
*/
static void
mono_jit_info_free_or_queue (MonoDomain *domain, MonoJitInfo *ji)
{
if (domain->num_jit_info_tables <= 1) {
/* Can it actually happen that we only have one table
but ji is still hazardous? */
mono_thread_hazardous_free_or_queue (ji, g_free, TRUE, FALSE);
} else {
domain->jit_info_free_queue = g_slist_prepend (domain->jit_info_free_queue, ji);
}
}
static void
unregister_thread (void *arg)
{
MonoThreadInfo *info = (MonoThreadInfo *) arg;
int small_id = info->small_id;
g_assert (info);
THREADS_DEBUG ("unregistering info %p\n", info);
mono_native_tls_set_value (thread_exited_key, GUINT_TO_POINTER (1));
mono_threads_core_unregister (info);
/*
* TLS destruction order is not reliable so small_id might be cleaned up
* before us.
*/
#ifndef HAVE_KW_THREAD
mono_native_tls_set_value (small_id_key, GUINT_TO_POINTER (info->small_id + 1));
#endif
/*
First perform the callback that requires no locks.
This callback has the potential of taking other locks, so we do it before.
After it completes, the thread remains functional.
*/
if (threads_callbacks.thread_detach)
threads_callbacks.thread_detach (info);
mono_thread_info_suspend_lock ();
/*
Now perform the callback that must be done under locks.
This will render the thread useless and non-suspendable, so it must
be done while holding the suspend lock to give no other thread chance
to suspend it.
*/
if (threads_callbacks.thread_unregister)
threads_callbacks.thread_unregister (info);
mono_threads_unregister_current_thread (info);
mono_threads_transition_detach (info);
mono_thread_info_suspend_unlock ();
g_byte_array_free (info->stackdata, /*free_segment=*/TRUE);
/*now it's safe to free the thread info.*/
mono_thread_hazardous_free_or_queue (info, free_thread_info, TRUE, FALSE);
mono_thread_small_id_free (small_id);
}
static void
list_remove_empty_desc (MonoLockFreeAllocSizeClass *sc)
{
int num_non_empty = 0;
for (;;) {
Descriptor *desc = (Descriptor*) mono_lock_free_queue_dequeue (&sc->partial);
if (!desc)
return;
/*
* We don't need to read atomically because we're the
* only thread that references this descriptor.
*/
if (desc->anchor.data.state == STATE_EMPTY) {
desc_retire (desc);
} else {
g_assert (desc->heap->sc == sc);
mono_thread_hazardous_free_or_queue (desc, desc_put_partial, FALSE, TRUE);
if (++num_non_empty >= 2)
return;
}
}
}
static void
conc_table_lf_free (conc_table *table)
{
mono_thread_hazardous_free_or_queue (table, conc_table_free, TRUE, FALSE);
}
static void
list_put_partial (Descriptor *desc)
{
g_assert (desc->anchor.data.state != STATE_FULL);
mono_thread_hazardous_free_or_queue (desc, desc_put_partial, FALSE, TRUE);
}
/* We add elements to the table by first making space for them by
* shifting the elements at the back to the right, one at a time.
* This results in duplicate entries during the process, but during
* all the time the table is in a sorted state. Also, when an element
* is replaced by another one, the element that replaces it has an end
* address that is equal to or lower than that of the replaced
* element. That property is necessary to guarantee that when
* searching for an element we end up at a position not higher than
* the one we're looking for (i.e. we either find the element directly
* or we end up to the left of it).
*/
static void
jit_info_table_add (MonoDomain *domain, MonoJitInfoTable *volatile *table_ptr, MonoJitInfo *ji)
{
MonoJitInfoTable *table;
MonoJitInfoTableChunk *chunk;
int chunk_pos, pos;
int num_elements;
int i;
table = *table_ptr;
restart:
chunk_pos = jit_info_table_index (table, (gint8*)ji->code_start + ji->code_size);
g_assert (chunk_pos < table->num_chunks);
chunk = table->chunks [chunk_pos];
if (chunk->num_elements >= MONO_JIT_INFO_TABLE_CHUNK_SIZE) {
MonoJitInfoTable *new_table = jit_info_table_chunk_overflow (table, chunk);
/* Debugging code, should be removed. */
//jit_info_table_check (new_table);
*table_ptr = new_table;
mono_memory_barrier ();
domain->num_jit_info_tables++;
mono_thread_hazardous_free_or_queue (table, (MonoHazardousFreeFunc)mono_jit_info_table_free, TRUE, FALSE);
table = new_table;
goto restart;
}
/* Debugging code, should be removed. */
//jit_info_table_check (table);
num_elements = chunk->num_elements;
pos = jit_info_table_chunk_index (chunk, NULL, (gint8*)ji->code_start + ji->code_size);
/* First we need to size up the chunk by one, by copying the
last item, or inserting the first one, if the table is
empty. */
if (num_elements > 0)
chunk->data [num_elements] = chunk->data [num_elements - 1];
else
chunk->data [0] = ji;
mono_memory_write_barrier ();
chunk->num_elements = ++num_elements;
/* Shift the elements up one by one. */
for (i = num_elements - 2; i >= pos; --i) {
mono_memory_write_barrier ();
chunk->data [i + 1] = chunk->data [i];
}
/* Now we have room and can insert the new item. */
mono_memory_write_barrier ();
chunk->data [pos] = ji;
/* Set the high code end address chunk entry. */
chunk->last_code_end = (gint8*)chunk->data [chunk->num_elements - 1]->code_start
+ chunk->data [chunk->num_elements - 1]->code_size;
/* Debugging code, should be removed. */
//jit_info_table_check (table);
}
MonoLockFreeQueueNode*
mono_lock_free_queue_dequeue (MonoLockFreeQueue *q)
{
MonoThreadHazardPointers *hp = mono_hazard_pointer_get ();
MonoLockFreeQueueNode *head;
retry:
for (;;) {
MonoLockFreeQueueNode *tail, *next;
head = (MonoLockFreeQueueNode *) get_hazardous_pointer ((gpointer volatile*)&q->head, hp, 0);
tail = (MonoLockFreeQueueNode*)q->tail;
mono_memory_read_barrier ();
next = head->next;
mono_memory_read_barrier ();
/* Are head, tail and next consistent? */
if (head == q->head) {
g_assert (next != INVALID_NEXT && next != FREE_NEXT);
g_assert (next != head);
/* Is queue empty or tail behind? */
if (head == tail) {
if (next == END_MARKER) {
/* Queue is empty */
mono_hazard_pointer_clear (hp, 0);
/*
* We only continue if we
* reenqueue the dummy
* ourselves, so as not to
* wait for threads that might
* not actually run.
*/
if (!is_dummy (q, head) && try_reenqueue_dummy (q))
continue;
return NULL;
}
/* Try to advance tail */
InterlockedCompareExchangePointer ((gpointer volatile*)&q->tail, next, tail);
} else {
g_assert (next != END_MARKER);
/* Try to dequeue head */
if (InterlockedCompareExchangePointer ((gpointer volatile*)&q->head, next, head) == head)
break;
}
}
mono_memory_write_barrier ();
mono_hazard_pointer_clear (hp, 0);
}
/*
* The head is dequeued now, so we know it's this thread's
* responsibility to free it - no other thread can.
*/
mono_memory_write_barrier ();
mono_hazard_pointer_clear (hp, 0);
g_assert (head->next);
/*
* Setting next here isn't necessary for correctness, but we
* do it to make sure that we catch dereferencing next in a
* node that's not in the queue anymore.
*/
head->next = INVALID_NEXT;
#if QUEUE_DEBUG
g_assert (head->in_queue);
head->in_queue = FALSE;
mono_memory_write_barrier ();
#endif
if (is_dummy (q, head)) {
g_assert (q->has_dummy);
q->has_dummy = 0;
mono_memory_write_barrier ();
mono_thread_hazardous_free_or_queue (head, free_dummy, FALSE, TRUE);
if (try_reenqueue_dummy (q))
goto retry;
return NULL;
}
/* The caller must hazardously free the node. */
return head;
}
