static bool mm_event_dispatch_check_epoch(struct mm_event_dispatch *dispatch, uint32_t epoch) { ENTER(); bool rc = true; mm_thread_t n = dispatch->nlisteners; struct mm_event_listener *listeners = dispatch->listeners; for (mm_thread_t i = 0; i < n; i++) { struct mm_event_listener *listener = &listeners[i]; struct mm_event_receiver *receiver = &listener->receiver; uint32_t local = mm_memory_load(receiver->reclaim_epoch); if (local == epoch) continue; mm_memory_load_fence(); bool active = mm_memory_load(receiver->reclaim_active); if (active) { mm_thread_post_1(listener->thread, mm_event_dispatch_observe_req, (uintptr_t) receiver); rc = false; break; } } LEAVE(); return rc; }
mm_event_listener_notify(struct mm_event_listener *listener, mm_ring_seqno_t stamp UNUSED) { ENTER(); #if ENABLE_NOTIFY_STAMP uintptr_t state = mm_memory_load(listener->state); if ((stamp << 2) == (state & ~MM_EVENT_LISTENER_STATUS)) { // Get the current status of the listener. It might // become obsolete by the time the notification is // sent. This is not a problem however as it implies // the listener thread has woken up on its own and // seen all the sent data. // // Sometimes this might lead to an extra listener // wake up (if the listener makes a full cycle) or // a wrong listener being waken (if another listener // becomes polling). So listeners should be prepared // to get spurious wake up notifications. mm_event_listener_status_t status = state & MM_EVENT_LISTENER_STATUS; if (status == MM_EVENT_LISTENER_WAITING) mm_event_listener_signal(listener); else if (status == MM_EVENT_LISTENER_POLLING) mm_event_backend_notify(&listener->receiver.dispatch->backend); } #else // Compare the notify and listen stamps. If the notify stamp has not // been already bumped relative to the listen stamp then do it now. // Do it atomically so that only a thread that succeeds with this // is elected to send a wake-up notification to the target listener. const uint32_t listen_stamp = mm_memory_load(listener->listen_stamp); mm_memory_load_fence(); const uint32_t notify_stamp = mm_memory_load(listener->notify_stamp); if (notify_stamp == (listen_stamp & ~MM_EVENT_LISTENER_STATUS)) { uint32_t next = notify_stamp + MM_EVENT_LISTENER_STATUS + 1; uint32_t prev = mm_atomic_uint32_cas(&listener->notify_stamp, notify_stamp, next); if (prev == notify_stamp) { // Get the current state of the listener. It might // become obsolete by the time the notification is // sent. This is not a problem however as it implies // the listener thread has woken up on its own and // seen all the sent data. // // Sometimes this might lead to an extra listener // wake up (if the listener makes a full cycle) or // a wrong listener being waken (if another listener // becomes polling). So listeners should be prepared // to get spurious wake up notifications. mm_event_listener_status_t status = listen_stamp & MM_EVENT_LISTENER_STATUS; if (status == MM_EVENT_LISTENER_WAITING) mm_event_listener_signal(listener); else if (status == MM_EVENT_LISTENER_POLLING) mm_event_backend_notify(&listener->receiver.dispatch->backend); } } #endif LEAVE(); }
struct mm_lock_stat * mm_lock_get_thread_stat(struct mm_lock_stat_set *stat_set, struct mm_thread *thread) { struct mm_lock_thread_stat *thr_stat; // Look for a matching thread entry. struct mm_slink *link = mm_stack_atomic_load_head(&stat_set->thread_list); while (link != NULL) { thr_stat = containerof(link, struct mm_lock_thread_stat, link); if (thr_stat->thread == thread) return &thr_stat->stat; link = mm_memory_load(link->next); } // If not found create a new entry. thr_stat = mm_global_alloc(sizeof(struct mm_lock_thread_stat)); thr_stat->thread = thread; thr_stat->stat.lock_count = 0; thr_stat->stat.fail_count = 0; // Link the entry into list. struct mm_slink *head = mm_stack_atomic_load_head(&stat_set->thread_list); for (uint32_t b = 0; ; b = mm_thread_backoff(b)) { thr_stat->link.next = head; link = mm_stack_atomic_cas_head(&stat_set->thread_list, head, &thr_stat->link); if (link == head) break; head = link; } return &thr_stat->stat; }
struct mm_lock_domain_stat * mm_lock_find_domain_stat(struct mm_lock_stat_set *stat_set, struct mm_domain *domain) { // Go through domain entries trying to find a match. struct mm_slink *link = mm_stack_atomic_load_head(&stat_set->domain_list); while (link != NULL) { struct mm_lock_domain_stat *dom_stat = containerof(link, struct mm_lock_domain_stat, link); if (dom_stat->domain == domain) { // If the entry is not yet ready then wait a bit until // it becomes ready. It shouldn't take long. Really. while (mm_memory_load(dom_stat->ready) == 0) mm_spin_pause(); mm_memory_load_fence(); return dom_stat; } link = mm_memory_load(link->next); } return NULL; }
void mm_lock_stats(void) { #if ENABLE_LOCK_STATS struct mm_slink *set_link = mm_stack_atomic_load_head(&mm_lock_stat_list); while (set_link != NULL) { struct mm_lock_stat_set *stat_set = containerof(set_link, struct mm_lock_stat_set, common_link); mm_memory_load_fence(); struct mm_slink *dom_link = mm_stack_atomic_load_head(&stat_set->domain_list); while (dom_link != NULL) { struct mm_lock_domain_stat *dom_stat = containerof(dom_link, struct mm_lock_domain_stat, link); struct mm_domain *domain = dom_stat->domain; for (mm_thread_t c = 0; c < domain->nthreads; c++) { struct mm_lock_stat *stat = MM_THREAD_LOCAL_DEREF(c, dom_stat->stat); struct mm_thread *thread = domain->threads[c]; mm_lock_print_stat(thread, stat_set, stat); } dom_link = mm_memory_load(dom_link->next); } struct mm_slink *thr_link = mm_stack_atomic_load_head(&stat_set->thread_list); while (thr_link != NULL) { struct mm_lock_thread_stat *thr_stat = containerof(thr_link, struct mm_lock_thread_stat, link); struct mm_thread *thread = thr_stat->thread; mm_lock_print_stat(thread, stat_set, &thr_stat->stat); thr_link = mm_memory_load(thr_link->next); } set_link = mm_memory_load(set_link->next); } #endif }
mm_event_dispatch_advance_epoch(struct mm_event_dispatch *dispatch) { ENTER(); uint32_t epoch = mm_memory_load(dispatch->reclaim_epoch); bool rc = mm_event_dispatch_check_epoch(dispatch, epoch); if (rc) { mm_memory_fence(); // TODO: load_store fence mm_memory_store(dispatch->reclaim_epoch, epoch + 1); DEBUG("advance epoch %u", epoch + 1); } LEAVE(); return rc; }
mm_lock_getstat(struct mm_lock_stat_info *info) { // Get statistics collection pertinent to the lock in question. struct mm_lock_stat_set *stat_set = mm_memory_load(info->stat); if (stat_set == NULL) stat_set = mm_lock_get_stat_set(info); // Get a statistic entry specific to the calling thread. struct mm_thread *thread = mm_thread_selfptr(); struct mm_domain *domain = mm_thread_getdomain(thread); if (domain != NULL) return mm_lock_get_domain_stat(stat_set, thread, domain); else return mm_lock_get_thread_stat(stat_set, thread); }
mm_thread_barrier_wait(struct mm_thread_barrier *const barrier, struct mm_thread_barrier_local *local) { uint32_t sense = ~local->sense; if (mm_atomic_uint32_dec_and_test(&barrier->value) == 0) { mm_memory_store(barrier->value, barrier->count); mm_memory_store_fence(); mm_memory_store(barrier->sense, sense); } else { mm_memory_fence(); // TODO: atomic_load fence while (mm_memory_load(barrier->sense) != sense) mm_cpu_backoff(); } local->sense = sense; }
mm_event_listener_wait(struct mm_event_listener *listener, mm_timeout_t timeout) { ENTER(); #if ENABLE_NOTIFY_STAMP // Get the next expected notify stamp. const mm_ring_seqno_t stamp = mm_event_listener_dequeue_stamp(listener); if (timeout != 0) { // Advertise that the thread is about to sleep. uintptr_t state = (stamp << 2) | MM_EVENT_LISTENER_WAITING; mm_memory_store(listener->state, state); mm_memory_strict_fence(); // TODO: store_load fence // Wait for a wake-up notification or timeout unless // an already pending notification is detected. if (stamp == mm_event_listener_enqueue_stamp(listener)) mm_event_listener_timedwait(listener, stamp, timeout); } // Advertise the start of another working cycle. mm_event_listener_finish(listener); #else // Get the current listener state. const uint32_t listen_stamp = listener->listen_stamp; ASSERT((listen_stamp & MM_EVENT_LISTENER_STATE) == MM_EVENT_LISTENER_RUNNING); if (timeout != 0) { // Advertise that the thread is about to sleep. uint32_t wait_stamp = listen_stamp | MM_EVENT_LISTENER_WAITING; mm_memory_store(listener->listen_stamp, wait_stamp); mm_memory_strict_fence(); // TODO: store_load fence // Wait for a wake-up notification or timeout unless // an already pending notification is detected. if (listen_stamp == mm_memory_load(listener->notify_stamp)) mm_event_listener_timedwait(listener, listen_stamp, timeout); } // Advertise the start of another working cycle. mm_event_listener_finish(listener, listen_stamp); #endif LEAVE(); }
static struct mm_lock_stat_set * mm_lock_find_stat_set(uint32_t bucket, const char *location, const char *moreinfo) { // Go through bucket entries trying to find a match. struct mm_slink *link = mm_stack_atomic_load_head(&mm_lock_stat_table[bucket]); while (link != NULL) { struct mm_lock_stat_set *stat_set = containerof(link, struct mm_lock_stat_set, bucket_link); mm_memory_load_fence(); // Match the current entry. if (mm_lock_match_stat_set(stat_set, location, moreinfo)) return stat_set; link = mm_memory_load(link->next); } return NULL; }
static uint32_t mc_action_get_exp_time(void) { return mm_memory_load(mc_action_exp_time); }
mm_event_listener_poll(struct mm_event_listener *listener, mm_timeout_t timeout) { ENTER(); // Prepare to receive events. mm_event_receiver_start(&listener->receiver); #if ENABLE_NOTIFY_STAMP // Get the next expected notify stamp. const mm_ring_seqno_t stamp = mm_event_listener_dequeue_stamp(listener); if (timeout != 0) { // Cleanup stale event notifications. mm_event_backend_dampen(&listener->receiver.dispatch->backend); // Advertise that the thread is about to sleep. uintptr_t state = (stamp << 2) | MM_EVENT_LISTENER_POLLING; mm_memory_store(listener->state, state); mm_memory_strict_fence(); // TODO: store_load fence // Wait for a wake-up notification or timeout unless // an already pending notification is detected. if (stamp != mm_event_listener_enqueue_stamp(listener)) timeout = 0; } // Check incoming events and wait for notification/timeout. mm_event_backend_listen(&listener->receiver.dispatch->backend, &listener->changes, &listener->receiver, timeout); // Advertise the start of another working cycle. mm_event_listener_finish(listener); #else // Get the current listener state. const uint32_t listen_stamp = listener->listen_stamp; ASSERT((listen_stamp & MM_EVENT_LISTENER_STATE) == MM_EVENT_LISTENER_RUNNING); if (timeout != 0) { // Cleanup stale event notifications. mm_event_backend_dampen(&listener->receiver.dispatch->backend); // Advertise that the thread is about to sleep. uint32_t poll_stamp = listen_stamp | MM_EVENT_LISTENER_POLLING; mm_memory_store(listener->listen_stamp, poll_stamp); mm_memory_strict_fence(); // TODO: store_load fence // Wait for a wake-up notification or timeout unless // an already pending notification is detected. if (listen_stamp != mm_memory_load(listener->notify_stamp)) timeout = 0; } // Check incoming events and wait for notification/timeout. mm_event_backend_listen(&listener->receiver.dispatch->backend, &listener->changes, &listener->receiver, timeout); // Advertise the start of another working cycle. mm_event_listener_finish(listener, listen_stamp); #endif // Flush received events. mm_event_receiver_finish(&listener->receiver); LEAVE(); }