bool ponyint_messageq_push(messageq_t* q, pony_msg_t* first, pony_msg_t* last)
{
atomic_store_explicit(&last->next, NULL, memory_order_relaxed);
// Without that fence, the store to last->next above could be reordered after
// the exchange on the head and after the store to prev->next done by the
// next push, which would result in the pop incorrectly seeing the queue as
// empty.
// Also synchronise with the pop on prev->next.
atomic_thread_fence(memory_order_release);
pony_msg_t* prev = atomic_exchange_explicit(&q->head, last,
memory_order_relaxed);
bool was_empty = ((uintptr_t)prev & 1) != 0;
prev = (pony_msg_t*)((uintptr_t)prev & ~(uintptr_t)1);
#ifdef USE_VALGRIND
// Double fence with Valgrind since we need to have prev in scope for the
// synchronisation annotation.
ANNOTATE_HAPPENS_BEFORE(&prev->next);
atomic_thread_fence(memory_order_release);
#endif
atomic_store_explicit(&prev->next, first, memory_order_relaxed);
return was_empty;
}
int starpu_profiling_status_get(void)
{
int ret;
ANNOTATE_HAPPENS_AFTER(&_starpu_profiling);
ret = _starpu_profiling;
ANNOTATE_HAPPENS_BEFORE(&_starpu_profiling);
return ret;
}
static int attach_thread(struct PupHeap *heap, pthread_t thread)
{
struct PupThreadInfo *tinfo = malloc(sizeof(struct PupThreadInfo));
if (tinfo == NULL) {
return -1;
}
AO_store(&tinfo->tid, thread);
ANNOTATE_HAPPENS_BEFORE(&tinfo->tid);
AO_store(&tinfo->next, 0);
ANNOTATE_HAPPENS_BEFORE(&tinfo->next);
tinfo->gc_waiting = false;
if (set_thread_info(heap, tinfo)) {
return -1;
}
attach_thread_to_heap(heap, tinfo);
ANNOTATE_THREAD_NAME("mutator");
return 0;
}
/*@-internalglobs@*/
rpmioItem rpmioLinkPoolItem(rpmioItem item, const char * msg,
const char * fn, unsigned ln)
{
rpmioPool pool;
if (item == NULL) return NULL;
yarnPossess(item->use);
if ((pool = (rpmioPool) item->pool) != NULL && pool->flags && msg != NULL) {
const char * imsg = (pool->dbg ? (*pool->dbg)((void *)item) : "");
/*@-modfilesys@*/
fprintf(stderr, "--> %s %p ++ %ld %s at %s:%u%s\n", pool->name,
item, yarnPeekLock(item->use)+1, msg, fn, ln, imsg);
/*@=modfilesys@*/
}
ANNOTATE_HAPPENS_BEFORE(item);
yarnTwist(item->use, BY, 1);
return item;
}
int starpu_profiling_status_set(int status)
{
ANNOTATE_HAPPENS_AFTER(&_starpu_profiling);
int prev_value = _starpu_profiling;
_starpu_profiling = status;
ANNOTATE_HAPPENS_BEFORE(&_starpu_profiling);
_STARPU_TRACE_SET_PROFILING(status);
/* If we enable profiling, we reset the counters. */
if (status == STARPU_PROFILING_ENABLE)
{
_starpu_profiling_reset_counters();
}
return prev_value;
}
bool ponyint_messageq_markempty(messageq_t* q)
{
pony_msg_t* tail = q->tail;
pony_msg_t* head = atomic_load_explicit(&q->head, memory_order_relaxed);
if(((uintptr_t)head & 1) != 0)
return true;
if(head != tail)
return false;
head = (pony_msg_t*)((uintptr_t)head | 1);
#ifdef USE_VALGRIND
ANNOTATE_HAPPENS_BEFORE(&q->head);
#endif
return atomic_compare_exchange_strong_explicit(&q->head, &tail, head,
memory_order_release, memory_order_relaxed);
}
void _starpu_profiling_init(void)
{
const char *env;
int worker;
for (worker = 0; worker < STARPU_NMAXWORKERS; worker++)
{
STARPU_PTHREAD_MUTEX_INIT(&worker_info_mutex[worker], NULL);
}
_starpu_profiling_reset_counters();
if ((env = getenv("STARPU_PROFILING")) && atoi(env))
{
ANNOTATE_HAPPENS_AFTER(&_starpu_profiling);
_starpu_profiling = STARPU_PROFILING_ENABLE;
ANNOTATE_HAPPENS_BEFORE(&_starpu_profiling);
}
}
bool ponyint_messageq_push_single(messageq_t* q, pony_msg_t* first,
pony_msg_t* last)
{
atomic_store_explicit(&last->next, NULL, memory_order_relaxed);
// If we have a single producer, the swap of the head need not be atomic RMW.
pony_msg_t* prev = atomic_load_explicit(&q->head, memory_order_relaxed);
atomic_store_explicit(&q->head, last, memory_order_relaxed);
bool was_empty = ((uintptr_t)prev & 1) != 0;
prev = (pony_msg_t*)((uintptr_t)prev & ~(uintptr_t)1);
// If we have a single producer, the fence can be replaced with a store
// release on prev->next.
#ifdef USE_VALGRIND
ANNOTATE_HAPPENS_BEFORE(&prev->next);
#endif
atomic_store_explicit(&prev->next, first, memory_order_release);
return was_empty;
}
PONY_API void pony_asio_event_unsubscribe(asio_event_t* ev)
{
if((ev == NULL) ||
(ev->flags == ASIO_DISPOSABLE) ||
(ev->flags == ASIO_DESTROYED))
{
pony_assert(0);
return;
}
asio_backend_t* b = ponyint_asio_get_backend();
pony_assert(b != NULL);
if(ev->noisy)
{
uint64_t old_count = ponyint_asio_noisy_remove();
// tell scheduler threads that asio has no noisy actors
// if the old_count was 1
if (old_count == 1)
{
ponyint_sched_unnoisy_asio(SPECIAL_THREADID_EPOLL);
// maybe wake up a scheduler thread if they've all fallen asleep
ponyint_sched_maybe_wakeup_if_all_asleep(-1);
}
ev->noisy = false;
}
epoll_ctl(b->epfd, EPOLL_CTL_DEL, ev->fd, NULL);
if(ev->flags & ASIO_TIMER)
{
if(ev->fd != -1)
{
close(ev->fd);
ev->fd = -1;
}
}
if(ev->flags & ASIO_SIGNAL)
{
int sig = (int)ev->nsec;
asio_event_t* prev = ev;
#ifdef USE_VALGRIND
ANNOTATE_HAPPENS_BEFORE(&b->sighandlers[sig]);
#endif
if((sig < MAX_SIGNAL) &&
atomic_compare_exchange_strong_explicit(&b->sighandlers[sig], &prev, NULL,
memory_order_release, memory_order_relaxed))
{
struct sigaction new_action;
#if !defined(USE_SCHEDULER_SCALING_PTHREADS)
// Make sure we ignore signals related to scheduler sleeping/waking
// as the default for those signals is termination
if(sig == PONY_SCHED_SLEEP_WAKE_SIGNAL)
new_action.sa_handler = empty_signal_handler;
else
#endif
new_action.sa_handler = SIG_DFL;
sigemptyset (&new_action.sa_mask);
// ask to restart interrupted syscalls to match `signal` behavior
new_action.sa_flags = SA_RESTART;
sigaction(sig, &new_action, NULL);
close(ev->fd);
ev->fd = -1;
}
}
ev->flags = ASIO_DISPOSABLE;
send_request(ev, ASIO_DISPOSABLE);
}
PONY_API void pony_asio_event_subscribe(asio_event_t* ev)
{
if((ev == NULL) ||
(ev->flags == ASIO_DISPOSABLE) ||
(ev->flags == ASIO_DESTROYED))
{
pony_assert(0);
return;
}
asio_backend_t* b = ponyint_asio_get_backend();
pony_assert(b != NULL);
if(ev->noisy)
{
uint64_t old_count = ponyint_asio_noisy_add();
// tell scheduler threads that asio has at least one noisy actor
// if the old_count was 0
if (old_count == 0)
ponyint_sched_noisy_asio(SPECIAL_THREADID_EPOLL);
}
struct epoll_event ep;
ep.data.ptr = ev;
ep.events = EPOLLRDHUP;
if(ev->flags & ASIO_READ)
ep.events |= EPOLLIN;
if(ev->flags & ASIO_WRITE)
ep.events |= EPOLLOUT;
if(ev->flags & ASIO_TIMER)
{
ev->fd = timerfd_create(CLOCK_MONOTONIC, TFD_NONBLOCK);
timer_set_nsec(ev->fd, ev->nsec);
ep.events |= EPOLLIN;
}
if(ev->flags & ASIO_SIGNAL)
{
int sig = (int)ev->nsec;
asio_event_t* prev = NULL;
#ifdef USE_VALGRIND
ANNOTATE_HAPPENS_BEFORE(&b->sighandlers[sig]);
#endif
if((sig < MAX_SIGNAL) &&
atomic_compare_exchange_strong_explicit(&b->sighandlers[sig], &prev, ev,
memory_order_release, memory_order_relaxed))
{
struct sigaction new_action;
new_action.sa_handler = signal_handler;
sigemptyset (&new_action.sa_mask);
// ask to restart interrupted syscalls to match `signal` behavior
new_action.sa_flags = SA_RESTART;
sigaction(sig, &new_action, NULL);
ev->fd = eventfd(0, EFD_NONBLOCK);
ep.events |= EPOLLIN;
} else {
return;
}
}
if(ev->flags & ASIO_ONESHOT)
{
ep.events |= EPOLLONESHOT;
} else {
// Only use edge triggering if one shot isn't enabled.
// This is because of how the runtime gets notifications
// from epoll in this ASIO thread and then notifies the
// appropriate actor to read/write as necessary.
// specifically, it seems there's an edge case/race condition
// with edge triggering where if there is already data waiting
// on the socket, then epoll might not be triggering immediately
// when an edge triggered epoll request is made.
ep.events |= EPOLLET;
}
epoll_ctl(b->epfd, EPOLL_CTL_ADD, ev->fd, &ep);
}
static void set_region_next(struct PupHeapRegion *region,
struct PupHeapRegion *next)
{
AO_store(®ion->next, (AO_t)next);
ANNOTATE_HAPPENS_BEFORE(®ion->next);
}
static void set_gc_state(struct PupHeap *heap,
struct PupGCState *gc_state)
{
AO_store((AO_t *)&heap->gc_state, (AO_t)gc_state);
ANNOTATE_HAPPENS_BEFORE(&heap->gc_state);
}
