void thorium_worker_run(struct thorium_worker *worker) { struct thorium_actor *actor; struct thorium_message other_message; #ifdef THORIUM_NODE_INJECT_CLEAN_WORKER_BUFFERS void *buffer; #endif #ifdef THORIUM_NODE_ENABLE_INSTRUMENTATION time_t current_time; int elapsed; int period; uint64_t current_nanoseconds; uint64_t elapsed_nanoseconds; #endif #ifdef THORIUM_WORKER_DEBUG int tag; int destination; struct thorium_message *message; #endif #ifdef THORIUM_WORKER_ENABLE_LOCK thorium_worker_lock(worker); #endif #ifdef THORIUM_NODE_ENABLE_INSTRUMENTATION period = THORIUM_NODE_LOAD_PERIOD; current_time = time(NULL); elapsed = current_time - worker->last_report; if (elapsed >= period) { current_nanoseconds = core_timer_get_nanoseconds(&worker->timer); #ifdef THORIUM_WORKER_DEBUG_LOAD printf("DEBUG Updating load report\n"); #endif elapsed_nanoseconds = current_nanoseconds - worker->epoch_start_in_nanoseconds; if (elapsed_nanoseconds > 0) { worker->epoch_load = (0.0 + worker->epoch_used_nanoseconds) / elapsed_nanoseconds; worker->epoch_used_nanoseconds = 0; worker->last_wake_up_count = core_thread_get_wake_up_count(&worker->thread); /* \see http://stackoverflow.com/questions/9657993/negative-zero-in-c */ if (worker->epoch_load == 0) { worker->epoch_load = 0; } worker->epoch_start_in_nanoseconds = current_nanoseconds; worker->last_report = current_time; } #ifdef THORIUM_WORKER_PRINT_SCHEDULING_QUEUE /* if (thorium_node_name(worker->node) == 0 && worker->name == 0) { */ thorium_scheduler_print(&worker->scheduler, thorium_node_name(worker->node), worker->name); /* } */ #endif if (core_bitmap_get_bit_uint32_t(&worker->flags, FLAG_DEBUG_ACTORS)) { thorium_worker_print_actors(worker, NULL); } } #endif #ifdef THORIUM_WORKER_DEBUG if (core_bitmap_get_bit_uint32_t(&worker->flags, FLAG_DEBUG)) { printf("DEBUG worker/%d thorium_worker_run\n", thorium_worker_name(worker)); } #endif /* check for messages in inbound FIFO */ if (thorium_worker_dequeue_actor(worker, &actor)) { #ifdef THORIUM_WORKER_DEBUG message = biosal_work_message(&work); tag = thorium_message_action(message); destination = thorium_message_destination(message); if (tag == ACTION_ASK_TO_STOP) { printf("DEBUG pulled ACTION_ASK_TO_STOP for %d\n", destination); } #endif /* * Update the priority of the actor * before starting the timer because this is part of the * runtime system (RTS). */ #ifdef THORIUM_UPDATE_SCHEDULING_PRIORITIES thorium_priority_assigner_update(&worker->scheduler, actor); #endif #ifdef THORIUM_NODE_ENABLE_INSTRUMENTATION core_timer_start(&worker->timer); #endif core_bitmap_set_bit_uint32_t(&worker->flags, FLAG_BUSY); /* * Dispatch message to a worker */ thorium_worker_work(worker, actor); core_bitmap_clear_bit_uint32_t(&worker->flags, FLAG_BUSY); #ifdef THORIUM_NODE_ENABLE_INSTRUMENTATION core_timer_stop(&worker->timer); elapsed_nanoseconds = core_timer_get_elapsed_nanoseconds(&worker->timer); if (elapsed_nanoseconds >= THORIUM_GRANULARITY_WARNING_THRESHOLD) { } worker->epoch_used_nanoseconds += elapsed_nanoseconds; worker->loop_used_nanoseconds += elapsed_nanoseconds; worker->scheduling_epoch_used_nanoseconds += elapsed_nanoseconds; worker->last_elapsed_nanoseconds = elapsed_nanoseconds; #endif } /* queue buffered message */ if (core_fast_queue_dequeue(&worker->outbound_message_queue_buffer, &other_message)) { if (!core_fast_ring_push_from_producer(&worker->outbound_message_queue, &other_message)) { #ifdef SHOW_FULL_RING_WARNINGS printf("thorium_worker: Warning: ring is full => outbound_message_queue\n"); #endif core_fast_queue_enqueue(&worker->outbound_message_queue_buffer, &other_message); } } #ifdef THORIUM_NODE_INJECT_CLEAN_WORKER_BUFFERS /* * Free outbound buffers, if any */ if (thorium_worker_fetch_clean_outbound_buffer(worker, &buffer)) { core_memory_pool_free(&worker->outbound_message_memory_pool, buffer); #ifdef THORIUM_WORKER_DEBUG_INJECTION ++worker->counter_freed_outbound_buffers_from_other_workers; #endif } #endif /* * Transfer messages for triage */ if (core_fast_queue_dequeue(&worker->clean_message_queue_for_triage, &other_message)) { CORE_DEBUGGER_ASSERT(thorium_message_buffer(&other_message) != NULL); thorium_worker_enqueue_message_for_triage(worker, &other_message); } #ifdef THORIUM_WORKER_ENABLE_LOCK thorium_worker_unlock(worker); #endif }
int thorium_balancer_select_worker_least_busy( struct thorium_balancer *self, int *worker_score) { int to_check; int score; int best_score; struct thorium_worker *worker; struct thorium_worker *best_worker; int selected_worker; #if 0 int last_worker_score; #endif #ifdef THORIUM_WORKER_DEBUG int tag; int destination; struct thorium_message *message; #endif best_worker = NULL; best_score = 99; to_check = THORIUM_SCHEDULER_WORK_SCHEDULING_WINDOW; while (to_check--) { /* * get the worker to test for this iteration. */ worker = thorium_worker_pool_get_worker(self->pool, self->worker_for_work); score = thorium_worker_get_epoch_load(worker); #ifdef THORIUM_WORKER_POOL_DEBUG_ISSUE_334 if (score >= THORIUM_WORKER_WARNING_THRESHOLD && (self->last_scheduling_warning == 0 || score >= self->last_scheduling_warning + THORIUM_WORKER_WARNING_THRESHOLD_STRIDE)) { printf("Warning: node %d worker %d has a scheduling score of %d\n", thorium_node_name(thorium_worker_pool_get_node(self->pool)), self->worker_for_work, score); self->last_scheduling_warning = score; } #endif /* if the worker is not busy and it has no work to do, * select it right away... */ if (score == 0) { best_worker = worker; best_score = 0; break; } /* Otherwise, test the worker */ if (best_worker == NULL || score < best_score) { best_worker = worker; best_score = score; } /* * assign the next worker */ self->worker_for_work = thorium_worker_pool_next_worker(self->pool, self->worker_for_work); } #ifdef THORIUM_WORKER_POOL_DEBUG message = biosal_work_message(work); tag = thorium_message_action(message); destination = thorium_message_destination(message); if (tag == ACTION_ASK_TO_STOP) { printf("DEBUG dispatching ACTION_ASK_TO_STOP for actor %d to worker %d\n", destination, *start); } #endif selected_worker = self->worker_for_work; /* * assign the next worker */ self->worker_for_work = thorium_worker_pool_next_worker(self->pool, self->worker_for_work); *worker_score = best_score; /* This is a best effort algorithm */ return selected_worker; }