int thorium_worker_pool_give_message_to_actor(struct thorium_worker_pool *pool, struct thorium_message *message)
{
int destination;
struct thorium_actor *actor;
struct thorium_worker *affinity_worker;
int worker_index;
int name;
int dead;
/*
void *buffer;
buffer = thorium_message_buffer(message);
*/
destination = thorium_message_destination(message);
actor = thorium_node_get_actor_from_name(pool->node, destination);
if (actor == NULL) {
#ifdef THORIUM_WORKER_POOL_DEBUG_DEAD_CHANNEL
printf("DEAD LETTER CHANNEL...\n");
#endif
core_fast_queue_enqueue(&pool->messages_for_triage, message);
return 0;
}
dead = thorium_actor_dead(actor);
/* If the actor is dead, don't use it.
*/
if (dead) {
core_fast_queue_enqueue(&pool->messages_for_triage, message);
return 0;
}
name = thorium_actor_name(actor);
/* give the message to the actor
*/
if (!thorium_actor_enqueue_mailbox_message(actor, message)) {
#ifdef THORIUM_WORKER_POOL_DEBUG_MESSAGE_BUFFERING
printf("DEBUG897 could not enqueue message, buffering...\n");
#endif
core_fast_queue_enqueue(&pool->inbound_message_queue_buffer, message);
} else {
/*
* At this point, the message has been pushed to the actor.
* Now, the actor must be scheduled on a worker.
*/
/*
printf("DEBUG message was enqueued in actor mailbox\n");
*/
/* Check if the actor is already assigned to a worker
*/
worker_index = thorium_balancer_get_actor_worker(&pool->balancer, name);
/* If not, ask the scheduler to assign the actor to a worker
*/
if (worker_index < 0) {
thorium_worker_pool_assign_worker_to_actor(pool, name);
worker_index = thorium_balancer_get_actor_worker(&pool->balancer, name);
}
affinity_worker = thorium_worker_pool_get_worker(pool, worker_index);
/*
printf("DEBUG actor has an assigned worker\n");
*/
/*
* Push the actor on the scheduling queue of the worker.
* If that fails, queue the actor.
*/
if (!thorium_worker_enqueue_actor(affinity_worker, actor)) {
core_fast_queue_enqueue(&pool->scheduled_actor_queue_buffer, &actor);
}
}
return 1;
}
void thorium_worker_pool_work(struct thorium_worker_pool *pool)
{
struct thorium_message other_message;
struct thorium_actor *actor;
int worker_index;
struct thorium_worker *worker;
int name;
#ifdef THORIUM_WORKER_POOL_BALANCE
#endif
/* If there are messages in the inbound message buffer,
* Try to give them too.
*/
if (core_fast_queue_dequeue(&pool->inbound_message_queue_buffer, &other_message)) {
thorium_worker_pool_give_message_to_actor(pool, &other_message);
}
/* Try to dequeue an actor for scheduling
*/
if (core_fast_queue_dequeue(&pool->scheduled_actor_queue_buffer, &actor)) {
name = thorium_actor_name(actor);
worker_index = thorium_balancer_get_actor_worker(&pool->balancer, name);
if (worker_index < 0) {
/*
* This case is very rare, but will happen since the number of
* messages and the number of actors are both very large.
*
* This case will happen if these 3 conditions are satisfied:
*
* A message was not queued in the mailbox of the actor X because its mailbox was full.
* The message, in that case, will be queued in the actor mailbox at a later time.
* But this code path will not queue the actor on any worker since the actor has yet.
* But the actor do have messages, but the problem is that all the actor scheduling queue
* on every worker are full.
*/
#ifdef THORIUM_WORKER_POOL_DEBUG_ACTOR_ASSIGNMENT_PROBLEM
printf("Notice: actor %d has no assigned worker\n", name);
#endif
thorium_worker_pool_assign_worker_to_actor(pool, name);
worker_index = thorium_balancer_get_actor_worker(&pool->balancer, name);
}
worker = thorium_worker_pool_get_worker(pool, worker_index);
if (!thorium_worker_enqueue_actor(worker, actor)) {
core_fast_queue_enqueue(&pool->scheduled_actor_queue_buffer, &actor);
}
}
#if 0
printf("DEBUG pool receives message for actor %d\n",
destination);
#endif
#ifdef THORIUM_WORKER_POOL_BALANCE
/* balance the pool regularly
*/
if (current_time - pool->last_balancing >= pool->balance_period) {
thorium_balancer_balance(&pool->scheduler);
pool->last_balancing = current_time;
}
#endif
/*
* If waiting is enabled, it is required to wake up workers
* once in a while because of the ordering
* of events for the actor scheduling queue
*/
/* Example, thorium_worker_signal is called just a bit before
* thorium_worker_wait.
*
*/
if (pool->waiting_is_enabled) {
thorium_worker_pool_wake_up_workers(pool);
}
}
void thorium_balancer_generate_symmetric_migrations(struct thorium_balancer *self, struct core_map *symmetric_actor_scripts,
struct core_vector *migrations)
{
int i;
int worker_count;
struct thorium_worker *worker;
struct core_map *set;
struct core_map_iterator iterator;
struct thorium_migration migration;
struct core_map script_current_worker;
struct core_map script_current_worker_actor_count;
int frequency;
int current_worker;
int current_worker_actor_count;
int old_worker;
#ifdef THORIUM_SCHEDULER_ENABLE_VERBOSITY
struct thorium_script *actual_script;
#endif
struct thorium_node *node;
int actor_name;
int script;
int new_worker;
struct thorium_actor *actor;
int enabled;
/* Gather symmetric actors:
*/
#ifdef THORIUM_SCHEDULER_ENABLE_SYMMETRIC_SCHEDULING
enabled = 1;
#else
enabled = 0;
#endif
core_map_init(&script_current_worker, sizeof(int), sizeof(int));
core_map_init(&script_current_worker_actor_count, sizeof(int), sizeof(int));
node = thorium_worker_pool_get_node(self->pool);
worker_count = thorium_worker_pool_worker_count(self->pool);
for (i = 0; i < worker_count; i++) {
worker = thorium_worker_pool_get_worker(self->pool, i);
set = thorium_worker_get_actors(worker);
core_map_iterator_init(&iterator, set);
while (core_map_iterator_get_next_key_and_value(&iterator, &actor_name, NULL)) {
actor = thorium_node_get_actor_from_name(node, actor_name);
if (actor == NULL) {
continue;
}
script = thorium_actor_script(actor);
/*
* Check if the actor is symmetric
*/
if (core_map_get_value(symmetric_actor_scripts, &script, &frequency)) {
current_worker = 0;
if (!core_map_get_value(&script_current_worker, &script, ¤t_worker)) {
core_map_add_value(&script_current_worker, &script, ¤t_worker);
}
current_worker_actor_count = 0;
if (!core_map_get_value(&script_current_worker_actor_count, &script, ¤t_worker_actor_count)) {
core_map_add_value(&script_current_worker_actor_count, &script, ¤t_worker_actor_count);
}
/*
* Emit migration instruction
*/
old_worker = thorium_balancer_get_actor_worker(self, actor_name);
new_worker = current_worker;
#ifdef THORIUM_SCHEDULER_ENABLE_VERBOSITY
actual_script = thorium_node_find_script(node, script);
#endif
if (enabled && old_worker != new_worker) {
thorium_migration_init(&migration, actor_name, old_worker, new_worker);
core_vector_push_back(migrations, &migration);
thorium_migration_destroy(&migration);
#ifdef THORIUM_SCHEDULER_ENABLE_VERBOSITY
printf("[EMIT] ");
#endif
} else {
#ifdef THORIUM_SCHEDULER_ENABLE_VERBOSITY
printf("[MOCK] ");
#endif
}
#ifdef THORIUM_SCHEDULER_ENABLE_VERBOSITY
printf("SCHEDULER -> symmetric placement... %s/%d scheduled for execution on worker/%d of node/%d\n",
thorium_script_description(actual_script),
actor_name,
new_worker,
thorium_node_name(node));
#endif
++current_worker_actor_count;
core_map_update_value(&script_current_worker_actor_count, &script, ¤t_worker_actor_count);
/* The current worker is full.
* Increment the current worker and set the
* worker actor count to 0.
*/
if (current_worker_actor_count == frequency) {
++current_worker;
core_map_update_value(&script_current_worker, &script, ¤t_worker);
current_worker_actor_count = 0;
core_map_update_value(&script_current_worker_actor_count, &script, ¤t_worker_actor_count);
}
}
}
core_map_iterator_destroy(&iterator);
}
core_map_destroy(&script_current_worker);
core_map_destroy(&script_current_worker_actor_count);
}
