void thorium_message_multiplexer_flush(struct thorium_message_multiplexer *self, int index, int force)
{
char *buffer;
struct thorium_message message;
int tag;
int count;
int current_size;
int maximum_size;
struct thorium_multiplexed_buffer *multiplexed_buffer;
int destination_node;
/*
int elapsed;
int message_count;
*/
if (CORE_BITMAP_GET_FLAG(self->flags, FLAG_DISABLED)) {
return;
}
#ifdef THORIUM_MULTIPLEXER_TRACK_BUFFERS_WITH_CONTENT
#ifdef CORE_DEBUGGER_ASSERT_ENABLED
if (!(core_set_find(&self->buffers_with_content, &index))) {
multiplexed_buffer = core_vector_at(&self->buffers, index);
thorium_printf("index %d has no content\n", index);
thorium_multiplexed_buffer_print(multiplexed_buffer);
}
#endif
CORE_DEBUGGER_ASSERT(core_set_find(&self->buffers_with_content, &index));
#endif
multiplexed_buffer = core_vector_at(&self->buffers, index);
current_size = thorium_multiplexed_buffer_current_size(multiplexed_buffer);
maximum_size = thorium_multiplexed_buffer_maximum_size(multiplexed_buffer);
/*
* The buffer was still in the timeline, but it was flushed elsewhere.
*/
if (current_size == 0) {
return;
/*
if (force == FORCE_NO && current_size < maximum_size) {
return;
} else if (force == FORCE_YES_TIME) {
elapsed = core_timer_get_nanoseconds(&self->timer) - multiplexed_buffer->timestamp_;
if (elapsed < self->timeout_in_nanoseconds) {
return;
}
} else if (force == FORCE_YES_DOA) {
message_count = multiplexed_buffer->message_count_;
if (message_count < self->degree_of_aggregation_limit) {
return;
}
*/
}
#ifdef CORE_DEBUGGER_ASSERT_ENABLED
if (current_size <= 0)
thorium_printf("current_size %d maximum_size %d\n", current_size, maximum_size);
#endif
CORE_DEBUGGER_ASSERT(current_size > 0);
buffer = thorium_multiplexed_buffer_buffer(multiplexed_buffer);
count = current_size + THORIUM_MESSAGE_METADATA_SIZE;
tag = ACTION_MULTIPLEXER_MESSAGE;
/*
* This count does not include metadata for the final big message.
*
* Avoid this copy by using an array of pointers in the first place.
*/
destination_node = index;
thorium_message_init(&message, tag, count, buffer);
thorium_message_set_destination(&message,
destination_node);
thorium_message_set_source(&message,
thorium_node_name(self->node));
/*
* Mark the message so that the buffer is eventually sent back here
* for recycling.
*/
thorium_message_set_worker(&message, thorium_worker_name(self->worker));
thorium_message_write_metadata(&message);
#ifdef DEBUG_MULTIPLEXER_FLUSH
thorium_printf("DEBUG_MULTIPLEXER thorium_message_multiplexer_flush index %d buffer %p force %d current_size %d maximum_size %d"
" destination_node %d\n",
index, buffer, force,
current_size, maximum_size,
thorium_message_destination_node(&message));
thorium_printf("message in flush\n");
thorium_multiplexed_buffer_print(multiplexed_buffer);
thorium_message_print(&message);
#endif
CORE_DEBUGGER_ASSERT_NOT_NULL(self->worker);
/*
* Make a copy of the buffer because the multiplexer does not have communication buffers.
*/
thorium_worker_enqueue_outbound_message(self->worker, &message);
/*
thorium_printf("MULTIPLEXER FLUSH\n");
*/
++self->real_message_count;
thorium_message_destroy(&message);
thorium_multiplexed_buffer_reset(multiplexed_buffer);
#ifdef THORIUM_MULTIPLEXER_TRACK_BUFFERS_WITH_CONTENT
core_set_delete(&self->buffers_with_content, &index);
#endif
}
int thorium_multiplexer_policy_is_action_to_skip(struct thorium_multiplexer_policy *self, int action)
{
return core_set_find(&self->actions_to_skip, &action);
}
/*
* This is O(n), that is if all thorium nodes have a buffer to flush.
*/
void thorium_message_multiplexer_test(struct thorium_message_multiplexer *self)
{
/*
* Check if the multiplexer has waited enough.
*/
int index;
struct thorium_multiplexed_buffer *multiplexed_buffer;
int buffer_is_ready;
#if defined(THORIUM_MULTIPLEXER_USE_TREE) || defined(THORIUM_MULTIPLEXER_USE_HEAP)
uint64_t *lowest_key;
int *index_bucket;
#endif
if (CORE_BITMAP_GET_FLAG(self->flags, FLAG_DISABLED)) {
return;
}
#ifdef THORIUM_MULTIPLEXER_TRACK_BUFFERS_WITH_CONTENT
/*
* Nothing to do, there is nothing to flush
* in the system.
*/
if (core_set_empty(&self->buffers_with_content)) {
return;
}
#endif
#if 1
/*
* Don't flush anything if the outbound ring is full,
* which means there is congestion.
*
* This means that the throughput is at its maximum value. In that case,
* it is better to generate even larger messages.
*/
if (thorium_worker_has_outbound_traffic_congestion(self->worker)) {
return;
}
#endif
#ifdef DEBUG_MULTIPLEXER_TEST
if (size >= DEBUG_MINIMUM_COUNT)
thorium_printf("DEBUG multiplexer_test buffers with content: %d\n",
size);
#endif
#ifdef CHECK_PREDICTED_TRAFFIC_REDUCTION
/*
* 0.95 corresponds to 20 actor messages in 1 network message.
* 0.90 corresponds to 10 actor messages in 1 network message.
*/
acceptable_traffic_reduction = 0.90;
#endif
/*
* Get the destination with the oldest buffer.
* If this one has not waited enough, then any other more recent
* buffer has not waited enough neither.
*/
while (1) {
#ifdef THORIUM_MULTIPLEXER_USE_TREE
lowest_key = core_red_black_tree_get_lowest_key(&self->timeline);
/*
* The timeline is empty.
*/
if (lowest_key == NULL)
return;
#elif defined(THORIUM_MULTIPLEXER_USE_HEAP)
lowest_key = NULL;
core_binary_heap_get_root(&self->timeline, (void **)&lowest_key,
(void **)&index_bucket);
/*
* The timeline is empty.
*/
if (lowest_key == NULL)
return;
#elif defined(THORIUM_MULTIPLEXER_USE_QUEUE)
if (!core_queue_dequeue(&self->timeline, &index)) {
return;
}
#endif
/*
* Get the index.
* The index is the destination.
*/
#ifdef THORIUM_MULTIPLEXER_USE_TREE
index_bucket = core_red_black_tree_get(&self->timeline, lowest_key);
index = *index_bucket;
#endif
multiplexed_buffer = core_vector_at(&self->buffers, index);
buffer_is_ready = thorium_message_multiplexer_buffer_is_ready(self, multiplexed_buffer);
/*
* The oldest item is too recent.
* Therefore, all the others are too recent too
* because the timeline is ordered.
*/
if (!buffer_is_ready) {
#ifdef THORIUM_MULTIPLEXER_USE_QUEUE
core_queue_enqueue(&self->timeline, &index);
#endif
return;
}
#ifdef CHECK_OUTBOUND_THROUGHPUT
/*
* Don't flush now since the transport layer has already reached its maximum
* throughput.
*/
if (thorium_worker_has_reached_maximum_outbound_throughput(self->worker)
&& traffic_reduction < acceptable_traffic_reduction) {
#ifdef THORIUM_MULTIPLEXER_USE_QUEUE
core_queue_enqueue(&self->timeline, &index);
#endif
return;
}
#endif
/*
* Remove the object from the timeline.
*/
#ifdef THORIUM_MULTIPLEXER_USE_TREE
core_red_black_tree_delete(&self->timeline, lowest_key);
#elif defined(THORIUM_MULTIPLEXER_USE_HEAP)
core_binary_heap_delete_root(&self->timeline);
#endif
#ifdef VERIFY_TARGET
/*
* Verify if the buffer can grow more.
*/
if (!thorium_multiplexed_buffer_has_reached_target(multiplexed_buffer)) {
thorium_multiplexed_buffer_set_time(multiplexed_buffer, time);
core_red_black_tree_add_key_and_value(&self->timeline, &time, &index);
return;
}
#endif
/*
* The item won't have content in the case were _flush()
* was called elsewhere with FORCE_YES_SIZE.
*/
#ifdef THORIUM_MULTIPLEXER_TRACK_BUFFERS_WITH_CONTENT
if (core_set_find(&self->buffers_with_content, &index)) {
#endif
thorium_message_multiplexer_flush(self, index, FORCE_YES_TIME);
#ifdef THORIUM_MULTIPLEXER_TRACK_BUFFERS_WITH_CONTENT
}
#endif
/*
* Otherwise, keep flushing stuff.
* This will eventually end anyway.
*/
}
}
/* This can only be called from the CONSUMER
*/
int thorium_worker_dequeue_actor(struct thorium_worker *worker, struct thorium_actor **actor)
{
int value;
int name;
struct thorium_actor *other_actor;
int other_name;
int operations;
int status;
int mailbox_size;
operations = 4;
other_actor = NULL;
/* Move an actor from the ring to the real actor scheduling queue
*/
while (operations--
&& core_fast_ring_pop_from_consumer(&worker->actors_to_schedule, &other_actor)) {
#ifdef CORE_DEBUGGER_ENABLE_ASSERT
if (other_actor == NULL) {
printf("NULL pointer pulled from ring, operations %d ring size %d\n",
operations, core_fast_ring_size_from_consumer(&worker->actors_to_schedule));
}
#endif
CORE_DEBUGGER_ASSERT(other_actor != NULL);
other_name = thorium_actor_name(other_actor);
#ifdef THORIUM_WORKER_DEBUG_SCHEDULER
printf("ring.DEQUEUE %d\n", other_name);
#endif
if (core_set_find(&worker->evicted_actors, &other_name)) {
#ifdef THORIUM_WORKER_DEBUG_SCHEDULER
printf("ALREADY EVICTED\n");
#endif
continue;
}
if (!core_map_get_value(&worker->actors, &other_name, &status)) {
/* Add the actor to the list of actors.
* This does nothing if it is already in the list.
*/
status = STATUS_IDLE;
core_map_add_value(&worker->actors, &other_name, &status);
core_map_iterator_destroy(&worker->actor_iterator);
core_map_iterator_init(&worker->actor_iterator, &worker->actors);
}
/* If the actor is not queued, queue it
*/
if (status == STATUS_IDLE) {
status = STATUS_QUEUED;
core_map_update_value(&worker->actors, &other_name, &status);
thorium_scheduler_enqueue(&worker->scheduler, other_actor);
} else {
#ifdef THORIUM_WORKER_DEBUG_SCHEDULER
printf("SCHEDULER %d already scheduled to run, scheduled: %d\n", other_name,
(int)core_set_size(&worker->queued_actors));
#endif
}
}
/* Now, dequeue an actor from the real queue.
* If it has more than 1 message, re-enqueue it
*/
value = thorium_scheduler_dequeue(&worker->scheduler, actor);
/* Setting name to nobody;
* check_production at the end uses the value and the name;
* if value is false, check_production is not using name anyway.
*/
name = THORIUM_ACTOR_NOBODY;
/* an actor is ready to be run and it was dequeued from the scheduling queue.
*/
if (value) {
name = thorium_actor_name(*actor);
#ifdef THORIUM_WORKER_DEBUG_SCHEDULER
printf("scheduler.DEQUEUE actor %d, removed from queued actors...\n", name);
#endif
mailbox_size = thorium_actor_get_mailbox_size(*actor);
/* The actor has only one message and it is going to
* be processed now.
*/
if (mailbox_size == 1) {
#ifdef THORIUM_WORKER_DEBUG_SCHEDULER
printf("SCHEDULER %d has no message to schedule...\n", name);
#endif
/* Set the status of the worker to STATUS_IDLE
*
* TODO: the ring new tail might not be visible too.
* That could possibly be a problem...
*/
status = STATUS_IDLE;
core_map_update_value(&worker->actors, &name, &status);
/* The actor still has a lot of messages
* to process. Keep them coming.
*/
} else if (mailbox_size >= 2) {
/* Add the actor to the scheduling queue if it
* still has messages
*/
#ifdef THORIUM_WORKER_DEBUG_SCHEDULER
printf("Scheduling actor %d again, messages: %d\n",
name,
thorium_actor_get_mailbox_size(*actor));
#endif
/* The status is still STATUS_QUEUED
*/
thorium_scheduler_enqueue(&worker->scheduler, *actor);
/* The actor is scheduled to run, but the new tail is not
* yet visible apparently.
*
* Solution, push back the actor in the scheduler queue, it can take a few cycles to see cache changes across cores. (MESIF protocol)
*
* This is done below.
*/
} else /* if (mailbox_size == 0) */ {
status = STATUS_IDLE;
core_map_update_value(&worker->actors, &name, &status);
value = 0;
}
}
thorium_worker_check_production(worker, value, name);
return value;
}