void thorium_actor_send_range_default(struct thorium_actor *actor, struct core_vector *actors,
int first, int last,
struct thorium_message *message)
{
#ifdef USE_BINOMIAL_TREE
struct core_vector destinations;
struct core_memory_pool *ephemeral_memory;
int name;
ephemeral_memory = thorium_actor_get_ephemeral_memory(actor);
core_vector_init(&destinations, sizeof(int));
core_vector_set_memory_pool(&destinations, ephemeral_memory);
core_vector_copy_range(actors, first, last, &destinations);
/*
* Set the source now.
*/
name = thorium_actor_name(actor);
thorium_message_set_source(message, name);
thorium_actor_send_range_binomial_tree(actor, &destinations, message);
core_vector_destroy(&destinations);
#else
thorium_actor_send_range_loop(actor, actors, first, last, message);
#endif
}
void thorium_actor_pack_proxy_message(struct thorium_actor *self, struct thorium_message *message,
int real_source)
{
int real_tag;
int count;
int new_count;
void *buffer;
void *new_buffer;
int offset;
struct core_memory_pool *ephemeral_memory;
/*
* pack data in this order:
*
* - data (count bytes)
* - real_source
* - real_tag
*/
ephemeral_memory = thorium_actor_get_ephemeral_memory(self);
real_tag = thorium_message_action(message);
buffer = thorium_message_buffer(message);
count = thorium_message_count(message);
#ifdef DEBUG_BINOMIAL_TREE
printf("DEBUG_BINOMIAL_TREE pack_proxy_message count %d source %d action %x\n", count,
thorium_actor_name(self),
real_tag);
thorium_message_print(message);
#endif
new_count = count + sizeof(real_source) + sizeof(real_tag);
/* use slab allocator */
new_buffer = core_memory_pool_allocate(ephemeral_memory, new_count);
#ifdef THORIUM_ACTOR_DEBUG
printf("DEBUG12 core_memory_pool_allocate %p (pack proxy message)\n",
new_buffer);
#endif
if (count > 0)
core_memory_copy(new_buffer, buffer, count);
offset = count;
core_memory_copy((char *)new_buffer + offset, &real_source, sizeof(real_source));
offset += sizeof(real_source);
core_memory_copy((char *)new_buffer + offset, &real_tag, sizeof(real_tag));
offset += sizeof(real_tag);
thorium_message_init(message, ACTION_PROXY_MESSAGE, new_count, new_buffer);
thorium_message_set_source(message, real_source);
#if 0
/* free the old buffer
*/
core_memory_free(buffer);
buffer = NULL;
#endif
}
void thorium_actor_send_range_default(struct thorium_actor *actor, struct core_vector *actors,
int first, int last,
struct thorium_message *message)
{
int use_binomial_tree;
struct core_vector destinations;
struct core_memory_pool *ephemeral_memory;
int name;
int action;
action = thorium_message_action(message);
use_binomial_tree = 0;
#ifdef USE_BINOMIAL_TREE
/*
* ACTION_ASK_TO_STOP basically kills actors (if they agree to).
* It is a bad idea to use a binomial tree to send this death signal
* since intermediate actors can die before acting as relays.
*/
if (action != ACTION_ASK_TO_STOP)
use_binomial_tree = 1;
#endif
if (!use_binomial_tree) {
thorium_actor_send_range_loop(actor, actors, first, last, message);
return;
}
tracepoint(thorium_binomial_tree, send_range, message, (int)core_vector_size(actors));
/*
* Otherwise, use the binomial tree code path. This algorithm is better since it distributed
* the sending operations intot a binomial tree where there are a lot of intermediate
* actors (to be exact, the number of intermediate actors is close to log2(actors.size)).
*/
ephemeral_memory = thorium_actor_get_ephemeral_memory(actor);
core_vector_init(&destinations, sizeof(int));
core_vector_set_memory_pool(&destinations, ephemeral_memory);
CORE_DEBUGGER_ASSERT(core_vector_empty(&destinations));
core_vector_copy_range(actors, first, last, &destinations);
/*
* Set the source now.
*/
name = thorium_actor_name(actor);
thorium_message_set_source(message, name);
thorium_actor_send_range_binomial_tree(actor, &destinations, message);
core_vector_destroy(&destinations);
}
void thorium_message_init_copy(struct thorium_message *self, struct thorium_message *old_message)
{
thorium_message_init(self,
thorium_message_action(old_message),
thorium_message_count(old_message),
thorium_message_buffer(old_message));
thorium_message_set_source(self,
thorium_message_source(old_message));
thorium_message_set_destination(self,
thorium_message_destination(old_message));
}
void thorium_actor_unpack_proxy_message(struct thorium_actor *self,
struct thorium_message *message)
{
int new_count;
int tag;
int source;
void *buffer;
int offset;
buffer = thorium_message_buffer(message);
new_count = thorium_message_count(message);
new_count -= sizeof(source);
new_count -= sizeof(tag);
offset = new_count;
source = *(int *)((char *)buffer + offset);
offset += sizeof(source);
tag = *(int *)((char *)buffer + offset);
#ifdef DEBUG_BINOMIAL_TREE
printf("DEBUG_BINOMIAL_TREE unpack_proxy_message real_count %d real_source %d real_action %x\n",
new_count, source, tag);
#endif
/*
* The action can not be ACTION_INVALID because it is invalid
* by convention.
*/
CORE_DEBUGGER_ASSERT(tag != ACTION_INVALID);
offset += sizeof(tag);
/*thorium_message_init(message, tag, new_count, buffer);*/
/*
* Change the tag, source, and count.
*/
thorium_message_set_source(message, source);
thorium_message_set_action(message, tag);
thorium_message_set_count(message, new_count);
#if 0
printf("DEBUG unpack_proxy_message... source %d tag %d count %d\n",
source, tag, new_count);
#endif
}
void thorium_message_init_with_nodes(struct thorium_message *self, int count, void *buffer, int source,
int destination)
{
int action;
action = -1;
thorium_message_init(self, action, count, buffer);
/*
* Initially assign the MPI source rank and MPI destination
* rank for the actor source and actor destination, respectively.
* Then, read the metadata and resolve the MPI rank from
* that. The resolved MPI ranks should be the same in all cases
*/
thorium_message_set_source(self, source);
thorium_message_set_destination(self, destination);
thorium_message_set_source_node(self, source);
thorium_message_set_destination_node(self, destination);
thorium_message_set_type(self, THORIUM_MESSAGE_TYPE_NODE_INBOUND);
}
void thorium_message_multiplexer_flush(struct thorium_message_multiplexer *self, int index, int force)
{
char *buffer;
struct thorium_message message;
int tag;
int count;
int current_size;
int maximum_size;
struct thorium_multiplexed_buffer *multiplexed_buffer;
int destination_node;
/*
int elapsed;
int message_count;
*/
if (CORE_BITMAP_GET_FLAG(self->flags, FLAG_DISABLED)) {
return;
}
#ifdef THORIUM_MULTIPLEXER_TRACK_BUFFERS_WITH_CONTENT
#ifdef CORE_DEBUGGER_ASSERT_ENABLED
if (!(core_set_find(&self->buffers_with_content, &index))) {
multiplexed_buffer = core_vector_at(&self->buffers, index);
thorium_printf("index %d has no content\n", index);
thorium_multiplexed_buffer_print(multiplexed_buffer);
}
#endif
CORE_DEBUGGER_ASSERT(core_set_find(&self->buffers_with_content, &index));
#endif
multiplexed_buffer = core_vector_at(&self->buffers, index);
current_size = thorium_multiplexed_buffer_current_size(multiplexed_buffer);
maximum_size = thorium_multiplexed_buffer_maximum_size(multiplexed_buffer);
/*
* The buffer was still in the timeline, but it was flushed elsewhere.
*/
if (current_size == 0) {
return;
/*
if (force == FORCE_NO && current_size < maximum_size) {
return;
} else if (force == FORCE_YES_TIME) {
elapsed = core_timer_get_nanoseconds(&self->timer) - multiplexed_buffer->timestamp_;
if (elapsed < self->timeout_in_nanoseconds) {
return;
}
} else if (force == FORCE_YES_DOA) {
message_count = multiplexed_buffer->message_count_;
if (message_count < self->degree_of_aggregation_limit) {
return;
}
*/
}
#ifdef CORE_DEBUGGER_ASSERT_ENABLED
if (current_size <= 0)
thorium_printf("current_size %d maximum_size %d\n", current_size, maximum_size);
#endif
CORE_DEBUGGER_ASSERT(current_size > 0);
buffer = thorium_multiplexed_buffer_buffer(multiplexed_buffer);
count = current_size + THORIUM_MESSAGE_METADATA_SIZE;
tag = ACTION_MULTIPLEXER_MESSAGE;
/*
* This count does not include metadata for the final big message.
*
* Avoid this copy by using an array of pointers in the first place.
*/
destination_node = index;
thorium_message_init(&message, tag, count, buffer);
thorium_message_set_destination(&message,
destination_node);
thorium_message_set_source(&message,
thorium_node_name(self->node));
/*
* Mark the message so that the buffer is eventually sent back here
* for recycling.
*/
thorium_message_set_worker(&message, thorium_worker_name(self->worker));
thorium_message_write_metadata(&message);
#ifdef DEBUG_MULTIPLEXER_FLUSH
thorium_printf("DEBUG_MULTIPLEXER thorium_message_multiplexer_flush index %d buffer %p force %d current_size %d maximum_size %d"
" destination_node %d\n",
index, buffer, force,
current_size, maximum_size,
thorium_message_destination_node(&message));
thorium_printf("message in flush\n");
thorium_multiplexed_buffer_print(multiplexed_buffer);
thorium_message_print(&message);
#endif
CORE_DEBUGGER_ASSERT_NOT_NULL(self->worker);
/*
* Make a copy of the buffer because the multiplexer does not have communication buffers.
*/
thorium_worker_enqueue_outbound_message(self->worker, &message);
/*
thorium_printf("MULTIPLEXER FLUSH\n");
*/
++self->real_message_count;
thorium_message_destroy(&message);
thorium_multiplexed_buffer_reset(multiplexed_buffer);
#ifdef THORIUM_MULTIPLEXER_TRACK_BUFFERS_WITH_CONTENT
core_set_delete(&self->buffers_with_content, &index);
#endif
}