void process_init(struct thorium_actor *self)
{
struct process *concrete_self;
int argc;
char **argv;
argc = thorium_actor_argc(self);
argv = thorium_actor_argv(self);
concrete_self = thorium_actor_concrete_actor(self);
core_vector_init(&concrete_self->actors, sizeof(int));
thorium_actor_add_action(self, ACTION_START, process_start);
thorium_actor_add_action(self, ACTION_ASK_TO_STOP, process_stop);
thorium_actor_add_action(self, ACTION_PING, process_ping);
thorium_actor_add_action(self, ACTION_PING_REPLY, process_ping_reply);
thorium_actor_add_action(self, ACTION_NOTIFY, process_notify);
concrete_self->passed = 0;
concrete_self->failed = 0;
concrete_self->events = 0;
concrete_self->minimum_buffer_size = 16;
concrete_self->maximum_buffer_size = 512*1024;
if (core_command_has_argument(argc, argv, MIN_BUFFER_SIZE_OPTION)) {
concrete_self->maximum_buffer_size = core_command_get_argument_value_int(argc, argv, MIN_BUFFER_SIZE_OPTION);
}
if (core_command_has_argument(argc, argv, MAX_BUFFER_SIZE_OPTION)) {
concrete_self->maximum_buffer_size = core_command_get_argument_value_int(argc, argv, MAX_BUFFER_SIZE_OPTION);
}
concrete_self->event_count = 100000;
if (core_command_has_argument(argc, argv, EVENT_COUNT_OPTION)) {
concrete_self->event_count = core_command_get_argument_value_int(argc, argv, EVENT_COUNT_OPTION);
}
concrete_self->concurrent_event_count = 8;
if (core_command_has_argument(argc, argv, CONCURRENT_EVENT_COUNT_OPTION)) {
concrete_self->concurrent_event_count = core_command_get_argument_value_int(argc, argv, CONCURRENT_EVENT_COUNT_OPTION);
}
concrete_self->active_messages = 0;
printf("%s/%d using %s %d %s %d %s %d %s %d\n",
thorium_actor_script_name(self),
thorium_actor_name(self),
MIN_BUFFER_SIZE_OPTION,
concrete_self->minimum_buffer_size,
MAX_BUFFER_SIZE_OPTION,
concrete_self->maximum_buffer_size,
EVENT_COUNT_OPTION,
concrete_self->event_count,
CONCURRENT_EVENT_COUNT_OPTION,
concrete_self->concurrent_event_count);
}
void biosal_input_stream_spawn_reply(struct thorium_actor *self, struct thorium_message *message)
{
int stream;
struct biosal_input_stream *concrete_self;
int i;
int size;
uint64_t start_offset;
uint64_t end_offset;
concrete_self = (struct biosal_input_stream *)thorium_actor_concrete_actor(self);
thorium_message_unpack_int(message, 0, &stream);
core_vector_push_back_int(&concrete_self->parallel_streams, stream);
#ifdef DEBUG_ISSUE_594
printf("DEBUG biosal_input_stream_spawn_reply %d/%d\n",
(int)core_vector_size(&concrete_self->parallel_streams),
(int)core_vector_size(&concrete_self->start_offsets));
#endif
if (core_vector_size(&concrete_self->parallel_streams) ==
core_vector_size(&concrete_self->start_offsets)) {
/* Set offsets
*/
thorium_actor_add_action(self, ACTION_INPUT_STREAM_SET_START_OFFSET_REPLY,
biosal_input_stream_set_offset_reply);
thorium_actor_add_action(self, ACTION_INPUT_STREAM_SET_END_OFFSET_REPLY,
biosal_input_stream_set_offset_reply);
concrete_self->finished_parallel_stream_count = 0;
size = core_vector_size(&concrete_self->parallel_streams);
for (i = 0; i < size; i++) {
start_offset = core_vector_at_as_uint64_t(&concrete_self->start_offsets, i);
end_offset = core_vector_at_as_uint64_t(&concrete_self->end_offsets, i);
stream = core_vector_at_as_int(&concrete_self->parallel_streams, i);
#ifdef DEBUG_ISSUE_594
printf("actor %d send ACTION_INPUT_STREAM_SET_START_OFFSET to %d\n",
name, stream);
printf("actor %d send ACTION_INPUT_STREAM_SET_END_OFFSET to %d\n",
name, stream);
#endif
thorium_actor_send_uint64_t(self, stream, ACTION_INPUT_STREAM_SET_START_OFFSET,
start_offset);
thorium_actor_send_uint64_t(self, stream, ACTION_INPUT_STREAM_SET_END_OFFSET,
end_offset);
}
}
}
void framr_init(actor_t *actor)
{
framr_t *self;
self = thorium_actor_concrete_actor(actor);
core_vector_init(&self->spawners, sizeof(int));
self->completed = 0;
thorium_actor_add_action(actor, ACTION_START, framr_start);
thorium_actor_add_action(actor, ACTION_FRAMR_HELLO, framr_hello);
thorium_actor_add_action(actor, ACTION_FRAMR_HELLO_REPLY, framr_hello_reply);
thorium_actor_add_action(actor, ACTION_NOTIFY, framr_notify);
thorium_actor_add_action(actor, ACTION_ASK_TO_STOP, framr_ask_to_stop);
}
void biosal_input_stream_set_offset_reply(struct thorium_actor *self, struct thorium_message *message)
{
struct biosal_input_stream *concrete_self;
concrete_self = (struct biosal_input_stream *)thorium_actor_concrete_actor(self);
++concrete_self->finished_parallel_stream_count;
#ifdef DEBUG_ISSUE_594
printf("DEBUG biosal_input_stream_set_offset_reply %d/%d\n",
concrete_self->finished_parallel_stream_count,
2 * core_vector_size(&concrete_self->parallel_streams));
#endif
if (concrete_self->finished_parallel_stream_count ==
2 * core_vector_size(&concrete_self->parallel_streams)) {
/*
* Assign files to input streams
*/
thorium_actor_add_action(self, ACTION_INPUT_OPEN_REPLY,
biosal_input_stream_open_reply);
concrete_self->finished_parallel_stream_count = 0;
thorium_actor_send_range_buffer(self,
&concrete_self->parallel_streams,
ACTION_INPUT_OPEN,
strlen(concrete_self->file_name) + 1,
concrete_self->file_name);
printf("DEBUG SEND ACTION_INPUT_OPEN to %d actors with file %s\n",
(int)core_vector_size(&concrete_self->parallel_streams),
concrete_self->file_name);
}
}
void biosal_assembly_arc_classifier_init(struct thorium_actor *self)
{
struct biosal_assembly_arc_classifier *concrete_self;
concrete_self = (struct biosal_assembly_arc_classifier *)thorium_actor_concrete_actor(self);
concrete_self->kmer_length = -1;
thorium_actor_add_action(self, ACTION_ASK_TO_STOP,
thorium_actor_ask_to_stop);
thorium_actor_add_action(self, ACTION_SET_KMER_LENGTH,
biosal_assembly_arc_classifier_set_kmer_length);
/*
*
* Configure the codec.
*/
biosal_dna_codec_init(&concrete_self->codec);
if (biosal_dna_codec_must_use_two_bit_encoding(&concrete_self->codec,
thorium_actor_get_node_count(self))) {
biosal_dna_codec_enable_two_bit_encoding(&concrete_self->codec);
}
core_vector_init(&concrete_self->consumers, sizeof(int));
thorium_actor_add_action(self, ACTION_ASSEMBLY_PUSH_ARC_BLOCK,
biosal_assembly_arc_classifier_push_arc_block);
concrete_self->received_blocks = 0;
core_vector_init(&concrete_self->pending_requests, sizeof(int));
concrete_self->active_requests = 0;
concrete_self->producer_is_waiting = 0;
concrete_self->maximum_pending_request_count = thorium_actor_active_message_limit(self);
concrete_self->consumer_count_above_threshold = 0;
printf("%s/%d is now active, ACTIVE_MESSAGE_LIMIT %d\n",
thorium_actor_script_name(self),
thorium_actor_name(self),
concrete_self->maximum_pending_request_count);
}
void biosal_input_controller_init(struct thorium_actor *actor)
{
struct biosal_input_controller *concrete_actor;
concrete_actor = (struct biosal_input_controller *)thorium_actor_concrete_actor(actor);
core_map_init(&concrete_actor->mega_blocks, sizeof(int), sizeof(struct core_vector));
core_map_init(&concrete_actor->assigned_blocks, sizeof(int), sizeof(int));
core_vector_init(&concrete_actor->mega_block_vector, sizeof(struct biosal_mega_block));
core_vector_init(&concrete_actor->counting_streams, sizeof(int));
core_vector_init(&concrete_actor->reading_streams, sizeof(int));
core_vector_init(&concrete_actor->partition_commands, sizeof(int));
core_vector_init(&concrete_actor->stream_consumers, sizeof(int));
core_vector_init(&concrete_actor->consumer_active_requests, sizeof(int));
core_vector_init(&concrete_actor->files, sizeof(char *));
core_vector_init(&concrete_actor->spawners, sizeof(int));
core_vector_init(&concrete_actor->counts, sizeof(int64_t));
core_vector_init(&concrete_actor->consumers, sizeof(int));
core_vector_init(&concrete_actor->stores_per_spawner, sizeof(int));
core_timer_init(&concrete_actor->input_timer);
core_timer_init(&concrete_actor->counting_timer);
core_timer_init(&concrete_actor->distribution_timer);
biosal_dna_codec_init(&concrete_actor->codec);
if (biosal_dna_codec_must_use_two_bit_encoding(&concrete_actor->codec,
thorium_actor_get_node_count(actor))) {
biosal_dna_codec_enable_two_bit_encoding(&concrete_actor->codec);
}
core_queue_init(&concrete_actor->unprepared_spawners, sizeof(int));
concrete_actor->opened_streams = 0;
concrete_actor->state = BIOSAL_INPUT_CONTROLLER_STATE_NONE;
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_10355
printf("DEBUG actor %d register ACTION_INPUT_CONTROLLER_CREATE_STORES\n",
thorium_actor_name(actor));
#endif
thorium_actor_add_action(actor, ACTION_INPUT_CONTROLLER_CREATE_STORES,
biosal_input_controller_create_stores);
thorium_actor_add_action(actor, ACTION_GET_NODE_NAME_REPLY,
biosal_input_controller_get_node_name_reply);
thorium_actor_add_action(actor, ACTION_GET_NODE_WORKER_COUNT_REPLY,
biosal_input_controller_get_node_worker_count_reply);
thorium_actor_add_action(actor, ACTION_INPUT_CONTROLLER_PREPARE_SPAWNERS,
biosal_input_controller_prepare_spawners);
thorium_actor_add_action(actor, ACTION_INPUT_CONTROLLER_SPAWN_READING_STREAMS,
biosal_input_controller_spawn_streams);
thorium_actor_add_action(actor, ACTION_INPUT_STREAM_SET_START_OFFSET_REPLY,
biosal_input_controller_set_offset_reply);
thorium_actor_add_script(actor, SCRIPT_INPUT_STREAM, &biosal_input_stream_script);
thorium_actor_add_script(actor, SCRIPT_SEQUENCE_STORE, &biosal_sequence_store_script);
thorium_actor_add_script(actor, SCRIPT_SEQUENCE_PARTITIONER,
&biosal_sequence_partitioner_script);
/* configuration for the input controller
* other values for block size: 512, 1024, 2048, 4096, 8192 * /
*/
concrete_actor->block_size = 4096;
concrete_actor->stores_per_worker_per_spawner = 0;
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG
printf("DEBUG %d init controller\n",
thorium_actor_name(actor));
#endif
concrete_actor->ready_spawners = 0;
concrete_actor->ready_consumers = 0;
concrete_actor->partitioner = THORIUM_ACTOR_NOBODY;
concrete_actor->filled_consumers = 0;
concrete_actor->counted = 0;
}
void spate_init(struct thorium_actor *self)
{
struct spate *concrete_self;
concrete_self = (struct spate *)thorium_actor_concrete_actor(self);
core_vector_init(&concrete_self->initial_actors, sizeof(int));
core_vector_init(&concrete_self->sequence_stores, sizeof(int));
core_vector_init(&concrete_self->graph_stores, sizeof(int));
concrete_self->is_leader = 0;
concrete_self->input_controller = THORIUM_ACTOR_NOBODY;
concrete_self->manager_for_sequence_stores = THORIUM_ACTOR_NOBODY;
concrete_self->assembly_graph = THORIUM_ACTOR_NOBODY;
concrete_self->assembly_graph_builder = THORIUM_ACTOR_NOBODY;
core_timer_init(&concrete_self->timer);
thorium_actor_add_action(self,
ACTION_START, spate_start);
thorium_actor_add_action(self,
ACTION_ASK_TO_STOP, spate_ask_to_stop);
thorium_actor_add_action(self,
ACTION_SPAWN_REPLY, spate_spawn_reply);
thorium_actor_add_action(self,
ACTION_MANAGER_SET_SCRIPT_REPLY, spate_set_script_reply);
thorium_actor_add_action(self,
ACTION_SET_CONSUMERS_REPLY, spate_set_consumers_reply);
thorium_actor_add_action(self,
ACTION_SET_BLOCK_SIZE_REPLY, spate_set_block_size_reply);
thorium_actor_add_action(self,
ACTION_INPUT_DISTRIBUTE_REPLY, spate_distribute_reply);
thorium_actor_add_action(self,
ACTION_SPATE_ADD_FILES, spate_add_files);
thorium_actor_add_action(self,
ACTION_SPATE_ADD_FILES_REPLY, spate_add_files_reply);
thorium_actor_add_action(self,
ACTION_ADD_FILE_REPLY, spate_add_file_reply);
thorium_actor_add_action(self,
ACTION_START_REPLY, spate_start_reply);
/*
* Register required actor scripts now
*/
thorium_actor_add_script(self, SCRIPT_INPUT_CONTROLLER,
&biosal_input_controller_script);
thorium_actor_add_script(self, SCRIPT_DNA_KMER_COUNTER_KERNEL,
&biosal_dna_kmer_counter_kernel_script);
thorium_actor_add_script(self, SCRIPT_MANAGER,
&core_manager_script);
thorium_actor_add_script(self, SCRIPT_WRITER_PROCESS,
&core_writer_process_script);
thorium_actor_add_script(self, SCRIPT_AGGREGATOR,
&biosal_aggregator_script);
thorium_actor_add_script(self, SCRIPT_KMER_STORE,
&biosal_kmer_store_script);
thorium_actor_add_script(self, SCRIPT_SEQUENCE_STORE,
&biosal_sequence_store_script);
thorium_actor_add_script(self, SCRIPT_COVERAGE_DISTRIBUTION,
&biosal_coverage_distribution_script);
thorium_actor_add_script(self, SCRIPT_ASSEMBLY_GRAPH_BUILDER,
&biosal_assembly_graph_builder_script);
thorium_actor_add_script(self, SCRIPT_ASSEMBLY_GRAPH_STORE,
&biosal_assembly_graph_store_script);
thorium_actor_add_script(self, SCRIPT_ASSEMBLY_SLIDING_WINDOW,
&biosal_assembly_sliding_window_script);
thorium_actor_add_script(self, SCRIPT_ASSEMBLY_BLOCK_CLASSIFIER,
&biosal_assembly_block_classifier_script);
thorium_actor_add_script(self, SCRIPT_COVERAGE_DISTRIBUTION,
&biosal_coverage_distribution_script);
thorium_actor_add_script(self, SCRIPT_ASSEMBLY_ARC_KERNEL,
&biosal_assembly_arc_kernel_script);
thorium_actor_add_script(self, SCRIPT_ASSEMBLY_ARC_CLASSIFIER,
&biosal_assembly_arc_classifier_script);
thorium_actor_add_script(self, SCRIPT_UNITIG_WALKER,
&biosal_unitig_walker_script);
thorium_actor_add_script(self, SCRIPT_UNITIG_VISITOR,
&biosal_unitig_visitor_script);
thorium_actor_add_script(self, SCRIPT_UNITIG_MANAGER,
&biosal_unitig_manager_script);
/*
* This is the I/O controller block size.
* This is a number of sequences.
*/
concrete_self->block_size = 16 * 4096;
concrete_self->file_index = 0;
}
void biosal_input_stream_count_in_parallel(struct thorium_actor *self, struct thorium_message *message)
{
struct biosal_input_stream *concrete_self;
char *file;
uint64_t file_size;
uint64_t start_offset;
uint64_t end_offset;
void *buffer;
int spawner;
int i;
int size;
uint64_t parallel_block_size;
/*
* The block size for deciding when to spawn new actors for
* I/O.
*/
parallel_block_size = 536870912;
/*
parallel_block_size = 0;
--parallel_block_size;
*/
/*
* Approach:
*
* 1. Check number of bytes.
* 2. Determine the number of streams.
* 3. Spawn streams using a manager.
* 4. Set the offsets for each stream
* 5. Set ask them to count
*/
concrete_self = (struct biosal_input_stream *)thorium_actor_concrete_actor(self);
buffer = thorium_message_buffer(message);
core_vector_unpack(&concrete_self->spawners, buffer);
file = concrete_self->file_name;
file_size = core_file_get_size(file);
printf("COUNT_IN_PARALLEL %s %" PRIu64 "\n",
file, file_size);
start_offset = 0;
i = 0;
while (start_offset < file_size) {
end_offset = start_offset + parallel_block_size - 1;
if (end_offset > file_size - 1 || end_offset < start_offset) {
end_offset = file_size - 1;
}
core_vector_push_back_uint64_t(&concrete_self->start_offsets, start_offset);
core_vector_push_back_uint64_t(&concrete_self->end_offsets, end_offset);
printf("DEBUG PARALLEL BLOCK %s %i %" PRIu64 " %" PRIu64 "\n",
file,
i,
start_offset,
end_offset);
start_offset = end_offset + 1;
++i;
}
size = core_vector_size(&concrete_self->start_offsets);
thorium_actor_add_action(self, ACTION_SPAWN_REPLY, biosal_input_stream_spawn_reply);
printf("DEBUG stream/%d spawns %d streams for counting\n",
thorium_actor_name(self),
size);
for (i = 0; i < size; i++) {
spawner = thorium_actor_get_random_spawner(self, &concrete_self->spawners);
thorium_actor_send_int(self, spawner, ACTION_SPAWN,
SCRIPT_INPUT_STREAM);
}
}
void biosal_input_stream_init(struct thorium_actor *actor)
{
struct biosal_input_stream *input;
struct biosal_input_stream *concrete_self;
input = (struct biosal_input_stream *)thorium_actor_concrete_actor(actor);
concrete_self = input;
concrete_self->proxy_ready = 0;
concrete_self->buffer_for_sequence = NULL;
concrete_self->maximum_sequence_length = 0;
concrete_self->open = 0;
concrete_self->error = BIOSAL_INPUT_ERROR_NO_ERROR;
concrete_self->file_name = NULL;
biosal_dna_codec_init(&concrete_self->codec);
if (biosal_dna_codec_must_use_two_bit_encoding(&concrete_self->codec,
thorium_actor_get_node_count(actor))) {
biosal_dna_codec_enable_two_bit_encoding(&concrete_self->codec);
}
/*concrete_self->mega_block_size = 2097152*/
/*
* This is the mega block size in number of sequences.
*/
concrete_self->mega_block_size = 2097152;
concrete_self->granularity = 1024;
concrete_self->last_offset = 0;
concrete_self->last_entries = 0;
concrete_self->starting_offset = 0;
/*
* Use a large ending offset.
*/
concrete_self->ending_offset = 0;
--concrete_self->ending_offset;
core_vector_init(&concrete_self->mega_blocks, sizeof(struct biosal_mega_block));
thorium_actor_add_action(actor, ACTION_INPUT_STREAM_SET_START_OFFSET, biosal_input_stream_set_start_offset);
thorium_actor_add_action(actor, ACTION_INPUT_STREAM_SET_END_OFFSET, biosal_input_stream_set_end_offset);
#ifdef ENABLE_PARALLEL_COUNT
thorium_actor_add_action(actor, ACTION_INPUT_COUNT_IN_PARALLEL, biosal_input_stream_count_in_parallel);
thorium_actor_add_action(actor, ACTION_INPUT_COUNT_REPLY, biosal_input_stream_count_reply);
#else
thorium_actor_add_action(actor, ACTION_INPUT_COUNT_IN_PARALLEL, biosal_input_stream_count_in_parallel_mock);
thorium_actor_add_action(actor, ACTION_INPUT_COUNT_REPLY, biosal_input_stream_count_reply_mock);
#endif
concrete_self->count_customer = THORIUM_ACTOR_NOBODY;
#if 0
core_string_init(&concrete_self->file_for_parallel_counting, NULL);
#endif
/*
* Parallel counting.
*/
concrete_self->total_entries = 0;
/*
* Disable parallel counting.
*/
concrete_self->finished_parallel_stream_count = 0;
core_vector_init(&concrete_self->spawners, sizeof(int));
core_vector_init(&concrete_self->parallel_streams, sizeof(int));
core_vector_init(&concrete_self->start_offsets, sizeof(uint64_t));
core_vector_init(&concrete_self->end_offsets, sizeof(uint64_t));
core_vector_init(&concrete_self->parallel_mega_blocks, sizeof(struct core_vector));
printf("%s/%d is now online on node %d\n",
thorium_actor_script_name(actor),
thorium_actor_name(actor),
thorium_actor_node_name(actor));
}
void biosal_assembly_graph_store_init(struct thorium_actor *self)
{
struct biosal_assembly_graph_store *concrete_self;
#if 0
thorium_actor_set_priority(self, THORIUM_PRIORITY_MAX);
#endif
concrete_self = thorium_actor_concrete_actor(self);
core_memory_pool_init(&concrete_self->persistent_memory, 0,
MEMORY_POOL_NAME_GRAPH_STORE);
concrete_self->consumed_canonical_vertex_count = 0;
concrete_self->kmer_length = -1;
concrete_self->received = 0;
biosal_assembly_graph_summary_init(&concrete_self->graph_summary);
biosal_dna_codec_init(&concrete_self->transport_codec);
/*
* When running with only 1 node, the transport codec and storage codec
* are different.
*/
concrete_self->codec_are_different = 1;
if (biosal_dna_codec_must_use_two_bit_encoding(&concrete_self->transport_codec,
thorium_actor_get_node_count(self))) {
concrete_self->codec_are_different = 0;
biosal_dna_codec_enable_two_bit_encoding(&concrete_self->transport_codec);
}
biosal_dna_codec_init(&concrete_self->storage_codec);
/* This option enables 2-bit encoding
* for kmers.
*/
biosal_dna_codec_enable_two_bit_encoding(&concrete_self->storage_codec);
thorium_actor_add_action(self, ACTION_YIELD_REPLY, biosal_assembly_graph_store_yield_reply);
thorium_actor_add_action(self, ACTION_PUSH_KMER_BLOCK,
biosal_assembly_graph_store_push_kmer_block);
thorium_actor_add_action(self, ACTION_ASSEMBLY_PUSH_ARC_BLOCK,
biosal_assembly_graph_store_push_arc_block);
thorium_actor_add_action(self, ACTION_ASSEMBLY_GET_SUMMARY,
biosal_assembly_graph_store_get_summary);
thorium_actor_add_action(self, ACTION_ASSEMBLY_GET_VERTEX,
biosal_assembly_graph_store_get_vertex);
thorium_actor_add_action(self, ACTION_ASSEMBLY_GET_STARTING_KMER,
biosal_assembly_graph_store_get_starting_vertex);
thorium_actor_add_action(self, ACTION_MARK_VERTEX_AS_VISITED,
biosal_assembly_graph_store_mark_vertex_as_visited);
thorium_actor_add_action(self, ACTION_SET_VERTEX_FLAG,
biosal_assembly_graph_store_set_vertex_flag);
concrete_self->printed_vertex_size = 0;
concrete_self->printed_arc_size = 0;
concrete_self->last_received = 0;
concrete_self->received_arc_block_count = 0;
concrete_self->received_arc_count = 0;
concrete_self->summary_in_progress = 0;
concrete_self->unitig_vertex_count = 0;
}
