void biosal_assembly_graph_store_push_kmer_block(struct thorium_actor *self, struct thorium_message *message) { struct core_memory_pool *ephemeral_memory; struct biosal_dna_kmer_frequency_block block; struct biosal_assembly_vertex *bucket; void *packed_kmer; struct core_map_iterator iterator; struct biosal_assembly_graph_store *concrete_self; /*int tag;*/ void *key; struct core_map *kmers; struct biosal_dna_kmer kmer; void *buffer; int count; struct biosal_dna_kmer encoded_kmer; char *raw_kmer; int period; struct biosal_dna_kmer *kmer_pointer; int *frequency; ephemeral_memory = thorium_actor_get_ephemeral_memory(self); concrete_self = thorium_actor_concrete_actor(self); /*tag = thorium_message_action(message);*/ buffer = thorium_message_buffer(message); count = thorium_message_count(message); /* * Handler for PUSH_DATA */ biosal_dna_kmer_frequency_block_init(&block, concrete_self->kmer_length, ephemeral_memory, &concrete_self->transport_codec, 0); biosal_dna_kmer_frequency_block_unpack(&block, buffer, ephemeral_memory, &concrete_self->transport_codec); key = core_memory_pool_allocate(ephemeral_memory, concrete_self->key_length_in_bytes); kmers = biosal_dna_kmer_frequency_block_kmers(&block); core_map_iterator_init(&iterator, kmers); period = 2500000; raw_kmer = core_memory_pool_allocate(thorium_actor_get_ephemeral_memory(self), concrete_self->kmer_length + 1); if (!concrete_self->printed_vertex_size) { printf("DEBUG VERTEX DELIVERY %d bytes\n", count); concrete_self->printed_vertex_size = 1; } while (core_map_iterator_has_next(&iterator)) { /* * add kmers to store */ core_map_iterator_next(&iterator, (void **)&packed_kmer, (void **)&frequency); /* Store the kmer in 2 bit encoding */ biosal_dna_kmer_init_empty(&kmer); biosal_dna_kmer_unpack(&kmer, packed_kmer, concrete_self->kmer_length, ephemeral_memory, &concrete_self->transport_codec); kmer_pointer = &kmer; if (concrete_self->codec_are_different) { /* * Get a copy of the sequence */ biosal_dna_kmer_get_sequence(kmer_pointer, raw_kmer, concrete_self->kmer_length, &concrete_self->transport_codec); biosal_dna_kmer_init(&encoded_kmer, raw_kmer, &concrete_self->storage_codec, thorium_actor_get_ephemeral_memory(self)); kmer_pointer = &encoded_kmer; } biosal_dna_kmer_pack_store_key(kmer_pointer, key, concrete_self->kmer_length, &concrete_self->storage_codec, thorium_actor_get_ephemeral_memory(self)); #ifdef BIOSAL_DEBUG_ISSUE_540 if (strcmp(raw_kmer, "AGCTGGTAGTCATCACCAGACTGGAACAG") == 0 || strcmp(raw_kmer, "CGCGATCTGTTGCTGGGCCTAACGTGGTA") == 0 || strcmp(raw_kmer, "TACCACGTTAGGCCCAGCAACAGATCGCG") == 0) { printf("Examine store key for %s\n", raw_kmer); core_debugger_examine(key, concrete_self->key_length_in_bytes); } #endif bucket = core_map_get(&concrete_self->table, key); if (bucket == NULL) { /* This is the first time that this kmer is seen. */ bucket = core_map_add(&concrete_self->table, key); biosal_assembly_vertex_init(bucket); #if 0 printf("DEBUG303 ADD_KEY"); biosal_dna_kmer_print(&encoded_kmer, concrete_self->kmer_length, &concrete_self->storage_codec, ephemeral_memory); #endif } if (concrete_self->codec_are_different) { biosal_dna_kmer_destroy(&encoded_kmer, thorium_actor_get_ephemeral_memory(self)); } biosal_dna_kmer_destroy(&kmer, ephemeral_memory); biosal_assembly_vertex_increase_coverage_depth(bucket, *frequency); if (concrete_self->received >= concrete_self->last_received + period) { printf("%s/%d received %" PRIu64 " kmers so far," " store has %" PRIu64 " canonical kmers, %" PRIu64 " kmers\n", thorium_actor_script_name(self), thorium_actor_name(self), concrete_self->received, core_map_size(&concrete_self->table), 2 * core_map_size(&concrete_self->table)); concrete_self->last_received = concrete_self->received; } concrete_self->received += *frequency; } core_memory_pool_free(ephemeral_memory, key); core_memory_pool_free(ephemeral_memory, raw_kmer); core_map_iterator_destroy(&iterator); biosal_dna_kmer_frequency_block_destroy(&block, thorium_actor_get_ephemeral_memory(self)); thorium_actor_send_reply_empty(self, ACTION_PUSH_KMER_BLOCK_REPLY); }
void biosal_assembly_graph_store_init(struct thorium_actor *self) { struct biosal_assembly_graph_store *concrete_self; #if 0 thorium_actor_set_priority(self, THORIUM_PRIORITY_MAX); #endif concrete_self = thorium_actor_concrete_actor(self); core_memory_pool_init(&concrete_self->persistent_memory, 0, MEMORY_POOL_NAME_GRAPH_STORE); concrete_self->consumed_canonical_vertex_count = 0; concrete_self->kmer_length = -1; concrete_self->received = 0; biosal_assembly_graph_summary_init(&concrete_self->graph_summary); biosal_dna_codec_init(&concrete_self->transport_codec); /* * When running with only 1 node, the transport codec and storage codec * are different. */ concrete_self->codec_are_different = 1; if (biosal_dna_codec_must_use_two_bit_encoding(&concrete_self->transport_codec, thorium_actor_get_node_count(self))) { concrete_self->codec_are_different = 0; biosal_dna_codec_enable_two_bit_encoding(&concrete_self->transport_codec); } biosal_dna_codec_init(&concrete_self->storage_codec); /* This option enables 2-bit encoding * for kmers. */ biosal_dna_codec_enable_two_bit_encoding(&concrete_self->storage_codec); thorium_actor_add_action(self, ACTION_YIELD_REPLY, biosal_assembly_graph_store_yield_reply); thorium_actor_add_action(self, ACTION_PUSH_KMER_BLOCK, biosal_assembly_graph_store_push_kmer_block); thorium_actor_add_action(self, ACTION_ASSEMBLY_PUSH_ARC_BLOCK, biosal_assembly_graph_store_push_arc_block); thorium_actor_add_action(self, ACTION_ASSEMBLY_GET_SUMMARY, biosal_assembly_graph_store_get_summary); thorium_actor_add_action(self, ACTION_ASSEMBLY_GET_VERTEX, biosal_assembly_graph_store_get_vertex); thorium_actor_add_action(self, ACTION_ASSEMBLY_GET_STARTING_KMER, biosal_assembly_graph_store_get_starting_vertex); thorium_actor_add_action(self, ACTION_MARK_VERTEX_AS_VISITED, biosal_assembly_graph_store_mark_vertex_as_visited); thorium_actor_add_action(self, ACTION_SET_VERTEX_FLAG, biosal_assembly_graph_store_set_vertex_flag); concrete_self->printed_vertex_size = 0; concrete_self->printed_arc_size = 0; concrete_self->last_received = 0; concrete_self->received_arc_block_count = 0; concrete_self->received_arc_count = 0; concrete_self->summary_in_progress = 0; concrete_self->unitig_vertex_count = 0; }
void biosal_assembly_graph_store_print(struct thorium_actor *self) { struct core_map_iterator iterator; struct biosal_dna_kmer kmer; void *key; struct biosal_assembly_vertex *value; int coverage; char *sequence; struct biosal_assembly_graph_store *concrete_self; int maximum_length; int length; struct core_memory_pool *ephemeral_memory; ephemeral_memory = thorium_actor_get_ephemeral_memory(self); concrete_self = thorium_actor_concrete_actor(self); core_map_iterator_init(&iterator, &concrete_self->table); printf("map size %d\n", (int)core_map_size(&concrete_self->table)); maximum_length = 0; while (core_map_iterator_has_next(&iterator)) { core_map_iterator_next(&iterator, (void **)&key, (void **)&value); biosal_dna_kmer_init_empty(&kmer); biosal_dna_kmer_unpack(&kmer, key, concrete_self->kmer_length, thorium_actor_get_ephemeral_memory(self), &concrete_self->storage_codec); length = biosal_dna_kmer_length(&kmer, concrete_self->kmer_length); /* printf("length %d\n", length); */ if (length > maximum_length) { maximum_length = length; } biosal_dna_kmer_destroy(&kmer, thorium_actor_get_ephemeral_memory(self)); } /* printf("MAx length %d\n", maximum_length); */ sequence = core_memory_pool_allocate(ephemeral_memory, maximum_length + 1); sequence[0] = '\0'; core_map_iterator_destroy(&iterator); core_map_iterator_init(&iterator, &concrete_self->table); while (core_map_iterator_has_next(&iterator)) { core_map_iterator_next(&iterator, (void **)&key, (void **)&value); biosal_dna_kmer_init_empty(&kmer); biosal_dna_kmer_unpack(&kmer, key, concrete_self->kmer_length, thorium_actor_get_ephemeral_memory(self), &concrete_self->storage_codec); biosal_dna_kmer_get_sequence(&kmer, sequence, concrete_self->kmer_length, &concrete_self->storage_codec); coverage = biosal_assembly_vertex_coverage_depth(value); printf("Sequence %s Coverage %d\n", sequence, coverage); biosal_dna_kmer_destroy(&kmer, thorium_actor_get_ephemeral_memory(self)); } core_map_iterator_destroy(&iterator); core_memory_pool_free(ephemeral_memory, sequence); }
void biosal_assembly_graph_store_yield_reply(struct thorium_actor *self, struct thorium_message *message) { struct biosal_dna_kmer kmer; void *key; struct biosal_assembly_vertex *value; int coverage; int customer; uint64_t *count; int new_count; void *new_buffer; struct thorium_message new_message; struct core_memory_pool *ephemeral_memory; struct biosal_assembly_graph_store *concrete_self; int i; int max; ephemeral_memory = thorium_actor_get_ephemeral_memory(self); concrete_self = thorium_actor_concrete_actor(self); customer = concrete_self->customer; #if 0 printf("YIELD REPLY\n"); #endif i = 0; max = 1024; key = NULL; value = NULL; while (i < max && core_map_iterator_has_next(&concrete_self->iterator)) { core_map_iterator_next(&concrete_self->iterator, (void **)&key, (void **)&value); biosal_dna_kmer_init_empty(&kmer); biosal_dna_kmer_unpack(&kmer, key, concrete_self->kmer_length, ephemeral_memory, &concrete_self->storage_codec); coverage = biosal_assembly_vertex_coverage_depth(value); count = (uint64_t *)core_map_get(&concrete_self->coverage_distribution, &coverage); if (count == NULL) { count = (uint64_t *)core_map_add(&concrete_self->coverage_distribution, &coverage); (*count) = 0; } /* increment for the lowest kmer (canonical) */ (*count)++; biosal_dna_kmer_destroy(&kmer, ephemeral_memory); ++i; } /* yield again if the iterator is not at the end */ if (core_map_iterator_has_next(&concrete_self->iterator)) { #if 0 printf("yield ! %d\n", i); #endif thorium_actor_send_to_self_empty(self, ACTION_YIELD); return; } /* printf("ready...\n"); */ core_map_iterator_destroy(&concrete_self->iterator); new_count = core_map_pack_size(&concrete_self->coverage_distribution); new_buffer = thorium_actor_allocate(self, new_count); core_map_pack(&concrete_self->coverage_distribution, new_buffer); printf("SENDING %s/%d sends map to %d, %d bytes / %d entries\n", thorium_actor_script_name(self), thorium_actor_name(self), customer, new_count, (int)core_map_size(&concrete_self->coverage_distribution)); thorium_message_init(&new_message, ACTION_PUSH_DATA, new_count, new_buffer); thorium_actor_send(self, customer, &new_message); thorium_message_destroy(&new_message); core_map_destroy(&concrete_self->coverage_distribution); thorium_actor_send_empty(self, concrete_self->source, ACTION_PUSH_DATA_REPLY); }
void biosal_assembly_graph_store_mark_vertex_as_visited(struct thorium_actor *self, struct thorium_message *message) { struct biosal_assembly_graph_store *concrete_self; char *buffer; int source; int path_index; char *sequence; struct core_memory_pool *ephemeral_memory; struct biosal_dna_kmer kmer; struct biosal_dna_kmer storage_kmer; struct biosal_assembly_vertex *canonical_vertex; int position; void *key; int force; force = 1; position = 0; concrete_self = thorium_actor_concrete_actor(self); source = thorium_message_source(message); buffer = thorium_message_buffer(message); ephemeral_memory = thorium_actor_get_ephemeral_memory(self); /* * Get the kmer. */ biosal_dna_kmer_init_empty(&kmer); position += biosal_dna_kmer_unpack(&kmer, buffer, concrete_self->kmer_length, ephemeral_memory, &concrete_self->transport_codec); sequence = core_memory_pool_allocate(ephemeral_memory, concrete_self->kmer_length + 1); biosal_dna_kmer_get_sequence(&kmer, sequence, concrete_self->kmer_length, &concrete_self->transport_codec); biosal_dna_kmer_init(&storage_kmer, sequence, &concrete_self->storage_codec, ephemeral_memory); /* * Get store key */ key = core_memory_pool_allocate(ephemeral_memory, concrete_self->key_length_in_bytes); biosal_dna_kmer_pack_store_key(&storage_kmer, key, concrete_self->kmer_length, &concrete_self->storage_codec, ephemeral_memory); /* Get vertex. */ canonical_vertex = core_map_get(&concrete_self->table, key); biosal_dna_kmer_destroy(&kmer, ephemeral_memory); biosal_dna_kmer_destroy(&storage_kmer, ephemeral_memory); core_memory_pool_free(ephemeral_memory, key); core_memory_pool_free(ephemeral_memory, sequence); position += thorium_message_unpack_int(message, position, &path_index); /* * At this point, mark the vertex with flag BIOSAL_VERTEX_FLAG_USED * so that any other actor that attempt to grab it will have to communicate * with the actor. */ /* * This is a good idea to always update with the last one. */ if (force || !biosal_assembly_vertex_get_flag(canonical_vertex, BIOSAL_VERTEX_FLAG_USED)) { biosal_assembly_graph_store_mark_as_used(self, canonical_vertex, source, path_index); } #if 0 #endif thorium_actor_send_reply_empty(self, ACTION_MARK_VERTEX_AS_VISITED_REPLY); }
void biosal_assembly_graph_store_receive(struct thorium_actor *self, struct thorium_message *message) { int tag; /*void *buffer;*/ struct biosal_assembly_graph_store *concrete_self; double value; struct biosal_dna_kmer kmer; /*struct core_memory_pool *ephemeral_memory;*/ int customer; int big_key_size; int big_value_size; if (thorium_actor_take_action(self, message)) { return; } /*ephemeral_memory = thorium_actor_get_ephemeral_memory(self);*/ concrete_self = thorium_actor_concrete_actor(self); tag = thorium_message_action(message); /*buffer = thorium_message_buffer(message);*/ if (tag == ACTION_SET_KMER_LENGTH) { thorium_message_unpack_int(message, 0, &concrete_self->kmer_length); biosal_dna_kmer_init_mock(&kmer, concrete_self->kmer_length, &concrete_self->storage_codec, thorium_actor_get_ephemeral_memory(self)); concrete_self->key_length_in_bytes = biosal_dna_kmer_pack_size(&kmer, concrete_self->kmer_length, &concrete_self->storage_codec); biosal_dna_kmer_destroy(&kmer, thorium_actor_get_ephemeral_memory(self)); big_key_size = concrete_self->key_length_in_bytes; big_value_size = sizeof(struct biosal_assembly_vertex); core_map_init(&concrete_self->table, big_key_size, big_value_size); core_map_set_memory_pool(&concrete_self->table, &concrete_self->persistent_memory); printf("DEBUG big_key_size %d big_value_size %d\n", big_key_size, big_value_size); /* * Configure the map for better performance. */ core_map_disable_deletion_support(&concrete_self->table); /* * The threshold of the map is not very important because * requests that hit the map have to first arrive as messages, * which are slow. */ core_map_set_threshold(&concrete_self->table, 0.95); thorium_actor_send_reply_empty(self, ACTION_SET_KMER_LENGTH_REPLY); } else if (tag == ACTION_ASSEMBLY_GET_KMER_LENGTH) { thorium_actor_send_reply_int(self, ACTION_ASSEMBLY_GET_KMER_LENGTH_REPLY, concrete_self->kmer_length); } else if (tag == ACTION_RESET) { /* * Reset the iterator. */ core_map_iterator_init(&concrete_self->iterator, &concrete_self->table); printf("DEBUG unitig_vertex_count %d\n", concrete_self->unitig_vertex_count); thorium_actor_send_reply_empty(self, ACTION_RESET_REPLY); } else if (tag == ACTION_SEQUENCE_STORE_REQUEST_PROGRESS_REPLY) { thorium_message_unpack_double(message, 0, &value); core_map_set_current_size_estimate(&concrete_self->table, value); } else if (tag == ACTION_ASK_TO_STOP) { printf("%s/%d received %d arc blocks\n", thorium_actor_script_name(self), thorium_actor_name(self), concrete_self->received_arc_block_count); thorium_actor_ask_to_stop(self, message); } else if (tag == ACTION_SET_CONSUMER) { thorium_message_unpack_int(message, 0, &customer); printf("%s/%d will use coverage distribution %d\n", thorium_actor_script_name(self), thorium_actor_name(self), customer); concrete_self->customer = customer; thorium_actor_send_reply_empty(self, ACTION_SET_CONSUMER_REPLY); } else if (tag == ACTION_PUSH_DATA) { printf("%s/%d receives ACTION_PUSH_DATA\n", thorium_actor_script_name(self), thorium_actor_name(self)); biosal_assembly_graph_store_push_data(self, message); } else if (tag == ACTION_STORE_GET_ENTRY_COUNT) { thorium_actor_send_reply_uint64_t(self, ACTION_STORE_GET_ENTRY_COUNT_REPLY, concrete_self->received); } else if (tag == ACTION_GET_RECEIVED_ARC_COUNT) { thorium_actor_send_reply_uint64_t(self, ACTION_GET_RECEIVED_ARC_COUNT_REPLY, concrete_self->received_arc_count); } }
void biosal_sequence_partitioner_receive(struct thorium_actor *actor, struct thorium_message *message) { int tag; int source; int count; void *buffer; int bytes; struct biosal_sequence_partitioner *concrete_actor; struct biosal_partition_command command; struct thorium_message response; int command_number; struct biosal_partition_command *active_command; int stream_index; struct biosal_partition_command *command_bucket; int i; thorium_message_get_all(message, &tag, &count, &buffer, &source); concrete_actor = (struct biosal_sequence_partitioner *)thorium_actor_concrete_actor(actor); if (tag == ACTION_SEQUENCE_PARTITIONER_SET_BLOCK_SIZE) { thorium_message_unpack_int(message, 0, &concrete_actor->block_size); thorium_actor_send_reply_empty(actor, ACTION_SEQUENCE_PARTITIONER_SET_BLOCK_SIZE_REPLY); biosal_sequence_partitioner_verify(actor); /* printf("DEBUG biosal_sequence_partitioner_receive received block size\n"); */ } else if (tag == ACTION_SEQUENCE_PARTITIONER_SET_ENTRY_VECTOR) { /* printf("DEBUG biosal_sequence_partitioner_receive unpacking vector, %d bytes\n", count); */ core_vector_init(&concrete_actor->stream_entries, 0); core_vector_unpack(&concrete_actor->stream_entries, buffer); /* printf("DEBUG after unpack\n"); */ thorium_actor_send_reply_empty(actor, ACTION_SEQUENCE_PARTITIONER_SET_ENTRY_VECTOR_REPLY); /* printf("DEBUG biosal_sequence_partitioner_receive received received entry vector\n"); */ biosal_sequence_partitioner_verify(actor); } else if (tag == ACTION_SEQUENCE_PARTITIONER_SET_ACTOR_COUNT) { thorium_message_unpack_int(message, 0, &concrete_actor->store_count); thorium_actor_send_reply_empty(actor, ACTION_SEQUENCE_PARTITIONER_SET_ACTOR_COUNT_REPLY); biosal_sequence_partitioner_verify(actor); /* printf("DEBUG biosal_sequence_partitioner_receive received received store count\n"); */ } else if (tag == ACTION_SEQUENCE_PARTITIONER_GET_COMMAND) { if (core_queue_dequeue(&concrete_actor->available_commands, &command)) { bytes = biosal_partition_command_pack_size(&command); /* printf("DEBUG partitioner has command, packing %d bytes!\n", bytes); */ buffer = thorium_actor_allocate(actor, bytes); biosal_partition_command_pack(&command, buffer); thorium_message_init(&response, ACTION_SEQUENCE_PARTITIONER_GET_COMMAND_REPLY, bytes, buffer); thorium_actor_send_reply(actor, &response); /* store the active command */ command_number = biosal_partition_command_name(&command); command_bucket = (struct biosal_partition_command *)core_map_add(&concrete_actor->active_commands, &command_number); *command_bucket = command; /* there may be other command available too ! */ } } else if (tag == ACTION_SEQUENCE_PARTITIONER_GET_COMMAND_REPLY_REPLY) { /* * take the name of the command, find it in the active * command, generate a new command, and send ACTION_SEQUENCE_PARTITIONER_COMMAND_IS_READY * as a reply */ thorium_message_unpack_int(message, 0, &command_number); active_command = core_map_get(&concrete_actor->active_commands, &command_number); if (active_command == NULL) { return; } stream_index = biosal_partition_command_stream_index(active_command); active_command = NULL; core_map_delete(&concrete_actor->active_commands, &command_number); biosal_sequence_partitioner_generate_command(actor, stream_index); if (core_map_size(&concrete_actor->active_commands) == 0 && core_queue_size(&concrete_actor->available_commands) == 0) { thorium_actor_send_reply_empty(actor, ACTION_SEQUENCE_PARTITIONER_FINISHED); } } else if (tag == ACTION_ASK_TO_STOP && source == thorium_actor_supervisor(actor)) { #ifdef BIOSAL_SEQUENCE_PARTITIONER_DEBUG printf("DEBUG biosal_sequence_partitioner_receive ACTION_ASK_TO_STOP\n"); #endif thorium_actor_send_to_self_empty(actor, ACTION_STOP); } else if (tag == ACTION_SEQUENCE_PARTITIONER_PROVIDE_STORE_ENTRY_COUNTS_REPLY) { /* generate commands */ for (i = 0; i < core_vector_size(&concrete_actor->stream_entries); i++) { biosal_sequence_partitioner_generate_command(actor, i); } } }
void framr_destroy(actor_t *actor) { framr_t *self; self = thorium_actor_concrete_actor(actor); core_vector_destroy(&self->spawners); }
void biosal_sequence_partitioner_verify(struct thorium_actor *actor) { struct biosal_sequence_partitioner *concrete_actor; int i; int64_t entries; uint64_t position; uint64_t stream_entries; int bytes; void *buffer; struct thorium_message message; int64_t remaining; int remainder; uint64_t *bucket_for_store_count; struct core_vector_iterator iterator; concrete_actor = (struct biosal_sequence_partitioner *)thorium_actor_concrete_actor(actor); /* * check if parameters are * initialized */ if (concrete_actor->block_size == -1) { return; } if (concrete_actor->store_count == -1) { return; } if (core_vector_size(&concrete_actor->stream_entries) == 0) { return; } /* at this point, all parameters are ready. * prepare <stream_entries.size> commands */ position = 0; entries = 0; /* printf("DEBUG generating initial positions\n"); */ /* generate stream positions, stream global positions, and total */ for (i = 0; i < core_vector_size(&concrete_actor->stream_entries); i++) { core_vector_push_back(&concrete_actor->stream_positions, &position); core_vector_push_back(&concrete_actor->stream_global_positions, &entries); stream_entries = *(uint64_t *)core_vector_at(&concrete_actor->stream_entries, i); #ifdef BIOSAL_SEQUENCE_PARTITIONER_DEBUG printf("DEBUG stream_entries %i %" PRIu64 "\n", i, stream_entries); #endif entries += stream_entries; } concrete_actor->total = entries; /* compute the number of entries for each store */ entries = concrete_actor->total / concrete_actor->store_count; /* make sure that this is a multiple of block size * examples: * total= 20000 * store_count= 2 * block_size= 8192 * 20000 / 2 = 10000 * 10000 % 8192 = 1808 * difference = 8192 - 1808 = 6384 * 10000 + 6384 = 16384 */ if (entries % concrete_actor->block_size != 0) { remainder = entries % concrete_actor->block_size; entries -= remainder; } /* make sure that at most one store has less * than block size */ if (entries < concrete_actor->block_size) { entries = concrete_actor->block_size; } #ifdef BIOSAL_SEQUENCE_PARTITIONER_DEBUG printf("DEBUG93 entries for stores %d\n", (int)entries); #endif remaining = concrete_actor->total; if (remaining <= entries) { entries = remaining; } /* example: 10000, block_size 4096, 3 stores * * total entries remaining * 10000 4096 5904 * 10000 4096 1808 * 10000 1808 0 */ for (i = 0; i < concrete_actor->store_count; i++) { core_vector_push_back(&concrete_actor->store_entries, &entries); remaining -= entries; if (remaining < entries) { entries = remaining; } } core_vector_iterator_init(&iterator, &concrete_actor->store_entries); while (core_vector_iterator_has_next(&iterator)) { core_vector_iterator_next(&iterator, (void **)&bucket_for_store_count); if (remaining >= concrete_actor->block_size) { *bucket_for_store_count += concrete_actor->block_size; remaining -= concrete_actor->block_size; } else if (remaining == 0) { break; } else { /* between 1 and block_size - 1 inclusively */ *bucket_for_store_count += remaining; remaining = 0; } } core_vector_iterator_destroy(&iterator); #ifdef BIOSAL_SEQUENCE_PARTITIONER_DEBUG printf("DEBUG biosal_sequence_partitioner_verify sending store counts\n"); #endif bytes = core_vector_pack_size(&concrete_actor->store_entries); buffer = thorium_actor_allocate(actor, bytes); core_vector_pack(&concrete_actor->store_entries, buffer); thorium_message_init(&message, ACTION_SEQUENCE_PARTITIONER_PROVIDE_STORE_ENTRY_COUNTS, bytes, buffer); thorium_actor_send_reply(actor, &message); }
void biosal_sequence_partitioner_generate_command(struct thorium_actor *actor, int stream_index) { uint64_t *bucket_for_stream_position; uint64_t *bucket_for_global_position; struct biosal_partition_command command; int store_index; uint64_t stream_entries; uint64_t stream_first; uint64_t stream_last; int64_t available_in_stream; uint64_t store_first; uint64_t store_last; struct biosal_sequence_partitioner *concrete_actor; int64_t actual_block_size; int distance; uint64_t global_first; uint64_t global_last; uint64_t next_block_first; int command_name; int blocks; float progress; concrete_actor = (struct biosal_sequence_partitioner *)thorium_actor_concrete_actor(actor); /* printf("DEBUG biosal_sequence_partitioner_generate_command %d\n", stream_index); */ bucket_for_stream_position = (uint64_t *)core_vector_at(&concrete_actor->stream_positions, stream_index); bucket_for_global_position = (uint64_t *)core_vector_at(&concrete_actor->stream_global_positions, stream_index); /* printf("DEBUG got buckets.\n"); */ /* compute feasible block size given the stream and the store. */ global_first = *bucket_for_global_position; actual_block_size = concrete_actor->block_size; next_block_first = global_first + (concrete_actor->block_size - global_first % concrete_actor->block_size); distance = next_block_first - global_first; if (distance < actual_block_size) { actual_block_size = distance; } store_index = biosal_sequence_partitioner_get_store(global_first, concrete_actor->block_size, concrete_actor->store_count); stream_entries = *(uint64_t *)core_vector_at(&concrete_actor->stream_entries, stream_index); stream_first = *bucket_for_stream_position; /* check out what is left in the stream */ available_in_stream = stream_entries - *bucket_for_stream_position; #ifdef BIOSAL_SEQUENCE_PARTITIONER_DEBUG printf("DEBUG stream_entries %" PRIu64 " !\n", stream_entries); printf("DEBUG bucket_for_stream_position %" PRIu64 " !\n", *bucket_for_stream_position); printf("DEBUG available_in_stream %" PRIu64 " !\n", available_in_stream); #endif if (available_in_stream < actual_block_size) { #ifdef BIOSAL_SEQUENCE_PARTITIONER_DEBUG #endif actual_block_size = available_in_stream; } /* can't do that */ if (actual_block_size == 0) { #ifdef BIOSAL_SEQUENCE_PARTITIONER_DEBUG printf("DEBUG stream %d actual_block_size %d\n", stream_index, actual_block_size); #endif return; } stream_first = *bucket_for_stream_position; stream_last = stream_first + actual_block_size - 1; store_first = biosal_sequence_partitioner_get_index_in_store(global_first, concrete_actor->block_size, concrete_actor->store_count); store_last = store_first + actual_block_size - 1; global_last = global_first + actual_block_size - 1; /* printf("DEBUG %" PRIu64 " goes in store %d\n", global_first, store_index); */ biosal_partition_command_init(&command, concrete_actor->command_number, stream_index, stream_first, stream_last, store_index, store_first, store_last, global_first, global_last); concrete_actor->command_number++; core_queue_enqueue(&concrete_actor->available_commands, &command); #ifdef BIOSAL_SEQUENCE_PARTITIONER_DEBUG printf("DEBUG906 in partitioner:\n"); biosal_partition_command_print(&command); #endif /* update positions */ *bucket_for_stream_position = stream_last + 1; *bucket_for_global_position = global_last + 1; /* printf("DEBUG command is ready\n"); */ /* emit a signal */ thorium_actor_send_reply_empty(actor, ACTION_SEQUENCE_PARTITIONER_COMMAND_IS_READY); command_name = biosal_partition_command_name(&command); blocks = (int)(concrete_actor->total / concrete_actor->block_size); if (concrete_actor->total % concrete_actor->block_size != 0) { blocks++; } progress = (0.0 + command_name) / blocks; if (concrete_actor->last_progress < 0 || progress >= concrete_actor->last_progress + 0.04 || command_name == blocks - 1) { printf("partitioner/%d generated partition command # %d (total %" PRIu64 ", block_size %d, blocks %d, progress %.2f)\n", thorium_actor_name(actor), command_name, concrete_actor->total, concrete_actor->block_size, blocks, progress); concrete_actor->last_progress = progress; } biosal_partition_command_destroy(&command); }
void core_writer_process_receive(struct thorium_actor *self, struct thorium_message *message) { int action; int count; int source; char *buffer; char *file_name; struct core_writer_process *concrete_self; concrete_self = thorium_actor_concrete_actor(self); action = thorium_message_action(message); count = thorium_message_count(message); buffer = thorium_message_buffer(message); source = thorium_message_source(message); if (action == ACTION_OPEN) { if (concrete_self->has_file) { thorium_actor_log(self, "actor error, already open, can not open\n"); return; } file_name = buffer; core_buffered_file_writer_init(&concrete_self->writer, file_name); concrete_self->has_file = 1; thorium_actor_send_reply_empty(self, ACTION_OPEN_REPLY); } else if (action == ACTION_WRITE) { core_buffered_file_writer_write(&concrete_self->writer, buffer, count); thorium_actor_send_reply_empty(self, ACTION_WRITE_REPLY); } else if (action == ACTION_CLOSE) { if (!concrete_self->has_file) { thorium_actor_log(self, "Error, can not close a file that is not open\n"); return; } core_buffered_file_writer_destroy(&concrete_self->writer); concrete_self->has_file = 0; thorium_actor_send_reply_empty(self, ACTION_CLOSE_REPLY); } else if (action == ACTION_ASK_TO_STOP && source == thorium_actor_supervisor(self)) { /* * Close the file if it is open * right now. */ if (concrete_self->has_file) { core_buffered_file_writer_destroy(&concrete_self->writer); concrete_self->has_file = 0; } thorium_actor_send_to_self_empty(self, ACTION_STOP); thorium_actor_send_reply_empty(self, ACTION_ASK_TO_STOP_REPLY); } }
void biosal_coverage_distribution_receive(struct thorium_actor *self, struct thorium_message *message) { int tag; struct core_map map; struct core_map_iterator iterator; int *coverage_from_message; uint64_t *count_from_message; uint64_t *frequency; int count; void *buffer; struct biosal_coverage_distribution *concrete_actor; int name; int source; struct core_memory_pool *ephemeral_memory; ephemeral_memory = thorium_actor_get_ephemeral_memory(self); name = thorium_actor_name(self); source = thorium_message_source(message); concrete_actor = (struct biosal_coverage_distribution *)thorium_actor_concrete_actor(self); tag = thorium_message_action(message); count = thorium_message_count(message); buffer = thorium_message_buffer(message); if (tag == ACTION_PUSH_DATA) { core_map_init(&map, 0, 0); core_map_set_memory_pool(&map, ephemeral_memory); core_map_unpack(&map, buffer); core_map_iterator_init(&iterator, &map); while (core_map_iterator_has_next(&iterator)) { core_map_iterator_next(&iterator, (void **)&coverage_from_message, (void **)&count_from_message); #ifdef BIOSAL_COVERAGE_DISTRIBUTION_DEBUG thorium_actor_log(self, "DEBUG DATA %d %d\n", (int)*coverage_from_message, (int)*count_from_message); #endif frequency = core_map_get(&concrete_actor->distribution, coverage_from_message); if (frequency == NULL) { frequency = core_map_add(&concrete_actor->distribution, coverage_from_message); (*frequency) = 0; } (*frequency) += (*count_from_message); } core_map_iterator_destroy(&iterator); thorium_actor_send_reply_empty(self, ACTION_PUSH_DATA_REPLY); concrete_actor->actual++; thorium_actor_log(self, "distribution/%d receives coverage data from producer/%d, %d entries / %d bytes %d/%d\n", name, source, (int)core_map_size(&map), count, concrete_actor->actual, concrete_actor->expected); if (concrete_actor->expected != 0 && concrete_actor->expected == concrete_actor->actual) { thorium_actor_log(self, "received everything %d/%d\n", concrete_actor->actual, concrete_actor->expected); biosal_coverage_distribution_write_distribution(self); thorium_actor_send_empty(self, concrete_actor->source, ACTION_NOTIFY); } core_map_destroy(&map); } else if (tag == ACTION_ASK_TO_STOP) { biosal_coverage_distribution_ask_to_stop(self, message); } else if (tag == ACTION_SET_EXPECTED_MESSAGE_COUNT) { concrete_actor->source = source; thorium_message_unpack_int(message, 0, &concrete_actor->expected); thorium_actor_log(self, "distribution %d expects %d messages\n", thorium_actor_name(self), concrete_actor->expected); thorium_actor_send_reply_empty(self, ACTION_SET_EXPECTED_MESSAGE_COUNT_REPLY); } }
void biosal_coverage_distribution_write_distribution(struct thorium_actor *self) { struct core_map_iterator iterator; int *coverage; uint64_t *canonical_frequency; uint64_t frequency; struct biosal_coverage_distribution *concrete_actor; struct core_vector coverage_values; struct core_vector_iterator vector_iterator; struct core_buffered_file_writer descriptor; struct core_buffered_file_writer descriptor_canonical; struct core_string file_name; struct core_string canonical_file_name; int argc; char **argv; int name; char *directory_name; name = thorium_actor_name(self); argc = thorium_actor_argc(self); argv = thorium_actor_argv(self); directory_name = biosal_command_get_output_directory(argc, argv); /* Create the directory if it does not exist */ if (!core_directory_verify_existence(directory_name)) { core_directory_create(directory_name); } core_string_init(&file_name, ""); core_string_append(&file_name, directory_name); core_string_append(&file_name, "/"); core_string_append(&file_name, BIOSAL_COVERAGE_DISTRIBUTION_DEFAULT_OUTPUT_FILE); core_string_init(&canonical_file_name, ""); core_string_append(&canonical_file_name, directory_name); core_string_append(&canonical_file_name, "/"); core_string_append(&canonical_file_name, BIOSAL_COVERAGE_DISTRIBUTION_DEFAULT_OUTPUT_FILE_CANONICAL); core_buffered_file_writer_init(&descriptor, core_string_get(&file_name)); core_buffered_file_writer_init(&descriptor_canonical, core_string_get(&canonical_file_name)); concrete_actor = (struct biosal_coverage_distribution *)thorium_actor_concrete_actor(self); core_vector_init(&coverage_values, sizeof(int)); core_map_iterator_init(&iterator, &concrete_actor->distribution); #ifdef BIOSAL_COVERAGE_DISTRIBUTION_DEBUG thorium_actor_log(self, "map size %d\n", (int)core_map_size(&concrete_actor->distribution)); #endif while (core_map_iterator_has_next(&iterator)) { core_map_iterator_next(&iterator, (void **)&coverage, (void **)&canonical_frequency); #ifdef BIOSAL_COVERAGE_DISTRIBUTION_DEBUG thorium_actor_log(self, "DEBUG COVERAGE %d FREQUENCY %" PRIu64 "\n", *coverage, *frequency); #endif core_vector_push_back(&coverage_values, coverage); } core_map_iterator_destroy(&iterator); core_vector_sort_int(&coverage_values); #ifdef BIOSAL_COVERAGE_DISTRIBUTION_DEBUG thorium_actor_log(self, "after sort "); core_vector_print_int(&coverage_values); thorium_actor_log(self, "\n"); #endif core_vector_iterator_init(&vector_iterator, &coverage_values); #if 0 core_buffered_file_writer_printf(&descriptor_canonical, "Coverage\tFrequency\n"); #endif core_buffered_file_writer_printf(&descriptor, "Coverage\tFrequency\n"); #ifdef BIOSAL_COVERAGE_DISTRIBUTION_DEBUG #endif while (core_vector_iterator_has_next(&vector_iterator)) { core_vector_iterator_next(&vector_iterator, (void **)&coverage); canonical_frequency = (uint64_t *)core_map_get(&concrete_actor->distribution, coverage); frequency = 2 * *canonical_frequency; core_buffered_file_writer_printf(&descriptor_canonical, "%d %" PRIu64 "\n", *coverage, *canonical_frequency); core_buffered_file_writer_printf(&descriptor, "%d\t%" PRIu64 "\n", *coverage, frequency); } core_vector_destroy(&coverage_values); core_vector_iterator_destroy(&vector_iterator); thorium_actor_log(self, "distribution %d wrote %s\n", name, core_string_get(&file_name)); thorium_actor_log(self, "distribution %d wrote %s\n", name, core_string_get(&canonical_file_name)); core_buffered_file_writer_destroy(&descriptor); core_buffered_file_writer_destroy(&descriptor_canonical); core_string_destroy(&file_name); core_string_destroy(&canonical_file_name); }
static void source_receive(struct thorium_actor *self, struct thorium_message *message) { int action; void *buffer; int leader; int source; int count; struct source *concrete_self; int name; concrete_self = (struct source *)thorium_actor_concrete_actor(self); action = thorium_message_action(message); buffer = thorium_message_buffer(message); source = thorium_message_source(message); name = thorium_actor_name(self); count = thorium_message_count(message); if (action == ACTION_ASK_TO_STOP) { thorium_actor_log(self, "sent %d ACTION_PING messages\n", concrete_self->message_count); thorium_actor_send_to_self_empty(self, ACTION_STOP); } else if (action == ACTION_NOTIFY) { #ifdef LATENCY_PROBE_USE_MULTIPLEXER thorium_actor_send_to_self_empty(self, ACTION_ENABLE_MULTIPLEXER); #endif CORE_DEBUGGER_ASSERT(core_vector_empty(&concrete_self->targets)); core_vector_unpack(&concrete_self->targets, buffer); if (source_is_important(self)) { printf("%d (node %d worker %d) has %d targets\n", thorium_actor_name(self), thorium_actor_node_name(self), thorium_actor_worker_name(self), (int)core_vector_size(&concrete_self->targets)); } concrete_self->leader = source; source_send_ping(self); } else if (action == ACTION_PING_REPLY) { CORE_DEBUGGER_ASSERT(count == 0); ++concrete_self->message_count; CORE_DEBUGGER_ASSERT_IS_EQUAL_INT(count, 0); CORE_DEBUGGER_ASSERT_IS_NULL(buffer); if (concrete_self->message_count % PERIOD == 0 || concrete_self->event_count < 500) { if (source_is_important(self)) { printf("progress %d %d/%d\n", name, concrete_self->message_count, concrete_self->event_count); } } if (concrete_self->message_count == concrete_self->event_count) { leader = concrete_self->leader; thorium_actor_send_empty(self, leader, ACTION_NOTIFY_REPLY); if (source_is_important(self)) printf("%d (ACTION_PING sent: %d)" " sends ACTION_NOTIFY_REPLY to %d\n", thorium_actor_name(self), concrete_self->message_count, leader); } else { source_send_ping(self); } } }
void ring_receive(struct thorium_actor *actor, struct thorium_message *message) { int tag; int new_actor; int previous; int name; struct ring *concrete_actor; int messages; int previous_actor; char *buffer; int destination; concrete_actor = (struct ring *)thorium_actor_concrete_actor(actor); tag = thorium_message_action(message); buffer = thorium_message_buffer(message); name = thorium_actor_name(actor); if (tag == ACTION_START) { core_vector_init(&concrete_actor->spawners, 0); core_vector_unpack(&concrete_actor->spawners, buffer); printf("actor %d ACTION_START, %d spawners\n", name, (int)core_vector_size(&concrete_actor->spawners)); destination = *(int *)core_vector_at(&concrete_actor->spawners, 0); thorium_actor_send_empty(actor, destination, ACTION_RING_READY); } else if (tag == ACTION_RING_READY && concrete_actor->step == RING_STEP_RECEIVE_SPAWNERS) { concrete_actor->ready_rings++; if (concrete_actor->ready_rings == (int)core_vector_size(&concrete_actor->spawners)) { thorium_actor_send_range_empty(actor, &concrete_actor->spawners, ACTION_RING_SPAWN); concrete_actor->step = RING_STEP_SPAWN; concrete_actor->ready_rings = 0; } } else if (tag == ACTION_RING_SPAWN) { printf("actor/%d is spawning %d senders\n", thorium_actor_name(actor), concrete_actor->senders); concrete_actor->step = RING_STEP_SPAWN; new_actor = thorium_actor_spawn(actor, SCRIPT_SENDER); concrete_actor->first = new_actor; previous = new_actor; new_actor = thorium_actor_spawn(actor, SCRIPT_SENDER); concrete_actor->previous = new_actor; thorium_message_init(message, ACTION_SENDER_SET_NEXT, sizeof(new_actor), &new_actor); thorium_actor_send(actor, previous, message); ++concrete_actor->spawned_senders; ++concrete_actor->spawned_senders; } else if (tag == ACTION_RING_READY && concrete_actor->step == RING_STEP_SPAWN) { concrete_actor->ready_rings++; #if 0 printf("READY: %d/%d\n", concrete_actor->ready_rings, (int)core_vector_size(&concrete_actor->spawners)); #endif if (concrete_actor->ready_rings == core_vector_size(&concrete_actor->spawners)) { thorium_actor_send_range_empty(actor, &concrete_actor->spawners, ACTION_RING_PUSH_NEXT); concrete_actor->ready_rings = 0; concrete_actor->step = RING_STEP_PUSH_NEXT; } } else if (tag == ACTION_RING_PUSH_NEXT) { previous_actor = core_vector_index_of(&concrete_actor->spawners, &name) - 1; if (previous_actor < 0) { previous_actor = core_vector_size(&concrete_actor->spawners)- 1; } printf("%d received ACTION_RING_PUSH_NEXT\n", name); thorium_message_init(message, ACTION_RING_SET_NEXT, sizeof(concrete_actor->first), &concrete_actor->first); thorium_actor_send(actor, *(int *)core_vector_at(&concrete_actor->spawners, previous_actor), message); } else if (tag == ACTION_RING_SET_NEXT) { concrete_actor->step = RING_STEP_PUSH_NEXT; thorium_message_set_action(message, ACTION_SENDER_SET_NEXT); thorium_actor_send(actor, concrete_actor->last, message); } else if (tag == ACTION_SENDER_SET_NEXT_REPLY && concrete_actor->step == RING_STEP_SPAWN) { #if 0 printf("ready senders %d/%d\n", concrete_actor->ready_senders, concrete_actor->senders); #endif if (concrete_actor->spawned_senders % 10000 == 0) { printf("spawned %d/%d\n", concrete_actor->spawned_senders, concrete_actor->senders); } if (concrete_actor->spawned_senders == concrete_actor->senders) { printf("RING_STEP_SPAWN completed.\n"); thorium_actor_send_empty(actor, *(int *)core_vector_at(&concrete_actor->spawners, 0), ACTION_RING_READY); concrete_actor->ready_senders = 0; concrete_actor->last = concrete_actor->previous; } else { new_actor = thorium_actor_spawn(actor, SCRIPT_SENDER); ++concrete_actor->spawned_senders; previous = concrete_actor->previous; thorium_message_init(message, ACTION_SENDER_SET_NEXT, sizeof(new_actor), &new_actor); thorium_actor_send(actor, previous, message); concrete_actor->previous = new_actor; } } else if (tag == ACTION_SENDER_SET_NEXT_REPLY && concrete_actor->step == RING_STEP_PUSH_NEXT) { concrete_actor->ready_senders++; printf("ACTION_SENDER_SET_NEXT_REPLY %d/%d\n", concrete_actor->ready_senders, 1); if (concrete_actor->ready_senders == 1) { thorium_actor_send_empty(actor, *(int *)core_vector_at(&concrete_actor->spawners, 0), ACTION_RING_READY); printf("RING_STEP_PUSH_NEXT completed.\n"); concrete_actor->ready_senders = 0; } } else if (tag == ACTION_RING_READY && concrete_actor->step == RING_STEP_PUSH_NEXT) { concrete_actor->ready_rings++; if (concrete_actor->ready_rings == core_vector_size(&concrete_actor->spawners)) { printf("system is ready...\n"); messages = 2000007; thorium_message_init(message, ACTION_SENDER_HELLO, sizeof(messages), &messages); thorium_actor_send(actor, concrete_actor->first, message); concrete_actor->ready_rings = 0; } } else if (tag == ACTION_SENDER_HELLO_REPLY) { thorium_actor_send_range_empty(actor, &concrete_actor->spawners, ACTION_RING_KILL); thorium_actor_send_empty(actor, concrete_actor->first, ACTION_SENDER_KILL); } else if (tag == ACTION_RING_KILL) { thorium_actor_send_to_self_empty(actor, ACTION_STOP); } }