int core_map_add_value(struct core_map *self, void *key, void *value) { void *bucket; int value_size; #ifdef CORE_MAP_ALIGNMENT_ENABLED key = core_map_pad_key(self, key); #endif bucket = core_map_get(self, key); /* it's already there... */ if (bucket != NULL) { return 0; } bucket = core_map_add(self, key); #ifdef CORE_MAP_ALIGNMENT_ENABLED value_size = self->original_value_size; #else value_size = core_map_get_value_size(self); #endif core_memory_copy(bucket, value, value_size); return 1; }
int thorium_balancer_select_worker_script_round_robin(struct thorium_balancer *self, int script) { int worker; int worker_count; int next_worker; int *bucket; bucket = core_map_get(&self->current_script_workers, &script); worker_count = thorium_worker_pool_worker_count(self->pool); /* First time for this script */ if (bucket == NULL) { bucket = core_map_add(&self->current_script_workers, &script); *bucket = self->first_worker; self->first_worker = (self->first_worker + 1) % worker_count; } worker = *bucket; next_worker = (worker + 1) % worker_count; *bucket = next_worker; return worker; }
void biosal_input_controller_receive(struct thorium_actor *actor, struct thorium_message *message) { int tag; int count; char *file; void *buffer; struct biosal_input_controller *controller; struct biosal_input_controller *concrete_actor; int destination; int script; int stream; char *local_file; int i; int name; int source; int destination_index; struct thorium_message new_message; int error; int stream_index; int64_t entries; int64_t *bucket; int *int_bucket; int spawner; int command_name; int stream_name; int consumer; int consumer_index; int *bucket_for_requests; char *new_buffer; int new_count; int file_index; struct core_vector mega_blocks; struct core_vector_iterator vector_iterator; struct biosal_mega_block *mega_block; struct core_vector *vector_bucket; struct core_vector block_counts; uint64_t block_entries; int mega_block_index; uint64_t offset; struct biosal_mega_block *block; int acquaintance_index; if (thorium_actor_take_action(actor, message)) { return; } thorium_message_get_all(message, &tag, &count, &buffer, &source); name = thorium_actor_name(actor); controller = (struct biosal_input_controller *)thorium_actor_concrete_actor(actor); concrete_actor = controller; if (tag == ACTION_START) { core_vector_init(&concrete_actor->spawners, 0); core_vector_unpack(&concrete_actor->spawners, buffer); core_vector_resize(&concrete_actor->stores_per_spawner, core_vector_size(&concrete_actor->spawners)); for (i = 0; i < core_vector_size(&concrete_actor->spawners); i++) { int_bucket = (int *)core_vector_at(&concrete_actor->stores_per_spawner, i); *int_bucket = 0; spawner = core_vector_at_as_int(&concrete_actor->spawners, i); core_queue_enqueue(&concrete_actor->unprepared_spawners, &spawner); } concrete_actor->state = BIOSAL_INPUT_CONTROLLER_STATE_PREPARE_SPAWNERS; #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL printf("DEBUG preparing first spawner\n"); #endif thorium_actor_send_to_self_empty(actor, ACTION_INPUT_CONTROLLER_PREPARE_SPAWNERS); /* thorium_dispatcher_print(thorium_actor_dispatcher(actor)); */ } else if (tag == ACTION_ADD_FILE) { file = (char *)buffer; local_file = core_memory_allocate(strlen(file) + 1, MEMORY_CONTROLLER); strcpy(local_file, file); printf("controller %d ACTION_ADD_FILE %s\n", thorium_actor_name(actor), local_file); core_vector_push_back(&concrete_actor->files, &local_file); bucket = core_vector_at(&concrete_actor->files, core_vector_size(&concrete_actor->files) - 1); local_file = *(char **)bucket; #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2 printf("DEBUG11 ACTION_ADD_FILE %s %p bucket %p index %d\n", local_file, local_file, (void *)bucket, core_vector_size(&concrete_actor->files) - 1); #endif thorium_actor_send_reply_empty(actor, ACTION_ADD_FILE_REPLY); } else if (tag == ACTION_SPAWN_REPLY) { if (concrete_actor->state == BIOSAL_INPUT_CONTROLLER_STATE_SPAWN_STORES) { biosal_input_controller_add_store(actor, message); return; } else if (concrete_actor->state == BIOSAL_INPUT_CONTROLLER_STATE_PREPARE_SPAWNERS) { concrete_actor->ready_spawners++; thorium_message_unpack_int(message, 0, &name); thorium_actor_send_empty(actor, name, ACTION_ASK_TO_STOP); thorium_actor_send_to_self_empty(actor, ACTION_INPUT_CONTROLLER_PREPARE_SPAWNERS); if (concrete_actor->ready_spawners == (int)core_vector_size(&concrete_actor->spawners)) { #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG printf("DEBUG all spawners are prepared\n"); #endif thorium_actor_send_to_supervisor_empty(actor, ACTION_START_REPLY); } return; } else if (concrete_actor->state == BIOSAL_INPUT_CONTROLLER_STATE_SPAWN_PARTITIONER) { #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG printf("DEBUG received spawn reply, state is spawn_partitioner\n"); #endif thorium_message_unpack_int(message, 0, &concrete_actor->partitioner); /* configure the partitioner */ destination = concrete_actor->partitioner; thorium_actor_send_int(actor, destination, ACTION_SEQUENCE_PARTITIONER_SET_BLOCK_SIZE, concrete_actor->block_size); thorium_actor_send_int(actor, destination, ACTION_SEQUENCE_PARTITIONER_SET_ACTOR_COUNT, core_vector_size(&concrete_actor->consumers)); core_vector_init(&block_counts, sizeof(uint64_t)); for (i = 0; i < core_vector_size(&concrete_actor->mega_block_vector); i++) { block = (struct biosal_mega_block *)core_vector_at(&concrete_actor->mega_block_vector, i); block_entries = biosal_mega_block_get_entries(block); core_vector_push_back_uint64_t(&block_counts, block_entries); } new_count = core_vector_pack_size(&block_counts); new_buffer = thorium_actor_allocate(actor, new_count); #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG printf("DEBUG packed counts, %d bytes\n", count); #endif core_vector_pack(&block_counts, new_buffer); thorium_message_init(&new_message, ACTION_SEQUENCE_PARTITIONER_SET_ENTRY_VECTOR, new_count, new_buffer); thorium_actor_send(actor, destination, &new_message); core_vector_destroy(&block_counts); return; } else if (concrete_actor->state == BIOSAL_INPUT_CONTROLLER_STATE_SPAWN_READING_STREAMS) { thorium_message_unpack_int(message, 0, &stream); stream_index = stream; mega_block_index = core_vector_size(&concrete_actor->reading_streams); core_vector_push_back_int(&concrete_actor->reading_streams, stream_index); core_vector_push_back_int(&concrete_actor->partition_commands, -1); core_vector_push_back_int(&concrete_actor->stream_consumers, -1); stream_index = core_vector_size(&concrete_actor->reading_streams) - 1; mega_block = (struct biosal_mega_block *)core_vector_at(&concrete_actor->mega_block_vector, mega_block_index); offset = biosal_mega_block_get_offset(mega_block); core_map_add_value(&concrete_actor->assigned_blocks, &stream_index, &mega_block_index); #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_READING_STREAMS printf("DEBUG setting offset to %" PRIu64 " for stream/%d\n", offset, stream); #endif thorium_actor_send_uint64_t(actor, stream, ACTION_INPUT_STREAM_SET_START_OFFSET, offset); return; } stream = *(int *)buffer; file_index = core_vector_size(&concrete_actor->counting_streams); local_file = *(char **)core_vector_at(&concrete_actor->files, file_index); #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_READING_STREAMS printf("DEBUG actor %d receives stream %d from spawner %d for file %s\n", name, stream, source, local_file); #endif core_vector_push_back(&concrete_actor->counting_streams, &stream); #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_READING_STREAMS printf("asking stream/%d to open %s\n", stream, local_file); #endif thorium_message_init(&new_message, ACTION_INPUT_OPEN, strlen(local_file) + 1, local_file); #ifdef DEBUG_ISSUE_594 thorium_message_print(&new_message); printf("SEND Buffer %s\n", local_file); #endif thorium_actor_send(actor, stream, &new_message); thorium_message_destroy(&new_message); if (core_vector_size(&concrete_actor->counting_streams) != core_vector_size(&concrete_actor->files)) { thorium_actor_send_to_self_empty(actor, ACTION_INPUT_SPAWN); } #ifdef DEBUG_ISSUE_594 printf("EXIT Buffer %s\n", local_file); #endif } else if (tag == ACTION_INPUT_OPEN_REPLY) { if (concrete_actor->state == BIOSAL_INPUT_CONTROLLER_STATE_SPAWN_READING_STREAMS) { #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_READING_STREAMS printf("DEBUG receives open.reply for reading stream/%d\n", source); #endif concrete_actor->opened_streams++; if (concrete_actor->opened_streams == core_vector_size(&concrete_actor->mega_block_vector)) { thorium_actor_send_to_self_empty(actor, ACTION_INPUT_CONTROLLER_CREATE_STORES); } return; } concrete_actor->opened_streams++; stream = source; thorium_message_unpack_int(message, 0, &error); if (error == BIOSAL_INPUT_ERROR_NO_ERROR) { #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2 printf("DEBUG actor %d asks %d ACTION_INPUT_COUNT_IN_PARALLEL\n", name, stream); #endif thorium_actor_send_vector(actor, stream, ACTION_INPUT_COUNT_IN_PARALLEL, &concrete_actor->spawners); } else { #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2 printf("DEBUG actor %d received error %d from %d\n", name, error, stream); #endif concrete_actor->counted++; } /* if all streams failed, notice supervisor */ if (concrete_actor->counted == core_vector_size(&concrete_actor->files)) { #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2 #endif printf("DEBUG %d: Error all streams failed.\n", thorium_actor_name(actor)); thorium_actor_send_to_supervisor_empty(actor, ACTION_INPUT_DISTRIBUTE_REPLY); } /* if (concrete_actor->opened_streams == core_vector_size(&concrete_actor->files)) { #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG printf("DEBUG controller %d sends ACTION_INPUT_DISTRIBUTE_REPLY to supervisor %d [%d/%d]\n", name, thorium_actor_supervisor(actor), concrete_actor->opened_streams, core_vector_size(&concrete_actor->files)); #endif } */ } else if (tag == ACTION_INPUT_COUNT_PROGRESS) { stream_index = core_vector_index_of(&concrete_actor->counting_streams, &source); local_file = core_vector_at_as_char_pointer(&concrete_actor->files, stream_index); thorium_message_unpack_int64_t(message, 0, &entries); bucket = (int64_t *)core_vector_at(&concrete_actor->counts, stream_index); printf("controller/%d receives progress from stream/%d file %s %" PRIu64 " entries so far\n", name, source, local_file, entries); *bucket = entries; } else if (tag == ACTION_INPUT_COUNT_IN_PARALLEL_REPLY) { stream_index = core_vector_index_of(&concrete_actor->counting_streams, &source); local_file = core_vector_at_as_char_pointer(&concrete_actor->files, stream_index); core_vector_init(&mega_blocks, 0); core_vector_unpack(&mega_blocks, buffer); printf("DEBUG receive mega blocks from %d\n", source); /* * Update the file index for every mega block. */ core_vector_iterator_init(&vector_iterator, &mega_blocks); bucket = (int64_t*)core_vector_at(&concrete_actor->counts, stream_index); (*bucket) = 0; while (core_vector_iterator_has_next(&vector_iterator)) { core_vector_iterator_next(&vector_iterator, (void **)&mega_block); printf("SETTING setting file to %d for mega block\n", stream_index); biosal_mega_block_set_file(mega_block, stream_index); entries = biosal_mega_block_get_entries_from_start(mega_block); printf("Cataloging %d ENTRIES\n", (int)entries); (*bucket) = entries; biosal_mega_block_print(mega_block); } core_vector_iterator_destroy(&vector_iterator); vector_bucket = (struct core_vector *)core_map_add(&concrete_actor->mega_blocks, &stream_index); core_vector_init_copy(vector_bucket, &mega_blocks); core_vector_destroy(&mega_blocks); concrete_actor->counted++; printf("controller/%d received from stream/%d for file %s %" PRIu64 " entries (final) %d/%d\n", name, source, local_file, entries, concrete_actor->counted, (int)core_vector_size(&concrete_actor->files)); thorium_actor_send_reply_empty(actor, ACTION_INPUT_CLOSE); /* continue work here, tell supervisor about it */ if (concrete_actor->counted == core_vector_size(&concrete_actor->files)) { thorium_actor_send_to_self_empty(actor, ACTION_INPUT_CONTROLLER_SPAWN_READING_STREAMS); } } else if (tag == ACTION_INPUT_DISTRIBUTE) { core_timer_start(&concrete_actor->input_timer); core_timer_start(&concrete_actor->counting_timer); /* for each file, spawn a stream to count */ /* no files, return immediately */ if (core_vector_size(&concrete_actor->files) == 0) { printf("Error: no file to distribute...\n"); thorium_actor_send_reply_empty(actor, ACTION_INPUT_DISTRIBUTE_REPLY); return; } #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2 printf("DEBUG actor %d receives ACTION_INPUT_DISTRIBUTE\n", name); #endif #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2 printf("DEBUG send ACTION_INPUT_SPAWN to self\n"); #endif thorium_actor_send_to_self_empty(actor, ACTION_INPUT_SPAWN); #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2 printf("DEBUG resizing counts to %d\n", core_vector_size(&concrete_actor->files)); #endif core_vector_resize(&concrete_actor->counts, core_vector_size(&concrete_actor->files)); for (i = 0; i < core_vector_size(&concrete_actor->counts); i++) { bucket = (int64_t*)core_vector_at(&concrete_actor->counts, i); *bucket = 0; } } else if (tag == ACTION_INPUT_SPAWN && source == name) { #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2 printf("DEBUG ACTION_INPUT_SPAWN\n"); #endif script = SCRIPT_INPUT_STREAM; concrete_actor->state = BIOSAL_INPUT_CONTROLLER_STATE_SPAWN_STREAMS; /* the next file name to send is the current number of streams */ i = core_vector_size(&concrete_actor->counting_streams); destination_index = i % core_vector_size(&concrete_actor->spawners); destination = *(int *)core_vector_at(&concrete_actor->spawners, destination_index); thorium_message_init(message, ACTION_SPAWN, sizeof(script), &script); thorium_actor_send(actor, destination, message); bucket = core_vector_at(&concrete_actor->files, i); local_file = *(char **)core_vector_at(&concrete_actor->files, i); #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2 printf("DEBUG890 local_file %p bucket %p index %d\n", local_file, (void *)bucket, i); #endif #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG printf("DEBUG actor %d spawns a stream for file %d/%d via spawner %d\n", name, i, core_vector_size(&concrete_actor->files), destination); #endif /* also, spawn 4 stores on each node */ } else if (tag == ACTION_ASK_TO_STOP && ( source == thorium_actor_supervisor(actor) || source == thorium_actor_name(actor))) { #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2 #endif /* stop streams */ for (i = 0; i < core_vector_size(&concrete_actor->counting_streams); i++) { stream = *(int *)core_vector_at(&concrete_actor->counting_streams, i); thorium_actor_send_empty(actor, stream, ACTION_ASK_TO_STOP); } for (i = 0; i < core_vector_size(&concrete_actor->reading_streams); i++) { stream = *(int *)core_vector_at(&concrete_actor->reading_streams, i); thorium_actor_send_empty(actor, stream, ACTION_ASK_TO_STOP); } #if 0 /* stop data stores */ for (i = 0; i < core_vector_size(&concrete_actor->consumers); i++) { store = core_vector_at_as_int(&concrete_actor->consumers, i); thorium_actor_send_empty(actor, store, ACTION_ASK_TO_STOP); } #endif /* stop partitioner */ if (concrete_actor->partitioner != THORIUM_ACTOR_NOBODY) { thorium_actor_send_empty(actor, concrete_actor->partitioner, ACTION_ASK_TO_STOP); #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG printf("DEBUG controller %d sends ACTION_ASK_TO_STOP_REPLY to %d\n", thorium_actor_name(actor), thorium_message_source(message)); #endif } thorium_actor_send_reply_empty(actor, ACTION_ASK_TO_STOP_REPLY); /* stop self */ thorium_actor_send_to_self_empty(actor, ACTION_STOP); thorium_actor_ask_to_stop(actor, message); printf("DEBUG controller %d dies\n", name); #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG #endif } else if (tag == ACTION_INPUT_CONTROLLER_CREATE_PARTITION && source == name) { spawner = *(int *)core_vector_at(&concrete_actor->spawners, core_vector_size(&concrete_actor->spawners) / 2); thorium_actor_send_int(actor, spawner, ACTION_SPAWN, SCRIPT_SEQUENCE_PARTITIONER); concrete_actor->state = BIOSAL_INPUT_CONTROLLER_STATE_SPAWN_PARTITIONER; #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG printf("DEBUG input controller %d spawns a partitioner via spawner %d\n", name, spawner); #endif } else if (tag == ACTION_SEQUENCE_PARTITIONER_COMMAND_IS_READY) { #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG printf("DEBUG controller receives ACTION_SEQUENCE_PARTITIONER_COMMAND_IS_READY, asks for command\n"); #endif thorium_actor_send_reply_empty(actor, ACTION_SEQUENCE_PARTITIONER_GET_COMMAND); } else if (tag == ACTION_SEQUENCE_PARTITIONER_GET_COMMAND_REPLY) { biosal_input_controller_receive_command(actor, message); } else if (tag == ACTION_SEQUENCE_PARTITIONER_FINISHED) { thorium_actor_send_empty(actor, concrete_actor->partitioner, ACTION_ASK_TO_STOP); biosal_input_controller_verify_requests(actor, message); } else if (tag == ACTION_SEQUENCE_PARTITIONER_PROVIDE_STORE_ENTRY_COUNTS) { biosal_input_controller_receive_store_entry_counts(actor, message); } else if (tag == ACTION_RESERVE_REPLY) { concrete_actor->ready_consumers++; printf("DEBUG marker ACTION_RESERVE_REPLY %d/%d\n", concrete_actor->ready_consumers, (int)core_vector_size(&concrete_actor->consumers)); if (concrete_actor->ready_consumers == core_vector_size(&concrete_actor->consumers)) { concrete_actor->ready_consumers = 0; printf("DEBUG all consumers are ready\n"); thorium_actor_send_empty(actor, concrete_actor->partitioner, ACTION_SEQUENCE_PARTITIONER_PROVIDE_STORE_ENTRY_COUNTS_REPLY); } } else if (tag == ACTION_INPUT_PUSH_SEQUENCES_READY) { #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG printf("DEBUG biosal_input_controller_receive received ACTION_INPUT_PUSH_SEQUENCES_READY\n"); #endif stream_name = source; acquaintance_index = stream_name; stream_index = core_vector_index_of(&concrete_actor->reading_streams, &acquaintance_index); command_name = *(int *)core_vector_at(&concrete_actor->partition_commands, stream_index); thorium_actor_send_int(actor, concrete_actor->partitioner, ACTION_SEQUENCE_PARTITIONER_GET_COMMAND_REPLY_REPLY, command_name); } else if (tag == ACTION_INPUT_PUSH_SEQUENCES_REPLY) { stream_name = source; thorium_message_unpack_int(message, 0, &consumer); consumer_index = core_vector_index_of(&concrete_actor->consumers, &consumer); bucket_for_requests = (int *)core_vector_at(&concrete_actor->consumer_active_requests, consumer_index); (*bucket_for_requests)--; biosal_input_controller_verify_requests(actor, message); #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_CONSUMERS printf("DEBUG consumer # %d has %d active requests\n", consumer_index, *bucket_for_requests); #endif } else if (tag == ACTION_SET_CONSUMERS) { core_vector_init(&concrete_actor->consumers, 0); core_vector_unpack(&concrete_actor->consumers, buffer); printf("controller %d receives %d consumers\n", thorium_actor_name(actor), (int)core_vector_size(&concrete_actor->consumers)); for (i = 0; i < core_vector_size(&concrete_actor->consumers); i++) { core_vector_push_back_int(&concrete_actor->consumer_active_requests, 0); } #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG core_vector_print_int(&concrete_actor->consumers); printf("\n"); #endif thorium_actor_send_reply_empty(actor, ACTION_SET_CONSUMERS_REPLY); } else if (tag == ACTION_SET_BLOCK_SIZE) { thorium_message_unpack_int(message, 0, &concrete_actor->block_size); thorium_actor_send_reply_empty(actor, ACTION_SET_BLOCK_SIZE_REPLY); } else if (tag == ACTION_SEQUENCE_STORE_READY) { concrete_actor->filled_consumers++; #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG printf("DEBUG ACTION_SEQUENCE_STORE_READY %d/%d\n", concrete_actor->filled_consumers, (int)core_vector_size(&concrete_actor->consumers)); #endif if (concrete_actor->filled_consumers == core_vector_size(&concrete_actor->consumers)) { concrete_actor->filled_consumers = 0; printf("DEBUG: all consumers are filled, sending ACTION_INPUT_DISTRIBUTE_REPLY\n"); core_timer_stop(&concrete_actor->input_timer); core_timer_stop(&concrete_actor->distribution_timer); core_timer_print_with_description(&concrete_actor->distribution_timer, "Load input / Distribute input data"); core_timer_print_with_description(&concrete_actor->input_timer, "Load input"); thorium_actor_send_to_supervisor_empty(actor, ACTION_INPUT_DISTRIBUTE_REPLY); } } }
void biosal_assembly_graph_store_push_kmer_block(struct thorium_actor *self, struct thorium_message *message) { struct core_memory_pool *ephemeral_memory; struct biosal_dna_kmer_frequency_block block; struct biosal_assembly_vertex *bucket; void *packed_kmer; struct core_map_iterator iterator; struct biosal_assembly_graph_store *concrete_self; /*int tag;*/ void *key; struct core_map *kmers; struct biosal_dna_kmer kmer; void *buffer; int count; struct biosal_dna_kmer encoded_kmer; char *raw_kmer; int period; struct biosal_dna_kmer *kmer_pointer; int *frequency; ephemeral_memory = thorium_actor_get_ephemeral_memory(self); concrete_self = thorium_actor_concrete_actor(self); /*tag = thorium_message_action(message);*/ buffer = thorium_message_buffer(message); count = thorium_message_count(message); /* * Handler for PUSH_DATA */ biosal_dna_kmer_frequency_block_init(&block, concrete_self->kmer_length, ephemeral_memory, &concrete_self->transport_codec, 0); biosal_dna_kmer_frequency_block_unpack(&block, buffer, ephemeral_memory, &concrete_self->transport_codec); key = core_memory_pool_allocate(ephemeral_memory, concrete_self->key_length_in_bytes); kmers = biosal_dna_kmer_frequency_block_kmers(&block); core_map_iterator_init(&iterator, kmers); period = 2500000; raw_kmer = core_memory_pool_allocate(thorium_actor_get_ephemeral_memory(self), concrete_self->kmer_length + 1); if (!concrete_self->printed_vertex_size) { printf("DEBUG VERTEX DELIVERY %d bytes\n", count); concrete_self->printed_vertex_size = 1; } while (core_map_iterator_has_next(&iterator)) { /* * add kmers to store */ core_map_iterator_next(&iterator, (void **)&packed_kmer, (void **)&frequency); /* Store the kmer in 2 bit encoding */ biosal_dna_kmer_init_empty(&kmer); biosal_dna_kmer_unpack(&kmer, packed_kmer, concrete_self->kmer_length, ephemeral_memory, &concrete_self->transport_codec); kmer_pointer = &kmer; if (concrete_self->codec_are_different) { /* * Get a copy of the sequence */ biosal_dna_kmer_get_sequence(kmer_pointer, raw_kmer, concrete_self->kmer_length, &concrete_self->transport_codec); biosal_dna_kmer_init(&encoded_kmer, raw_kmer, &concrete_self->storage_codec, thorium_actor_get_ephemeral_memory(self)); kmer_pointer = &encoded_kmer; } biosal_dna_kmer_pack_store_key(kmer_pointer, key, concrete_self->kmer_length, &concrete_self->storage_codec, thorium_actor_get_ephemeral_memory(self)); #ifdef BIOSAL_DEBUG_ISSUE_540 if (strcmp(raw_kmer, "AGCTGGTAGTCATCACCAGACTGGAACAG") == 0 || strcmp(raw_kmer, "CGCGATCTGTTGCTGGGCCTAACGTGGTA") == 0 || strcmp(raw_kmer, "TACCACGTTAGGCCCAGCAACAGATCGCG") == 0) { printf("Examine store key for %s\n", raw_kmer); core_debugger_examine(key, concrete_self->key_length_in_bytes); } #endif bucket = core_map_get(&concrete_self->table, key); if (bucket == NULL) { /* This is the first time that this kmer is seen. */ bucket = core_map_add(&concrete_self->table, key); biosal_assembly_vertex_init(bucket); #if 0 printf("DEBUG303 ADD_KEY"); biosal_dna_kmer_print(&encoded_kmer, concrete_self->kmer_length, &concrete_self->storage_codec, ephemeral_memory); #endif } if (concrete_self->codec_are_different) { biosal_dna_kmer_destroy(&encoded_kmer, thorium_actor_get_ephemeral_memory(self)); } biosal_dna_kmer_destroy(&kmer, ephemeral_memory); biosal_assembly_vertex_increase_coverage_depth(bucket, *frequency); if (concrete_self->received >= concrete_self->last_received + period) { printf("%s/%d received %" PRIu64 " kmers so far," " store has %" PRIu64 " canonical kmers, %" PRIu64 " kmers\n", thorium_actor_script_name(self), thorium_actor_name(self), concrete_self->received, core_map_size(&concrete_self->table), 2 * core_map_size(&concrete_self->table)); concrete_self->last_received = concrete_self->received; } concrete_self->received += *frequency; } core_memory_pool_free(ephemeral_memory, key); core_memory_pool_free(ephemeral_memory, raw_kmer); core_map_iterator_destroy(&iterator); biosal_dna_kmer_frequency_block_destroy(&block, thorium_actor_get_ephemeral_memory(self)); thorium_actor_send_reply_empty(self, ACTION_PUSH_KMER_BLOCK_REPLY); }
void biosal_assembly_graph_store_yield_reply(struct thorium_actor *self, struct thorium_message *message) { struct biosal_dna_kmer kmer; void *key; struct biosal_assembly_vertex *value; int coverage; int customer; uint64_t *count; int new_count; void *new_buffer; struct thorium_message new_message; struct core_memory_pool *ephemeral_memory; struct biosal_assembly_graph_store *concrete_self; int i; int max; ephemeral_memory = thorium_actor_get_ephemeral_memory(self); concrete_self = thorium_actor_concrete_actor(self); customer = concrete_self->customer; #if 0 printf("YIELD REPLY\n"); #endif i = 0; max = 1024; key = NULL; value = NULL; while (i < max && core_map_iterator_has_next(&concrete_self->iterator)) { core_map_iterator_next(&concrete_self->iterator, (void **)&key, (void **)&value); biosal_dna_kmer_init_empty(&kmer); biosal_dna_kmer_unpack(&kmer, key, concrete_self->kmer_length, ephemeral_memory, &concrete_self->storage_codec); coverage = biosal_assembly_vertex_coverage_depth(value); count = (uint64_t *)core_map_get(&concrete_self->coverage_distribution, &coverage); if (count == NULL) { count = (uint64_t *)core_map_add(&concrete_self->coverage_distribution, &coverage); (*count) = 0; } /* increment for the lowest kmer (canonical) */ (*count)++; biosal_dna_kmer_destroy(&kmer, ephemeral_memory); ++i; } /* yield again if the iterator is not at the end */ if (core_map_iterator_has_next(&concrete_self->iterator)) { #if 0 printf("yield ! %d\n", i); #endif thorium_actor_send_to_self_empty(self, ACTION_YIELD); return; } /* printf("ready...\n"); */ core_map_iterator_destroy(&concrete_self->iterator); new_count = core_map_pack_size(&concrete_self->coverage_distribution); new_buffer = thorium_actor_allocate(self, new_count); core_map_pack(&concrete_self->coverage_distribution, new_buffer); printf("SENDING %s/%d sends map to %d, %d bytes / %d entries\n", thorium_actor_script_name(self), thorium_actor_name(self), customer, new_count, (int)core_map_size(&concrete_self->coverage_distribution)); thorium_message_init(&new_message, ACTION_PUSH_DATA, new_count, new_buffer); thorium_actor_send(self, customer, &new_message); thorium_message_destroy(&new_message); core_map_destroy(&concrete_self->coverage_distribution); thorium_actor_send_empty(self, concrete_self->source, ACTION_PUSH_DATA_REPLY); }
void biosal_sequence_partitioner_receive(struct thorium_actor *actor, struct thorium_message *message) { int tag; int source; int count; void *buffer; int bytes; struct biosal_sequence_partitioner *concrete_actor; struct biosal_partition_command command; struct thorium_message response; int command_number; struct biosal_partition_command *active_command; int stream_index; struct biosal_partition_command *command_bucket; int i; thorium_message_get_all(message, &tag, &count, &buffer, &source); concrete_actor = (struct biosal_sequence_partitioner *)thorium_actor_concrete_actor(actor); if (tag == ACTION_SEQUENCE_PARTITIONER_SET_BLOCK_SIZE) { thorium_message_unpack_int(message, 0, &concrete_actor->block_size); thorium_actor_send_reply_empty(actor, ACTION_SEQUENCE_PARTITIONER_SET_BLOCK_SIZE_REPLY); biosal_sequence_partitioner_verify(actor); /* printf("DEBUG biosal_sequence_partitioner_receive received block size\n"); */ } else if (tag == ACTION_SEQUENCE_PARTITIONER_SET_ENTRY_VECTOR) { /* printf("DEBUG biosal_sequence_partitioner_receive unpacking vector, %d bytes\n", count); */ core_vector_init(&concrete_actor->stream_entries, 0); core_vector_unpack(&concrete_actor->stream_entries, buffer); /* printf("DEBUG after unpack\n"); */ thorium_actor_send_reply_empty(actor, ACTION_SEQUENCE_PARTITIONER_SET_ENTRY_VECTOR_REPLY); /* printf("DEBUG biosal_sequence_partitioner_receive received received entry vector\n"); */ biosal_sequence_partitioner_verify(actor); } else if (tag == ACTION_SEQUENCE_PARTITIONER_SET_ACTOR_COUNT) { thorium_message_unpack_int(message, 0, &concrete_actor->store_count); thorium_actor_send_reply_empty(actor, ACTION_SEQUENCE_PARTITIONER_SET_ACTOR_COUNT_REPLY); biosal_sequence_partitioner_verify(actor); /* printf("DEBUG biosal_sequence_partitioner_receive received received store count\n"); */ } else if (tag == ACTION_SEQUENCE_PARTITIONER_GET_COMMAND) { if (core_queue_dequeue(&concrete_actor->available_commands, &command)) { bytes = biosal_partition_command_pack_size(&command); /* printf("DEBUG partitioner has command, packing %d bytes!\n", bytes); */ buffer = thorium_actor_allocate(actor, bytes); biosal_partition_command_pack(&command, buffer); thorium_message_init(&response, ACTION_SEQUENCE_PARTITIONER_GET_COMMAND_REPLY, bytes, buffer); thorium_actor_send_reply(actor, &response); /* store the active command */ command_number = biosal_partition_command_name(&command); command_bucket = (struct biosal_partition_command *)core_map_add(&concrete_actor->active_commands, &command_number); *command_bucket = command; /* there may be other command available too ! */ } } else if (tag == ACTION_SEQUENCE_PARTITIONER_GET_COMMAND_REPLY_REPLY) { /* * take the name of the command, find it in the active * command, generate a new command, and send ACTION_SEQUENCE_PARTITIONER_COMMAND_IS_READY * as a reply */ thorium_message_unpack_int(message, 0, &command_number); active_command = core_map_get(&concrete_actor->active_commands, &command_number); if (active_command == NULL) { return; } stream_index = biosal_partition_command_stream_index(active_command); active_command = NULL; core_map_delete(&concrete_actor->active_commands, &command_number); biosal_sequence_partitioner_generate_command(actor, stream_index); if (core_map_size(&concrete_actor->active_commands) == 0 && core_queue_size(&concrete_actor->available_commands) == 0) { thorium_actor_send_reply_empty(actor, ACTION_SEQUENCE_PARTITIONER_FINISHED); } } else if (tag == ACTION_ASK_TO_STOP && source == thorium_actor_supervisor(actor)) { #ifdef BIOSAL_SEQUENCE_PARTITIONER_DEBUG printf("DEBUG biosal_sequence_partitioner_receive ACTION_ASK_TO_STOP\n"); #endif thorium_actor_send_to_self_empty(actor, ACTION_STOP); } else if (tag == ACTION_SEQUENCE_PARTITIONER_PROVIDE_STORE_ENTRY_COUNTS_REPLY) { /* generate commands */ for (i = 0; i < core_vector_size(&concrete_actor->stream_entries); i++) { biosal_sequence_partitioner_generate_command(actor, i); } } }
int main(int argc, char **argv) { BEGIN_TESTS(); { struct core_map big_map; int kmer_length = 43; struct biosal_dna_kmer kmer; int count; int run_test; int coverage; void *key; int key_length; int *bucket; int i; struct biosal_dna_codec codec; struct core_memory_pool memory; core_memory_pool_init(&memory, 1048576, -1); biosal_dna_codec_init(&codec); run_test = 1; count = 100000000; printf("STRESS TEST\n"); biosal_dna_kmer_init_mock(&kmer, kmer_length, &codec, &memory); key_length = biosal_dna_kmer_pack_size(&kmer, kmer_length, &codec); biosal_dna_kmer_destroy(&kmer, &memory); core_map_init(&big_map, key_length, sizeof(coverage)); key = core_memory_allocate(key_length, -1); i = 0; while (i < count && run_test) { biosal_dna_kmer_init_random(&kmer, kmer_length, &codec, &memory); biosal_dna_kmer_pack_store_key(&kmer, key, kmer_length, &codec, &memory); bucket = core_map_add(&big_map, key); coverage = 99; (*bucket) = coverage; biosal_dna_kmer_destroy(&kmer, &memory); if (i % 100000 == 0) { printf("ADD %d/%d %" PRIu64 "\n", i, count, core_map_size(&big_map)); } i++; } core_map_destroy(&big_map); core_memory_free(key, -1); biosal_dna_codec_destroy(&codec); core_memory_pool_destroy(&memory); } END_TESTS(); return 0; }
void biosal_coverage_distribution_receive(struct thorium_actor *self, struct thorium_message *message) { int tag; struct core_map map; struct core_map_iterator iterator; int *coverage_from_message; uint64_t *count_from_message; uint64_t *frequency; int count; void *buffer; struct biosal_coverage_distribution *concrete_actor; int name; int source; struct core_memory_pool *ephemeral_memory; ephemeral_memory = thorium_actor_get_ephemeral_memory(self); name = thorium_actor_name(self); source = thorium_message_source(message); concrete_actor = (struct biosal_coverage_distribution *)thorium_actor_concrete_actor(self); tag = thorium_message_action(message); count = thorium_message_count(message); buffer = thorium_message_buffer(message); if (tag == ACTION_PUSH_DATA) { core_map_init(&map, 0, 0); core_map_set_memory_pool(&map, ephemeral_memory); core_map_unpack(&map, buffer); core_map_iterator_init(&iterator, &map); while (core_map_iterator_has_next(&iterator)) { core_map_iterator_next(&iterator, (void **)&coverage_from_message, (void **)&count_from_message); #ifdef BIOSAL_COVERAGE_DISTRIBUTION_DEBUG thorium_actor_log(self, "DEBUG DATA %d %d\n", (int)*coverage_from_message, (int)*count_from_message); #endif frequency = core_map_get(&concrete_actor->distribution, coverage_from_message); if (frequency == NULL) { frequency = core_map_add(&concrete_actor->distribution, coverage_from_message); (*frequency) = 0; } (*frequency) += (*count_from_message); } core_map_iterator_destroy(&iterator); thorium_actor_send_reply_empty(self, ACTION_PUSH_DATA_REPLY); concrete_actor->actual++; thorium_actor_log(self, "distribution/%d receives coverage data from producer/%d, %d entries / %d bytes %d/%d\n", name, source, (int)core_map_size(&map), count, concrete_actor->actual, concrete_actor->expected); if (concrete_actor->expected != 0 && concrete_actor->expected == concrete_actor->actual) { thorium_actor_log(self, "received everything %d/%d\n", concrete_actor->actual, concrete_actor->expected); biosal_coverage_distribution_write_distribution(self); thorium_actor_send_empty(self, concrete_actor->source, ACTION_NOTIFY); } core_map_destroy(&map); } else if (tag == ACTION_ASK_TO_STOP) { biosal_coverage_distribution_ask_to_stop(self, message); } else if (tag == ACTION_SET_EXPECTED_MESSAGE_COUNT) { concrete_actor->source = source; thorium_message_unpack_int(message, 0, &concrete_actor->expected); thorium_actor_log(self, "distribution %d expects %d messages\n", thorium_actor_name(self), concrete_actor->expected); thorium_actor_send_reply_empty(self, ACTION_SET_EXPECTED_MESSAGE_COUNT_REPLY); } }