void core_string_combine(struct core_string *string, const char *data, int operation) { int old_length; int length; int new_length; char *new_data; /* * Nothing to do. */ if (data == NULL) { return; } length = strlen(data); /* Nothing to combine */ if (length == 0) { return; } old_length = 0; if (string->data != NULL) { old_length = strlen(string->data); } new_length = old_length + length; new_data = (char *)core_memory_allocate(new_length + 1, MEMORY_STRING); strcpy(new_data, ""); if (operation == CORE_STRING_APPEND) { if (string->data != NULL) { strcat(new_data, string->data); } strcat(new_data, data); } else if (operation == CORE_STRING_PREPEND) { strcat(new_data, data); if (string->data != NULL) { strcat(new_data, string->data); } } if (string->data != NULL) { core_memory_free(string->data, MEMORY_STRING); } string->data = new_data; }
void core_vector_sort(struct core_vector *self, core_compare_fn_t compare) { void *saved_pivot_value; int element_size; element_size = core_vector_element_size(self); saved_pivot_value = core_memory_allocate(element_size, MEMORY_VECTOR_HELPER); core_vector_quicksort(self, 0, core_vector_size(self) - 1, compare, saved_pivot_value); core_memory_free(saved_pivot_value, MEMORY_VECTOR_HELPER); }
void core_string_init(struct core_string *string, const char *data) { int length; string->data = NULL; if (data != NULL) { length = strlen(data); string->data = (char *)core_memory_allocate(length + 1, MEMORY_STRING); strcpy(string->data, data); } }
void test_allocator(struct core_memory_pool *memory) { int i; int size; void *pointer; struct core_vector vector; struct core_timer timer; uint64_t elapsed; i = 1000000; size = 45; core_vector_init(&vector, sizeof(void *)); core_timer_init(&timer); core_timer_start(&timer); while (i--) { if (memory != NULL) { pointer = core_memory_pool_allocate(memory, size); } else { pointer = core_memory_allocate(size, -1); } core_vector_push_back(&vector, &pointer); } core_timer_stop(&timer); elapsed = core_timer_get_elapsed_nanoseconds(&timer); if (memory == NULL) { printf("Not using memory pool... "); } else { printf("Using memory pool... "); } printf("Elapsed : %" PRIu64 " milliseconds\n", elapsed / 1000 / 1000); size = core_vector_size(&vector); for (i = 0; i < size; ++i) { pointer = core_vector_at_as_void_pointer(&vector, i); if (memory != NULL) { core_memory_pool_free(memory, pointer); } else { core_memory_free(pointer, -1); } } core_vector_destroy(&vector); core_timer_destroy(&timer); }
void core_map_init_with_capacity(struct core_map *self, int key_size, int value_size, uint64_t buckets) { #ifdef CORE_MAP_ALIGNMENT_ENABLED self->original_key_size = key_size; self->original_value_size = value_size; key_size = core_memory_align(key_size); value_size = core_memory_align(value_size); self->key_padding = key_size - self->original_key_size; self->key_buffer = core_memory_allocate(key_size); #endif core_dynamic_hash_table_init(&self->table, buckets, key_size, value_size); }
int core_map_pack_unpack(struct core_map *self, int operation, void *buffer) { struct core_packer packer; int offset; #ifdef CORE_MAP_ALIGNMENT_ENABLED int key_size; #endif CORE_DEBUGGER_ASSERT(self != NULL); core_packer_init(&packer, operation, buffer); #ifdef CORE_MAP_ALIGNMENT_ENABLED core_packer_process(&packer, &self->original_key_size, sizeof(self->original_key_size)); core_packer_process(&packer, &self->original_value_size, sizeof(self->original_value_size)); #endif offset = core_packer_get_byte_count(&packer); core_packer_destroy(&packer); if (operation == CORE_PACKER_OPERATION_PACK_SIZE) { offset += core_dynamic_hash_table_pack_size(&self->table); } else if (operation == CORE_PACKER_OPERATION_PACK) { offset += core_dynamic_hash_table_pack(&self->table, (char *)buffer + offset); } else if (operation == CORE_PACKER_OPERATION_UNPACK) { offset += core_dynamic_hash_table_unpack(&self->table, (char *)buffer + offset); #ifdef CORE_MAP_ALIGNMENT_ENABLED key_size = core_dynamic_hash_table_get_key_size(&self->table); self->key_padding = key_size - self->original_key_size; self->key_buffer = core_memory_allocate(key_size); #endif } return offset; }
void core_fast_ring_init(struct core_fast_ring *self, int capacity, int cell_size) { /* +1 because an empty cell is needed */ self->number_of_cells = core_fast_ring_get_next_power_of_two(capacity + 1); self->mask = self->number_of_cells - 1; #ifdef CORE_FAST_RING_DEBUG printf("DEBUG RING CELLS %" PRIu64 "\n", self->number_of_cells); #endif self->cell_size = cell_size; self->head = 0; self->tail = 0; #ifdef CORE_FAST_RING_USE_CACHE self->head_cache = 0; self->tail_cache = 0; #endif self->cells = core_memory_allocate(self->number_of_cells * self->cell_size, MEMORY_FAST_RING); #ifdef CORE_FAST_RING_USE_PADDING /* assign values to the padding */ self->consumer_padding_0 = 0; self->consumer_padding_1 = 0; self->consumer_padding_2 = 0; self->consumer_padding_3 = 0; self->consumer_padding_4 = 0; self->consumer_padding_5 = 0; self->producer_padding_0 = 0; self->producer_padding_1 = 0; self->producer_padding_2 = 0; self->producer_padding_3 = 0; self->producer_padding_4 = 0; self->producer_padding_5 = 0; #endif }
int core_string_pack_unpack(struct core_string *self, int operation, void *buffer) { struct core_packer packer; int bytes; int length; length = core_string_length(self); core_packer_init(&packer, operation, buffer); core_packer_process(&packer, &length, sizeof(length)); if (operation == CORE_PACKER_OPERATION_UNPACK) { self->data = core_memory_allocate(length + 1, MEMORY_STRING); } core_packer_process(&packer, self->data, length + 1); bytes = core_packer_get_byte_count(&packer); core_packer_destroy(&packer); return bytes; }
void biosal_input_controller_receive(struct thorium_actor *actor, struct thorium_message *message) { int tag; int count; char *file; void *buffer; struct biosal_input_controller *controller; struct biosal_input_controller *concrete_actor; int destination; int script; int stream; char *local_file; int i; int name; int source; int destination_index; struct thorium_message new_message; int error; int stream_index; int64_t entries; int64_t *bucket; int *int_bucket; int spawner; int command_name; int stream_name; int consumer; int consumer_index; int *bucket_for_requests; char *new_buffer; int new_count; int file_index; struct core_vector mega_blocks; struct core_vector_iterator vector_iterator; struct biosal_mega_block *mega_block; struct core_vector *vector_bucket; struct core_vector block_counts; uint64_t block_entries; int mega_block_index; uint64_t offset; struct biosal_mega_block *block; int acquaintance_index; if (thorium_actor_take_action(actor, message)) { return; } thorium_message_get_all(message, &tag, &count, &buffer, &source); name = thorium_actor_name(actor); controller = (struct biosal_input_controller *)thorium_actor_concrete_actor(actor); concrete_actor = controller; if (tag == ACTION_START) { core_vector_init(&concrete_actor->spawners, 0); core_vector_unpack(&concrete_actor->spawners, buffer); core_vector_resize(&concrete_actor->stores_per_spawner, core_vector_size(&concrete_actor->spawners)); for (i = 0; i < core_vector_size(&concrete_actor->spawners); i++) { int_bucket = (int *)core_vector_at(&concrete_actor->stores_per_spawner, i); *int_bucket = 0; spawner = core_vector_at_as_int(&concrete_actor->spawners, i); core_queue_enqueue(&concrete_actor->unprepared_spawners, &spawner); } concrete_actor->state = BIOSAL_INPUT_CONTROLLER_STATE_PREPARE_SPAWNERS; #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL printf("DEBUG preparing first spawner\n"); #endif thorium_actor_send_to_self_empty(actor, ACTION_INPUT_CONTROLLER_PREPARE_SPAWNERS); /* thorium_dispatcher_print(thorium_actor_dispatcher(actor)); */ } else if (tag == ACTION_ADD_FILE) { file = (char *)buffer; local_file = core_memory_allocate(strlen(file) + 1, MEMORY_CONTROLLER); strcpy(local_file, file); printf("controller %d ACTION_ADD_FILE %s\n", thorium_actor_name(actor), local_file); core_vector_push_back(&concrete_actor->files, &local_file); bucket = core_vector_at(&concrete_actor->files, core_vector_size(&concrete_actor->files) - 1); local_file = *(char **)bucket; #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2 printf("DEBUG11 ACTION_ADD_FILE %s %p bucket %p index %d\n", local_file, local_file, (void *)bucket, core_vector_size(&concrete_actor->files) - 1); #endif thorium_actor_send_reply_empty(actor, ACTION_ADD_FILE_REPLY); } else if (tag == ACTION_SPAWN_REPLY) { if (concrete_actor->state == BIOSAL_INPUT_CONTROLLER_STATE_SPAWN_STORES) { biosal_input_controller_add_store(actor, message); return; } else if (concrete_actor->state == BIOSAL_INPUT_CONTROLLER_STATE_PREPARE_SPAWNERS) { concrete_actor->ready_spawners++; thorium_message_unpack_int(message, 0, &name); thorium_actor_send_empty(actor, name, ACTION_ASK_TO_STOP); thorium_actor_send_to_self_empty(actor, ACTION_INPUT_CONTROLLER_PREPARE_SPAWNERS); if (concrete_actor->ready_spawners == (int)core_vector_size(&concrete_actor->spawners)) { #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG printf("DEBUG all spawners are prepared\n"); #endif thorium_actor_send_to_supervisor_empty(actor, ACTION_START_REPLY); } return; } else if (concrete_actor->state == BIOSAL_INPUT_CONTROLLER_STATE_SPAWN_PARTITIONER) { #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG printf("DEBUG received spawn reply, state is spawn_partitioner\n"); #endif thorium_message_unpack_int(message, 0, &concrete_actor->partitioner); /* configure the partitioner */ destination = concrete_actor->partitioner; thorium_actor_send_int(actor, destination, ACTION_SEQUENCE_PARTITIONER_SET_BLOCK_SIZE, concrete_actor->block_size); thorium_actor_send_int(actor, destination, ACTION_SEQUENCE_PARTITIONER_SET_ACTOR_COUNT, core_vector_size(&concrete_actor->consumers)); core_vector_init(&block_counts, sizeof(uint64_t)); for (i = 0; i < core_vector_size(&concrete_actor->mega_block_vector); i++) { block = (struct biosal_mega_block *)core_vector_at(&concrete_actor->mega_block_vector, i); block_entries = biosal_mega_block_get_entries(block); core_vector_push_back_uint64_t(&block_counts, block_entries); } new_count = core_vector_pack_size(&block_counts); new_buffer = thorium_actor_allocate(actor, new_count); #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG printf("DEBUG packed counts, %d bytes\n", count); #endif core_vector_pack(&block_counts, new_buffer); thorium_message_init(&new_message, ACTION_SEQUENCE_PARTITIONER_SET_ENTRY_VECTOR, new_count, new_buffer); thorium_actor_send(actor, destination, &new_message); core_vector_destroy(&block_counts); return; } else if (concrete_actor->state == BIOSAL_INPUT_CONTROLLER_STATE_SPAWN_READING_STREAMS) { thorium_message_unpack_int(message, 0, &stream); stream_index = stream; mega_block_index = core_vector_size(&concrete_actor->reading_streams); core_vector_push_back_int(&concrete_actor->reading_streams, stream_index); core_vector_push_back_int(&concrete_actor->partition_commands, -1); core_vector_push_back_int(&concrete_actor->stream_consumers, -1); stream_index = core_vector_size(&concrete_actor->reading_streams) - 1; mega_block = (struct biosal_mega_block *)core_vector_at(&concrete_actor->mega_block_vector, mega_block_index); offset = biosal_mega_block_get_offset(mega_block); core_map_add_value(&concrete_actor->assigned_blocks, &stream_index, &mega_block_index); #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_READING_STREAMS printf("DEBUG setting offset to %" PRIu64 " for stream/%d\n", offset, stream); #endif thorium_actor_send_uint64_t(actor, stream, ACTION_INPUT_STREAM_SET_START_OFFSET, offset); return; } stream = *(int *)buffer; file_index = core_vector_size(&concrete_actor->counting_streams); local_file = *(char **)core_vector_at(&concrete_actor->files, file_index); #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_READING_STREAMS printf("DEBUG actor %d receives stream %d from spawner %d for file %s\n", name, stream, source, local_file); #endif core_vector_push_back(&concrete_actor->counting_streams, &stream); #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_READING_STREAMS printf("asking stream/%d to open %s\n", stream, local_file); #endif thorium_message_init(&new_message, ACTION_INPUT_OPEN, strlen(local_file) + 1, local_file); #ifdef DEBUG_ISSUE_594 thorium_message_print(&new_message); printf("SEND Buffer %s\n", local_file); #endif thorium_actor_send(actor, stream, &new_message); thorium_message_destroy(&new_message); if (core_vector_size(&concrete_actor->counting_streams) != core_vector_size(&concrete_actor->files)) { thorium_actor_send_to_self_empty(actor, ACTION_INPUT_SPAWN); } #ifdef DEBUG_ISSUE_594 printf("EXIT Buffer %s\n", local_file); #endif } else if (tag == ACTION_INPUT_OPEN_REPLY) { if (concrete_actor->state == BIOSAL_INPUT_CONTROLLER_STATE_SPAWN_READING_STREAMS) { #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_READING_STREAMS printf("DEBUG receives open.reply for reading stream/%d\n", source); #endif concrete_actor->opened_streams++; if (concrete_actor->opened_streams == core_vector_size(&concrete_actor->mega_block_vector)) { thorium_actor_send_to_self_empty(actor, ACTION_INPUT_CONTROLLER_CREATE_STORES); } return; } concrete_actor->opened_streams++; stream = source; thorium_message_unpack_int(message, 0, &error); if (error == BIOSAL_INPUT_ERROR_NO_ERROR) { #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2 printf("DEBUG actor %d asks %d ACTION_INPUT_COUNT_IN_PARALLEL\n", name, stream); #endif thorium_actor_send_vector(actor, stream, ACTION_INPUT_COUNT_IN_PARALLEL, &concrete_actor->spawners); } else { #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2 printf("DEBUG actor %d received error %d from %d\n", name, error, stream); #endif concrete_actor->counted++; } /* if all streams failed, notice supervisor */ if (concrete_actor->counted == core_vector_size(&concrete_actor->files)) { #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2 #endif printf("DEBUG %d: Error all streams failed.\n", thorium_actor_name(actor)); thorium_actor_send_to_supervisor_empty(actor, ACTION_INPUT_DISTRIBUTE_REPLY); } /* if (concrete_actor->opened_streams == core_vector_size(&concrete_actor->files)) { #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG printf("DEBUG controller %d sends ACTION_INPUT_DISTRIBUTE_REPLY to supervisor %d [%d/%d]\n", name, thorium_actor_supervisor(actor), concrete_actor->opened_streams, core_vector_size(&concrete_actor->files)); #endif } */ } else if (tag == ACTION_INPUT_COUNT_PROGRESS) { stream_index = core_vector_index_of(&concrete_actor->counting_streams, &source); local_file = core_vector_at_as_char_pointer(&concrete_actor->files, stream_index); thorium_message_unpack_int64_t(message, 0, &entries); bucket = (int64_t *)core_vector_at(&concrete_actor->counts, stream_index); printf("controller/%d receives progress from stream/%d file %s %" PRIu64 " entries so far\n", name, source, local_file, entries); *bucket = entries; } else if (tag == ACTION_INPUT_COUNT_IN_PARALLEL_REPLY) { stream_index = core_vector_index_of(&concrete_actor->counting_streams, &source); local_file = core_vector_at_as_char_pointer(&concrete_actor->files, stream_index); core_vector_init(&mega_blocks, 0); core_vector_unpack(&mega_blocks, buffer); printf("DEBUG receive mega blocks from %d\n", source); /* * Update the file index for every mega block. */ core_vector_iterator_init(&vector_iterator, &mega_blocks); bucket = (int64_t*)core_vector_at(&concrete_actor->counts, stream_index); (*bucket) = 0; while (core_vector_iterator_has_next(&vector_iterator)) { core_vector_iterator_next(&vector_iterator, (void **)&mega_block); printf("SETTING setting file to %d for mega block\n", stream_index); biosal_mega_block_set_file(mega_block, stream_index); entries = biosal_mega_block_get_entries_from_start(mega_block); printf("Cataloging %d ENTRIES\n", (int)entries); (*bucket) = entries; biosal_mega_block_print(mega_block); } core_vector_iterator_destroy(&vector_iterator); vector_bucket = (struct core_vector *)core_map_add(&concrete_actor->mega_blocks, &stream_index); core_vector_init_copy(vector_bucket, &mega_blocks); core_vector_destroy(&mega_blocks); concrete_actor->counted++; printf("controller/%d received from stream/%d for file %s %" PRIu64 " entries (final) %d/%d\n", name, source, local_file, entries, concrete_actor->counted, (int)core_vector_size(&concrete_actor->files)); thorium_actor_send_reply_empty(actor, ACTION_INPUT_CLOSE); /* continue work here, tell supervisor about it */ if (concrete_actor->counted == core_vector_size(&concrete_actor->files)) { thorium_actor_send_to_self_empty(actor, ACTION_INPUT_CONTROLLER_SPAWN_READING_STREAMS); } } else if (tag == ACTION_INPUT_DISTRIBUTE) { core_timer_start(&concrete_actor->input_timer); core_timer_start(&concrete_actor->counting_timer); /* for each file, spawn a stream to count */ /* no files, return immediately */ if (core_vector_size(&concrete_actor->files) == 0) { printf("Error: no file to distribute...\n"); thorium_actor_send_reply_empty(actor, ACTION_INPUT_DISTRIBUTE_REPLY); return; } #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2 printf("DEBUG actor %d receives ACTION_INPUT_DISTRIBUTE\n", name); #endif #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2 printf("DEBUG send ACTION_INPUT_SPAWN to self\n"); #endif thorium_actor_send_to_self_empty(actor, ACTION_INPUT_SPAWN); #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2 printf("DEBUG resizing counts to %d\n", core_vector_size(&concrete_actor->files)); #endif core_vector_resize(&concrete_actor->counts, core_vector_size(&concrete_actor->files)); for (i = 0; i < core_vector_size(&concrete_actor->counts); i++) { bucket = (int64_t*)core_vector_at(&concrete_actor->counts, i); *bucket = 0; } } else if (tag == ACTION_INPUT_SPAWN && source == name) { #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2 printf("DEBUG ACTION_INPUT_SPAWN\n"); #endif script = SCRIPT_INPUT_STREAM; concrete_actor->state = BIOSAL_INPUT_CONTROLLER_STATE_SPAWN_STREAMS; /* the next file name to send is the current number of streams */ i = core_vector_size(&concrete_actor->counting_streams); destination_index = i % core_vector_size(&concrete_actor->spawners); destination = *(int *)core_vector_at(&concrete_actor->spawners, destination_index); thorium_message_init(message, ACTION_SPAWN, sizeof(script), &script); thorium_actor_send(actor, destination, message); bucket = core_vector_at(&concrete_actor->files, i); local_file = *(char **)core_vector_at(&concrete_actor->files, i); #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2 printf("DEBUG890 local_file %p bucket %p index %d\n", local_file, (void *)bucket, i); #endif #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG printf("DEBUG actor %d spawns a stream for file %d/%d via spawner %d\n", name, i, core_vector_size(&concrete_actor->files), destination); #endif /* also, spawn 4 stores on each node */ } else if (tag == ACTION_ASK_TO_STOP && ( source == thorium_actor_supervisor(actor) || source == thorium_actor_name(actor))) { #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2 #endif /* stop streams */ for (i = 0; i < core_vector_size(&concrete_actor->counting_streams); i++) { stream = *(int *)core_vector_at(&concrete_actor->counting_streams, i); thorium_actor_send_empty(actor, stream, ACTION_ASK_TO_STOP); } for (i = 0; i < core_vector_size(&concrete_actor->reading_streams); i++) { stream = *(int *)core_vector_at(&concrete_actor->reading_streams, i); thorium_actor_send_empty(actor, stream, ACTION_ASK_TO_STOP); } #if 0 /* stop data stores */ for (i = 0; i < core_vector_size(&concrete_actor->consumers); i++) { store = core_vector_at_as_int(&concrete_actor->consumers, i); thorium_actor_send_empty(actor, store, ACTION_ASK_TO_STOP); } #endif /* stop partitioner */ if (concrete_actor->partitioner != THORIUM_ACTOR_NOBODY) { thorium_actor_send_empty(actor, concrete_actor->partitioner, ACTION_ASK_TO_STOP); #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG printf("DEBUG controller %d sends ACTION_ASK_TO_STOP_REPLY to %d\n", thorium_actor_name(actor), thorium_message_source(message)); #endif } thorium_actor_send_reply_empty(actor, ACTION_ASK_TO_STOP_REPLY); /* stop self */ thorium_actor_send_to_self_empty(actor, ACTION_STOP); thorium_actor_ask_to_stop(actor, message); printf("DEBUG controller %d dies\n", name); #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG #endif } else if (tag == ACTION_INPUT_CONTROLLER_CREATE_PARTITION && source == name) { spawner = *(int *)core_vector_at(&concrete_actor->spawners, core_vector_size(&concrete_actor->spawners) / 2); thorium_actor_send_int(actor, spawner, ACTION_SPAWN, SCRIPT_SEQUENCE_PARTITIONER); concrete_actor->state = BIOSAL_INPUT_CONTROLLER_STATE_SPAWN_PARTITIONER; #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG printf("DEBUG input controller %d spawns a partitioner via spawner %d\n", name, spawner); #endif } else if (tag == ACTION_SEQUENCE_PARTITIONER_COMMAND_IS_READY) { #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG printf("DEBUG controller receives ACTION_SEQUENCE_PARTITIONER_COMMAND_IS_READY, asks for command\n"); #endif thorium_actor_send_reply_empty(actor, ACTION_SEQUENCE_PARTITIONER_GET_COMMAND); } else if (tag == ACTION_SEQUENCE_PARTITIONER_GET_COMMAND_REPLY) { biosal_input_controller_receive_command(actor, message); } else if (tag == ACTION_SEQUENCE_PARTITIONER_FINISHED) { thorium_actor_send_empty(actor, concrete_actor->partitioner, ACTION_ASK_TO_STOP); biosal_input_controller_verify_requests(actor, message); } else if (tag == ACTION_SEQUENCE_PARTITIONER_PROVIDE_STORE_ENTRY_COUNTS) { biosal_input_controller_receive_store_entry_counts(actor, message); } else if (tag == ACTION_RESERVE_REPLY) { concrete_actor->ready_consumers++; printf("DEBUG marker ACTION_RESERVE_REPLY %d/%d\n", concrete_actor->ready_consumers, (int)core_vector_size(&concrete_actor->consumers)); if (concrete_actor->ready_consumers == core_vector_size(&concrete_actor->consumers)) { concrete_actor->ready_consumers = 0; printf("DEBUG all consumers are ready\n"); thorium_actor_send_empty(actor, concrete_actor->partitioner, ACTION_SEQUENCE_PARTITIONER_PROVIDE_STORE_ENTRY_COUNTS_REPLY); } } else if (tag == ACTION_INPUT_PUSH_SEQUENCES_READY) { #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG printf("DEBUG biosal_input_controller_receive received ACTION_INPUT_PUSH_SEQUENCES_READY\n"); #endif stream_name = source; acquaintance_index = stream_name; stream_index = core_vector_index_of(&concrete_actor->reading_streams, &acquaintance_index); command_name = *(int *)core_vector_at(&concrete_actor->partition_commands, stream_index); thorium_actor_send_int(actor, concrete_actor->partitioner, ACTION_SEQUENCE_PARTITIONER_GET_COMMAND_REPLY_REPLY, command_name); } else if (tag == ACTION_INPUT_PUSH_SEQUENCES_REPLY) { stream_name = source; thorium_message_unpack_int(message, 0, &consumer); consumer_index = core_vector_index_of(&concrete_actor->consumers, &consumer); bucket_for_requests = (int *)core_vector_at(&concrete_actor->consumer_active_requests, consumer_index); (*bucket_for_requests)--; biosal_input_controller_verify_requests(actor, message); #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_CONSUMERS printf("DEBUG consumer # %d has %d active requests\n", consumer_index, *bucket_for_requests); #endif } else if (tag == ACTION_SET_CONSUMERS) { core_vector_init(&concrete_actor->consumers, 0); core_vector_unpack(&concrete_actor->consumers, buffer); printf("controller %d receives %d consumers\n", thorium_actor_name(actor), (int)core_vector_size(&concrete_actor->consumers)); for (i = 0; i < core_vector_size(&concrete_actor->consumers); i++) { core_vector_push_back_int(&concrete_actor->consumer_active_requests, 0); } #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG core_vector_print_int(&concrete_actor->consumers); printf("\n"); #endif thorium_actor_send_reply_empty(actor, ACTION_SET_CONSUMERS_REPLY); } else if (tag == ACTION_SET_BLOCK_SIZE) { thorium_message_unpack_int(message, 0, &concrete_actor->block_size); thorium_actor_send_reply_empty(actor, ACTION_SET_BLOCK_SIZE_REPLY); } else if (tag == ACTION_SEQUENCE_STORE_READY) { concrete_actor->filled_consumers++; #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG printf("DEBUG ACTION_SEQUENCE_STORE_READY %d/%d\n", concrete_actor->filled_consumers, (int)core_vector_size(&concrete_actor->consumers)); #endif if (concrete_actor->filled_consumers == core_vector_size(&concrete_actor->consumers)) { concrete_actor->filled_consumers = 0; printf("DEBUG: all consumers are filled, sending ACTION_INPUT_DISTRIBUTE_REPLY\n"); core_timer_stop(&concrete_actor->input_timer); core_timer_stop(&concrete_actor->distribution_timer); core_timer_print_with_description(&concrete_actor->distribution_timer, "Load input / Distribute input data"); core_timer_print_with_description(&concrete_actor->input_timer, "Load input"); thorium_actor_send_to_supervisor_empty(actor, ACTION_INPUT_DISTRIBUTE_REPLY); } } }
uint64_t core_fasta_input_get_sequence(struct biosal_input_format *input, char *sequence) { struct core_fasta_input *fasta; /* TODO use a dynamic buffer to accept long reads... */ int maximum_sequence_length = BIOSAL_INPUT_MAXIMUM_SEQUENCE_LENGTH; int value; int lines; int total; int position_in_sequence; int is_header; int block_length; fasta = (struct core_fasta_input *)biosal_input_format_implementation(input); if (fasta->buffer == NULL) { fasta->buffer = core_memory_allocate(maximum_sequence_length + 1, MEMORY_FASTA); fasta->next_header= core_memory_allocate(maximum_sequence_length + 1, MEMORY_FASTA); fasta->buffer[0] = '\0'; fasta->next_header[0] = '\0'; } value = 0; total = 0; lines = 0; /* * Read name */ if (fasta->has_header) { strcpy(fasta->buffer, fasta->next_header); value = strlen(fasta->buffer); fasta->has_header = 0; } else { value = core_buffered_reader_read_line(&fasta->reader, fasta->buffer, maximum_sequence_length); /* Make sure that this is an identifier. */ if (!fasta->has_first) { while (!core_fasta_input_check_header(input, fasta->buffer)) { value = core_buffered_reader_read_line(&fasta->reader, fasta->buffer, maximum_sequence_length); } fasta->has_first = 1; } } /* * Add new line. */ if (value) { ++lines; } total += value; /* * Read sequence. * * Discard any new line symbol too. */ position_in_sequence = 0; while (1) { value = core_buffered_reader_read_line(&fasta->reader, fasta->buffer, maximum_sequence_length); if (value == 0) { break; } is_header = 0; if (strlen(fasta->buffer) > 0 && fasta->buffer[0] == '>') { is_header = 1; } if (is_header) { sequence[position_in_sequence] = '\0'; strcpy(fasta->next_header, fasta->buffer); fasta->has_header = 1; break; } /* * Otherwise, add the sequence. */ if (value) { ++lines; } block_length = strlen(fasta->buffer); /* * Remove the new line. */ if (fasta->buffer[block_length - 1] == '\n') { --block_length; } core_memory_copy(sequence + position_in_sequence, fasta->buffer, block_length); position_in_sequence += block_length; } return total; }
void thorium_worker_pool_print_load(struct thorium_worker_pool *self, int type) { int count; int i; float epoch_load; struct thorium_worker *worker; float loop_load; uint64_t epoch_wake_up_count; uint64_t loop_wake_up_count; /* int scheduling_score; */ int node_name; char *buffer; char *buffer_for_wake_up_events; char *buffer_for_future_timeline; int allocated; int offset; int offset_for_wake_up; int offset_for_future; int extra; time_t current_time; int elapsed; float selected_load; uint64_t selected_wake_up_count; float sum; char loop[] = "COMPUTATION"; char epoch[] = "EPOCH"; char *description; float load; description = NULL; if (type == THORIUM_WORKER_POOL_LOAD_LOOP) { description = loop; } else if (type == THORIUM_WORKER_POOL_LOAD_EPOCH) { description = epoch; } else { return; } current_time = time(NULL); elapsed = current_time - self->starting_time; extra = 100; count = thorium_worker_pool_worker_count(self); allocated = count * 20 + 20 + extra; buffer = core_memory_allocate(allocated, MEMORY_WORKER_POOL_KEY); buffer_for_wake_up_events = core_memory_allocate(allocated, MEMORY_WORKER_POOL_KEY); buffer_for_future_timeline = core_memory_allocate(allocated, MEMORY_WORKER_POOL_KEY); node_name = thorium_node_name(self->node); offset = 0; offset_for_wake_up = 0; offset_for_future = 0; i = 0; sum = 0; while (i < count && offset + extra < allocated) { worker = thorium_worker_pool_get_worker(self, i); epoch_load = thorium_worker_get_epoch_load(worker); loop_load = thorium_worker_get_loop_load(worker); epoch_wake_up_count = thorium_worker_get_epoch_wake_up_count(worker); loop_wake_up_count = thorium_worker_get_loop_wake_up_count(worker); selected_load = epoch_load; selected_wake_up_count = epoch_wake_up_count; if (type == THORIUM_WORKER_POOL_LOAD_EPOCH) { selected_load = epoch_load; selected_wake_up_count = epoch_wake_up_count; } else if (type == THORIUM_WORKER_POOL_LOAD_LOOP) { selected_load = loop_load; selected_wake_up_count = loop_wake_up_count; } /* offset += sprintf(buffer + offset, " [%d %d %.2f]", i, scheduling_score, selected_load); */ offset += sprintf(buffer + offset, " %.2f", selected_load); offset_for_wake_up += sprintf(buffer_for_wake_up_events + offset_for_wake_up, " %" PRIu64 "", selected_wake_up_count); offset_for_future += sprintf(buffer_for_future_timeline + offset_for_future, " %d", thorium_worker_get_scheduled_actor_count(worker)); sum += selected_load; ++i; } load = sum / count; printf("thorium_worker_pool: node/%d %s LOAD %d s %.2f/%d (%.2f)%s\n", node_name, description, elapsed, sum, count, load, buffer); printf("thorium_worker_pool: node/%d %s FUTURE_TIMELINE %d s %s\n", node_name, description, elapsed, buffer_for_future_timeline); printf("thorium_worker_pool: node/%d %s WAKE_UP_COUNT %d s %s\n", node_name, description, elapsed, buffer_for_wake_up_events); core_memory_free(buffer, MEMORY_WORKER_POOL_KEY); core_memory_free(buffer_for_wake_up_events, MEMORY_WORKER_POOL_KEY); core_memory_free(buffer_for_future_timeline, MEMORY_WORKER_POOL_KEY); }
void biosal_input_stream_receive(struct thorium_actor *actor, struct thorium_message *message) { int tag; int source; uint64_t count; struct biosal_input_stream *concrete_self; int i; int has_sequence; int sequences; int sequence_index; int buffer_size; char *buffer; char *read_buffer; struct biosal_mega_block mega_block; char *file_name_in_buffer; if (thorium_actor_take_action(actor, message)) { return; } concrete_self = thorium_actor_concrete_actor(actor); tag = thorium_message_action(message); source = thorium_message_source(message); buffer = (char *)thorium_message_buffer(message); /* Do nothing if there is an error. * has_error returns the error to the source. */ /* if (biosal_input_stream_has_error(actor, message)) { return; } */ if (tag == ACTION_INPUT_OPEN) { #ifdef BIOSAL_INPUT_STREAM_DEBUG printf("DEBUG ACTION_INPUT_OPEN\n"); #endif if (concrete_self->open) { concrete_self->error = BIOSAL_INPUT_ERROR_ALREADY_OPEN; thorium_actor_send_reply_int(actor, ACTION_INPUT_OPEN_REPLY, concrete_self->error); thorium_actor_send_to_self_empty(actor, ACTION_ASK_TO_STOP); return; } concrete_self->open = 1; /* TODO: find out the maximum read length in some way */ concrete_self->maximum_sequence_length = BIOSAL_INPUT_MAXIMUM_SEQUENCE_LENGTH; concrete_self->buffer_for_sequence = (char *)core_memory_allocate(concrete_self->maximum_sequence_length, MEMORY_INPUT_STREAM); /*biosal_input_stream_init(actor);*/ #ifdef BIOSAL_INPUT_STREAM_DEBUG printf("DEBUG biosal_input_stream_receive open %s\n", buffer); #endif /*core_memory_copy(&concrete_self->file_index, buffer, sizeof(concrete_self->file_index));*/ file_name_in_buffer = buffer; printf("stream/%d (node/%d) opens file %s offset %" PRIu64 "\n", thorium_actor_name(actor), thorium_actor_node_name(actor), file_name_in_buffer, concrete_self->starting_offset); #ifdef DEBUG_ISSUE_594 thorium_message_print(message); printf("Buffer %s\n", buffer); #endif concrete_self->file_name = core_memory_allocate(strlen(file_name_in_buffer) + 1, MEMORY_INPUT_STREAM); strcpy(concrete_self->file_name, file_name_in_buffer); biosal_input_proxy_init(&concrete_self->proxy, concrete_self->file_name, concrete_self->starting_offset, concrete_self->ending_offset); concrete_self->proxy_ready = 1; /* Die if there is an error... */ if (biosal_input_stream_has_error(actor, message)) { #ifdef BIOSAL_INPUT_STREAM_DEBUG printf("DEBUG has error\n"); #endif thorium_actor_send_reply_int(actor, ACTION_INPUT_OPEN_REPLY, concrete_self->error); thorium_actor_send_to_self_empty(actor, ACTION_ASK_TO_STOP); return; } concrete_self->controller = source; /* no error here... */ thorium_actor_send_reply_int(actor, ACTION_INPUT_OPEN_REPLY, concrete_self->error); } else if (tag == ACTION_INPUT_COUNT) { /* count a little bit and yield the worker */ if (concrete_self->count_customer == THORIUM_ACTOR_NOBODY) { concrete_self->count_customer = source; } if (biosal_input_stream_check_open_error(actor, message)) { thorium_actor_send_reply_int64_t(actor, ACTION_INPUT_COUNT_REPLY, concrete_self->error); thorium_actor_send_to_self_empty(actor, ACTION_ASK_TO_STOP); return; } #ifdef BIOSAL_INPUT_STREAM_DEBUG printf("DEBUG ACTION_INPUT_COUNT received...\n"); #endif i = 0; /* continue counting ... */ has_sequence = 1; while (i < concrete_self->granularity && has_sequence) { has_sequence = biosal_input_proxy_get_sequence(&concrete_self->proxy, concrete_self->buffer_for_sequence); #if 0 printf("Sequence= %s\n", concrete_self->buffer_for_sequence); #endif i++; } sequences = biosal_input_proxy_size(&concrete_self->proxy); #ifdef BIOSAL_INPUT_STREAM_DEBUG printf("DEBUG ACTION_INPUT_COUNT sequences %d...\n", sequences); #endif if (!has_sequence || sequences % concrete_self->mega_block_size == 0) { biosal_mega_block_init(&mega_block, -1, concrete_self->last_offset, sequences - concrete_self->last_entries, sequences); core_vector_push_back(&concrete_self->mega_blocks, &mega_block); concrete_self->last_entries = sequences; concrete_self->last_offset = biosal_input_proxy_offset(&concrete_self->proxy); thorium_actor_send_int64_t(actor, concrete_self->controller, ACTION_INPUT_COUNT_PROGRESS, sequences); } if (has_sequence) { /*printf("DEBUG yield\n");*/ thorium_actor_send_to_self_empty(actor, ACTION_YIELD); /* notify the controller of our progress... */ } else { thorium_actor_send_to_self_empty(actor, ACTION_INPUT_COUNT_READY); } } else if (tag == ACTION_YIELD_REPLY) { if (biosal_input_stream_check_open_error(actor, message)) { /* * it is not clear that there can be an error when receiving YIELD. error = concrete_self->error; thorium_actor_send_reply_int(actor, ACTION_INPUT_COUNT_REPLY, error); thorium_actor_send_to_self(actor, ACTION_ASK_TO_STOP); */ return; } thorium_actor_send_to_self_empty(actor, ACTION_INPUT_COUNT); } else if (tag == ACTION_INPUT_COUNT_READY) { if (biosal_input_stream_check_open_error(actor, message)) { return; } count = biosal_input_proxy_size(&concrete_self->proxy); thorium_actor_send_vector(actor, concrete_self->count_customer, ACTION_INPUT_COUNT_REPLY, &concrete_self->mega_blocks); printf("input_stream/%d on node/%d counted entries in %s, %" PRIu64 "\n", thorium_actor_name(actor), thorium_actor_node_name(actor), concrete_self->file_name, count); } else if (tag == ACTION_INPUT_CLOSE) { #ifdef BIOSAL_INPUT_STREAM_DEBUG printf("DEBUG destroy proxy\n"); #endif concrete_self->error = BIOSAL_INPUT_ERROR_NO_ERROR; if (biosal_input_stream_check_open_error(actor, message)) { concrete_self->error = BIOSAL_INPUT_ERROR_FILE_NOT_OPEN; thorium_message_init(message, ACTION_INPUT_CLOSE_REPLY, sizeof(concrete_self->error), &concrete_self->error); thorium_actor_send(actor, source, message); thorium_actor_send_to_self_empty(actor, ACTION_ASK_TO_STOP); return; } thorium_message_init(message, ACTION_INPUT_CLOSE_REPLY, sizeof(concrete_self->error), &concrete_self->error); thorium_actor_send(actor, source, message); thorium_actor_send_to_self_empty(actor, ACTION_ASK_TO_STOP); #ifdef BIOSAL_INPUT_STREAM_DEBUG printf("actor %d sending ACTION_INPUT_CLOSE_REPLY to %d\n", thorium_actor_name(actor), source); #endif } else if (tag == ACTION_INPUT_GET_SEQUENCE) { if (biosal_input_stream_check_open_error(actor, message)) { /* the error management could be better. */ concrete_self->error = BIOSAL_INPUT_ERROR_FILE_NOT_OPEN; thorium_message_init(message, ACTION_INPUT_GET_SEQUENCE_REPLY, sizeof(concrete_self->error), &concrete_self->error); thorium_actor_send(actor, source, message); return; } sequence_index = biosal_input_proxy_size(&concrete_self->proxy); /* TODO it would be clearer to use a struct to pack a int and a char [] * then to use the code below. */ read_buffer = concrete_self->buffer_for_sequence + sizeof(sequence_index); has_sequence = biosal_input_proxy_get_sequence(&concrete_self->proxy, read_buffer); if (!has_sequence) { thorium_message_init(message, ACTION_INPUT_GET_SEQUENCE_END, 0, NULL); thorium_actor_send(actor, source, message); return; } buffer_size = sizeof(sequence_index) + strlen(read_buffer) + 1; core_memory_copy(concrete_self->buffer_for_sequence, &sequence_index, sizeof(sequence_index)); thorium_message_init(message, ACTION_INPUT_GET_SEQUENCE_REPLY, buffer_size, concrete_self->buffer_for_sequence); thorium_actor_send(actor, source, message); } else if (tag == ACTION_INPUT_PUSH_SEQUENCES) { biosal_input_stream_push_sequences(actor, message); } else if (tag == ACTION_ASK_TO_STOP) { thorium_actor_send_to_self_empty(actor, ACTION_STOP); thorium_actor_send_range_empty(actor, &concrete_self->parallel_streams, ACTION_ASK_TO_STOP); thorium_actor_send_reply_empty(actor, ACTION_ASK_TO_STOP_REPLY); } else if (tag == ACTION_INPUT_STREAM_RESET) { /* fail silently */ if (!concrete_self->open) { thorium_actor_send_reply_empty(actor, ACTION_INPUT_STREAM_RESET_REPLY); return; } #ifdef BIOSAL_INPUT_STREAM_DEBUG printf("DEBUG ACTION_INPUT_STREAM_RESET\n"); #endif biosal_input_proxy_destroy(&concrete_self->proxy); biosal_input_proxy_init(&concrete_self->proxy, concrete_self->file_name, concrete_self->starting_offset, concrete_self->ending_offset); thorium_actor_send_reply_empty(actor, ACTION_INPUT_STREAM_RESET_REPLY); } else if (tag == ACTION_PUSH_SEQUENCE_DATA_BLOCK_REPLY) { thorium_actor_send_to_supervisor_int(actor, ACTION_INPUT_PUSH_SEQUENCES_REPLY, source); } }
uint64_t core_fastq_input_get_sequence(struct biosal_input_format *input, char *sequence) { struct core_fastq_input *fastq; /* * Input sequence has at least BIOSAL_INPUT_MAXIMUM_SEQUENCE_LENGTH * which is currently 512k */ /* TODO use a dynamic buffer to accept long reads... */ int maximum_sequence_length = BIOSAL_INPUT_MAXIMUM_SEQUENCE_LENGTH; int value; int length; fastq = (struct core_fastq_input *)biosal_input_format_implementation(input); if (fastq->buffer == NULL) { fastq->buffer = (char *)core_memory_allocate(maximum_sequence_length + 1, MEMORY_FASTQ); } value = 0; /* * Read name */ value += core_buffered_reader_read_line(&fastq->reader, fastq->buffer, maximum_sequence_length); #ifdef FIND_IDENTIFIER /* * If we do not have the first entry yet, * make sure that the line is a good line. */ if (!fastq->has_first) { while (!core_fastq_input_is_identifier(input, fastq->buffer)) { value += core_buffered_reader_read_line(&fastq->reader, fastq->buffer, maximum_sequence_length); } fastq->has_first = 1; } #endif /* * Read DNA sequence */ length = core_buffered_reader_read_line(&fastq->reader, sequence, maximum_sequence_length); #ifdef BIOSAL_FASTQ_INPUT_DEBUG_READ_LINE printf("FASTQ ReadLine <<%s>>\n", sequence); #endif if (sequence[length - 1] == '\n') { /* * Remove new line symbol. */ sequence[length - 1] = '\0'; } value += length; #ifdef BIOSAL_FASTQ_INPUT_DEBUG2 printf("DEBUG core_fastq_input_get_sequence %s\n", buffer); #endif /* * Read the + symbol */ value += core_buffered_reader_read_line(&fastq->reader, fastq->buffer, maximum_sequence_length); /* * Read quality string. */ value += core_buffered_reader_read_line(&fastq->reader, fastq->buffer, maximum_sequence_length); return value; }
void thorium_transport_init(struct thorium_transport *self, struct thorium_node *node, int *argc, char ***argv, struct core_memory_pool *inbound_message_memory_pool, struct core_memory_pool *outbound_message_memory_pool) { int actual_argc; char **actual_argv; self->active_request_count = 0; actual_argc = *argc; actual_argv = *argv; self->flags = 0; core_bitmap_clear_bit_uint32_t(&self->flags, FLAG_PROFILE); core_bitmap_clear_bit_uint32_t(&self->flags, FLAG_PRINT_TRANSPORT_EVENTS); self->transport_interface = NULL; self->concrete_transport = NULL; /* printf("DEBUG Initiating transport\n"); */ /* Select the transport layer */ /* * Assign functions */ thorium_transport_select_implementation(self, actual_argc, actual_argv); self->node = node; self->rank = -1; self->size = -1; if (self->transport_interface != NULL) { self->concrete_transport = core_memory_allocate(self->transport_interface->size, MEMORY_TRANSPORT); self->transport_interface->init(self, argc, argv); } CORE_DEBUGGER_ASSERT(self->rank >= 0); CORE_DEBUGGER_ASSERT(self->size >= 1); CORE_DEBUGGER_ASSERT(self->node != NULL); self->inbound_message_memory_pool = inbound_message_memory_pool; self->outbound_message_memory_pool = outbound_message_memory_pool; thorium_transport_profiler_init(&self->transport_profiler); if (core_command_has_argument(actual_argc, actual_argv, "-enable-transport-profiler")) { printf("Enable transport profiler\n"); core_bitmap_set_bit_uint32_t(&self->flags, FLAG_PROFILE); } if (self->rank == 0) { printf("thorium_transport: type %s\n", self->transport_interface->name); } if (core_command_has_argument(actual_argc, actual_argv, "-print-transport-events")) { core_bitmap_set_bit_uint32_t(&self->flags, FLAG_PRINT_TRANSPORT_EVENTS); } core_timer_init(&self->timer); self->start_time = core_timer_get_nanoseconds(&self->timer); }
int main(int argc, char **argv) { BEGIN_TESTS(); { struct core_map big_map; int kmer_length = 43; struct biosal_dna_kmer kmer; int count; int run_test; int coverage; void *key; int key_length; int *bucket; int i; struct biosal_dna_codec codec; struct core_memory_pool memory; core_memory_pool_init(&memory, 1048576, -1); biosal_dna_codec_init(&codec); run_test = 1; count = 100000000; printf("STRESS TEST\n"); biosal_dna_kmer_init_mock(&kmer, kmer_length, &codec, &memory); key_length = biosal_dna_kmer_pack_size(&kmer, kmer_length, &codec); biosal_dna_kmer_destroy(&kmer, &memory); core_map_init(&big_map, key_length, sizeof(coverage)); key = core_memory_allocate(key_length, -1); i = 0; while (i < count && run_test) { biosal_dna_kmer_init_random(&kmer, kmer_length, &codec, &memory); biosal_dna_kmer_pack_store_key(&kmer, key, kmer_length, &codec, &memory); bucket = core_map_add(&big_map, key); coverage = 99; (*bucket) = coverage; biosal_dna_kmer_destroy(&kmer, &memory); if (i % 100000 == 0) { printf("ADD %d/%d %" PRIu64 "\n", i, count, core_map_size(&big_map)); } i++; } core_map_destroy(&big_map); core_memory_free(key, -1); biosal_dna_codec_destroy(&codec); core_memory_pool_destroy(&memory); } END_TESTS(); return 0; }