void test_allocator(struct core_memory_pool *memory) { int i; int size; void *pointer; struct core_vector vector; struct core_timer timer; uint64_t elapsed; i = 1000000; size = 45; core_vector_init(&vector, sizeof(void *)); core_timer_init(&timer); core_timer_start(&timer); while (i--) { if (memory != NULL) { pointer = core_memory_pool_allocate(memory, size); } else { pointer = core_memory_allocate(size, -1); } core_vector_push_back(&vector, &pointer); } core_timer_stop(&timer); elapsed = core_timer_get_elapsed_nanoseconds(&timer); if (memory == NULL) { printf("Not using memory pool... "); } else { printf("Using memory pool... "); } printf("Elapsed : %" PRIu64 " milliseconds\n", elapsed / 1000 / 1000); size = core_vector_size(&vector); for (i = 0; i < size; ++i) { pointer = core_vector_at_as_void_pointer(&vector, i); if (memory != NULL) { core_memory_pool_free(memory, pointer); } else { core_memory_free(pointer, -1); } } core_vector_destroy(&vector); core_timer_destroy(&timer); }
void spate_ask_to_stop(struct thorium_actor *self, struct thorium_message *message) { int source; struct spate *concrete_self; concrete_self = (struct spate *)thorium_actor_concrete_actor(self); source = thorium_message_source(message); if (!spate_must_print_help(self)) { thorium_actor_log(self, "spate %d stops", thorium_actor_name(self)); } #if 0 thorium_actor_ask_to_stop(self, message); #endif /* * Check if the source is an initial actor because each initial actor * is its own supervisor. */ if (core_vector_index_of(&concrete_self->initial_actors, &source) >= 0) { thorium_actor_send_to_self_empty(self, ACTION_STOP); } if (concrete_self->is_leader) { if (concrete_self->assembly_graph_builder != THORIUM_ACTOR_NOBODY) thorium_actor_send_empty(self, concrete_self->assembly_graph_builder, ACTION_ASK_TO_STOP); if (concrete_self->assembly_graph_builder != THORIUM_ACTOR_NOBODY) thorium_actor_send_empty(self, concrete_self->manager_for_sequence_stores, ACTION_ASK_TO_STOP); if (!spate_must_print_help(self)) { core_timer_stop(&concrete_self->timer); core_timer_print_with_description(&concrete_self->timer, "Total"); } } }
void biosal_input_controller_create_stores(struct thorium_actor *actor, struct thorium_message *message) { int tag; int source; void *buffer; int count; int i; struct biosal_input_controller *concrete_actor; int value; int spawner; uint64_t total; int block_size; int blocks; uint64_t entries; char *local_file; int name; concrete_actor = (struct biosal_input_controller *)thorium_actor_concrete_actor(actor); thorium_message_get_all(message, &tag, &count, &buffer, &source); /* printf("DEBUG biosal_input_controller_create_stores\n"); */ for (i = 0; i < core_vector_size(&concrete_actor->stores_per_spawner); i++) { value = core_vector_at_as_int(&concrete_actor->stores_per_spawner, i); if (value == -1) { /* printf("DEBUG need more information about spawner at %i\n", i); */ spawner = core_vector_at_as_int(&concrete_actor->spawners, i); thorium_actor_send_empty(actor, spawner, ACTION_GET_NODE_NAME); return; } } concrete_actor->state = BIOSAL_INPUT_CONTROLLER_STATE_SPAWN_STORES; /* at this point, we know the worker count of every node corresponding * to each spawner */ for (i = 0; i < core_vector_size(&concrete_actor->stores_per_spawner); i++) { /* printf("DEBUG polling spawner %i/%d\n", i, core_vector_size(&concrete_actor->stores_per_spawner)); */ value = core_vector_at_as_int(&concrete_actor->stores_per_spawner, i); if (value != 0) { spawner = core_vector_at_as_int(&concrete_actor->spawners, i); /* printf("DEBUG spawner %d is %d\n", i, spawner); */ thorium_actor_send_int(actor, spawner, ACTION_SPAWN, SCRIPT_SEQUENCE_STORE); return; } /* printf("DEBUG spawner %i spawned all its stores\n", i); */ } printf("DEBUG controller %d: consumers are ready (%d)\n", thorium_actor_name(actor), (int)core_vector_size(&concrete_actor->consumers)); for (i = 0; i < core_vector_size(&concrete_actor->consumers); i++) { value = core_vector_at_as_int(&concrete_actor->consumers, i); printf("DEBUG controller %d: consumer %i is %d\n", thorium_actor_name(actor), i, value); } printf("DEBUG controller %d: streams are\n", thorium_actor_name(actor)); total = 0; block_size = concrete_actor->block_size; for (i = 0; i < core_vector_size(&concrete_actor->files); i++) { entries = *(uint64_t*)core_vector_at(&concrete_actor->counts, i); local_file = core_vector_at_as_char_pointer(&concrete_actor->files, i); name = *(int *)core_vector_at(&concrete_actor->counting_streams, i); printf("stream %d, %d/%d %s %" PRIu64 "\n", name, i, (int)core_vector_size(&concrete_actor->files), local_file, entries); total += entries; } blocks = total / block_size; if (total % block_size != 0) { blocks++; } core_timer_stop(&concrete_actor->counting_timer); core_timer_start(&concrete_actor->distribution_timer); core_timer_print_with_description(&concrete_actor->counting_timer, "Load input / Count input data"); printf("DEBUG controller %d: Partition Total: %" PRIu64 ", block_size: %d, blocks: %d\n", thorium_actor_name(actor), total, block_size, blocks); #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_10355 printf("DEBUG send ACTION_INPUT_CONTROLLER_CREATE_STORES to self %d\n", thorium_actor_name(actor)); #endif #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG printf("DEBUG biosal_input_controller_create_stores send ACTION_INPUT_CONTROLLER_CREATE_PARTITION\n"); #endif /* no sequences at all ! */ if (total == 0) { printf("Error, total is 0, can not distribute\n"); thorium_actor_send_to_supervisor_empty(actor, ACTION_INPUT_DISTRIBUTE_REPLY); return; } else { thorium_actor_send_to_self_empty(actor, ACTION_INPUT_CONTROLLER_CREATE_PARTITION); } /* thorium_actor_send_to_self_empty(actor, ACTION_STOP); */ }
void biosal_input_controller_receive(struct thorium_actor *actor, struct thorium_message *message) { int tag; int count; char *file; void *buffer; struct biosal_input_controller *controller; struct biosal_input_controller *concrete_actor; int destination; int script; int stream; char *local_file; int i; int name; int source; int destination_index; struct thorium_message new_message; int error; int stream_index; int64_t entries; int64_t *bucket; int *int_bucket; int spawner; int command_name; int stream_name; int consumer; int consumer_index; int *bucket_for_requests; char *new_buffer; int new_count; int file_index; struct core_vector mega_blocks; struct core_vector_iterator vector_iterator; struct biosal_mega_block *mega_block; struct core_vector *vector_bucket; struct core_vector block_counts; uint64_t block_entries; int mega_block_index; uint64_t offset; struct biosal_mega_block *block; int acquaintance_index; if (thorium_actor_take_action(actor, message)) { return; } thorium_message_get_all(message, &tag, &count, &buffer, &source); name = thorium_actor_name(actor); controller = (struct biosal_input_controller *)thorium_actor_concrete_actor(actor); concrete_actor = controller; if (tag == ACTION_START) { core_vector_init(&concrete_actor->spawners, 0); core_vector_unpack(&concrete_actor->spawners, buffer); core_vector_resize(&concrete_actor->stores_per_spawner, core_vector_size(&concrete_actor->spawners)); for (i = 0; i < core_vector_size(&concrete_actor->spawners); i++) { int_bucket = (int *)core_vector_at(&concrete_actor->stores_per_spawner, i); *int_bucket = 0; spawner = core_vector_at_as_int(&concrete_actor->spawners, i); core_queue_enqueue(&concrete_actor->unprepared_spawners, &spawner); } concrete_actor->state = BIOSAL_INPUT_CONTROLLER_STATE_PREPARE_SPAWNERS; #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL printf("DEBUG preparing first spawner\n"); #endif thorium_actor_send_to_self_empty(actor, ACTION_INPUT_CONTROLLER_PREPARE_SPAWNERS); /* thorium_dispatcher_print(thorium_actor_dispatcher(actor)); */ } else if (tag == ACTION_ADD_FILE) { file = (char *)buffer; local_file = core_memory_allocate(strlen(file) + 1, MEMORY_CONTROLLER); strcpy(local_file, file); printf("controller %d ACTION_ADD_FILE %s\n", thorium_actor_name(actor), local_file); core_vector_push_back(&concrete_actor->files, &local_file); bucket = core_vector_at(&concrete_actor->files, core_vector_size(&concrete_actor->files) - 1); local_file = *(char **)bucket; #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2 printf("DEBUG11 ACTION_ADD_FILE %s %p bucket %p index %d\n", local_file, local_file, (void *)bucket, core_vector_size(&concrete_actor->files) - 1); #endif thorium_actor_send_reply_empty(actor, ACTION_ADD_FILE_REPLY); } else if (tag == ACTION_SPAWN_REPLY) { if (concrete_actor->state == BIOSAL_INPUT_CONTROLLER_STATE_SPAWN_STORES) { biosal_input_controller_add_store(actor, message); return; } else if (concrete_actor->state == BIOSAL_INPUT_CONTROLLER_STATE_PREPARE_SPAWNERS) { concrete_actor->ready_spawners++; thorium_message_unpack_int(message, 0, &name); thorium_actor_send_empty(actor, name, ACTION_ASK_TO_STOP); thorium_actor_send_to_self_empty(actor, ACTION_INPUT_CONTROLLER_PREPARE_SPAWNERS); if (concrete_actor->ready_spawners == (int)core_vector_size(&concrete_actor->spawners)) { #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG printf("DEBUG all spawners are prepared\n"); #endif thorium_actor_send_to_supervisor_empty(actor, ACTION_START_REPLY); } return; } else if (concrete_actor->state == BIOSAL_INPUT_CONTROLLER_STATE_SPAWN_PARTITIONER) { #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG printf("DEBUG received spawn reply, state is spawn_partitioner\n"); #endif thorium_message_unpack_int(message, 0, &concrete_actor->partitioner); /* configure the partitioner */ destination = concrete_actor->partitioner; thorium_actor_send_int(actor, destination, ACTION_SEQUENCE_PARTITIONER_SET_BLOCK_SIZE, concrete_actor->block_size); thorium_actor_send_int(actor, destination, ACTION_SEQUENCE_PARTITIONER_SET_ACTOR_COUNT, core_vector_size(&concrete_actor->consumers)); core_vector_init(&block_counts, sizeof(uint64_t)); for (i = 0; i < core_vector_size(&concrete_actor->mega_block_vector); i++) { block = (struct biosal_mega_block *)core_vector_at(&concrete_actor->mega_block_vector, i); block_entries = biosal_mega_block_get_entries(block); core_vector_push_back_uint64_t(&block_counts, block_entries); } new_count = core_vector_pack_size(&block_counts); new_buffer = thorium_actor_allocate(actor, new_count); #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG printf("DEBUG packed counts, %d bytes\n", count); #endif core_vector_pack(&block_counts, new_buffer); thorium_message_init(&new_message, ACTION_SEQUENCE_PARTITIONER_SET_ENTRY_VECTOR, new_count, new_buffer); thorium_actor_send(actor, destination, &new_message); core_vector_destroy(&block_counts); return; } else if (concrete_actor->state == BIOSAL_INPUT_CONTROLLER_STATE_SPAWN_READING_STREAMS) { thorium_message_unpack_int(message, 0, &stream); stream_index = stream; mega_block_index = core_vector_size(&concrete_actor->reading_streams); core_vector_push_back_int(&concrete_actor->reading_streams, stream_index); core_vector_push_back_int(&concrete_actor->partition_commands, -1); core_vector_push_back_int(&concrete_actor->stream_consumers, -1); stream_index = core_vector_size(&concrete_actor->reading_streams) - 1; mega_block = (struct biosal_mega_block *)core_vector_at(&concrete_actor->mega_block_vector, mega_block_index); offset = biosal_mega_block_get_offset(mega_block); core_map_add_value(&concrete_actor->assigned_blocks, &stream_index, &mega_block_index); #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_READING_STREAMS printf("DEBUG setting offset to %" PRIu64 " for stream/%d\n", offset, stream); #endif thorium_actor_send_uint64_t(actor, stream, ACTION_INPUT_STREAM_SET_START_OFFSET, offset); return; } stream = *(int *)buffer; file_index = core_vector_size(&concrete_actor->counting_streams); local_file = *(char **)core_vector_at(&concrete_actor->files, file_index); #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_READING_STREAMS printf("DEBUG actor %d receives stream %d from spawner %d for file %s\n", name, stream, source, local_file); #endif core_vector_push_back(&concrete_actor->counting_streams, &stream); #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_READING_STREAMS printf("asking stream/%d to open %s\n", stream, local_file); #endif thorium_message_init(&new_message, ACTION_INPUT_OPEN, strlen(local_file) + 1, local_file); #ifdef DEBUG_ISSUE_594 thorium_message_print(&new_message); printf("SEND Buffer %s\n", local_file); #endif thorium_actor_send(actor, stream, &new_message); thorium_message_destroy(&new_message); if (core_vector_size(&concrete_actor->counting_streams) != core_vector_size(&concrete_actor->files)) { thorium_actor_send_to_self_empty(actor, ACTION_INPUT_SPAWN); } #ifdef DEBUG_ISSUE_594 printf("EXIT Buffer %s\n", local_file); #endif } else if (tag == ACTION_INPUT_OPEN_REPLY) { if (concrete_actor->state == BIOSAL_INPUT_CONTROLLER_STATE_SPAWN_READING_STREAMS) { #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_READING_STREAMS printf("DEBUG receives open.reply for reading stream/%d\n", source); #endif concrete_actor->opened_streams++; if (concrete_actor->opened_streams == core_vector_size(&concrete_actor->mega_block_vector)) { thorium_actor_send_to_self_empty(actor, ACTION_INPUT_CONTROLLER_CREATE_STORES); } return; } concrete_actor->opened_streams++; stream = source; thorium_message_unpack_int(message, 0, &error); if (error == BIOSAL_INPUT_ERROR_NO_ERROR) { #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2 printf("DEBUG actor %d asks %d ACTION_INPUT_COUNT_IN_PARALLEL\n", name, stream); #endif thorium_actor_send_vector(actor, stream, ACTION_INPUT_COUNT_IN_PARALLEL, &concrete_actor->spawners); } else { #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2 printf("DEBUG actor %d received error %d from %d\n", name, error, stream); #endif concrete_actor->counted++; } /* if all streams failed, notice supervisor */ if (concrete_actor->counted == core_vector_size(&concrete_actor->files)) { #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2 #endif printf("DEBUG %d: Error all streams failed.\n", thorium_actor_name(actor)); thorium_actor_send_to_supervisor_empty(actor, ACTION_INPUT_DISTRIBUTE_REPLY); } /* if (concrete_actor->opened_streams == core_vector_size(&concrete_actor->files)) { #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG printf("DEBUG controller %d sends ACTION_INPUT_DISTRIBUTE_REPLY to supervisor %d [%d/%d]\n", name, thorium_actor_supervisor(actor), concrete_actor->opened_streams, core_vector_size(&concrete_actor->files)); #endif } */ } else if (tag == ACTION_INPUT_COUNT_PROGRESS) { stream_index = core_vector_index_of(&concrete_actor->counting_streams, &source); local_file = core_vector_at_as_char_pointer(&concrete_actor->files, stream_index); thorium_message_unpack_int64_t(message, 0, &entries); bucket = (int64_t *)core_vector_at(&concrete_actor->counts, stream_index); printf("controller/%d receives progress from stream/%d file %s %" PRIu64 " entries so far\n", name, source, local_file, entries); *bucket = entries; } else if (tag == ACTION_INPUT_COUNT_IN_PARALLEL_REPLY) { stream_index = core_vector_index_of(&concrete_actor->counting_streams, &source); local_file = core_vector_at_as_char_pointer(&concrete_actor->files, stream_index); core_vector_init(&mega_blocks, 0); core_vector_unpack(&mega_blocks, buffer); printf("DEBUG receive mega blocks from %d\n", source); /* * Update the file index for every mega block. */ core_vector_iterator_init(&vector_iterator, &mega_blocks); bucket = (int64_t*)core_vector_at(&concrete_actor->counts, stream_index); (*bucket) = 0; while (core_vector_iterator_has_next(&vector_iterator)) { core_vector_iterator_next(&vector_iterator, (void **)&mega_block); printf("SETTING setting file to %d for mega block\n", stream_index); biosal_mega_block_set_file(mega_block, stream_index); entries = biosal_mega_block_get_entries_from_start(mega_block); printf("Cataloging %d ENTRIES\n", (int)entries); (*bucket) = entries; biosal_mega_block_print(mega_block); } core_vector_iterator_destroy(&vector_iterator); vector_bucket = (struct core_vector *)core_map_add(&concrete_actor->mega_blocks, &stream_index); core_vector_init_copy(vector_bucket, &mega_blocks); core_vector_destroy(&mega_blocks); concrete_actor->counted++; printf("controller/%d received from stream/%d for file %s %" PRIu64 " entries (final) %d/%d\n", name, source, local_file, entries, concrete_actor->counted, (int)core_vector_size(&concrete_actor->files)); thorium_actor_send_reply_empty(actor, ACTION_INPUT_CLOSE); /* continue work here, tell supervisor about it */ if (concrete_actor->counted == core_vector_size(&concrete_actor->files)) { thorium_actor_send_to_self_empty(actor, ACTION_INPUT_CONTROLLER_SPAWN_READING_STREAMS); } } else if (tag == ACTION_INPUT_DISTRIBUTE) { core_timer_start(&concrete_actor->input_timer); core_timer_start(&concrete_actor->counting_timer); /* for each file, spawn a stream to count */ /* no files, return immediately */ if (core_vector_size(&concrete_actor->files) == 0) { printf("Error: no file to distribute...\n"); thorium_actor_send_reply_empty(actor, ACTION_INPUT_DISTRIBUTE_REPLY); return; } #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2 printf("DEBUG actor %d receives ACTION_INPUT_DISTRIBUTE\n", name); #endif #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2 printf("DEBUG send ACTION_INPUT_SPAWN to self\n"); #endif thorium_actor_send_to_self_empty(actor, ACTION_INPUT_SPAWN); #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2 printf("DEBUG resizing counts to %d\n", core_vector_size(&concrete_actor->files)); #endif core_vector_resize(&concrete_actor->counts, core_vector_size(&concrete_actor->files)); for (i = 0; i < core_vector_size(&concrete_actor->counts); i++) { bucket = (int64_t*)core_vector_at(&concrete_actor->counts, i); *bucket = 0; } } else if (tag == ACTION_INPUT_SPAWN && source == name) { #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2 printf("DEBUG ACTION_INPUT_SPAWN\n"); #endif script = SCRIPT_INPUT_STREAM; concrete_actor->state = BIOSAL_INPUT_CONTROLLER_STATE_SPAWN_STREAMS; /* the next file name to send is the current number of streams */ i = core_vector_size(&concrete_actor->counting_streams); destination_index = i % core_vector_size(&concrete_actor->spawners); destination = *(int *)core_vector_at(&concrete_actor->spawners, destination_index); thorium_message_init(message, ACTION_SPAWN, sizeof(script), &script); thorium_actor_send(actor, destination, message); bucket = core_vector_at(&concrete_actor->files, i); local_file = *(char **)core_vector_at(&concrete_actor->files, i); #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2 printf("DEBUG890 local_file %p bucket %p index %d\n", local_file, (void *)bucket, i); #endif #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG printf("DEBUG actor %d spawns a stream for file %d/%d via spawner %d\n", name, i, core_vector_size(&concrete_actor->files), destination); #endif /* also, spawn 4 stores on each node */ } else if (tag == ACTION_ASK_TO_STOP && ( source == thorium_actor_supervisor(actor) || source == thorium_actor_name(actor))) { #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2 #endif /* stop streams */ for (i = 0; i < core_vector_size(&concrete_actor->counting_streams); i++) { stream = *(int *)core_vector_at(&concrete_actor->counting_streams, i); thorium_actor_send_empty(actor, stream, ACTION_ASK_TO_STOP); } for (i = 0; i < core_vector_size(&concrete_actor->reading_streams); i++) { stream = *(int *)core_vector_at(&concrete_actor->reading_streams, i); thorium_actor_send_empty(actor, stream, ACTION_ASK_TO_STOP); } #if 0 /* stop data stores */ for (i = 0; i < core_vector_size(&concrete_actor->consumers); i++) { store = core_vector_at_as_int(&concrete_actor->consumers, i); thorium_actor_send_empty(actor, store, ACTION_ASK_TO_STOP); } #endif /* stop partitioner */ if (concrete_actor->partitioner != THORIUM_ACTOR_NOBODY) { thorium_actor_send_empty(actor, concrete_actor->partitioner, ACTION_ASK_TO_STOP); #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG printf("DEBUG controller %d sends ACTION_ASK_TO_STOP_REPLY to %d\n", thorium_actor_name(actor), thorium_message_source(message)); #endif } thorium_actor_send_reply_empty(actor, ACTION_ASK_TO_STOP_REPLY); /* stop self */ thorium_actor_send_to_self_empty(actor, ACTION_STOP); thorium_actor_ask_to_stop(actor, message); printf("DEBUG controller %d dies\n", name); #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG #endif } else if (tag == ACTION_INPUT_CONTROLLER_CREATE_PARTITION && source == name) { spawner = *(int *)core_vector_at(&concrete_actor->spawners, core_vector_size(&concrete_actor->spawners) / 2); thorium_actor_send_int(actor, spawner, ACTION_SPAWN, SCRIPT_SEQUENCE_PARTITIONER); concrete_actor->state = BIOSAL_INPUT_CONTROLLER_STATE_SPAWN_PARTITIONER; #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG printf("DEBUG input controller %d spawns a partitioner via spawner %d\n", name, spawner); #endif } else if (tag == ACTION_SEQUENCE_PARTITIONER_COMMAND_IS_READY) { #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG printf("DEBUG controller receives ACTION_SEQUENCE_PARTITIONER_COMMAND_IS_READY, asks for command\n"); #endif thorium_actor_send_reply_empty(actor, ACTION_SEQUENCE_PARTITIONER_GET_COMMAND); } else if (tag == ACTION_SEQUENCE_PARTITIONER_GET_COMMAND_REPLY) { biosal_input_controller_receive_command(actor, message); } else if (tag == ACTION_SEQUENCE_PARTITIONER_FINISHED) { thorium_actor_send_empty(actor, concrete_actor->partitioner, ACTION_ASK_TO_STOP); biosal_input_controller_verify_requests(actor, message); } else if (tag == ACTION_SEQUENCE_PARTITIONER_PROVIDE_STORE_ENTRY_COUNTS) { biosal_input_controller_receive_store_entry_counts(actor, message); } else if (tag == ACTION_RESERVE_REPLY) { concrete_actor->ready_consumers++; printf("DEBUG marker ACTION_RESERVE_REPLY %d/%d\n", concrete_actor->ready_consumers, (int)core_vector_size(&concrete_actor->consumers)); if (concrete_actor->ready_consumers == core_vector_size(&concrete_actor->consumers)) { concrete_actor->ready_consumers = 0; printf("DEBUG all consumers are ready\n"); thorium_actor_send_empty(actor, concrete_actor->partitioner, ACTION_SEQUENCE_PARTITIONER_PROVIDE_STORE_ENTRY_COUNTS_REPLY); } } else if (tag == ACTION_INPUT_PUSH_SEQUENCES_READY) { #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG printf("DEBUG biosal_input_controller_receive received ACTION_INPUT_PUSH_SEQUENCES_READY\n"); #endif stream_name = source; acquaintance_index = stream_name; stream_index = core_vector_index_of(&concrete_actor->reading_streams, &acquaintance_index); command_name = *(int *)core_vector_at(&concrete_actor->partition_commands, stream_index); thorium_actor_send_int(actor, concrete_actor->partitioner, ACTION_SEQUENCE_PARTITIONER_GET_COMMAND_REPLY_REPLY, command_name); } else if (tag == ACTION_INPUT_PUSH_SEQUENCES_REPLY) { stream_name = source; thorium_message_unpack_int(message, 0, &consumer); consumer_index = core_vector_index_of(&concrete_actor->consumers, &consumer); bucket_for_requests = (int *)core_vector_at(&concrete_actor->consumer_active_requests, consumer_index); (*bucket_for_requests)--; biosal_input_controller_verify_requests(actor, message); #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_CONSUMERS printf("DEBUG consumer # %d has %d active requests\n", consumer_index, *bucket_for_requests); #endif } else if (tag == ACTION_SET_CONSUMERS) { core_vector_init(&concrete_actor->consumers, 0); core_vector_unpack(&concrete_actor->consumers, buffer); printf("controller %d receives %d consumers\n", thorium_actor_name(actor), (int)core_vector_size(&concrete_actor->consumers)); for (i = 0; i < core_vector_size(&concrete_actor->consumers); i++) { core_vector_push_back_int(&concrete_actor->consumer_active_requests, 0); } #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG core_vector_print_int(&concrete_actor->consumers); printf("\n"); #endif thorium_actor_send_reply_empty(actor, ACTION_SET_CONSUMERS_REPLY); } else if (tag == ACTION_SET_BLOCK_SIZE) { thorium_message_unpack_int(message, 0, &concrete_actor->block_size); thorium_actor_send_reply_empty(actor, ACTION_SET_BLOCK_SIZE_REPLY); } else if (tag == ACTION_SEQUENCE_STORE_READY) { concrete_actor->filled_consumers++; #ifdef BIOSAL_INPUT_CONTROLLER_DEBUG printf("DEBUG ACTION_SEQUENCE_STORE_READY %d/%d\n", concrete_actor->filled_consumers, (int)core_vector_size(&concrete_actor->consumers)); #endif if (concrete_actor->filled_consumers == core_vector_size(&concrete_actor->consumers)) { concrete_actor->filled_consumers = 0; printf("DEBUG: all consumers are filled, sending ACTION_INPUT_DISTRIBUTE_REPLY\n"); core_timer_stop(&concrete_actor->input_timer); core_timer_stop(&concrete_actor->distribution_timer); core_timer_print_with_description(&concrete_actor->distribution_timer, "Load input / Distribute input data"); core_timer_print_with_description(&concrete_actor->input_timer, "Load input"); thorium_actor_send_to_supervisor_empty(actor, ACTION_INPUT_DISTRIBUTE_REPLY); } } }
/* * Basically, this actor does this: * - spawn visitors * - let them visit stuff * - kill them. * - spawn walkers * - let them walk * - kill the walkers * - return OK */ void biosal_unitig_manager_receive(struct thorium_actor *self, struct thorium_message *message) { struct biosal_unitig_manager *concrete_self; int tag; void *buffer; int spawner; int expected; int script; int actor_count; int source; struct core_string file_name; char *directory; int argc; char **argv; char *path; tag = thorium_message_action(message); source = thorium_message_source(message); buffer = thorium_message_buffer(message); concrete_self = (struct biosal_unitig_manager *)thorium_actor_concrete_actor(self); if (tag == ACTION_START) { core_vector_unpack(&concrete_self->spawners, buffer); spawner = thorium_actor_get_random_spawner(self, &concrete_self->spawners); concrete_self->state = STATE_SPAWN_WRITER; thorium_actor_send_int(self, spawner, ACTION_SPAWN, SCRIPT_WRITER_PROCESS); } else if (tag == ACTION_SPAWN_REPLY && concrete_self->state == STATE_SPAWN_WRITER) { thorium_message_unpack_int(message, 0, &concrete_self->writer_process); /* * open the file now. */ argc = thorium_actor_argc(self); argv = thorium_actor_argv(self); directory = core_command_get_output_directory(argc, argv); core_string_init(&file_name, directory); core_string_append(&file_name, "/"); core_string_append(&file_name, "unitigs.fasta"); path = core_string_get(&file_name); thorium_actor_send_buffer(self, concrete_self->writer_process, ACTION_OPEN, strlen(path) + 1, path); core_string_destroy(&file_name); } else if (tag == ACTION_OPEN_REPLY && source == concrete_self->writer_process) { /* * Spawn visitors. */ concrete_self->state = STATE_VISITORS; thorium_actor_send_to_self_empty(self, ACTION_PING); } else if (tag == ACTION_PING) { spawner = thorium_actor_get_random_spawner(self, &concrete_self->spawners); thorium_actor_send_int(self, spawner, ACTION_SPAWN, SCRIPT_MANAGER); } else if (tag == ACTION_SPAWN_REPLY) { thorium_message_unpack_int(message, 0, &concrete_self->manager); script = SCRIPT_UNITIG_VISITOR; if (concrete_self->state == STATE_WALKERS) { script = SCRIPT_UNITIG_WALKER; } thorium_actor_send_int(self, concrete_self->manager, ACTION_MANAGER_SET_SCRIPT, script); } else if (tag == ACTION_ASK_TO_STOP) { thorium_actor_send_empty(self, concrete_self->writer_process, ACTION_ASK_TO_STOP); thorium_actor_send_empty(self, concrete_self->manager, ACTION_ASK_TO_STOP); thorium_actor_send_to_self_empty(self, ACTION_STOP); thorium_actor_send_reply_empty(self, ACTION_ASK_TO_STOP_REPLY); } else if (tag == ACTION_MANAGER_SET_SCRIPT_REPLY) { actor_count = UNITIG_VISITOR_COUNT_PER_WORKER; if (concrete_self->state == STATE_WALKERS) actor_count = UNITIG_WALKER_COUNT_PER_WORKER; thorium_actor_send_reply_int(self, ACTION_MANAGER_SET_ACTORS_PER_WORKER, actor_count); } else if (tag == ACTION_MANAGER_SET_ACTORS_PER_WORKER_REPLY) { thorium_actor_send_reply_vector(self, ACTION_START, &concrete_self->spawners); } else if (tag == ACTION_START_REPLY && concrete_self->state == STATE_VISITORS && core_vector_size(&concrete_self->visitors) == 0) { core_vector_unpack(&concrete_self->visitors, buffer); printf("DEBUG the system has %d visitors\n", (int)core_vector_size(&concrete_self->visitors)); thorium_actor_send_to_supervisor_empty(self, ACTION_START_REPLY); } else if (tag == ACTION_START_REPLY && concrete_self->state == STATE_WALKERS && core_vector_size(&concrete_self->walkers) == 0) { core_vector_unpack(&concrete_self->walkers, buffer); printf("DEBUG the system has %d walkers\n", (int)core_vector_size(&concrete_self->walkers)); core_timer_start(&concrete_self->timer); concrete_self->completed = 0; thorium_actor_send_range_int(self, &concrete_self->walkers, ACTION_SET_CONSUMER, concrete_self->writer_process); thorium_actor_send_range_vector(self, &concrete_self->walkers, ACTION_START, &concrete_self->graph_stores); } else if (tag == ACTION_SET_PRODUCERS) { core_vector_unpack(&concrete_self->graph_stores, buffer); core_timer_start(&concrete_self->timer); concrete_self->completed = 0; thorium_actor_send_range_vector(self, &concrete_self->visitors, ACTION_START, &concrete_self->graph_stores); } else if (tag == ACTION_START_REPLY && concrete_self->state == STATE_VISITORS) { ++concrete_self->completed; expected = core_vector_size(&concrete_self->visitors); if (concrete_self->completed % UNITIG_VISITOR_COUNT_PER_WORKER == 0 || concrete_self->completed == expected) { printf("PROGRESS unitig visitors %d/%d\n", concrete_self->completed, expected); } if (concrete_self->completed == expected) { core_timer_stop(&concrete_self->timer); core_timer_print_with_description(&concrete_self->timer, "Visit vertices for unitigs"); /* * Stop the visitor manager and all visitors too. */ thorium_actor_send_empty(self, concrete_self->manager, ACTION_ASK_TO_STOP); /* * Reset graph stores. */ thorium_actor_send_range_empty(self, &concrete_self->graph_stores, ACTION_RESET); concrete_self->completed = 0; } } else if (tag == ACTION_RESET_REPLY) { ++concrete_self->completed; expected = core_vector_size(&concrete_self->graph_stores); if (concrete_self->completed == expected) { concrete_self->completed = 0; concrete_self->state = STATE_WALKERS; /* * Go back at the beginning. */ thorium_actor_send_to_self_empty(self, ACTION_PING); } } else if (tag == ACTION_START_REPLY && concrete_self->state == STATE_WALKERS) { ++concrete_self->completed; expected = core_vector_size(&concrete_self->walkers); if (concrete_self->completed % UNITIG_WALKER_COUNT_PER_WORKER == 0 || concrete_self->completed == expected) { printf("PROGRESS unitig walkers %d/%d\n", concrete_self->completed, expected); } if (concrete_self->completed == expected) { core_timer_stop(&concrete_self->timer); core_timer_print_with_description(&concrete_self->timer, "Walk for unitigs"); thorium_actor_send_to_supervisor_empty(self, ACTION_SET_PRODUCERS_REPLY); } } }
void thorium_worker_run(struct thorium_worker *worker) { struct thorium_actor *actor; struct thorium_message other_message; #ifdef THORIUM_NODE_INJECT_CLEAN_WORKER_BUFFERS void *buffer; #endif #ifdef THORIUM_NODE_ENABLE_INSTRUMENTATION time_t current_time; int elapsed; int period; uint64_t current_nanoseconds; uint64_t elapsed_nanoseconds; #endif #ifdef THORIUM_WORKER_DEBUG int tag; int destination; struct thorium_message *message; #endif #ifdef THORIUM_WORKER_ENABLE_LOCK thorium_worker_lock(worker); #endif #ifdef THORIUM_NODE_ENABLE_INSTRUMENTATION period = THORIUM_NODE_LOAD_PERIOD; current_time = time(NULL); elapsed = current_time - worker->last_report; if (elapsed >= period) { current_nanoseconds = core_timer_get_nanoseconds(&worker->timer); #ifdef THORIUM_WORKER_DEBUG_LOAD printf("DEBUG Updating load report\n"); #endif elapsed_nanoseconds = current_nanoseconds - worker->epoch_start_in_nanoseconds; if (elapsed_nanoseconds > 0) { worker->epoch_load = (0.0 + worker->epoch_used_nanoseconds) / elapsed_nanoseconds; worker->epoch_used_nanoseconds = 0; worker->last_wake_up_count = core_thread_get_wake_up_count(&worker->thread); /* \see http://stackoverflow.com/questions/9657993/negative-zero-in-c */ if (worker->epoch_load == 0) { worker->epoch_load = 0; } worker->epoch_start_in_nanoseconds = current_nanoseconds; worker->last_report = current_time; } #ifdef THORIUM_WORKER_PRINT_SCHEDULING_QUEUE /* if (thorium_node_name(worker->node) == 0 && worker->name == 0) { */ thorium_scheduler_print(&worker->scheduler, thorium_node_name(worker->node), worker->name); /* } */ #endif if (core_bitmap_get_bit_uint32_t(&worker->flags, FLAG_DEBUG_ACTORS)) { thorium_worker_print_actors(worker, NULL); } } #endif #ifdef THORIUM_WORKER_DEBUG if (core_bitmap_get_bit_uint32_t(&worker->flags, FLAG_DEBUG)) { printf("DEBUG worker/%d thorium_worker_run\n", thorium_worker_name(worker)); } #endif /* check for messages in inbound FIFO */ if (thorium_worker_dequeue_actor(worker, &actor)) { #ifdef THORIUM_WORKER_DEBUG message = biosal_work_message(&work); tag = thorium_message_action(message); destination = thorium_message_destination(message); if (tag == ACTION_ASK_TO_STOP) { printf("DEBUG pulled ACTION_ASK_TO_STOP for %d\n", destination); } #endif /* * Update the priority of the actor * before starting the timer because this is part of the * runtime system (RTS). */ #ifdef THORIUM_UPDATE_SCHEDULING_PRIORITIES thorium_priority_assigner_update(&worker->scheduler, actor); #endif #ifdef THORIUM_NODE_ENABLE_INSTRUMENTATION core_timer_start(&worker->timer); #endif core_bitmap_set_bit_uint32_t(&worker->flags, FLAG_BUSY); /* * Dispatch message to a worker */ thorium_worker_work(worker, actor); core_bitmap_clear_bit_uint32_t(&worker->flags, FLAG_BUSY); #ifdef THORIUM_NODE_ENABLE_INSTRUMENTATION core_timer_stop(&worker->timer); elapsed_nanoseconds = core_timer_get_elapsed_nanoseconds(&worker->timer); if (elapsed_nanoseconds >= THORIUM_GRANULARITY_WARNING_THRESHOLD) { } worker->epoch_used_nanoseconds += elapsed_nanoseconds; worker->loop_used_nanoseconds += elapsed_nanoseconds; worker->scheduling_epoch_used_nanoseconds += elapsed_nanoseconds; worker->last_elapsed_nanoseconds = elapsed_nanoseconds; #endif } /* queue buffered message */ if (core_fast_queue_dequeue(&worker->outbound_message_queue_buffer, &other_message)) { if (!core_fast_ring_push_from_producer(&worker->outbound_message_queue, &other_message)) { #ifdef SHOW_FULL_RING_WARNINGS printf("thorium_worker: Warning: ring is full => outbound_message_queue\n"); #endif core_fast_queue_enqueue(&worker->outbound_message_queue_buffer, &other_message); } } #ifdef THORIUM_NODE_INJECT_CLEAN_WORKER_BUFFERS /* * Free outbound buffers, if any */ if (thorium_worker_fetch_clean_outbound_buffer(worker, &buffer)) { core_memory_pool_free(&worker->outbound_message_memory_pool, buffer); #ifdef THORIUM_WORKER_DEBUG_INJECTION ++worker->counter_freed_outbound_buffers_from_other_workers; #endif } #endif /* * Transfer messages for triage */ if (core_fast_queue_dequeue(&worker->clean_message_queue_for_triage, &other_message)) { CORE_DEBUGGER_ASSERT(thorium_message_buffer(&other_message) != NULL); thorium_worker_enqueue_message_for_triage(worker, &other_message); } #ifdef THORIUM_WORKER_ENABLE_LOCK thorium_worker_unlock(worker); #endif }
void thorium_balancer_balance(struct thorium_balancer *self) { /* * The 95th percentile is useful: * \see http://en.wikipedia.org/wiki/Burstable_billing * \see http://www.init7.net/en/backbone/95-percent-rule */ int load_percentile_50; struct core_timer timer; int i; struct core_vector loads; struct core_vector loads_unsorted; struct core_vector burdened_workers; struct core_vector stalled_workers; struct thorium_worker *worker; struct thorium_node *node; /*struct core_set *set;*/ struct core_pair pair; struct core_vector_iterator vector_iterator; int old_worker; int actor_name; int messages; int maximum; int with_maximum; struct core_map *set; struct core_map_iterator set_iterator; int stalled_index; int stalled_count; int new_worker_index; struct core_vector migrations; struct thorium_migration migration; struct thorium_migration *migration_to_do; struct thorium_actor *actor; int candidates; int load_value; int remaining_load; int projected_load; struct core_vector actors_to_migrate; int total; int with_messages; int stalled_percentile; int burdened_percentile; int old_total; int old_load; int new_load; int predicted_new_load; struct core_pair *pair_pointer; struct thorium_worker *new_worker; /*int new_total;*/ int actor_load; int test_stalled_index; int tests; int found_match; int spawned_actors; int killed_actors; int perfect; #ifdef THORIUM_SCHEDULER_ENABLE_SYMMETRIC_SCHEDULING struct core_map symmetric_actor_scripts; int script; #endif node = thorium_worker_pool_get_node(self->pool); spawned_actors = thorium_node_get_counter(node, CORE_COUNTER_SPAWNED_ACTORS); /* There is nothing to balance... */ if (spawned_actors == 0) { return; } killed_actors = thorium_node_get_counter(node, CORE_COUNTER_KILLED_ACTORS); /* * The system can probably not be balanced to get in * a better shape anyway. */ if (spawned_actors == self->last_spawned_actors && killed_actors == self->last_killed_actors && self->last_migrations == 0) { printf("SCHEDULER: balance can not be improved because nothing changed.\n"); return; } /* Check if we have perfection */ perfect = 1; for (i = 0; i < thorium_worker_pool_worker_count(self->pool); i++) { worker = thorium_worker_pool_get_worker(self->pool, i); load_value = thorium_worker_get_epoch_load(worker) * 100; if (load_value != 100) { perfect = 0; break; } } if (perfect) { printf("SCHEDULER: perfect balance can not be improved.\n"); return; } /* update counters */ self->last_spawned_actors = spawned_actors; self->last_killed_actors = killed_actors; /* Otherwise, try to balance things */ core_timer_init(&timer); core_timer_start(&timer); #ifdef THORIUM_SCHEDULER_ENABLE_SYMMETRIC_SCHEDULING core_map_init(&symmetric_actor_scripts, sizeof(int), sizeof(int)); thorium_balancer_detect_symmetric_scripts(self, &symmetric_actor_scripts); #endif #ifdef THORIUM_WORKER_ENABLE_LOCK /* Lock all workers first */ for (i = 0; i < thorium_worker_pool_worker_count(self->pool); i++) { worker = thorium_worker_pool_get_worker(self->pool, i); thorium_worker_lock(worker); } #endif core_vector_init(&migrations, sizeof(struct thorium_migration)); #ifdef THORIUM_SCHEDULER_ENABLE_VERBOSITY printf("BALANCING\n"); #endif core_vector_init(&loads, sizeof(int)); core_vector_init(&loads_unsorted, sizeof(int)); core_vector_init(&burdened_workers, sizeof(struct core_pair)); core_vector_init(&stalled_workers, sizeof(struct core_pair)); core_vector_init(&actors_to_migrate, sizeof(struct core_pair)); for (i = 0; i < thorium_worker_pool_worker_count(self->pool); i++) { worker = thorium_worker_pool_get_worker(self->pool, i); load_value = thorium_worker_get_scheduling_epoch_load(worker) * SCHEDULER_PRECISION; #if 0 printf("DEBUG LOAD %d %d\n", i, load_value); #endif core_vector_push_back(&loads, &load_value); core_vector_push_back(&loads_unsorted, &load_value); } core_vector_sort_int(&loads); stalled_percentile = core_statistics_get_percentile_int(&loads, SCHEDULER_WINDOW); /*load_percentile_25 = core_statistics_get_percentile_int(&loads, 25);*/ load_percentile_50 = core_statistics_get_percentile_int(&loads, 50); /*load_percentile_75 = core_statistics_get_percentile_int(&loads, 75);*/ burdened_percentile = core_statistics_get_percentile_int(&loads, 100 - SCHEDULER_WINDOW); #ifdef THORIUM_SCHEDULER_ENABLE_VERBOSITY printf("Percentiles for epoch loads: "); core_statistics_print_percentiles_int(&loads); #endif for (i = 0; i < thorium_worker_pool_worker_count(self->pool); i++) { worker = thorium_worker_pool_get_worker(self->pool, i); load_value = core_vector_at_as_int(&loads_unsorted, i); set = thorium_worker_get_actors(worker); if (stalled_percentile == burdened_percentile) { #ifdef THORIUM_SCHEDULER_ENABLE_VERBOSITY printf("scheduling_class:%s ", THORIUM_CLASS_NORMAL_STRING); #endif } else if (load_value <= stalled_percentile) { #ifdef THORIUM_SCHEDULER_ENABLE_VERBOSITY printf("scheduling_class:%s ", THORIUM_CLASS_STALLED_STRING); #endif core_pair_init(&pair, load_value, i); core_vector_push_back(&stalled_workers, &pair); } else if (load_value >= burdened_percentile) { #ifdef THORIUM_SCHEDULER_ENABLE_VERBOSITY printf("scheduling_class:%s ", THORIUM_CLASS_BURDENED_STRING); #endif core_pair_init(&pair, load_value, i); core_vector_push_back(&burdened_workers, &pair); } else { #ifdef THORIUM_SCHEDULER_ENABLE_VERBOSITY printf("scheduling_class:%s ", THORIUM_CLASS_NORMAL_STRING); #endif } #ifdef THORIUM_SCHEDULER_ENABLE_VERBOSITY thorium_worker_print_actors(worker, self); #endif } core_vector_sort_int_reverse(&burdened_workers); core_vector_sort_int(&stalled_workers); stalled_count = core_vector_size(&stalled_workers); #ifdef THORIUM_SCHEDULER_ENABLE_VERBOSITY printf("MIGRATIONS (stalled: %d, burdened: %d)\n", (int)core_vector_size(&stalled_workers), (int)core_vector_size(&burdened_workers)); #endif stalled_index = 0; core_vector_iterator_init(&vector_iterator, &burdened_workers); while (stalled_count > 0 && core_vector_iterator_get_next_value(&vector_iterator, &pair)) { old_worker = core_pair_get_second(&pair); worker = thorium_worker_pool_get_worker(self->pool, old_worker); set = thorium_worker_get_actors(worker); /* thorium_worker_print_actors(worker); printf("\n"); */ /* * Lock the worker and try to select actors for migration */ core_map_iterator_init(&set_iterator, set); maximum = -1; with_maximum = 0; total = 0; with_messages = 0; while (core_map_iterator_get_next_key_and_value(&set_iterator, &actor_name, NULL)) { actor = thorium_node_get_actor_from_name(thorium_worker_pool_get_node(self->pool), actor_name); messages = thorium_balancer_get_actor_production(self, actor); if (maximum == -1 || messages > maximum) { maximum = messages; with_maximum = 1; } else if (messages == maximum) { with_maximum++; } if (messages > 0) { ++with_messages; } total += messages; } core_map_iterator_destroy(&set_iterator); core_map_iterator_init(&set_iterator, set); --with_maximum; candidates = 0; load_value = thorium_worker_get_scheduling_epoch_load(worker) * SCHEDULER_PRECISION; remaining_load = load_value; #if 0 printf("maximum %d with_maximum %d\n", maximum, with_maximum); #endif while (core_map_iterator_get_next_key_and_value(&set_iterator, &actor_name, NULL)) { actor = thorium_node_get_actor_from_name(thorium_worker_pool_get_node(self->pool), actor_name); if (actor == NULL) { continue; } messages = thorium_balancer_get_actor_production(self, actor); #ifdef THORIUM_SCHEDULER_ENABLE_SYMMETRIC_SCHEDULING script = thorium_actor_script(actor); /* symmetric actors are migrated elsewhere. */ if (core_map_get_value(&symmetric_actor_scripts, &script, NULL)) { continue; } #endif /* Simulate the remaining load */ projected_load = remaining_load; projected_load -= ((0.0 + messages) / total) * load_value; #ifdef THORIUM_SCHEDULER_DEBUG printf(" TESTING actor %d, production was %d, projected_load is %d (- %d * (1 - %d/%d)\n", actor_name, messages, projected_load, load_value, messages, total); #endif /* An actor without any queued messages should not be migrated */ if (messages > 0 && ((with_maximum > 0 && messages == maximum) || messages < maximum) /* * Avoid removing too many actors because * generating a stalled one is not desired */ && (projected_load >= load_percentile_50 /* * The previous rule does not apply when there * are 2 actors. */ || with_messages == 2) ) { remaining_load = projected_load; candidates++; if (messages == maximum) { --with_maximum; } core_pair_init(&pair, messages, actor_name); core_vector_push_back(&actors_to_migrate, &pair); #ifdef THORIUM_SCHEDULER_DEBUG printf("early CANDIDATE for migration: actor %d, worker %d\n", actor_name, old_worker); #endif } } core_map_iterator_destroy(&set_iterator); } core_vector_iterator_destroy(&vector_iterator); /* Sort the candidates */ /* core_vector_sort_int(&actors_to_migrate); printf("Percentiles for production: "); core_statistics_print_percentiles_int(&actors_to_migrate); */ /* Sort them in reverse order. */ core_vector_sort_int_reverse(&actors_to_migrate); core_vector_iterator_init(&vector_iterator, &actors_to_migrate); /* For each highly active actor, * try to match it with a stalled worker */ while (core_vector_iterator_get_next_value(&vector_iterator, &pair)) { actor_name = core_pair_get_second(&pair); actor = thorium_node_get_actor_from_name(thorium_worker_pool_get_node(self->pool), actor_name); if (actor == NULL) { continue; } messages = thorium_balancer_get_actor_production(self, actor); old_worker = thorium_actor_assigned_worker(actor); worker = thorium_worker_pool_get_worker(self->pool, old_worker); /* old_total can not be 0 because otherwise the would not * be burdened. */ old_total = thorium_worker_get_production(worker, self); with_messages = thorium_worker_get_producer_count(worker, self); old_load = thorium_worker_get_scheduling_epoch_load(worker) * SCHEDULER_PRECISION; actor_load = ((0.0 + messages) / old_total) * old_load; /* Try to find a stalled worker that can take it. */ test_stalled_index = stalled_index; tests = 0; predicted_new_load = 0; found_match = 0; while (tests < stalled_count) { core_vector_get_value(&stalled_workers, test_stalled_index, &pair); new_worker_index = core_pair_get_second(&pair); new_worker = thorium_worker_pool_get_worker(self->pool, new_worker_index); new_load = thorium_worker_get_scheduling_epoch_load(new_worker) * SCHEDULER_PRECISION; /*new_total = thorium_worker_get_production(new_worker);*/ predicted_new_load = new_load + actor_load; if (predicted_new_load > SCHEDULER_PRECISION /* && with_messages != 2 */) { #ifdef THORIUM_SCHEDULER_DEBUG printf("Scheduler: skipping actor %d, predicted load is %d >= 100\n", actor_name, predicted_new_load); #endif ++tests; ++test_stalled_index; if (test_stalled_index == stalled_count) { test_stalled_index = 0; } continue; } /* Otherwise, this stalled worker is fine... */ stalled_index = test_stalled_index; found_match = 1; break; } /* This actor can not be migrated to any stalled worker. */ if (!found_match) { continue; } /* Otherwise, update the load of the stalled one and go forward with the change. */ pair_pointer = (struct core_pair *)core_vector_at(&stalled_workers, stalled_index); core_pair_set_first(pair_pointer, predicted_new_load); ++stalled_index; if (stalled_index == stalled_count) { stalled_index = 0; } #if 0 new_worker = thorium_worker_pool_get_worker(pool, new_worker_index); printf(" CANDIDATE: actor %d old worker %d (%d - %d = %d) new worker %d (%d + %d = %d)\n", actor_name, old_worker, value, messages, 2new_score, new_worker_index, new_worker_old_score, messages, new_worker_new_score); #endif thorium_migration_init(&migration, actor_name, old_worker, new_worker_index); core_vector_push_back(&migrations, &migration); thorium_migration_destroy(&migration); } core_vector_iterator_destroy(&vector_iterator); core_vector_destroy(&stalled_workers); core_vector_destroy(&burdened_workers); core_vector_destroy(&loads); core_vector_destroy(&loads_unsorted); core_vector_destroy(&actors_to_migrate); /* Update the last values */ for (i = 0; i < thorium_worker_pool_worker_count(self->pool); i++) { worker = thorium_worker_pool_get_worker(self->pool, i); set = thorium_worker_get_actors(worker); core_map_iterator_init(&set_iterator, set); while (core_map_iterator_get_next_key_and_value(&set_iterator, &actor_name, NULL)) { actor = thorium_node_get_actor_from_name(thorium_worker_pool_get_node(self->pool), actor_name); thorium_balancer_update_actor_production(self, actor); } core_map_iterator_destroy(&set_iterator); thorium_worker_reset_scheduling_epoch(worker); } #ifdef THORIUM_SCHEDULER_ENABLE_SYMMETRIC_SCHEDULING /* Generate migrations for symmetric actors. */ thorium_balancer_generate_symmetric_migrations(self, &symmetric_actor_scripts, &migrations); #endif /* Actually do the migrations */ core_vector_iterator_init(&vector_iterator, &migrations); while (core_vector_iterator_next(&vector_iterator, (void **)&migration_to_do)) { thorium_balancer_migrate(self, migration_to_do); } core_vector_iterator_destroy(&vector_iterator); self->last_migrations = core_vector_size(&migrations); core_vector_destroy(&migrations); #ifdef THORIUM_WORKER_ENABLE_LOCK /* Unlock all workers */ for (i = 0; i < thorium_worker_pool_worker_count(self->pool); i++) { worker = thorium_worker_pool_get_worker(self->pool, i); thorium_worker_unlock(worker); } #endif #ifdef THORIUM_SCHEDULER_ENABLE_SYMMETRIC_SCHEDULING core_map_destroy(&symmetric_actor_scripts); #endif core_timer_stop(&timer); printf("SCHEDULER: elapsed time for balancing: %d us, %d migrations performed\n", (int)(core_timer_get_elapsed_nanoseconds(&timer) / 1000), self->last_migrations); }