void core_string_combine(struct core_string *string, const char *data, int operation)
{
int old_length;
int length;
int new_length;
char *new_data;
/*
* Nothing to do.
*/
if (data == NULL) {
return;
}
length = strlen(data);
/* Nothing to combine
*/
if (length == 0) {
return;
}
old_length = 0;
if (string->data != NULL) {
old_length = strlen(string->data);
}
new_length = old_length + length;
new_data = (char *)core_memory_allocate(new_length + 1, MEMORY_STRING);
strcpy(new_data, "");
if (operation == CORE_STRING_APPEND) {
if (string->data != NULL) {
strcat(new_data, string->data);
}
strcat(new_data, data);
} else if (operation == CORE_STRING_PREPEND) {
strcat(new_data, data);
if (string->data != NULL) {
strcat(new_data, string->data);
}
}
if (string->data != NULL) {
core_memory_free(string->data, MEMORY_STRING);
}
string->data = new_data;
}
void core_vector_sort(struct core_vector *self, core_compare_fn_t compare)
{
void *saved_pivot_value;
int element_size;
element_size = core_vector_element_size(self);
saved_pivot_value = core_memory_allocate(element_size, MEMORY_VECTOR_HELPER);
core_vector_quicksort(self, 0, core_vector_size(self) - 1, compare, saved_pivot_value);
core_memory_free(saved_pivot_value, MEMORY_VECTOR_HELPER);
}
void core_string_init(struct core_string *string, const char *data)
{
int length;
string->data = NULL;
if (data != NULL) {
length = strlen(data);
string->data = (char *)core_memory_allocate(length + 1, MEMORY_STRING);
strcpy(string->data, data);
}
}
void test_allocator(struct core_memory_pool *memory)
{
int i;
int size;
void *pointer;
struct core_vector vector;
struct core_timer timer;
uint64_t elapsed;
i = 1000000;
size = 45;
core_vector_init(&vector, sizeof(void *));
core_timer_init(&timer);
core_timer_start(&timer);
while (i--) {
if (memory != NULL) {
pointer = core_memory_pool_allocate(memory, size);
} else {
pointer = core_memory_allocate(size, -1);
}
core_vector_push_back(&vector, &pointer);
}
core_timer_stop(&timer);
elapsed = core_timer_get_elapsed_nanoseconds(&timer);
if (memory == NULL) {
printf("Not using memory pool... ");
} else {
printf("Using memory pool... ");
}
printf("Elapsed : %" PRIu64 " milliseconds\n", elapsed / 1000 / 1000);
size = core_vector_size(&vector);
for (i = 0; i < size; ++i) {
pointer = core_vector_at_as_void_pointer(&vector, i);
if (memory != NULL) {
core_memory_pool_free(memory, pointer);
} else {
core_memory_free(pointer, -1);
}
}
core_vector_destroy(&vector);
core_timer_destroy(&timer);
}
void core_map_init_with_capacity(struct core_map *self, int key_size, int value_size, uint64_t buckets)
{
#ifdef CORE_MAP_ALIGNMENT_ENABLED
self->original_key_size = key_size;
self->original_value_size = value_size;
key_size = core_memory_align(key_size);
value_size = core_memory_align(value_size);
self->key_padding = key_size - self->original_key_size;
self->key_buffer = core_memory_allocate(key_size);
#endif
core_dynamic_hash_table_init(&self->table, buckets, key_size, value_size);
}
int core_map_pack_unpack(struct core_map *self, int operation, void *buffer)
{
struct core_packer packer;
int offset;
#ifdef CORE_MAP_ALIGNMENT_ENABLED
int key_size;
#endif
CORE_DEBUGGER_ASSERT(self != NULL);
core_packer_init(&packer, operation, buffer);
#ifdef CORE_MAP_ALIGNMENT_ENABLED
core_packer_process(&packer, &self->original_key_size, sizeof(self->original_key_size));
core_packer_process(&packer, &self->original_value_size, sizeof(self->original_value_size));
#endif
offset = core_packer_get_byte_count(&packer);
core_packer_destroy(&packer);
if (operation == CORE_PACKER_OPERATION_PACK_SIZE) {
offset += core_dynamic_hash_table_pack_size(&self->table);
} else if (operation == CORE_PACKER_OPERATION_PACK) {
offset += core_dynamic_hash_table_pack(&self->table, (char *)buffer + offset);
} else if (operation == CORE_PACKER_OPERATION_UNPACK) {
offset += core_dynamic_hash_table_unpack(&self->table, (char *)buffer + offset);
#ifdef CORE_MAP_ALIGNMENT_ENABLED
key_size = core_dynamic_hash_table_get_key_size(&self->table);
self->key_padding = key_size - self->original_key_size;
self->key_buffer = core_memory_allocate(key_size);
#endif
}
return offset;
}
void core_fast_ring_init(struct core_fast_ring *self, int capacity, int cell_size)
{
/* +1 because an empty cell is needed
*/
self->number_of_cells = core_fast_ring_get_next_power_of_two(capacity + 1);
self->mask = self->number_of_cells - 1;
#ifdef CORE_FAST_RING_DEBUG
printf("DEBUG RING CELLS %" PRIu64 "\n", self->number_of_cells);
#endif
self->cell_size = cell_size;
self->head = 0;
self->tail = 0;
#ifdef CORE_FAST_RING_USE_CACHE
self->head_cache = 0;
self->tail_cache = 0;
#endif
self->cells = core_memory_allocate(self->number_of_cells * self->cell_size, MEMORY_FAST_RING);
#ifdef CORE_FAST_RING_USE_PADDING
/* assign values to the padding
*/
self->consumer_padding_0 = 0;
self->consumer_padding_1 = 0;
self->consumer_padding_2 = 0;
self->consumer_padding_3 = 0;
self->consumer_padding_4 = 0;
self->consumer_padding_5 = 0;
self->producer_padding_0 = 0;
self->producer_padding_1 = 0;
self->producer_padding_2 = 0;
self->producer_padding_3 = 0;
self->producer_padding_4 = 0;
self->producer_padding_5 = 0;
#endif
}
int core_string_pack_unpack(struct core_string *self, int operation, void *buffer)
{
struct core_packer packer;
int bytes;
int length;
length = core_string_length(self);
core_packer_init(&packer, operation, buffer);
core_packer_process(&packer, &length, sizeof(length));
if (operation == CORE_PACKER_OPERATION_UNPACK) {
self->data = core_memory_allocate(length + 1, MEMORY_STRING);
}
core_packer_process(&packer, self->data, length + 1);
bytes = core_packer_get_byte_count(&packer);
core_packer_destroy(&packer);
return bytes;
}
void biosal_input_controller_receive(struct thorium_actor *actor, struct thorium_message *message)
{
int tag;
int count;
char *file;
void *buffer;
struct biosal_input_controller *controller;
struct biosal_input_controller *concrete_actor;
int destination;
int script;
int stream;
char *local_file;
int i;
int name;
int source;
int destination_index;
struct thorium_message new_message;
int error;
int stream_index;
int64_t entries;
int64_t *bucket;
int *int_bucket;
int spawner;
int command_name;
int stream_name;
int consumer;
int consumer_index;
int *bucket_for_requests;
char *new_buffer;
int new_count;
int file_index;
struct core_vector mega_blocks;
struct core_vector_iterator vector_iterator;
struct biosal_mega_block *mega_block;
struct core_vector *vector_bucket;
struct core_vector block_counts;
uint64_t block_entries;
int mega_block_index;
uint64_t offset;
struct biosal_mega_block *block;
int acquaintance_index;
if (thorium_actor_take_action(actor, message)) {
return;
}
thorium_message_get_all(message, &tag, &count, &buffer, &source);
name = thorium_actor_name(actor);
controller = (struct biosal_input_controller *)thorium_actor_concrete_actor(actor);
concrete_actor = controller;
if (tag == ACTION_START) {
core_vector_init(&concrete_actor->spawners, 0);
core_vector_unpack(&concrete_actor->spawners, buffer);
core_vector_resize(&concrete_actor->stores_per_spawner,
core_vector_size(&concrete_actor->spawners));
for (i = 0; i < core_vector_size(&concrete_actor->spawners); i++) {
int_bucket = (int *)core_vector_at(&concrete_actor->stores_per_spawner, i);
*int_bucket = 0;
spawner = core_vector_at_as_int(&concrete_actor->spawners, i);
core_queue_enqueue(&concrete_actor->unprepared_spawners, &spawner);
}
concrete_actor->state = BIOSAL_INPUT_CONTROLLER_STATE_PREPARE_SPAWNERS;
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL
printf("DEBUG preparing first spawner\n");
#endif
thorium_actor_send_to_self_empty(actor, ACTION_INPUT_CONTROLLER_PREPARE_SPAWNERS);
/*
thorium_dispatcher_print(thorium_actor_dispatcher(actor));
*/
} else if (tag == ACTION_ADD_FILE) {
file = (char *)buffer;
local_file = core_memory_allocate(strlen(file) + 1, MEMORY_CONTROLLER);
strcpy(local_file, file);
printf("controller %d ACTION_ADD_FILE %s\n",
thorium_actor_name(actor),
local_file);
core_vector_push_back(&concrete_actor->files, &local_file);
bucket = core_vector_at(&concrete_actor->files, core_vector_size(&concrete_actor->files) - 1);
local_file = *(char **)bucket;
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2
printf("DEBUG11 ACTION_ADD_FILE %s %p bucket %p index %d\n",
local_file, local_file, (void *)bucket, core_vector_size(&concrete_actor->files) - 1);
#endif
thorium_actor_send_reply_empty(actor, ACTION_ADD_FILE_REPLY);
} else if (tag == ACTION_SPAWN_REPLY) {
if (concrete_actor->state == BIOSAL_INPUT_CONTROLLER_STATE_SPAWN_STORES) {
biosal_input_controller_add_store(actor, message);
return;
} else if (concrete_actor->state == BIOSAL_INPUT_CONTROLLER_STATE_PREPARE_SPAWNERS) {
concrete_actor->ready_spawners++;
thorium_message_unpack_int(message, 0, &name);
thorium_actor_send_empty(actor, name, ACTION_ASK_TO_STOP);
thorium_actor_send_to_self_empty(actor, ACTION_INPUT_CONTROLLER_PREPARE_SPAWNERS);
if (concrete_actor->ready_spawners == (int)core_vector_size(&concrete_actor->spawners)) {
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG
printf("DEBUG all spawners are prepared\n");
#endif
thorium_actor_send_to_supervisor_empty(actor, ACTION_START_REPLY);
}
return;
} else if (concrete_actor->state == BIOSAL_INPUT_CONTROLLER_STATE_SPAWN_PARTITIONER) {
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG
printf("DEBUG received spawn reply, state is spawn_partitioner\n");
#endif
thorium_message_unpack_int(message, 0, &concrete_actor->partitioner);
/* configure the partitioner
*/
destination = concrete_actor->partitioner;
thorium_actor_send_int(actor, destination,
ACTION_SEQUENCE_PARTITIONER_SET_BLOCK_SIZE,
concrete_actor->block_size);
thorium_actor_send_int(actor, destination,
ACTION_SEQUENCE_PARTITIONER_SET_ACTOR_COUNT,
core_vector_size(&concrete_actor->consumers));
core_vector_init(&block_counts, sizeof(uint64_t));
for (i = 0; i < core_vector_size(&concrete_actor->mega_block_vector); i++) {
block = (struct biosal_mega_block *)core_vector_at(&concrete_actor->mega_block_vector, i);
block_entries = biosal_mega_block_get_entries(block);
core_vector_push_back_uint64_t(&block_counts, block_entries);
}
new_count = core_vector_pack_size(&block_counts);
new_buffer = thorium_actor_allocate(actor, new_count);
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG
printf("DEBUG packed counts, %d bytes\n", count);
#endif
core_vector_pack(&block_counts, new_buffer);
thorium_message_init(&new_message, ACTION_SEQUENCE_PARTITIONER_SET_ENTRY_VECTOR,
new_count, new_buffer);
thorium_actor_send(actor, destination, &new_message);
core_vector_destroy(&block_counts);
return;
} else if (concrete_actor->state == BIOSAL_INPUT_CONTROLLER_STATE_SPAWN_READING_STREAMS) {
thorium_message_unpack_int(message, 0, &stream);
stream_index = stream;
mega_block_index = core_vector_size(&concrete_actor->reading_streams);
core_vector_push_back_int(&concrete_actor->reading_streams, stream_index);
core_vector_push_back_int(&concrete_actor->partition_commands, -1);
core_vector_push_back_int(&concrete_actor->stream_consumers, -1);
stream_index = core_vector_size(&concrete_actor->reading_streams) - 1;
mega_block = (struct biosal_mega_block *)core_vector_at(&concrete_actor->mega_block_vector,
mega_block_index);
offset = biosal_mega_block_get_offset(mega_block);
core_map_add_value(&concrete_actor->assigned_blocks,
&stream_index, &mega_block_index);
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_READING_STREAMS
printf("DEBUG setting offset to %" PRIu64 " for stream/%d\n",
offset, stream);
#endif
thorium_actor_send_uint64_t(actor, stream, ACTION_INPUT_STREAM_SET_START_OFFSET, offset);
return;
}
stream = *(int *)buffer;
file_index = core_vector_size(&concrete_actor->counting_streams);
local_file = *(char **)core_vector_at(&concrete_actor->files, file_index);
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_READING_STREAMS
printf("DEBUG actor %d receives stream %d from spawner %d for file %s\n",
name, stream, source,
local_file);
#endif
core_vector_push_back(&concrete_actor->counting_streams, &stream);
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_READING_STREAMS
printf("asking stream/%d to open %s\n", stream, local_file);
#endif
thorium_message_init(&new_message, ACTION_INPUT_OPEN, strlen(local_file) + 1, local_file);
#ifdef DEBUG_ISSUE_594
thorium_message_print(&new_message);
printf("SEND Buffer %s\n", local_file);
#endif
thorium_actor_send(actor, stream, &new_message);
thorium_message_destroy(&new_message);
if (core_vector_size(&concrete_actor->counting_streams) != core_vector_size(&concrete_actor->files)) {
thorium_actor_send_to_self_empty(actor, ACTION_INPUT_SPAWN);
}
#ifdef DEBUG_ISSUE_594
printf("EXIT Buffer %s\n", local_file);
#endif
} else if (tag == ACTION_INPUT_OPEN_REPLY) {
if (concrete_actor->state == BIOSAL_INPUT_CONTROLLER_STATE_SPAWN_READING_STREAMS) {
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_READING_STREAMS
printf("DEBUG receives open.reply for reading stream/%d\n", source);
#endif
concrete_actor->opened_streams++;
if (concrete_actor->opened_streams == core_vector_size(&concrete_actor->mega_block_vector)) {
thorium_actor_send_to_self_empty(actor, ACTION_INPUT_CONTROLLER_CREATE_STORES);
}
return;
}
concrete_actor->opened_streams++;
stream = source;
thorium_message_unpack_int(message, 0, &error);
if (error == BIOSAL_INPUT_ERROR_NO_ERROR) {
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2
printf("DEBUG actor %d asks %d ACTION_INPUT_COUNT_IN_PARALLEL\n", name, stream);
#endif
thorium_actor_send_vector(actor, stream, ACTION_INPUT_COUNT_IN_PARALLEL,
&concrete_actor->spawners);
} else {
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2
printf("DEBUG actor %d received error %d from %d\n", name, error, stream);
#endif
concrete_actor->counted++;
}
/* if all streams failed, notice supervisor */
if (concrete_actor->counted == core_vector_size(&concrete_actor->files)) {
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2
#endif
printf("DEBUG %d: Error all streams failed.\n",
thorium_actor_name(actor));
thorium_actor_send_to_supervisor_empty(actor, ACTION_INPUT_DISTRIBUTE_REPLY);
}
/*
if (concrete_actor->opened_streams == core_vector_size(&concrete_actor->files)) {
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG
printf("DEBUG controller %d sends ACTION_INPUT_DISTRIBUTE_REPLY to supervisor %d [%d/%d]\n",
name, thorium_actor_supervisor(actor),
concrete_actor->opened_streams, core_vector_size(&concrete_actor->files));
#endif
}
*/
} else if (tag == ACTION_INPUT_COUNT_PROGRESS) {
stream_index = core_vector_index_of(&concrete_actor->counting_streams, &source);
local_file = core_vector_at_as_char_pointer(&concrete_actor->files, stream_index);
thorium_message_unpack_int64_t(message, 0, &entries);
bucket = (int64_t *)core_vector_at(&concrete_actor->counts, stream_index);
printf("controller/%d receives progress from stream/%d file %s %" PRIu64 " entries so far\n",
name, source, local_file, entries);
*bucket = entries;
} else if (tag == ACTION_INPUT_COUNT_IN_PARALLEL_REPLY) {
stream_index = core_vector_index_of(&concrete_actor->counting_streams, &source);
local_file = core_vector_at_as_char_pointer(&concrete_actor->files, stream_index);
core_vector_init(&mega_blocks, 0);
core_vector_unpack(&mega_blocks, buffer);
printf("DEBUG receive mega blocks from %d\n", source);
/*
* Update the file index for every mega block.
*/
core_vector_iterator_init(&vector_iterator, &mega_blocks);
bucket = (int64_t*)core_vector_at(&concrete_actor->counts, stream_index);
(*bucket) = 0;
while (core_vector_iterator_has_next(&vector_iterator)) {
core_vector_iterator_next(&vector_iterator, (void **)&mega_block);
printf("SETTING setting file to %d for mega block\n", stream_index);
biosal_mega_block_set_file(mega_block, stream_index);
entries = biosal_mega_block_get_entries_from_start(mega_block);
printf("Cataloging %d ENTRIES\n", (int)entries);
(*bucket) = entries;
biosal_mega_block_print(mega_block);
}
core_vector_iterator_destroy(&vector_iterator);
vector_bucket = (struct core_vector *)core_map_add(&concrete_actor->mega_blocks, &stream_index);
core_vector_init_copy(vector_bucket, &mega_blocks);
core_vector_destroy(&mega_blocks);
concrete_actor->counted++;
printf("controller/%d received from stream/%d for file %s %" PRIu64 " entries (final) %d/%d\n",
name, source, local_file, entries,
concrete_actor->counted, (int)core_vector_size(&concrete_actor->files));
thorium_actor_send_reply_empty(actor, ACTION_INPUT_CLOSE);
/* continue work here, tell supervisor about it */
if (concrete_actor->counted == core_vector_size(&concrete_actor->files)) {
thorium_actor_send_to_self_empty(actor, ACTION_INPUT_CONTROLLER_SPAWN_READING_STREAMS);
}
} else if (tag == ACTION_INPUT_DISTRIBUTE) {
core_timer_start(&concrete_actor->input_timer);
core_timer_start(&concrete_actor->counting_timer);
/* for each file, spawn a stream to count */
/* no files, return immediately
*/
if (core_vector_size(&concrete_actor->files) == 0) {
printf("Error: no file to distribute...\n");
thorium_actor_send_reply_empty(actor, ACTION_INPUT_DISTRIBUTE_REPLY);
return;
}
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2
printf("DEBUG actor %d receives ACTION_INPUT_DISTRIBUTE\n", name);
#endif
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2
printf("DEBUG send ACTION_INPUT_SPAWN to self\n");
#endif
thorium_actor_send_to_self_empty(actor, ACTION_INPUT_SPAWN);
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2
printf("DEBUG resizing counts to %d\n", core_vector_size(&concrete_actor->files));
#endif
core_vector_resize(&concrete_actor->counts, core_vector_size(&concrete_actor->files));
for (i = 0; i < core_vector_size(&concrete_actor->counts); i++) {
bucket = (int64_t*)core_vector_at(&concrete_actor->counts, i);
*bucket = 0;
}
} else if (tag == ACTION_INPUT_SPAWN && source == name) {
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2
printf("DEBUG ACTION_INPUT_SPAWN\n");
#endif
script = SCRIPT_INPUT_STREAM;
concrete_actor->state = BIOSAL_INPUT_CONTROLLER_STATE_SPAWN_STREAMS;
/* the next file name to send is the current number of streams */
i = core_vector_size(&concrete_actor->counting_streams);
destination_index = i % core_vector_size(&concrete_actor->spawners);
destination = *(int *)core_vector_at(&concrete_actor->spawners, destination_index);
thorium_message_init(message, ACTION_SPAWN, sizeof(script), &script);
thorium_actor_send(actor, destination, message);
bucket = core_vector_at(&concrete_actor->files, i);
local_file = *(char **)core_vector_at(&concrete_actor->files, i);
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2
printf("DEBUG890 local_file %p bucket %p index %d\n", local_file, (void *)bucket,
i);
#endif
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG
printf("DEBUG actor %d spawns a stream for file %d/%d via spawner %d\n",
name, i, core_vector_size(&concrete_actor->files), destination);
#endif
/* also, spawn 4 stores on each node */
} else if (tag == ACTION_ASK_TO_STOP && ( source == thorium_actor_supervisor(actor)
|| source == thorium_actor_name(actor))) {
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2
#endif
/* stop streams
*/
for (i = 0; i < core_vector_size(&concrete_actor->counting_streams); i++) {
stream = *(int *)core_vector_at(&concrete_actor->counting_streams, i);
thorium_actor_send_empty(actor, stream, ACTION_ASK_TO_STOP);
}
for (i = 0; i < core_vector_size(&concrete_actor->reading_streams); i++) {
stream = *(int *)core_vector_at(&concrete_actor->reading_streams, i);
thorium_actor_send_empty(actor, stream, ACTION_ASK_TO_STOP);
}
#if 0
/* stop data stores
*/
for (i = 0; i < core_vector_size(&concrete_actor->consumers); i++) {
store = core_vector_at_as_int(&concrete_actor->consumers, i);
thorium_actor_send_empty(actor, store, ACTION_ASK_TO_STOP);
}
#endif
/* stop partitioner
*/
if (concrete_actor->partitioner != THORIUM_ACTOR_NOBODY) {
thorium_actor_send_empty(actor,
concrete_actor->partitioner,
ACTION_ASK_TO_STOP);
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG
printf("DEBUG controller %d sends ACTION_ASK_TO_STOP_REPLY to %d\n",
thorium_actor_name(actor),
thorium_message_source(message));
#endif
}
thorium_actor_send_reply_empty(actor, ACTION_ASK_TO_STOP_REPLY);
/* stop self
*/
thorium_actor_send_to_self_empty(actor, ACTION_STOP);
thorium_actor_ask_to_stop(actor, message);
printf("DEBUG controller %d dies\n", name);
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG
#endif
} else if (tag == ACTION_INPUT_CONTROLLER_CREATE_PARTITION && source == name) {
spawner = *(int *)core_vector_at(&concrete_actor->spawners,
core_vector_size(&concrete_actor->spawners) / 2);
thorium_actor_send_int(actor, spawner, ACTION_SPAWN,
SCRIPT_SEQUENCE_PARTITIONER);
concrete_actor->state = BIOSAL_INPUT_CONTROLLER_STATE_SPAWN_PARTITIONER;
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG
printf("DEBUG input controller %d spawns a partitioner via spawner %d\n",
name, spawner);
#endif
} else if (tag == ACTION_SEQUENCE_PARTITIONER_COMMAND_IS_READY) {
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG
printf("DEBUG controller receives ACTION_SEQUENCE_PARTITIONER_COMMAND_IS_READY, asks for command\n");
#endif
thorium_actor_send_reply_empty(actor, ACTION_SEQUENCE_PARTITIONER_GET_COMMAND);
} else if (tag == ACTION_SEQUENCE_PARTITIONER_GET_COMMAND_REPLY) {
biosal_input_controller_receive_command(actor, message);
} else if (tag == ACTION_SEQUENCE_PARTITIONER_FINISHED) {
thorium_actor_send_empty(actor,
concrete_actor->partitioner,
ACTION_ASK_TO_STOP);
biosal_input_controller_verify_requests(actor, message);
} else if (tag == ACTION_SEQUENCE_PARTITIONER_PROVIDE_STORE_ENTRY_COUNTS) {
biosal_input_controller_receive_store_entry_counts(actor, message);
} else if (tag == ACTION_RESERVE_REPLY) {
concrete_actor->ready_consumers++;
printf("DEBUG marker ACTION_RESERVE_REPLY %d/%d\n",
concrete_actor->ready_consumers,
(int)core_vector_size(&concrete_actor->consumers));
if (concrete_actor->ready_consumers == core_vector_size(&concrete_actor->consumers)) {
concrete_actor->ready_consumers = 0;
printf("DEBUG all consumers are ready\n");
thorium_actor_send_empty(actor,
concrete_actor->partitioner,
ACTION_SEQUENCE_PARTITIONER_PROVIDE_STORE_ENTRY_COUNTS_REPLY);
}
} else if (tag == ACTION_INPUT_PUSH_SEQUENCES_READY) {
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG
printf("DEBUG biosal_input_controller_receive received ACTION_INPUT_PUSH_SEQUENCES_READY\n");
#endif
stream_name = source;
acquaintance_index = stream_name;
stream_index = core_vector_index_of(&concrete_actor->reading_streams, &acquaintance_index);
command_name = *(int *)core_vector_at(&concrete_actor->partition_commands,
stream_index);
thorium_actor_send_int(actor,
concrete_actor->partitioner,
ACTION_SEQUENCE_PARTITIONER_GET_COMMAND_REPLY_REPLY,
command_name);
} else if (tag == ACTION_INPUT_PUSH_SEQUENCES_REPLY) {
stream_name = source;
thorium_message_unpack_int(message, 0, &consumer);
consumer_index = core_vector_index_of(&concrete_actor->consumers,
&consumer);
bucket_for_requests = (int *)core_vector_at(&concrete_actor->consumer_active_requests, consumer_index);
(*bucket_for_requests)--;
biosal_input_controller_verify_requests(actor, message);
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_CONSUMERS
printf("DEBUG consumer # %d has %d active requests\n",
consumer_index, *bucket_for_requests);
#endif
} else if (tag == ACTION_SET_CONSUMERS) {
core_vector_init(&concrete_actor->consumers, 0);
core_vector_unpack(&concrete_actor->consumers, buffer);
printf("controller %d receives %d consumers\n",
thorium_actor_name(actor),
(int)core_vector_size(&concrete_actor->consumers));
for (i = 0; i < core_vector_size(&concrete_actor->consumers); i++) {
core_vector_push_back_int(&concrete_actor->consumer_active_requests, 0);
}
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG
core_vector_print_int(&concrete_actor->consumers);
printf("\n");
#endif
thorium_actor_send_reply_empty(actor, ACTION_SET_CONSUMERS_REPLY);
} else if (tag == ACTION_SET_BLOCK_SIZE) {
thorium_message_unpack_int(message, 0, &concrete_actor->block_size);
thorium_actor_send_reply_empty(actor, ACTION_SET_BLOCK_SIZE_REPLY);
} else if (tag == ACTION_SEQUENCE_STORE_READY) {
concrete_actor->filled_consumers++;
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG
printf("DEBUG ACTION_SEQUENCE_STORE_READY %d/%d\n", concrete_actor->filled_consumers,
(int)core_vector_size(&concrete_actor->consumers));
#endif
if (concrete_actor->filled_consumers == core_vector_size(&concrete_actor->consumers)) {
concrete_actor->filled_consumers = 0;
printf("DEBUG: all consumers are filled, sending ACTION_INPUT_DISTRIBUTE_REPLY\n");
core_timer_stop(&concrete_actor->input_timer);
core_timer_stop(&concrete_actor->distribution_timer);
core_timer_print_with_description(&concrete_actor->distribution_timer,
"Load input / Distribute input data");
core_timer_print_with_description(&concrete_actor->input_timer,
"Load input");
thorium_actor_send_to_supervisor_empty(actor, ACTION_INPUT_DISTRIBUTE_REPLY);
}
}
}
uint64_t core_fasta_input_get_sequence(struct biosal_input_format *input,
char *sequence)
{
struct core_fasta_input *fasta;
/* TODO use a dynamic buffer to accept long reads... */
int maximum_sequence_length = BIOSAL_INPUT_MAXIMUM_SEQUENCE_LENGTH;
int value;
int lines;
int total;
int position_in_sequence;
int is_header;
int block_length;
fasta = (struct core_fasta_input *)biosal_input_format_implementation(input);
if (fasta->buffer == NULL) {
fasta->buffer = core_memory_allocate(maximum_sequence_length + 1, MEMORY_FASTA);
fasta->next_header= core_memory_allocate(maximum_sequence_length + 1, MEMORY_FASTA);
fasta->buffer[0] = '\0';
fasta->next_header[0] = '\0';
}
value = 0;
total = 0;
lines = 0;
/*
* Read name
*/
if (fasta->has_header) {
strcpy(fasta->buffer, fasta->next_header);
value = strlen(fasta->buffer);
fasta->has_header = 0;
} else {
value = core_buffered_reader_read_line(&fasta->reader, fasta->buffer,
maximum_sequence_length);
/* Make sure that this is an identifier.
*/
if (!fasta->has_first) {
while (!core_fasta_input_check_header(input, fasta->buffer)) {
value = core_buffered_reader_read_line(&fasta->reader, fasta->buffer,
maximum_sequence_length);
}
fasta->has_first = 1;
}
}
/*
* Add new line.
*/
if (value) {
++lines;
}
total += value;
/*
* Read sequence.
*
* Discard any new line symbol too.
*/
position_in_sequence = 0;
while (1) {
value = core_buffered_reader_read_line(&fasta->reader, fasta->buffer,
maximum_sequence_length);
if (value == 0) {
break;
}
is_header = 0;
if (strlen(fasta->buffer) > 0
&& fasta->buffer[0] == '>') {
is_header = 1;
}
if (is_header) {
sequence[position_in_sequence] = '\0';
strcpy(fasta->next_header, fasta->buffer);
fasta->has_header = 1;
break;
}
/*
* Otherwise, add the sequence.
*/
if (value) {
++lines;
}
block_length = strlen(fasta->buffer);
/*
* Remove the new line.
*/
if (fasta->buffer[block_length - 1] == '\n') {
--block_length;
}
core_memory_copy(sequence + position_in_sequence,
fasta->buffer,
block_length);
position_in_sequence += block_length;
}
return total;
}
void thorium_worker_pool_print_load(struct thorium_worker_pool *self, int type)
{
int count;
int i;
float epoch_load;
struct thorium_worker *worker;
float loop_load;
uint64_t epoch_wake_up_count;
uint64_t loop_wake_up_count;
/*
int scheduling_score;
*/
int node_name;
char *buffer;
char *buffer_for_wake_up_events;
char *buffer_for_future_timeline;
int allocated;
int offset;
int offset_for_wake_up;
int offset_for_future;
int extra;
time_t current_time;
int elapsed;
float selected_load;
uint64_t selected_wake_up_count;
float sum;
char loop[] = "COMPUTATION";
char epoch[] = "EPOCH";
char *description;
float load;
description = NULL;
if (type == THORIUM_WORKER_POOL_LOAD_LOOP) {
description = loop;
} else if (type == THORIUM_WORKER_POOL_LOAD_EPOCH) {
description = epoch;
} else {
return;
}
current_time = time(NULL);
elapsed = current_time - self->starting_time;
extra = 100;
count = thorium_worker_pool_worker_count(self);
allocated = count * 20 + 20 + extra;
buffer = core_memory_allocate(allocated, MEMORY_WORKER_POOL_KEY);
buffer_for_wake_up_events = core_memory_allocate(allocated, MEMORY_WORKER_POOL_KEY);
buffer_for_future_timeline = core_memory_allocate(allocated, MEMORY_WORKER_POOL_KEY);
node_name = thorium_node_name(self->node);
offset = 0;
offset_for_wake_up = 0;
offset_for_future = 0;
i = 0;
sum = 0;
while (i < count && offset + extra < allocated) {
worker = thorium_worker_pool_get_worker(self, i);
epoch_load = thorium_worker_get_epoch_load(worker);
loop_load = thorium_worker_get_loop_load(worker);
epoch_wake_up_count = thorium_worker_get_epoch_wake_up_count(worker);
loop_wake_up_count = thorium_worker_get_loop_wake_up_count(worker);
selected_load = epoch_load;
selected_wake_up_count = epoch_wake_up_count;
if (type == THORIUM_WORKER_POOL_LOAD_EPOCH) {
selected_load = epoch_load;
selected_wake_up_count = epoch_wake_up_count;
} else if (type == THORIUM_WORKER_POOL_LOAD_LOOP) {
selected_load = loop_load;
selected_wake_up_count = loop_wake_up_count;
}
/*
offset += sprintf(buffer + offset, " [%d %d %.2f]", i,
scheduling_score,
selected_load);
*/
offset += sprintf(buffer + offset, " %.2f",
selected_load);
offset_for_wake_up += sprintf(buffer_for_wake_up_events + offset_for_wake_up, " %" PRIu64 "",
selected_wake_up_count);
offset_for_future += sprintf(buffer_for_future_timeline + offset_for_future, " %d",
thorium_worker_get_scheduled_actor_count(worker));
sum += selected_load;
++i;
}
load = sum / count;
printf("thorium_worker_pool: node/%d %s LOAD %d s %.2f/%d (%.2f)%s\n",
node_name,
description, elapsed,
sum, count, load, buffer);
printf("thorium_worker_pool: node/%d %s FUTURE_TIMELINE %d s %s\n",
node_name,
description, elapsed,
buffer_for_future_timeline);
printf("thorium_worker_pool: node/%d %s WAKE_UP_COUNT %d s %s\n",
node_name,
description, elapsed,
buffer_for_wake_up_events);
core_memory_free(buffer, MEMORY_WORKER_POOL_KEY);
core_memory_free(buffer_for_wake_up_events, MEMORY_WORKER_POOL_KEY);
core_memory_free(buffer_for_future_timeline, MEMORY_WORKER_POOL_KEY);
}
void biosal_input_stream_receive(struct thorium_actor *actor, struct thorium_message *message)
{
int tag;
int source;
uint64_t count;
struct biosal_input_stream *concrete_self;
int i;
int has_sequence;
int sequences;
int sequence_index;
int buffer_size;
char *buffer;
char *read_buffer;
struct biosal_mega_block mega_block;
char *file_name_in_buffer;
if (thorium_actor_take_action(actor, message)) {
return;
}
concrete_self = thorium_actor_concrete_actor(actor);
tag = thorium_message_action(message);
source = thorium_message_source(message);
buffer = (char *)thorium_message_buffer(message);
/* Do nothing if there is an error.
* has_error returns the error to the source.
*/
/*
if (biosal_input_stream_has_error(actor, message)) {
return;
}
*/
if (tag == ACTION_INPUT_OPEN) {
#ifdef BIOSAL_INPUT_STREAM_DEBUG
printf("DEBUG ACTION_INPUT_OPEN\n");
#endif
if (concrete_self->open) {
concrete_self->error = BIOSAL_INPUT_ERROR_ALREADY_OPEN;
thorium_actor_send_reply_int(actor, ACTION_INPUT_OPEN_REPLY, concrete_self->error);
thorium_actor_send_to_self_empty(actor, ACTION_ASK_TO_STOP);
return;
}
concrete_self->open = 1;
/* TODO: find out the maximum read length in some way */
concrete_self->maximum_sequence_length = BIOSAL_INPUT_MAXIMUM_SEQUENCE_LENGTH;
concrete_self->buffer_for_sequence = (char *)core_memory_allocate(concrete_self->maximum_sequence_length, MEMORY_INPUT_STREAM);
/*biosal_input_stream_init(actor);*/
#ifdef BIOSAL_INPUT_STREAM_DEBUG
printf("DEBUG biosal_input_stream_receive open %s\n",
buffer);
#endif
/*core_memory_copy(&concrete_self->file_index, buffer, sizeof(concrete_self->file_index));*/
file_name_in_buffer = buffer;
printf("stream/%d (node/%d) opens file %s offset %" PRIu64 "\n", thorium_actor_name(actor),
thorium_actor_node_name(actor), file_name_in_buffer,
concrete_self->starting_offset);
#ifdef DEBUG_ISSUE_594
thorium_message_print(message);
printf("Buffer %s\n", buffer);
#endif
concrete_self->file_name = core_memory_allocate(strlen(file_name_in_buffer) + 1, MEMORY_INPUT_STREAM);
strcpy(concrete_self->file_name, file_name_in_buffer);
biosal_input_proxy_init(&concrete_self->proxy, concrete_self->file_name,
concrete_self->starting_offset, concrete_self->ending_offset);
concrete_self->proxy_ready = 1;
/* Die if there is an error...
*/
if (biosal_input_stream_has_error(actor, message)) {
#ifdef BIOSAL_INPUT_STREAM_DEBUG
printf("DEBUG has error\n");
#endif
thorium_actor_send_reply_int(actor, ACTION_INPUT_OPEN_REPLY, concrete_self->error);
thorium_actor_send_to_self_empty(actor, ACTION_ASK_TO_STOP);
return;
}
concrete_self->controller = source;
/* no error here... */
thorium_actor_send_reply_int(actor, ACTION_INPUT_OPEN_REPLY, concrete_self->error);
} else if (tag == ACTION_INPUT_COUNT) {
/* count a little bit and yield the worker */
if (concrete_self->count_customer == THORIUM_ACTOR_NOBODY) {
concrete_self->count_customer = source;
}
if (biosal_input_stream_check_open_error(actor, message)) {
thorium_actor_send_reply_int64_t(actor, ACTION_INPUT_COUNT_REPLY, concrete_self->error);
thorium_actor_send_to_self_empty(actor, ACTION_ASK_TO_STOP);
return;
}
#ifdef BIOSAL_INPUT_STREAM_DEBUG
printf("DEBUG ACTION_INPUT_COUNT received...\n");
#endif
i = 0;
/* continue counting ... */
has_sequence = 1;
while (i < concrete_self->granularity && has_sequence) {
has_sequence = biosal_input_proxy_get_sequence(&concrete_self->proxy,
concrete_self->buffer_for_sequence);
#if 0
printf("Sequence= %s\n", concrete_self->buffer_for_sequence);
#endif
i++;
}
sequences = biosal_input_proxy_size(&concrete_self->proxy);
#ifdef BIOSAL_INPUT_STREAM_DEBUG
printf("DEBUG ACTION_INPUT_COUNT sequences %d...\n", sequences);
#endif
if (!has_sequence || sequences % concrete_self->mega_block_size == 0) {
biosal_mega_block_init(&mega_block, -1, concrete_self->last_offset,
sequences - concrete_self->last_entries, sequences);
core_vector_push_back(&concrete_self->mega_blocks, &mega_block);
concrete_self->last_entries = sequences;
concrete_self->last_offset = biosal_input_proxy_offset(&concrete_self->proxy);
thorium_actor_send_int64_t(actor, concrete_self->controller, ACTION_INPUT_COUNT_PROGRESS, sequences);
}
if (has_sequence) {
/*printf("DEBUG yield\n");*/
thorium_actor_send_to_self_empty(actor, ACTION_YIELD);
/* notify the controller of our progress...
*/
} else {
thorium_actor_send_to_self_empty(actor, ACTION_INPUT_COUNT_READY);
}
} else if (tag == ACTION_YIELD_REPLY) {
if (biosal_input_stream_check_open_error(actor, message)) {
/*
* it is not clear that there can be an error when receiving YIELD.
error = concrete_self->error;
thorium_actor_send_reply_int(actor, ACTION_INPUT_COUNT_REPLY, error);
thorium_actor_send_to_self(actor, ACTION_ASK_TO_STOP);
*/
return;
}
thorium_actor_send_to_self_empty(actor, ACTION_INPUT_COUNT);
} else if (tag == ACTION_INPUT_COUNT_READY) {
if (biosal_input_stream_check_open_error(actor, message)) {
return;
}
count = biosal_input_proxy_size(&concrete_self->proxy);
thorium_actor_send_vector(actor, concrete_self->count_customer, ACTION_INPUT_COUNT_REPLY,
&concrete_self->mega_blocks);
printf("input_stream/%d on node/%d counted entries in %s, %" PRIu64 "\n",
thorium_actor_name(actor), thorium_actor_node_name(actor),
concrete_self->file_name, count);
} else if (tag == ACTION_INPUT_CLOSE) {
#ifdef BIOSAL_INPUT_STREAM_DEBUG
printf("DEBUG destroy proxy\n");
#endif
concrete_self->error = BIOSAL_INPUT_ERROR_NO_ERROR;
if (biosal_input_stream_check_open_error(actor, message)) {
concrete_self->error = BIOSAL_INPUT_ERROR_FILE_NOT_OPEN;
thorium_message_init(message, ACTION_INPUT_CLOSE_REPLY, sizeof(concrete_self->error),
&concrete_self->error);
thorium_actor_send(actor, source, message);
thorium_actor_send_to_self_empty(actor, ACTION_ASK_TO_STOP);
return;
}
thorium_message_init(message, ACTION_INPUT_CLOSE_REPLY, sizeof(concrete_self->error),
&concrete_self->error);
thorium_actor_send(actor, source, message);
thorium_actor_send_to_self_empty(actor, ACTION_ASK_TO_STOP);
#ifdef BIOSAL_INPUT_STREAM_DEBUG
printf("actor %d sending ACTION_INPUT_CLOSE_REPLY to %d\n",
thorium_actor_name(actor), source);
#endif
} else if (tag == ACTION_INPUT_GET_SEQUENCE) {
if (biosal_input_stream_check_open_error(actor, message)) {
/* the error management could be better. */
concrete_self->error = BIOSAL_INPUT_ERROR_FILE_NOT_OPEN;
thorium_message_init(message, ACTION_INPUT_GET_SEQUENCE_REPLY, sizeof(concrete_self->error),
&concrete_self->error);
thorium_actor_send(actor, source, message);
return;
}
sequence_index = biosal_input_proxy_size(&concrete_self->proxy);
/* TODO it would be clearer to use a struct to pack a int and a char []
* then to use the code below.
*/
read_buffer = concrete_self->buffer_for_sequence + sizeof(sequence_index);
has_sequence = biosal_input_proxy_get_sequence(&concrete_self->proxy,
read_buffer);
if (!has_sequence) {
thorium_message_init(message, ACTION_INPUT_GET_SEQUENCE_END, 0, NULL);
thorium_actor_send(actor, source, message);
return;
}
buffer_size = sizeof(sequence_index) + strlen(read_buffer) + 1;
core_memory_copy(concrete_self->buffer_for_sequence, &sequence_index, sizeof(sequence_index));
thorium_message_init(message, ACTION_INPUT_GET_SEQUENCE_REPLY,
buffer_size, concrete_self->buffer_for_sequence);
thorium_actor_send(actor, source, message);
} else if (tag == ACTION_INPUT_PUSH_SEQUENCES) {
biosal_input_stream_push_sequences(actor, message);
} else if (tag == ACTION_ASK_TO_STOP) {
thorium_actor_send_to_self_empty(actor, ACTION_STOP);
thorium_actor_send_range_empty(actor, &concrete_self->parallel_streams,
ACTION_ASK_TO_STOP);
thorium_actor_send_reply_empty(actor, ACTION_ASK_TO_STOP_REPLY);
} else if (tag == ACTION_INPUT_STREAM_RESET) {
/* fail silently
*/
if (!concrete_self->open) {
thorium_actor_send_reply_empty(actor, ACTION_INPUT_STREAM_RESET_REPLY);
return;
}
#ifdef BIOSAL_INPUT_STREAM_DEBUG
printf("DEBUG ACTION_INPUT_STREAM_RESET\n");
#endif
biosal_input_proxy_destroy(&concrete_self->proxy);
biosal_input_proxy_init(&concrete_self->proxy, concrete_self->file_name,
concrete_self->starting_offset,
concrete_self->ending_offset);
thorium_actor_send_reply_empty(actor, ACTION_INPUT_STREAM_RESET_REPLY);
} else if (tag == ACTION_PUSH_SEQUENCE_DATA_BLOCK_REPLY) {
thorium_actor_send_to_supervisor_int(actor, ACTION_INPUT_PUSH_SEQUENCES_REPLY,
source);
}
}
uint64_t core_fastq_input_get_sequence(struct biosal_input_format *input,
char *sequence)
{
struct core_fastq_input *fastq;
/*
* Input sequence has at least BIOSAL_INPUT_MAXIMUM_SEQUENCE_LENGTH
* which is currently 512k
*/
/* TODO use a dynamic buffer to accept long reads... */
int maximum_sequence_length = BIOSAL_INPUT_MAXIMUM_SEQUENCE_LENGTH;
int value;
int length;
fastq = (struct core_fastq_input *)biosal_input_format_implementation(input);
if (fastq->buffer == NULL) {
fastq->buffer = (char *)core_memory_allocate(maximum_sequence_length + 1, MEMORY_FASTQ);
}
value = 0;
/*
* Read name
*/
value += core_buffered_reader_read_line(&fastq->reader, fastq->buffer,
maximum_sequence_length);
#ifdef FIND_IDENTIFIER
/*
* If we do not have the first entry yet,
* make sure that the line is a good line.
*/
if (!fastq->has_first) {
while (!core_fastq_input_is_identifier(input, fastq->buffer)) {
value += core_buffered_reader_read_line(&fastq->reader, fastq->buffer,
maximum_sequence_length);
}
fastq->has_first = 1;
}
#endif
/*
* Read DNA sequence
*/
length = core_buffered_reader_read_line(&fastq->reader, sequence,
maximum_sequence_length);
#ifdef BIOSAL_FASTQ_INPUT_DEBUG_READ_LINE
printf("FASTQ ReadLine <<%s>>\n", sequence);
#endif
if (sequence[length - 1] == '\n') {
/*
* Remove new line symbol.
*/
sequence[length - 1] = '\0';
}
value += length;
#ifdef BIOSAL_FASTQ_INPUT_DEBUG2
printf("DEBUG core_fastq_input_get_sequence %s\n", buffer);
#endif
/*
* Read the + symbol
*/
value += core_buffered_reader_read_line(&fastq->reader, fastq->buffer,
maximum_sequence_length);
/*
* Read quality string.
*/
value += core_buffered_reader_read_line(&fastq->reader, fastq->buffer,
maximum_sequence_length);
return value;
}
void thorium_transport_init(struct thorium_transport *self, struct thorium_node *node,
int *argc, char ***argv,
struct core_memory_pool *inbound_message_memory_pool,
struct core_memory_pool *outbound_message_memory_pool)
{
int actual_argc;
char **actual_argv;
self->active_request_count = 0;
actual_argc = *argc;
actual_argv = *argv;
self->flags = 0;
core_bitmap_clear_bit_uint32_t(&self->flags, FLAG_PROFILE);
core_bitmap_clear_bit_uint32_t(&self->flags, FLAG_PRINT_TRANSPORT_EVENTS);
self->transport_interface = NULL;
self->concrete_transport = NULL;
/*
printf("DEBUG Initiating transport\n");
*/
/* Select the transport layer
*/
/*
* Assign functions
*/
thorium_transport_select_implementation(self, actual_argc, actual_argv);
self->node = node;
self->rank = -1;
self->size = -1;
if (self->transport_interface != NULL) {
self->concrete_transport = core_memory_allocate(self->transport_interface->size, MEMORY_TRANSPORT);
self->transport_interface->init(self, argc, argv);
}
CORE_DEBUGGER_ASSERT(self->rank >= 0);
CORE_DEBUGGER_ASSERT(self->size >= 1);
CORE_DEBUGGER_ASSERT(self->node != NULL);
self->inbound_message_memory_pool = inbound_message_memory_pool;
self->outbound_message_memory_pool = outbound_message_memory_pool;
thorium_transport_profiler_init(&self->transport_profiler);
if (core_command_has_argument(actual_argc, actual_argv, "-enable-transport-profiler")) {
printf("Enable transport profiler\n");
core_bitmap_set_bit_uint32_t(&self->flags, FLAG_PROFILE);
}
if (self->rank == 0) {
printf("thorium_transport: type %s\n",
self->transport_interface->name);
}
if (core_command_has_argument(actual_argc, actual_argv, "-print-transport-events")) {
core_bitmap_set_bit_uint32_t(&self->flags, FLAG_PRINT_TRANSPORT_EVENTS);
}
core_timer_init(&self->timer);
self->start_time = core_timer_get_nanoseconds(&self->timer);
}
int main(int argc, char **argv)
{
BEGIN_TESTS();
{
struct core_map big_map;
int kmer_length = 43;
struct biosal_dna_kmer kmer;
int count;
int run_test;
int coverage;
void *key;
int key_length;
int *bucket;
int i;
struct biosal_dna_codec codec;
struct core_memory_pool memory;
core_memory_pool_init(&memory, 1048576, -1);
biosal_dna_codec_init(&codec);
run_test = 1;
count = 100000000;
printf("STRESS TEST\n");
biosal_dna_kmer_init_mock(&kmer, kmer_length, &codec, &memory);
key_length = biosal_dna_kmer_pack_size(&kmer, kmer_length, &codec);
biosal_dna_kmer_destroy(&kmer, &memory);
core_map_init(&big_map, key_length, sizeof(coverage));
key = core_memory_allocate(key_length, -1);
i = 0;
while (i < count && run_test) {
biosal_dna_kmer_init_random(&kmer, kmer_length, &codec, &memory);
biosal_dna_kmer_pack_store_key(&kmer, key, kmer_length, &codec, &memory);
bucket = core_map_add(&big_map, key);
coverage = 99;
(*bucket) = coverage;
biosal_dna_kmer_destroy(&kmer, &memory);
if (i % 100000 == 0) {
printf("ADD %d/%d %" PRIu64 "\n", i, count,
core_map_size(&big_map));
}
i++;
}
core_map_destroy(&big_map);
core_memory_free(key, -1);
biosal_dna_codec_destroy(&codec);
core_memory_pool_destroy(&memory);
}
END_TESTS();
return 0;
}
