void biosal_input_command_destroy(struct biosal_input_command *self, struct core_memory_pool *memory)
{
struct biosal_dna_sequence *sequence;
struct core_vector_iterator iterator;
self->store_name= -1;
self->store_first = 0;
self->store_last = 0;
core_vector_iterator_init(&iterator, &self->entries);
while (core_vector_iterator_has_next(&iterator)) {
core_vector_iterator_next(&iterator, (void **)&sequence);
biosal_dna_sequence_destroy(sequence, memory);
}
core_vector_iterator_destroy(&iterator);
core_vector_destroy(&self->entries);
}
void biosal_input_controller_receive(struct thorium_actor *actor, struct thorium_message *message)
{
int tag;
int count;
char *file;
void *buffer;
struct biosal_input_controller *controller;
struct biosal_input_controller *concrete_actor;
int destination;
int script;
int stream;
char *local_file;
int i;
int name;
int source;
int destination_index;
struct thorium_message new_message;
int error;
int stream_index;
int64_t entries;
int64_t *bucket;
int *int_bucket;
int spawner;
int command_name;
int stream_name;
int consumer;
int consumer_index;
int *bucket_for_requests;
char *new_buffer;
int new_count;
int file_index;
struct core_vector mega_blocks;
struct core_vector_iterator vector_iterator;
struct biosal_mega_block *mega_block;
struct core_vector *vector_bucket;
struct core_vector block_counts;
uint64_t block_entries;
int mega_block_index;
uint64_t offset;
struct biosal_mega_block *block;
int acquaintance_index;
if (thorium_actor_take_action(actor, message)) {
return;
}
thorium_message_get_all(message, &tag, &count, &buffer, &source);
name = thorium_actor_name(actor);
controller = (struct biosal_input_controller *)thorium_actor_concrete_actor(actor);
concrete_actor = controller;
if (tag == ACTION_START) {
core_vector_init(&concrete_actor->spawners, 0);
core_vector_unpack(&concrete_actor->spawners, buffer);
core_vector_resize(&concrete_actor->stores_per_spawner,
core_vector_size(&concrete_actor->spawners));
for (i = 0; i < core_vector_size(&concrete_actor->spawners); i++) {
int_bucket = (int *)core_vector_at(&concrete_actor->stores_per_spawner, i);
*int_bucket = 0;
spawner = core_vector_at_as_int(&concrete_actor->spawners, i);
core_queue_enqueue(&concrete_actor->unprepared_spawners, &spawner);
}
concrete_actor->state = BIOSAL_INPUT_CONTROLLER_STATE_PREPARE_SPAWNERS;
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL
printf("DEBUG preparing first spawner\n");
#endif
thorium_actor_send_to_self_empty(actor, ACTION_INPUT_CONTROLLER_PREPARE_SPAWNERS);
/*
thorium_dispatcher_print(thorium_actor_dispatcher(actor));
*/
} else if (tag == ACTION_ADD_FILE) {
file = (char *)buffer;
local_file = core_memory_allocate(strlen(file) + 1, MEMORY_CONTROLLER);
strcpy(local_file, file);
printf("controller %d ACTION_ADD_FILE %s\n",
thorium_actor_name(actor),
local_file);
core_vector_push_back(&concrete_actor->files, &local_file);
bucket = core_vector_at(&concrete_actor->files, core_vector_size(&concrete_actor->files) - 1);
local_file = *(char **)bucket;
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2
printf("DEBUG11 ACTION_ADD_FILE %s %p bucket %p index %d\n",
local_file, local_file, (void *)bucket, core_vector_size(&concrete_actor->files) - 1);
#endif
thorium_actor_send_reply_empty(actor, ACTION_ADD_FILE_REPLY);
} else if (tag == ACTION_SPAWN_REPLY) {
if (concrete_actor->state == BIOSAL_INPUT_CONTROLLER_STATE_SPAWN_STORES) {
biosal_input_controller_add_store(actor, message);
return;
} else if (concrete_actor->state == BIOSAL_INPUT_CONTROLLER_STATE_PREPARE_SPAWNERS) {
concrete_actor->ready_spawners++;
thorium_message_unpack_int(message, 0, &name);
thorium_actor_send_empty(actor, name, ACTION_ASK_TO_STOP);
thorium_actor_send_to_self_empty(actor, ACTION_INPUT_CONTROLLER_PREPARE_SPAWNERS);
if (concrete_actor->ready_spawners == (int)core_vector_size(&concrete_actor->spawners)) {
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG
printf("DEBUG all spawners are prepared\n");
#endif
thorium_actor_send_to_supervisor_empty(actor, ACTION_START_REPLY);
}
return;
} else if (concrete_actor->state == BIOSAL_INPUT_CONTROLLER_STATE_SPAWN_PARTITIONER) {
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG
printf("DEBUG received spawn reply, state is spawn_partitioner\n");
#endif
thorium_message_unpack_int(message, 0, &concrete_actor->partitioner);
/* configure the partitioner
*/
destination = concrete_actor->partitioner;
thorium_actor_send_int(actor, destination,
ACTION_SEQUENCE_PARTITIONER_SET_BLOCK_SIZE,
concrete_actor->block_size);
thorium_actor_send_int(actor, destination,
ACTION_SEQUENCE_PARTITIONER_SET_ACTOR_COUNT,
core_vector_size(&concrete_actor->consumers));
core_vector_init(&block_counts, sizeof(uint64_t));
for (i = 0; i < core_vector_size(&concrete_actor->mega_block_vector); i++) {
block = (struct biosal_mega_block *)core_vector_at(&concrete_actor->mega_block_vector, i);
block_entries = biosal_mega_block_get_entries(block);
core_vector_push_back_uint64_t(&block_counts, block_entries);
}
new_count = core_vector_pack_size(&block_counts);
new_buffer = thorium_actor_allocate(actor, new_count);
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG
printf("DEBUG packed counts, %d bytes\n", count);
#endif
core_vector_pack(&block_counts, new_buffer);
thorium_message_init(&new_message, ACTION_SEQUENCE_PARTITIONER_SET_ENTRY_VECTOR,
new_count, new_buffer);
thorium_actor_send(actor, destination, &new_message);
core_vector_destroy(&block_counts);
return;
} else if (concrete_actor->state == BIOSAL_INPUT_CONTROLLER_STATE_SPAWN_READING_STREAMS) {
thorium_message_unpack_int(message, 0, &stream);
stream_index = stream;
mega_block_index = core_vector_size(&concrete_actor->reading_streams);
core_vector_push_back_int(&concrete_actor->reading_streams, stream_index);
core_vector_push_back_int(&concrete_actor->partition_commands, -1);
core_vector_push_back_int(&concrete_actor->stream_consumers, -1);
stream_index = core_vector_size(&concrete_actor->reading_streams) - 1;
mega_block = (struct biosal_mega_block *)core_vector_at(&concrete_actor->mega_block_vector,
mega_block_index);
offset = biosal_mega_block_get_offset(mega_block);
core_map_add_value(&concrete_actor->assigned_blocks,
&stream_index, &mega_block_index);
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_READING_STREAMS
printf("DEBUG setting offset to %" PRIu64 " for stream/%d\n",
offset, stream);
#endif
thorium_actor_send_uint64_t(actor, stream, ACTION_INPUT_STREAM_SET_START_OFFSET, offset);
return;
}
stream = *(int *)buffer;
file_index = core_vector_size(&concrete_actor->counting_streams);
local_file = *(char **)core_vector_at(&concrete_actor->files, file_index);
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_READING_STREAMS
printf("DEBUG actor %d receives stream %d from spawner %d for file %s\n",
name, stream, source,
local_file);
#endif
core_vector_push_back(&concrete_actor->counting_streams, &stream);
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_READING_STREAMS
printf("asking stream/%d to open %s\n", stream, local_file);
#endif
thorium_message_init(&new_message, ACTION_INPUT_OPEN, strlen(local_file) + 1, local_file);
#ifdef DEBUG_ISSUE_594
thorium_message_print(&new_message);
printf("SEND Buffer %s\n", local_file);
#endif
thorium_actor_send(actor, stream, &new_message);
thorium_message_destroy(&new_message);
if (core_vector_size(&concrete_actor->counting_streams) != core_vector_size(&concrete_actor->files)) {
thorium_actor_send_to_self_empty(actor, ACTION_INPUT_SPAWN);
}
#ifdef DEBUG_ISSUE_594
printf("EXIT Buffer %s\n", local_file);
#endif
} else if (tag == ACTION_INPUT_OPEN_REPLY) {
if (concrete_actor->state == BIOSAL_INPUT_CONTROLLER_STATE_SPAWN_READING_STREAMS) {
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_READING_STREAMS
printf("DEBUG receives open.reply for reading stream/%d\n", source);
#endif
concrete_actor->opened_streams++;
if (concrete_actor->opened_streams == core_vector_size(&concrete_actor->mega_block_vector)) {
thorium_actor_send_to_self_empty(actor, ACTION_INPUT_CONTROLLER_CREATE_STORES);
}
return;
}
concrete_actor->opened_streams++;
stream = source;
thorium_message_unpack_int(message, 0, &error);
if (error == BIOSAL_INPUT_ERROR_NO_ERROR) {
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2
printf("DEBUG actor %d asks %d ACTION_INPUT_COUNT_IN_PARALLEL\n", name, stream);
#endif
thorium_actor_send_vector(actor, stream, ACTION_INPUT_COUNT_IN_PARALLEL,
&concrete_actor->spawners);
} else {
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2
printf("DEBUG actor %d received error %d from %d\n", name, error, stream);
#endif
concrete_actor->counted++;
}
/* if all streams failed, notice supervisor */
if (concrete_actor->counted == core_vector_size(&concrete_actor->files)) {
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2
#endif
printf("DEBUG %d: Error all streams failed.\n",
thorium_actor_name(actor));
thorium_actor_send_to_supervisor_empty(actor, ACTION_INPUT_DISTRIBUTE_REPLY);
}
/*
if (concrete_actor->opened_streams == core_vector_size(&concrete_actor->files)) {
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG
printf("DEBUG controller %d sends ACTION_INPUT_DISTRIBUTE_REPLY to supervisor %d [%d/%d]\n",
name, thorium_actor_supervisor(actor),
concrete_actor->opened_streams, core_vector_size(&concrete_actor->files));
#endif
}
*/
} else if (tag == ACTION_INPUT_COUNT_PROGRESS) {
stream_index = core_vector_index_of(&concrete_actor->counting_streams, &source);
local_file = core_vector_at_as_char_pointer(&concrete_actor->files, stream_index);
thorium_message_unpack_int64_t(message, 0, &entries);
bucket = (int64_t *)core_vector_at(&concrete_actor->counts, stream_index);
printf("controller/%d receives progress from stream/%d file %s %" PRIu64 " entries so far\n",
name, source, local_file, entries);
*bucket = entries;
} else if (tag == ACTION_INPUT_COUNT_IN_PARALLEL_REPLY) {
stream_index = core_vector_index_of(&concrete_actor->counting_streams, &source);
local_file = core_vector_at_as_char_pointer(&concrete_actor->files, stream_index);
core_vector_init(&mega_blocks, 0);
core_vector_unpack(&mega_blocks, buffer);
printf("DEBUG receive mega blocks from %d\n", source);
/*
* Update the file index for every mega block.
*/
core_vector_iterator_init(&vector_iterator, &mega_blocks);
bucket = (int64_t*)core_vector_at(&concrete_actor->counts, stream_index);
(*bucket) = 0;
while (core_vector_iterator_has_next(&vector_iterator)) {
core_vector_iterator_next(&vector_iterator, (void **)&mega_block);
printf("SETTING setting file to %d for mega block\n", stream_index);
biosal_mega_block_set_file(mega_block, stream_index);
entries = biosal_mega_block_get_entries_from_start(mega_block);
printf("Cataloging %d ENTRIES\n", (int)entries);
(*bucket) = entries;
biosal_mega_block_print(mega_block);
}
core_vector_iterator_destroy(&vector_iterator);
vector_bucket = (struct core_vector *)core_map_add(&concrete_actor->mega_blocks, &stream_index);
core_vector_init_copy(vector_bucket, &mega_blocks);
core_vector_destroy(&mega_blocks);
concrete_actor->counted++;
printf("controller/%d received from stream/%d for file %s %" PRIu64 " entries (final) %d/%d\n",
name, source, local_file, entries,
concrete_actor->counted, (int)core_vector_size(&concrete_actor->files));
thorium_actor_send_reply_empty(actor, ACTION_INPUT_CLOSE);
/* continue work here, tell supervisor about it */
if (concrete_actor->counted == core_vector_size(&concrete_actor->files)) {
thorium_actor_send_to_self_empty(actor, ACTION_INPUT_CONTROLLER_SPAWN_READING_STREAMS);
}
} else if (tag == ACTION_INPUT_DISTRIBUTE) {
core_timer_start(&concrete_actor->input_timer);
core_timer_start(&concrete_actor->counting_timer);
/* for each file, spawn a stream to count */
/* no files, return immediately
*/
if (core_vector_size(&concrete_actor->files) == 0) {
printf("Error: no file to distribute...\n");
thorium_actor_send_reply_empty(actor, ACTION_INPUT_DISTRIBUTE_REPLY);
return;
}
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2
printf("DEBUG actor %d receives ACTION_INPUT_DISTRIBUTE\n", name);
#endif
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2
printf("DEBUG send ACTION_INPUT_SPAWN to self\n");
#endif
thorium_actor_send_to_self_empty(actor, ACTION_INPUT_SPAWN);
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2
printf("DEBUG resizing counts to %d\n", core_vector_size(&concrete_actor->files));
#endif
core_vector_resize(&concrete_actor->counts, core_vector_size(&concrete_actor->files));
for (i = 0; i < core_vector_size(&concrete_actor->counts); i++) {
bucket = (int64_t*)core_vector_at(&concrete_actor->counts, i);
*bucket = 0;
}
} else if (tag == ACTION_INPUT_SPAWN && source == name) {
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2
printf("DEBUG ACTION_INPUT_SPAWN\n");
#endif
script = SCRIPT_INPUT_STREAM;
concrete_actor->state = BIOSAL_INPUT_CONTROLLER_STATE_SPAWN_STREAMS;
/* the next file name to send is the current number of streams */
i = core_vector_size(&concrete_actor->counting_streams);
destination_index = i % core_vector_size(&concrete_actor->spawners);
destination = *(int *)core_vector_at(&concrete_actor->spawners, destination_index);
thorium_message_init(message, ACTION_SPAWN, sizeof(script), &script);
thorium_actor_send(actor, destination, message);
bucket = core_vector_at(&concrete_actor->files, i);
local_file = *(char **)core_vector_at(&concrete_actor->files, i);
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2
printf("DEBUG890 local_file %p bucket %p index %d\n", local_file, (void *)bucket,
i);
#endif
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG
printf("DEBUG actor %d spawns a stream for file %d/%d via spawner %d\n",
name, i, core_vector_size(&concrete_actor->files), destination);
#endif
/* also, spawn 4 stores on each node */
} else if (tag == ACTION_ASK_TO_STOP && ( source == thorium_actor_supervisor(actor)
|| source == thorium_actor_name(actor))) {
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_LEVEL_2
#endif
/* stop streams
*/
for (i = 0; i < core_vector_size(&concrete_actor->counting_streams); i++) {
stream = *(int *)core_vector_at(&concrete_actor->counting_streams, i);
thorium_actor_send_empty(actor, stream, ACTION_ASK_TO_STOP);
}
for (i = 0; i < core_vector_size(&concrete_actor->reading_streams); i++) {
stream = *(int *)core_vector_at(&concrete_actor->reading_streams, i);
thorium_actor_send_empty(actor, stream, ACTION_ASK_TO_STOP);
}
#if 0
/* stop data stores
*/
for (i = 0; i < core_vector_size(&concrete_actor->consumers); i++) {
store = core_vector_at_as_int(&concrete_actor->consumers, i);
thorium_actor_send_empty(actor, store, ACTION_ASK_TO_STOP);
}
#endif
/* stop partitioner
*/
if (concrete_actor->partitioner != THORIUM_ACTOR_NOBODY) {
thorium_actor_send_empty(actor,
concrete_actor->partitioner,
ACTION_ASK_TO_STOP);
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG
printf("DEBUG controller %d sends ACTION_ASK_TO_STOP_REPLY to %d\n",
thorium_actor_name(actor),
thorium_message_source(message));
#endif
}
thorium_actor_send_reply_empty(actor, ACTION_ASK_TO_STOP_REPLY);
/* stop self
*/
thorium_actor_send_to_self_empty(actor, ACTION_STOP);
thorium_actor_ask_to_stop(actor, message);
printf("DEBUG controller %d dies\n", name);
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG
#endif
} else if (tag == ACTION_INPUT_CONTROLLER_CREATE_PARTITION && source == name) {
spawner = *(int *)core_vector_at(&concrete_actor->spawners,
core_vector_size(&concrete_actor->spawners) / 2);
thorium_actor_send_int(actor, spawner, ACTION_SPAWN,
SCRIPT_SEQUENCE_PARTITIONER);
concrete_actor->state = BIOSAL_INPUT_CONTROLLER_STATE_SPAWN_PARTITIONER;
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG
printf("DEBUG input controller %d spawns a partitioner via spawner %d\n",
name, spawner);
#endif
} else if (tag == ACTION_SEQUENCE_PARTITIONER_COMMAND_IS_READY) {
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG
printf("DEBUG controller receives ACTION_SEQUENCE_PARTITIONER_COMMAND_IS_READY, asks for command\n");
#endif
thorium_actor_send_reply_empty(actor, ACTION_SEQUENCE_PARTITIONER_GET_COMMAND);
} else if (tag == ACTION_SEQUENCE_PARTITIONER_GET_COMMAND_REPLY) {
biosal_input_controller_receive_command(actor, message);
} else if (tag == ACTION_SEQUENCE_PARTITIONER_FINISHED) {
thorium_actor_send_empty(actor,
concrete_actor->partitioner,
ACTION_ASK_TO_STOP);
biosal_input_controller_verify_requests(actor, message);
} else if (tag == ACTION_SEQUENCE_PARTITIONER_PROVIDE_STORE_ENTRY_COUNTS) {
biosal_input_controller_receive_store_entry_counts(actor, message);
} else if (tag == ACTION_RESERVE_REPLY) {
concrete_actor->ready_consumers++;
printf("DEBUG marker ACTION_RESERVE_REPLY %d/%d\n",
concrete_actor->ready_consumers,
(int)core_vector_size(&concrete_actor->consumers));
if (concrete_actor->ready_consumers == core_vector_size(&concrete_actor->consumers)) {
concrete_actor->ready_consumers = 0;
printf("DEBUG all consumers are ready\n");
thorium_actor_send_empty(actor,
concrete_actor->partitioner,
ACTION_SEQUENCE_PARTITIONER_PROVIDE_STORE_ENTRY_COUNTS_REPLY);
}
} else if (tag == ACTION_INPUT_PUSH_SEQUENCES_READY) {
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG
printf("DEBUG biosal_input_controller_receive received ACTION_INPUT_PUSH_SEQUENCES_READY\n");
#endif
stream_name = source;
acquaintance_index = stream_name;
stream_index = core_vector_index_of(&concrete_actor->reading_streams, &acquaintance_index);
command_name = *(int *)core_vector_at(&concrete_actor->partition_commands,
stream_index);
thorium_actor_send_int(actor,
concrete_actor->partitioner,
ACTION_SEQUENCE_PARTITIONER_GET_COMMAND_REPLY_REPLY,
command_name);
} else if (tag == ACTION_INPUT_PUSH_SEQUENCES_REPLY) {
stream_name = source;
thorium_message_unpack_int(message, 0, &consumer);
consumer_index = core_vector_index_of(&concrete_actor->consumers,
&consumer);
bucket_for_requests = (int *)core_vector_at(&concrete_actor->consumer_active_requests, consumer_index);
(*bucket_for_requests)--;
biosal_input_controller_verify_requests(actor, message);
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_CONSUMERS
printf("DEBUG consumer # %d has %d active requests\n",
consumer_index, *bucket_for_requests);
#endif
} else if (tag == ACTION_SET_CONSUMERS) {
core_vector_init(&concrete_actor->consumers, 0);
core_vector_unpack(&concrete_actor->consumers, buffer);
printf("controller %d receives %d consumers\n",
thorium_actor_name(actor),
(int)core_vector_size(&concrete_actor->consumers));
for (i = 0; i < core_vector_size(&concrete_actor->consumers); i++) {
core_vector_push_back_int(&concrete_actor->consumer_active_requests, 0);
}
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG
core_vector_print_int(&concrete_actor->consumers);
printf("\n");
#endif
thorium_actor_send_reply_empty(actor, ACTION_SET_CONSUMERS_REPLY);
} else if (tag == ACTION_SET_BLOCK_SIZE) {
thorium_message_unpack_int(message, 0, &concrete_actor->block_size);
thorium_actor_send_reply_empty(actor, ACTION_SET_BLOCK_SIZE_REPLY);
} else if (tag == ACTION_SEQUENCE_STORE_READY) {
concrete_actor->filled_consumers++;
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG
printf("DEBUG ACTION_SEQUENCE_STORE_READY %d/%d\n", concrete_actor->filled_consumers,
(int)core_vector_size(&concrete_actor->consumers));
#endif
if (concrete_actor->filled_consumers == core_vector_size(&concrete_actor->consumers)) {
concrete_actor->filled_consumers = 0;
printf("DEBUG: all consumers are filled, sending ACTION_INPUT_DISTRIBUTE_REPLY\n");
core_timer_stop(&concrete_actor->input_timer);
core_timer_stop(&concrete_actor->distribution_timer);
core_timer_print_with_description(&concrete_actor->distribution_timer,
"Load input / Distribute input data");
core_timer_print_with_description(&concrete_actor->input_timer,
"Load input");
thorium_actor_send_to_supervisor_empty(actor, ACTION_INPUT_DISTRIBUTE_REPLY);
}
}
}
void biosal_input_controller_spawn_streams(struct thorium_actor *actor, struct thorium_message *message)
{
int spawner;
struct biosal_input_controller *concrete_actor;
struct core_vector_iterator iterator;
int i;
int j;
int block_index;
struct core_vector *vector;
struct biosal_mega_block *block;
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_READING_STREAMS
printf("DEBUG biosal_input_controller_spawn_streams\n");
#endif
concrete_actor = (struct biosal_input_controller *)thorium_actor_concrete_actor(actor);
concrete_actor->opened_streams = 0;
concrete_actor->spawner = 0;
/* gather mega blocks
*/
block_index = 0;
printf("DEBUG received MEGA BLOCKS\n");
for (i = 0; i < core_vector_size(&concrete_actor->files); i++) {
vector = (struct core_vector *)core_map_get(&concrete_actor->mega_blocks, &i);
if (vector == NULL) {
continue;
}
for (j = 0; j < core_vector_size(vector); j++) {
block = (struct biosal_mega_block *)core_vector_at(vector, j);
printf("BLOCK # %d ", block_index);
block_index++;
biosal_mega_block_print(block);
core_vector_push_back(&concrete_actor->mega_block_vector, block);
}
}
printf("DEBUG MEGA BLOCKS (total: %d)\n", block_index);
core_vector_iterator_init(&iterator, &concrete_actor->mega_block_vector);
while (core_vector_iterator_has_next(&iterator)) {
core_vector_iterator_next(&iterator, NULL);
spawner = core_vector_at_as_int(&concrete_actor->spawners, concrete_actor->spawner);
concrete_actor->spawner++;
concrete_actor->spawner %= core_vector_size(&concrete_actor->spawners);
#ifdef BIOSAL_INPUT_CONTROLLER_DEBUG_READING_STREAMS
printf("DEBUG asking %d to spawn script %d\n", spawner, SCRIPT_INPUT_STREAM);
#endif
thorium_actor_send_int(actor, spawner, ACTION_SPAWN, SCRIPT_INPUT_STREAM);
}
core_vector_iterator_destroy(&iterator);
concrete_actor->state = BIOSAL_INPUT_CONTROLLER_STATE_SPAWN_READING_STREAMS;
}
void biosal_sequence_partitioner_verify(struct thorium_actor *actor)
{
struct biosal_sequence_partitioner *concrete_actor;
int i;
int64_t entries;
uint64_t position;
uint64_t stream_entries;
int bytes;
void *buffer;
struct thorium_message message;
int64_t remaining;
int remainder;
uint64_t *bucket_for_store_count;
struct core_vector_iterator iterator;
concrete_actor = (struct biosal_sequence_partitioner *)thorium_actor_concrete_actor(actor);
/*
* check if parameters are
* initialized
*/
if (concrete_actor->block_size == -1) {
return;
}
if (concrete_actor->store_count == -1) {
return;
}
if (core_vector_size(&concrete_actor->stream_entries) == 0) {
return;
}
/* at this point, all parameters are ready.
* prepare <stream_entries.size> commands
*/
position = 0;
entries = 0;
/*
printf("DEBUG generating initial positions\n");
*/
/* generate stream positions, stream global positions, and total
*/
for (i = 0; i < core_vector_size(&concrete_actor->stream_entries); i++) {
core_vector_push_back(&concrete_actor->stream_positions, &position);
core_vector_push_back(&concrete_actor->stream_global_positions, &entries);
stream_entries = *(uint64_t *)core_vector_at(&concrete_actor->stream_entries, i);
#ifdef BIOSAL_SEQUENCE_PARTITIONER_DEBUG
printf("DEBUG stream_entries %i %" PRIu64 "\n",
i, stream_entries);
#endif
entries += stream_entries;
}
concrete_actor->total = entries;
/* compute the number of entries for each store
*/
entries = concrete_actor->total / concrete_actor->store_count;
/* make sure that this is a multiple of block size
* examples:
* total= 20000
* store_count= 2
* block_size= 8192
* 20000 / 2 = 10000
* 10000 % 8192 = 1808
* difference = 8192 - 1808 = 6384
* 10000 + 6384 = 16384
*/
if (entries % concrete_actor->block_size != 0) {
remainder = entries % concrete_actor->block_size;
entries -= remainder;
}
/* make sure that at most one store has less
* than block size
*/
if (entries < concrete_actor->block_size) {
entries = concrete_actor->block_size;
}
#ifdef BIOSAL_SEQUENCE_PARTITIONER_DEBUG
printf("DEBUG93 entries for stores %d\n", (int)entries);
#endif
remaining = concrete_actor->total;
if (remaining <= entries) {
entries = remaining;
}
/* example: 10000, block_size 4096, 3 stores
*
* total entries remaining
* 10000 4096 5904
* 10000 4096 1808
* 10000 1808 0
*/
for (i = 0; i < concrete_actor->store_count; i++) {
core_vector_push_back(&concrete_actor->store_entries, &entries);
remaining -= entries;
if (remaining < entries) {
entries = remaining;
}
}
core_vector_iterator_init(&iterator, &concrete_actor->store_entries);
while (core_vector_iterator_has_next(&iterator)) {
core_vector_iterator_next(&iterator, (void **)&bucket_for_store_count);
if (remaining >= concrete_actor->block_size) {
*bucket_for_store_count += concrete_actor->block_size;
remaining -= concrete_actor->block_size;
} else if (remaining == 0) {
break;
} else {
/* between 1 and block_size - 1 inclusively
*/
*bucket_for_store_count += remaining;
remaining = 0;
}
}
core_vector_iterator_destroy(&iterator);
#ifdef BIOSAL_SEQUENCE_PARTITIONER_DEBUG
printf("DEBUG biosal_sequence_partitioner_verify sending store counts\n");
#endif
bytes = core_vector_pack_size(&concrete_actor->store_entries);
buffer = thorium_actor_allocate(actor, bytes);
core_vector_pack(&concrete_actor->store_entries, buffer);
thorium_message_init(&message, ACTION_SEQUENCE_PARTITIONER_PROVIDE_STORE_ENTRY_COUNTS,
bytes, buffer);
thorium_actor_send_reply(actor, &message);
}
void biosal_coverage_distribution_write_distribution(struct thorium_actor *self)
{
struct core_map_iterator iterator;
int *coverage;
uint64_t *canonical_frequency;
uint64_t frequency;
struct biosal_coverage_distribution *concrete_actor;
struct core_vector coverage_values;
struct core_vector_iterator vector_iterator;
struct core_buffered_file_writer descriptor;
struct core_buffered_file_writer descriptor_canonical;
struct core_string file_name;
struct core_string canonical_file_name;
int argc;
char **argv;
int name;
char *directory_name;
name = thorium_actor_name(self);
argc = thorium_actor_argc(self);
argv = thorium_actor_argv(self);
directory_name = biosal_command_get_output_directory(argc, argv);
/* Create the directory if it does not exist
*/
if (!core_directory_verify_existence(directory_name)) {
core_directory_create(directory_name);
}
core_string_init(&file_name, "");
core_string_append(&file_name, directory_name);
core_string_append(&file_name, "/");
core_string_append(&file_name, BIOSAL_COVERAGE_DISTRIBUTION_DEFAULT_OUTPUT_FILE);
core_string_init(&canonical_file_name, "");
core_string_append(&canonical_file_name, directory_name);
core_string_append(&canonical_file_name, "/");
core_string_append(&canonical_file_name, BIOSAL_COVERAGE_DISTRIBUTION_DEFAULT_OUTPUT_FILE_CANONICAL);
core_buffered_file_writer_init(&descriptor, core_string_get(&file_name));
core_buffered_file_writer_init(&descriptor_canonical, core_string_get(&canonical_file_name));
concrete_actor = (struct biosal_coverage_distribution *)thorium_actor_concrete_actor(self);
core_vector_init(&coverage_values, sizeof(int));
core_map_iterator_init(&iterator, &concrete_actor->distribution);
#ifdef BIOSAL_COVERAGE_DISTRIBUTION_DEBUG
thorium_actor_log(self, "map size %d\n", (int)core_map_size(&concrete_actor->distribution));
#endif
while (core_map_iterator_has_next(&iterator)) {
core_map_iterator_next(&iterator, (void **)&coverage, (void **)&canonical_frequency);
#ifdef BIOSAL_COVERAGE_DISTRIBUTION_DEBUG
thorium_actor_log(self, "DEBUG COVERAGE %d FREQUENCY %" PRIu64 "\n", *coverage, *frequency);
#endif
core_vector_push_back(&coverage_values, coverage);
}
core_map_iterator_destroy(&iterator);
core_vector_sort_int(&coverage_values);
#ifdef BIOSAL_COVERAGE_DISTRIBUTION_DEBUG
thorium_actor_log(self, "after sort ");
core_vector_print_int(&coverage_values);
thorium_actor_log(self, "\n");
#endif
core_vector_iterator_init(&vector_iterator, &coverage_values);
#if 0
core_buffered_file_writer_printf(&descriptor_canonical, "Coverage\tFrequency\n");
#endif
core_buffered_file_writer_printf(&descriptor, "Coverage\tFrequency\n");
#ifdef BIOSAL_COVERAGE_DISTRIBUTION_DEBUG
#endif
while (core_vector_iterator_has_next(&vector_iterator)) {
core_vector_iterator_next(&vector_iterator, (void **)&coverage);
canonical_frequency = (uint64_t *)core_map_get(&concrete_actor->distribution, coverage);
frequency = 2 * *canonical_frequency;
core_buffered_file_writer_printf(&descriptor_canonical, "%d %" PRIu64 "\n",
*coverage,
*canonical_frequency);
core_buffered_file_writer_printf(&descriptor, "%d\t%" PRIu64 "\n",
*coverage,
frequency);
}
core_vector_destroy(&coverage_values);
core_vector_iterator_destroy(&vector_iterator);
thorium_actor_log(self, "distribution %d wrote %s\n", name, core_string_get(&file_name));
thorium_actor_log(self, "distribution %d wrote %s\n", name, core_string_get(&canonical_file_name));
core_buffered_file_writer_destroy(&descriptor);
core_buffered_file_writer_destroy(&descriptor_canonical);
core_string_destroy(&file_name);
core_string_destroy(&canonical_file_name);
}
void thorium_balancer_balance(struct thorium_balancer *self)
{
/*
* The 95th percentile is useful:
* \see http://en.wikipedia.org/wiki/Burstable_billing
* \see http://www.init7.net/en/backbone/95-percent-rule
*/
int load_percentile_50;
struct core_timer timer;
int i;
struct core_vector loads;
struct core_vector loads_unsorted;
struct core_vector burdened_workers;
struct core_vector stalled_workers;
struct thorium_worker *worker;
struct thorium_node *node;
/*struct core_set *set;*/
struct core_pair pair;
struct core_vector_iterator vector_iterator;
int old_worker;
int actor_name;
int messages;
int maximum;
int with_maximum;
struct core_map *set;
struct core_map_iterator set_iterator;
int stalled_index;
int stalled_count;
int new_worker_index;
struct core_vector migrations;
struct thorium_migration migration;
struct thorium_migration *migration_to_do;
struct thorium_actor *actor;
int candidates;
int load_value;
int remaining_load;
int projected_load;
struct core_vector actors_to_migrate;
int total;
int with_messages;
int stalled_percentile;
int burdened_percentile;
int old_total;
int old_load;
int new_load;
int predicted_new_load;
struct core_pair *pair_pointer;
struct thorium_worker *new_worker;
/*int new_total;*/
int actor_load;
int test_stalled_index;
int tests;
int found_match;
int spawned_actors;
int killed_actors;
int perfect;
#ifdef THORIUM_SCHEDULER_ENABLE_SYMMETRIC_SCHEDULING
struct core_map symmetric_actor_scripts;
int script;
#endif
node = thorium_worker_pool_get_node(self->pool);
spawned_actors = thorium_node_get_counter(node, CORE_COUNTER_SPAWNED_ACTORS);
/* There is nothing to balance...
*/
if (spawned_actors == 0) {
return;
}
killed_actors = thorium_node_get_counter(node, CORE_COUNTER_KILLED_ACTORS);
/*
* The system can probably not be balanced to get in
* a better shape anyway.
*/
if (spawned_actors == self->last_spawned_actors
&& killed_actors == self->last_killed_actors
&& self->last_migrations == 0) {
printf("SCHEDULER: balance can not be improved because nothing changed.\n");
return;
}
/* Check if we have perfection
*/
perfect = 1;
for (i = 0; i < thorium_worker_pool_worker_count(self->pool); i++) {
worker = thorium_worker_pool_get_worker(self->pool, i);
load_value = thorium_worker_get_epoch_load(worker) * 100;
if (load_value != 100) {
perfect = 0;
break;
}
}
if (perfect) {
printf("SCHEDULER: perfect balance can not be improved.\n");
return;
}
/* update counters
*/
self->last_spawned_actors = spawned_actors;
self->last_killed_actors = killed_actors;
/* Otherwise, try to balance things
*/
core_timer_init(&timer);
core_timer_start(&timer);
#ifdef THORIUM_SCHEDULER_ENABLE_SYMMETRIC_SCHEDULING
core_map_init(&symmetric_actor_scripts, sizeof(int), sizeof(int));
thorium_balancer_detect_symmetric_scripts(self, &symmetric_actor_scripts);
#endif
#ifdef THORIUM_WORKER_ENABLE_LOCK
/* Lock all workers first
*/
for (i = 0; i < thorium_worker_pool_worker_count(self->pool); i++) {
worker = thorium_worker_pool_get_worker(self->pool, i);
thorium_worker_lock(worker);
}
#endif
core_vector_init(&migrations, sizeof(struct thorium_migration));
#ifdef THORIUM_SCHEDULER_ENABLE_VERBOSITY
printf("BALANCING\n");
#endif
core_vector_init(&loads, sizeof(int));
core_vector_init(&loads_unsorted, sizeof(int));
core_vector_init(&burdened_workers, sizeof(struct core_pair));
core_vector_init(&stalled_workers, sizeof(struct core_pair));
core_vector_init(&actors_to_migrate, sizeof(struct core_pair));
for (i = 0; i < thorium_worker_pool_worker_count(self->pool); i++) {
worker = thorium_worker_pool_get_worker(self->pool, i);
load_value = thorium_worker_get_scheduling_epoch_load(worker) * SCHEDULER_PRECISION;
#if 0
printf("DEBUG LOAD %d %d\n", i, load_value);
#endif
core_vector_push_back(&loads, &load_value);
core_vector_push_back(&loads_unsorted, &load_value);
}
core_vector_sort_int(&loads);
stalled_percentile = core_statistics_get_percentile_int(&loads, SCHEDULER_WINDOW);
/*load_percentile_25 = core_statistics_get_percentile_int(&loads, 25);*/
load_percentile_50 = core_statistics_get_percentile_int(&loads, 50);
/*load_percentile_75 = core_statistics_get_percentile_int(&loads, 75);*/
burdened_percentile = core_statistics_get_percentile_int(&loads, 100 - SCHEDULER_WINDOW);
#ifdef THORIUM_SCHEDULER_ENABLE_VERBOSITY
printf("Percentiles for epoch loads: ");
core_statistics_print_percentiles_int(&loads);
#endif
for (i = 0; i < thorium_worker_pool_worker_count(self->pool); i++) {
worker = thorium_worker_pool_get_worker(self->pool, i);
load_value = core_vector_at_as_int(&loads_unsorted, i);
set = thorium_worker_get_actors(worker);
if (stalled_percentile == burdened_percentile) {
#ifdef THORIUM_SCHEDULER_ENABLE_VERBOSITY
printf("scheduling_class:%s ",
THORIUM_CLASS_NORMAL_STRING);
#endif
} else if (load_value <= stalled_percentile) {
#ifdef THORIUM_SCHEDULER_ENABLE_VERBOSITY
printf("scheduling_class:%s ",
THORIUM_CLASS_STALLED_STRING);
#endif
core_pair_init(&pair, load_value, i);
core_vector_push_back(&stalled_workers, &pair);
} else if (load_value >= burdened_percentile) {
#ifdef THORIUM_SCHEDULER_ENABLE_VERBOSITY
printf("scheduling_class:%s ",
THORIUM_CLASS_BURDENED_STRING);
#endif
core_pair_init(&pair, load_value, i);
core_vector_push_back(&burdened_workers, &pair);
} else {
#ifdef THORIUM_SCHEDULER_ENABLE_VERBOSITY
printf("scheduling_class:%s ",
THORIUM_CLASS_NORMAL_STRING);
#endif
}
#ifdef THORIUM_SCHEDULER_ENABLE_VERBOSITY
thorium_worker_print_actors(worker, self);
#endif
}
core_vector_sort_int_reverse(&burdened_workers);
core_vector_sort_int(&stalled_workers);
stalled_count = core_vector_size(&stalled_workers);
#ifdef THORIUM_SCHEDULER_ENABLE_VERBOSITY
printf("MIGRATIONS (stalled: %d, burdened: %d)\n", (int)core_vector_size(&stalled_workers),
(int)core_vector_size(&burdened_workers));
#endif
stalled_index = 0;
core_vector_iterator_init(&vector_iterator, &burdened_workers);
while (stalled_count > 0
&& core_vector_iterator_get_next_value(&vector_iterator, &pair)) {
old_worker = core_pair_get_second(&pair);
worker = thorium_worker_pool_get_worker(self->pool, old_worker);
set = thorium_worker_get_actors(worker);
/*
thorium_worker_print_actors(worker);
printf("\n");
*/
/*
* Lock the worker and try to select actors for migration
*/
core_map_iterator_init(&set_iterator, set);
maximum = -1;
with_maximum = 0;
total = 0;
with_messages = 0;
while (core_map_iterator_get_next_key_and_value(&set_iterator, &actor_name, NULL)) {
actor = thorium_node_get_actor_from_name(thorium_worker_pool_get_node(self->pool), actor_name);
messages = thorium_balancer_get_actor_production(self, actor);
if (maximum == -1 || messages > maximum) {
maximum = messages;
with_maximum = 1;
} else if (messages == maximum) {
with_maximum++;
}
if (messages > 0) {
++with_messages;
}
total += messages;
}
core_map_iterator_destroy(&set_iterator);
core_map_iterator_init(&set_iterator, set);
--with_maximum;
candidates = 0;
load_value = thorium_worker_get_scheduling_epoch_load(worker) * SCHEDULER_PRECISION;
remaining_load = load_value;
#if 0
printf("maximum %d with_maximum %d\n", maximum, with_maximum);
#endif
while (core_map_iterator_get_next_key_and_value(&set_iterator, &actor_name, NULL)) {
actor = thorium_node_get_actor_from_name(thorium_worker_pool_get_node(self->pool), actor_name);
if (actor == NULL) {
continue;
}
messages = thorium_balancer_get_actor_production(self, actor);
#ifdef THORIUM_SCHEDULER_ENABLE_SYMMETRIC_SCHEDULING
script = thorium_actor_script(actor);
/* symmetric actors are migrated elsewhere.
*/
if (core_map_get_value(&symmetric_actor_scripts, &script, NULL)) {
continue;
}
#endif
/* Simulate the remaining load
*/
projected_load = remaining_load;
projected_load -= ((0.0 + messages) / total) * load_value;
#ifdef THORIUM_SCHEDULER_DEBUG
printf(" TESTING actor %d, production was %d, projected_load is %d (- %d * (1 - %d/%d)\n",
actor_name, messages, projected_load,
load_value, messages, total);
#endif
/* An actor without any queued messages should not be migrated
*/
if (messages > 0
&& ((with_maximum > 0 && messages == maximum) || messages < maximum)
/*
* Avoid removing too many actors because
* generating a stalled one is not desired
*/
&& (projected_load >= load_percentile_50
/*
* The previous rule does not apply when there
* are 2 actors.
*/
|| with_messages == 2) ) {
remaining_load = projected_load;
candidates++;
if (messages == maximum) {
--with_maximum;
}
core_pair_init(&pair, messages, actor_name);
core_vector_push_back(&actors_to_migrate, &pair);
#ifdef THORIUM_SCHEDULER_DEBUG
printf("early CANDIDATE for migration: actor %d, worker %d\n",
actor_name, old_worker);
#endif
}
}
core_map_iterator_destroy(&set_iterator);
}
core_vector_iterator_destroy(&vector_iterator);
/* Sort the candidates
*/
/*
core_vector_sort_int(&actors_to_migrate);
printf("Percentiles for production: ");
core_statistics_print_percentiles_int(&actors_to_migrate);
*/
/* Sort them in reverse order.
*/
core_vector_sort_int_reverse(&actors_to_migrate);
core_vector_iterator_init(&vector_iterator, &actors_to_migrate);
/* For each highly active actor,
* try to match it with a stalled worker
*/
while (core_vector_iterator_get_next_value(&vector_iterator, &pair)) {
actor_name = core_pair_get_second(&pair);
actor = thorium_node_get_actor_from_name(thorium_worker_pool_get_node(self->pool), actor_name);
if (actor == NULL) {
continue;
}
messages = thorium_balancer_get_actor_production(self, actor);
old_worker = thorium_actor_assigned_worker(actor);
worker = thorium_worker_pool_get_worker(self->pool, old_worker);
/* old_total can not be 0 because otherwise the would not
* be burdened.
*/
old_total = thorium_worker_get_production(worker, self);
with_messages = thorium_worker_get_producer_count(worker, self);
old_load = thorium_worker_get_scheduling_epoch_load(worker) * SCHEDULER_PRECISION;
actor_load = ((0.0 + messages) / old_total) * old_load;
/* Try to find a stalled worker that can take it.
*/
test_stalled_index = stalled_index;
tests = 0;
predicted_new_load = 0;
found_match = 0;
while (tests < stalled_count) {
core_vector_get_value(&stalled_workers, test_stalled_index, &pair);
new_worker_index = core_pair_get_second(&pair);
new_worker = thorium_worker_pool_get_worker(self->pool, new_worker_index);
new_load = thorium_worker_get_scheduling_epoch_load(new_worker) * SCHEDULER_PRECISION;
/*new_total = thorium_worker_get_production(new_worker);*/
predicted_new_load = new_load + actor_load;
if (predicted_new_load > SCHEDULER_PRECISION /* && with_messages != 2 */) {
#ifdef THORIUM_SCHEDULER_DEBUG
printf("Scheduler: skipping actor %d, predicted load is %d >= 100\n",
actor_name, predicted_new_load);
#endif
++tests;
++test_stalled_index;
if (test_stalled_index == stalled_count) {
test_stalled_index = 0;
}
continue;
}
/* Otherwise, this stalled worker is fine...
*/
stalled_index = test_stalled_index;
found_match = 1;
break;
}
/* This actor can not be migrated to any stalled worker.
*/
if (!found_match) {
continue;
}
/* Otherwise, update the load of the stalled one and go forward with the change.
*/
pair_pointer = (struct core_pair *)core_vector_at(&stalled_workers, stalled_index);
core_pair_set_first(pair_pointer, predicted_new_load);
++stalled_index;
if (stalled_index == stalled_count) {
stalled_index = 0;
}
#if 0
new_worker = thorium_worker_pool_get_worker(pool, new_worker_index);
printf(" CANDIDATE: actor %d old worker %d (%d - %d = %d) new worker %d (%d + %d = %d)\n",
actor_name,
old_worker, value, messages, 2new_score,
new_worker_index, new_worker_old_score, messages, new_worker_new_score);
#endif
thorium_migration_init(&migration, actor_name, old_worker, new_worker_index);
core_vector_push_back(&migrations, &migration);
thorium_migration_destroy(&migration);
}
core_vector_iterator_destroy(&vector_iterator);
core_vector_destroy(&stalled_workers);
core_vector_destroy(&burdened_workers);
core_vector_destroy(&loads);
core_vector_destroy(&loads_unsorted);
core_vector_destroy(&actors_to_migrate);
/* Update the last values
*/
for (i = 0; i < thorium_worker_pool_worker_count(self->pool); i++) {
worker = thorium_worker_pool_get_worker(self->pool, i);
set = thorium_worker_get_actors(worker);
core_map_iterator_init(&set_iterator, set);
while (core_map_iterator_get_next_key_and_value(&set_iterator, &actor_name, NULL)) {
actor = thorium_node_get_actor_from_name(thorium_worker_pool_get_node(self->pool), actor_name);
thorium_balancer_update_actor_production(self, actor);
}
core_map_iterator_destroy(&set_iterator);
thorium_worker_reset_scheduling_epoch(worker);
}
#ifdef THORIUM_SCHEDULER_ENABLE_SYMMETRIC_SCHEDULING
/* Generate migrations for symmetric actors.
*/
thorium_balancer_generate_symmetric_migrations(self, &symmetric_actor_scripts, &migrations);
#endif
/* Actually do the migrations
*/
core_vector_iterator_init(&vector_iterator, &migrations);
while (core_vector_iterator_next(&vector_iterator, (void **)&migration_to_do)) {
thorium_balancer_migrate(self, migration_to_do);
}
core_vector_iterator_destroy(&vector_iterator);
self->last_migrations = core_vector_size(&migrations);
core_vector_destroy(&migrations);
#ifdef THORIUM_WORKER_ENABLE_LOCK
/* Unlock all workers
*/
for (i = 0; i < thorium_worker_pool_worker_count(self->pool); i++) {
worker = thorium_worker_pool_get_worker(self->pool, i);
thorium_worker_unlock(worker);
}
#endif
#ifdef THORIUM_SCHEDULER_ENABLE_SYMMETRIC_SCHEDULING
core_map_destroy(&symmetric_actor_scripts);
#endif
core_timer_stop(&timer);
printf("SCHEDULER: elapsed time for balancing: %d us, %d migrations performed\n",
(int)(core_timer_get_elapsed_nanoseconds(&timer) / 1000),
self->last_migrations);
}
