int run_association_test(shared_options_data_t* shared_options_data, assoc_options_data_t* options_data) {
list_t *output_list = (list_t*) malloc (sizeof(list_t));
list_init("output", shared_options_data->num_threads, INT_MAX, output_list);
int ret_code = 0;
vcf_file_t *vcf_file = vcf_open(shared_options_data->vcf_filename, shared_options_data->max_batches);
if (!vcf_file) {
LOG_FATAL("VCF file does not exist!\n");
}
ped_file_t *ped_file = ped_open(shared_options_data->ped_filename);
if (!ped_file) {
LOG_FATAL("PED file does not exist!\n");
}
LOG_INFO("About to read PED file...\n");
// Read PED file before doing any processing
ret_code = ped_read(ped_file);
if (ret_code != 0) {
LOG_FATAL_F("Can't read PED file: %s\n", ped_file->filename);
}
// Try to create the directory where the output files will be stored
ret_code = create_directory(shared_options_data->output_directory);
if (ret_code != 0 && errno != EEXIST) {
LOG_FATAL_F("Can't create output directory: %s\n", shared_options_data->output_directory);
}
LOG_INFO("About to perform basic association test...\n");
#pragma omp parallel sections private(ret_code)
{
#pragma omp section
{
LOG_DEBUG_F("Level %d: number of threads in the team - %d\n", 0, omp_get_num_threads());
double start = omp_get_wtime();
ret_code = vcf_read(vcf_file, 0,
(shared_options_data->batch_bytes > 0) ? shared_options_data->batch_bytes : shared_options_data->batch_lines,
shared_options_data->batch_bytes <= 0);
double stop = omp_get_wtime();
double total = stop - start;
if (ret_code) {
LOG_FATAL_F("Error %d while reading the file %s\n", ret_code, vcf_file->filename);
}
LOG_INFO_F("[%dR] Time elapsed = %f s\n", omp_get_thread_num(), total);
LOG_INFO_F("[%dR] Time elapsed = %e ms\n", omp_get_thread_num(), total*1000);
notify_end_reading(vcf_file);
}
#pragma omp section
{
LOG_DEBUG_F("Level %d: number of threads in the team - %d\n", 10, omp_get_num_threads());
// Enable nested parallelism
omp_set_nested(1);
volatile int initialization_done = 0;
// Pedigree information
individual_t **individuals = NULL;
khash_t(ids) *sample_ids = NULL;
// Create chain of filters for the VCF file
filter_t **filters = NULL;
int num_filters = 0;
if (shared_options_data->chain != NULL) {
filters = sort_filter_chain(shared_options_data->chain, &num_filters);
}
FILE *passed_file = NULL, *failed_file = NULL;
get_filtering_output_files(shared_options_data, &passed_file, &failed_file);
double start = omp_get_wtime();
double *factorial_logarithms = NULL;
int i = 0;
#pragma omp parallel num_threads(shared_options_data->num_threads) shared(initialization_done, factorial_logarithms, filters, individuals)
{
LOG_DEBUG_F("Level %d: number of threads in the team - %d\n", 11, omp_get_num_threads());
char *text_begin, *text_end;
vcf_reader_status *status;
while(text_begin = fetch_vcf_text_batch(vcf_file)) {
text_end = text_begin + strlen(text_begin);
if (text_begin == text_end) { // EOF
free(text_begin);
break;
}
# pragma omp critical
{
status = vcf_reader_status_new(shared_options_data->batch_lines, i);
i++;
}
if (shared_options_data->batch_bytes > 0) {
ret_code = run_vcf_parser(text_begin, text_end, 0, vcf_file, status);
} else if (shared_options_data->batch_lines > 0) {
ret_code = run_vcf_parser(text_begin, text_end, shared_options_data->batch_lines, vcf_file, status);
}
// Initialize structures needed for association tests and write headers of output files
if (!initialization_done && vcf_file->samples_names->size > 0) {
# pragma omp critical
{
// Guarantee that just one thread performs this operation
if (!initialization_done) {
// Create map to associate the position of individuals in the list of samples defined in the VCF file
sample_ids = associate_samples_and_positions(vcf_file);
// Sort individuals in PED as defined in the VCF file
individuals = sort_individuals(vcf_file, ped_file);
/*
printf("num samples = %zu\n", get_num_vcf_samples(file));
printf("pos = { ");
for (int j = 0; j < get_num_vcf_samples(file); j++) {
assert(individuals[j]);
printf("%s ", individuals[j]->id);
}
printf("}\n");
*/
// Add headers associated to the defined filters
vcf_header_entry_t **filter_headers = get_filters_as_vcf_headers(filters, num_filters);
for (int j = 0; j < num_filters; j++) {
add_vcf_header_entry(filter_headers[j], vcf_file);
}
// Write file format, header entries and delimiter
if (passed_file != NULL) { write_vcf_header(vcf_file, passed_file); }
if (failed_file != NULL) { write_vcf_header(vcf_file, failed_file); }
LOG_DEBUG("VCF header written\n");
if (options_data->task == FISHER) {
factorial_logarithms = init_logarithm_array(get_num_vcf_samples(vcf_file) * 10);
}
initialization_done = 1;
}
}
}
// If it has not been initialized it means that header is not fully read
if (!initialization_done) {
continue;
}
vcf_batch_t *batch = fetch_vcf_batch(vcf_file);
if (i % 100 == 0) {
LOG_INFO_F("Batch %d reached by thread %d - %zu/%zu records \n",
i, omp_get_thread_num(),
batch->records->size, batch->records->capacity);
}
assert(batch);
// Launch association test over records that passed the filters
array_list_t *failed_records = NULL;
int num_variables = ped_file? get_num_variables(ped_file): 0;
array_list_t *passed_records = filter_records(filters, num_filters, individuals, sample_ids, num_variables, batch->records, &failed_records);
if (passed_records->size > 0) {
assoc_test(options_data->task, (vcf_record_t**) passed_records->items, passed_records->size,
individuals, get_num_vcf_samples(vcf_file), factorial_logarithms, output_list);
}
// Write records that passed and failed filters to separate files, and free them
write_filtering_output_files(passed_records, failed_records, passed_file, failed_file);
free_filtered_records(passed_records, failed_records, batch->records);
// Free batch and its contents
vcf_reader_status_free(status);
vcf_batch_free(batch);
}
notify_end_parsing(vcf_file);
}
double stop = omp_get_wtime();
double total = stop - start;
LOG_INFO_F("[%d] Time elapsed = %f s\n", omp_get_thread_num(), total);
LOG_INFO_F("[%d] Time elapsed = %e ms\n", omp_get_thread_num(), total*1000);
// Free resources
for (int i = 0; i < num_filters; i++) {
filter_t *filter = filters[i];
filter->free_func(filter);
}
free(filters);
if (sample_ids) { kh_destroy(ids, sample_ids); }
if (individuals) { free(individuals); }
// Decrease list writers count
for (int i = 0; i < shared_options_data->num_threads; i++) {
list_decr_writers(output_list);
}
}
#pragma omp section
{
// Thread that writes the results to the output file
LOG_DEBUG_F("Level %d: number of threads in the team - %d\n", 20, omp_get_num_threads());
double start = omp_get_wtime();
// Get the file descriptor
char *path;
FILE *fd = get_assoc_output_file(options_data->task, shared_options_data, &path);
LOG_INFO_F("Association test output filename = %s\n", path);
// Write data: header + one line per variant
write_output_header(options_data->task, fd);
write_output_body(options_data->task, output_list, fd);
fclose(fd);
// Sort resulting file
char *cmd = calloc (40 + strlen(path) * 4, sizeof(char));
sprintf(cmd, "sort -k1,1h -k2,2n %s > %s.tmp && mv %s.tmp %s", path, path, path, path);
int sort_ret = system(cmd);
if (sort_ret) {
LOG_WARN("Association results could not be sorted by chromosome and position, will be shown unsorted\n");
}
double stop = omp_get_wtime();
double total = stop - start;
LOG_INFO_F("[%dW] Time elapsed = %f s\n", omp_get_thread_num(), total);
LOG_INFO_F("[%dW] Time elapsed = %e ms\n", omp_get_thread_num(), total*1000);
}
}
free(output_list);
vcf_close(vcf_file);
ped_close(ped_file, 1,1);
return ret_code;
}
int run_merge(shared_options_data_t *shared_options_data, merge_options_data_t *options_data) {
if (options_data->num_files == 1) {
LOG_INFO("Just one VCF file specified, no need to merge");
return 0;
}
list_t *read_list[options_data->num_files];
memset(read_list, 0, options_data->num_files * sizeof(list_t*));
list_t *output_header_list = (list_t*) malloc (sizeof(list_t));
list_init("headers", shared_options_data->num_threads, INT_MAX, output_header_list);
list_t *output_list = (list_t*) malloc (sizeof(list_t));
list_init("output", shared_options_data->num_threads, shared_options_data->max_batches * shared_options_data->batch_lines, output_list);
list_t *merge_tokens = (list_t*) malloc (sizeof(list_t));
list_init("tokens", 1, INT_MAX, merge_tokens);
int ret_code = 0;
double start, stop, total;
vcf_file_t *files[options_data->num_files];
memset(files, 0, options_data->num_files * sizeof(vcf_file_t*));
// Initialize variables related to the different files
for (int i = 0; i < options_data->num_files; i++) {
files[i] = vcf_open(options_data->input_files[i], shared_options_data->max_batches);
if (!files[i]) {
LOG_FATAL_F("VCF file %s does not exist!\n", options_data->input_files[i]);
}
read_list[i] = (list_t*) malloc(sizeof(list_t));
list_init("text", 1, shared_options_data->max_batches, read_list[i]);
}
ret_code = create_directory(shared_options_data->output_directory);
if (ret_code != 0 && errno != EEXIST) {
LOG_FATAL_F("Can't create output directory: %s\n", shared_options_data->output_directory);
}
chromosome_order = get_chromosome_order(shared_options_data->host_url, shared_options_data->species,
shared_options_data->version, &num_chromosomes);
printf("Number of threads = %d\n", shared_options_data->num_threads);
#pragma omp parallel sections private(start, stop, total)
{
#pragma omp section
{
LOG_DEBUG_F("Thread %d reads the VCF file\n", omp_get_thread_num());
// Reading
start = omp_get_wtime();
ret_code = vcf_multiread_batches(read_list, shared_options_data->batch_lines, files, options_data->num_files);
stop = omp_get_wtime();
total = stop - start;
if (ret_code) {
LOG_ERROR_F("Error %d while reading VCF files\n", ret_code);
}
LOG_INFO_F("[%dR] Time elapsed = %f s\n", omp_get_thread_num(), total);
LOG_INFO_F("[%dR] Time elapsed = %e ms\n", omp_get_thread_num(), total*1000);
}
#pragma omp section
{
// Enable nested parallelism
omp_set_nested(1);
LOG_DEBUG_F("Thread %d processes data\n", omp_get_thread_num());
int num_eof_found = 0;
int eof_found[options_data->num_files];
memset(eof_found, 0, options_data->num_files * sizeof(int));
list_item_t *items[options_data->num_files];
memset(items, 0, options_data->num_files * sizeof(list_item_t*));
char *texts[options_data->num_files];
memset(texts, 0, options_data->num_files * sizeof(char*));
khash_t(pos) *positions_read = kh_init(pos);
long max_position_merged = LONG_MAX;
char *max_chromosome_merged = NULL;
int header_merged = 0;
int token = 0;
double start_parsing, start_insertion, total_parsing = 0, total_insertion = 0;
start = omp_get_wtime();
while (num_eof_found < options_data->num_files) {
/* Process:
* - N threads getting batches of VCF records and inserting them in a data structure. The common minimum
* position of each group of batches will also be stored.
* - If the data structure reaches certain size or the end of a chromosome, merge positions prior to the
* last minimum registered.
*/
// Getting text elements in a critical region guarantees that each thread gets variants in positions in the same range
for (int i = 0; i < options_data->num_files; i++) {
if (eof_found[i]) {
continue;
}
items[i] = list_remove_item(read_list[i]);
if (items[i] == NULL || !strcmp(items[i]->data_p, "")) {
LOG_INFO_F("[%d] EOF found in file %s\n", omp_get_thread_num(), options_data->input_files[i]);
eof_found[i] = 1;
num_eof_found++;
if(items[i] != NULL && !strcmp(items[i]->data_p, "")) {
free(items[i]->data_p);
list_item_free(items[i]);
LOG_DEBUG_F("[%d] Text batch freed\n", omp_get_thread_num());
} else {
LOG_DEBUG_F("[%d] No need to free text batch\n", omp_get_thread_num());
}
continue;
}
assert(items[i]->data_p != NULL);
texts[i] = items[i]->data_p;
// printf("[%d] text batch from file %d\tcontents = '%s'\n", omp_get_thread_num(), i, texts[i]);
}
for (int i = 0; i < options_data->num_files; i++) {
if (eof_found[i]) {
continue;
}
start_parsing = omp_get_wtime();
char *text_begin = texts[i];
char *text_end = text_begin + strlen(text_begin);
assert(text_end != NULL);
// printf("batch = '%.*s'\n", text_end - text_begin, text_begin);
// Get VCF batches from text batches
vcf_reader_status *status = vcf_reader_status_new(shared_options_data->batch_lines, 0);
ret_code = run_vcf_parser(text_begin, text_end, shared_options_data->batch_lines, files[i], status);
if (ret_code) {
// TODO stop?
LOG_ERROR_F("Error %d while reading the file %s\n", ret_code, files[i]->filename);
continue;
}
// printf("batches = %d\n", files[i]->record_batches->length);
vcf_batch_t *batch = fetch_vcf_batch_non_blocking(files[i]);
if (!batch) {
continue;
}
total_parsing += omp_get_wtime() - start_parsing;
start_insertion = omp_get_wtime();
// Insert records into hashtable
for (int j = 0; j < batch->records->size; j++) {
vcf_record_t *record = vcf_record_copy(array_list_get(j, batch->records));
vcf_record_file_link *link = vcf_record_file_link_new(record, files[i]);
char key[64];
compose_key_value(record->chromosome, record->position, key);
int ret = insert_position_read(key, link, positions_read);
assert(ret);
}
total_insertion += omp_get_wtime() - start_insertion;
// Update minimum position being a maximum of these batches
vcf_record_t *current_record = (vcf_record_t*) array_list_get(batch->records->size - 1, batch->records);
calculate_merge_interval(current_record, &max_chromosome_merged, &max_position_merged, chromosome_order, num_chromosomes);
// Free batch and its contents
vcf_reader_status_free(status);
vcf_batch_free(batch);
list_item_free(items[i]);
}
if (num_eof_found == options_data->num_files) {
max_chromosome_merged = chromosome_order[num_chromosomes-1];
max_position_merged = LONG_MAX;
}
// Merge headers, if not previously done
if (!header_merged) {
merge_vcf_headers(files, options_data->num_files, options_data, output_header_list);
header_merged = 1;
// Decrease list writers count
for (int i = 0; i < shared_options_data->num_threads; i++) {
list_decr_writers(output_header_list);
}
}
// If the data structure reaches certain size or the end of a chromosome,
// merge positions prior to the last minimum registered
if (num_eof_found < options_data->num_files && kh_size(positions_read) > TREE_LIMIT) {
LOG_INFO_F("Merging until position %s:%ld\n", max_chromosome_merged, max_position_merged);
token = merge_interval(positions_read, max_chromosome_merged, max_position_merged, chromosome_order, num_chromosomes,
files, shared_options_data, options_data, output_list);
}
// When reaching EOF for all files, merge the remaining entries
else if (num_eof_found == options_data->num_files && kh_size(positions_read) > 0) {
LOG_INFO_F("Merging remaining positions (last = %s:%ld)\n", chromosome_order[num_chromosomes - 1], LONG_MAX);
token = merge_remaining_interval(positions_read, files, shared_options_data, options_data, output_list);
}
if (token) {
int *token_ptr = malloc (sizeof(int)); *token_ptr = token;
list_item_t *item = list_item_new(1, 0, token_ptr);
list_insert_item(item, merge_tokens);
}
// Set variables ready for next iteration of the algorithm
if (max_chromosome_merged) {
free(max_chromosome_merged);
}
token = 0;
max_chromosome_merged = NULL;
max_position_merged = LONG_MAX;
}
kh_destroy(pos, positions_read);
stop = omp_get_wtime();
total = stop - start;
LOG_INFO_F("[%d] Time elapsed = %f s\n", omp_get_thread_num(), total);
LOG_INFO_F("[%d] Time elapsed = %e ms\n", omp_get_thread_num(), total*1000);
LOG_DEBUG_F("** Time in parsing = %f s\n", total_parsing);
LOG_DEBUG_F("** Time in insertion = %f s\n", total_insertion);
// for (int i = 0; i < shared_options_data->num_threads; i++) {
// printf("[%d] Time in searching = %f s\n", i, total_search[i]);
// printf("[%d] Time in merging = %f s\n", i, total_merge[i]);
// }
// Decrease list writers count
for (int i = 0; i < shared_options_data->num_threads; i++) {
list_decr_writers(output_list);
}
list_decr_writers(merge_tokens);
}
#pragma omp section
{
LOG_DEBUG_F("Thread %d writes the output\n", omp_get_thread_num());
start = omp_get_wtime();
// Create file streams for results
char aux_filename[32]; memset(aux_filename, 0, 32 * sizeof(char));
sprintf(aux_filename, "merge_from_%d_files.vcf", options_data->num_files);
char *merge_filename;
FILE *merge_fd = get_output_file(shared_options_data, aux_filename, &merge_filename);
LOG_INFO_F("Output filename = %s\n", merge_filename);
free(merge_filename);
list_item_t *item1 = NULL, *item2 = NULL;
vcf_header_entry_t *entry;
vcf_record_t *record;
int *num_records;
// Write headers
while ((item1 = list_remove_item(output_header_list)) != NULL) {
entry = item1->data_p;
write_vcf_header_entry(entry, merge_fd);
}
// Write delimiter
array_list_t *sample_names = merge_vcf_sample_names(files, options_data->num_files);
write_vcf_delimiter_from_samples((char**) sample_names->items, sample_names->size, merge_fd);
// Write records
// When a token is present, it means a set of batches has been merged. The token contains the number of records merged.
// In this case, the records must be sorted by chromosome and position, and written afterwards.
while ((item1 = list_remove_item(merge_tokens)) != NULL) {
num_records = item1->data_p;
vcf_record_t *records[*num_records];
for (int i = 0; i < *num_records; i++) {
item2 = list_remove_item(output_list);
if (!item2) {
break;
}
records[i] = item2->data_p;
list_item_free(item2);
}
// Sort records
qsort(records, *num_records, sizeof(vcf_record_t*), record_cmp);
// Write and free sorted records
for (int i = 0; i < *num_records; i++) {
record = records[i];
write_vcf_record(record, merge_fd);
vcf_record_free_deep(record);
}
free(num_records);
list_item_free(item1);
}
// Close file
if (merge_fd != NULL) { fclose(merge_fd); }
stop = omp_get_wtime();
total = stop - start;
LOG_INFO_F("[%dW] Time elapsed = %f s\n", omp_get_thread_num(), total);
LOG_INFO_F("[%dW] Time elapsed = %e ms\n", omp_get_thread_num(), total*1000);
}
}
// Free variables related to the different files
for (int i = 0; i < options_data->num_files; i++) {
if(files[i]) { vcf_close(files[i]); }
if(read_list[i]) { free(read_list[i]); }
}
free(output_list);
return ret_code;
}