int run_association_test(shared_options_data_t* shared_options_data, assoc_options_data_t* options_data) { list_t *output_list = (list_t*) malloc (sizeof(list_t)); list_init("output", shared_options_data->num_threads, INT_MAX, output_list); int ret_code = 0; vcf_file_t *vcf_file = vcf_open(shared_options_data->vcf_filename, shared_options_data->max_batches); if (!vcf_file) { LOG_FATAL("VCF file does not exist!\n"); } ped_file_t *ped_file = ped_open(shared_options_data->ped_filename); if (!ped_file) { LOG_FATAL("PED file does not exist!\n"); } LOG_INFO("About to read PED file...\n"); // Read PED file before doing any processing ret_code = ped_read(ped_file); if (ret_code != 0) { LOG_FATAL_F("Can't read PED file: %s\n", ped_file->filename); } // Try to create the directory where the output files will be stored ret_code = create_directory(shared_options_data->output_directory); if (ret_code != 0 && errno != EEXIST) { LOG_FATAL_F("Can't create output directory: %s\n", shared_options_data->output_directory); } LOG_INFO("About to perform basic association test...\n"); #pragma omp parallel sections private(ret_code) { #pragma omp section { LOG_DEBUG_F("Level %d: number of threads in the team - %d\n", 0, omp_get_num_threads()); double start = omp_get_wtime(); ret_code = vcf_read(vcf_file, 0, (shared_options_data->batch_bytes > 0) ? shared_options_data->batch_bytes : shared_options_data->batch_lines, shared_options_data->batch_bytes <= 0); double stop = omp_get_wtime(); double total = stop - start; if (ret_code) { LOG_FATAL_F("Error %d while reading the file %s\n", ret_code, vcf_file->filename); } LOG_INFO_F("[%dR] Time elapsed = %f s\n", omp_get_thread_num(), total); LOG_INFO_F("[%dR] Time elapsed = %e ms\n", omp_get_thread_num(), total*1000); notify_end_reading(vcf_file); } #pragma omp section { LOG_DEBUG_F("Level %d: number of threads in the team - %d\n", 10, omp_get_num_threads()); // Enable nested parallelism omp_set_nested(1); volatile int initialization_done = 0; // Pedigree information individual_t **individuals = NULL; khash_t(ids) *sample_ids = NULL; // Create chain of filters for the VCF file filter_t **filters = NULL; int num_filters = 0; if (shared_options_data->chain != NULL) { filters = sort_filter_chain(shared_options_data->chain, &num_filters); } FILE *passed_file = NULL, *failed_file = NULL; get_filtering_output_files(shared_options_data, &passed_file, &failed_file); double start = omp_get_wtime(); double *factorial_logarithms = NULL; int i = 0; #pragma omp parallel num_threads(shared_options_data->num_threads) shared(initialization_done, factorial_logarithms, filters, individuals) { LOG_DEBUG_F("Level %d: number of threads in the team - %d\n", 11, omp_get_num_threads()); char *text_begin, *text_end; vcf_reader_status *status; while(text_begin = fetch_vcf_text_batch(vcf_file)) { text_end = text_begin + strlen(text_begin); if (text_begin == text_end) { // EOF free(text_begin); break; } # pragma omp critical { status = vcf_reader_status_new(shared_options_data->batch_lines, i); i++; } if (shared_options_data->batch_bytes > 0) { ret_code = run_vcf_parser(text_begin, text_end, 0, vcf_file, status); } else if (shared_options_data->batch_lines > 0) { ret_code = run_vcf_parser(text_begin, text_end, shared_options_data->batch_lines, vcf_file, status); } // Initialize structures needed for association tests and write headers of output files if (!initialization_done && vcf_file->samples_names->size > 0) { # pragma omp critical { // Guarantee that just one thread performs this operation if (!initialization_done) { // Create map to associate the position of individuals in the list of samples defined in the VCF file sample_ids = associate_samples_and_positions(vcf_file); // Sort individuals in PED as defined in the VCF file individuals = sort_individuals(vcf_file, ped_file); /* printf("num samples = %zu\n", get_num_vcf_samples(file)); printf("pos = { "); for (int j = 0; j < get_num_vcf_samples(file); j++) { assert(individuals[j]); printf("%s ", individuals[j]->id); } printf("}\n"); */ // Add headers associated to the defined filters vcf_header_entry_t **filter_headers = get_filters_as_vcf_headers(filters, num_filters); for (int j = 0; j < num_filters; j++) { add_vcf_header_entry(filter_headers[j], vcf_file); } // Write file format, header entries and delimiter if (passed_file != NULL) { write_vcf_header(vcf_file, passed_file); } if (failed_file != NULL) { write_vcf_header(vcf_file, failed_file); } LOG_DEBUG("VCF header written\n"); if (options_data->task == FISHER) { factorial_logarithms = init_logarithm_array(get_num_vcf_samples(vcf_file) * 10); } initialization_done = 1; } } } // If it has not been initialized it means that header is not fully read if (!initialization_done) { continue; } vcf_batch_t *batch = fetch_vcf_batch(vcf_file); if (i % 100 == 0) { LOG_INFO_F("Batch %d reached by thread %d - %zu/%zu records \n", i, omp_get_thread_num(), batch->records->size, batch->records->capacity); } assert(batch); // Launch association test over records that passed the filters array_list_t *failed_records = NULL; int num_variables = ped_file? get_num_variables(ped_file): 0; array_list_t *passed_records = filter_records(filters, num_filters, individuals, sample_ids, num_variables, batch->records, &failed_records); if (passed_records->size > 0) { assoc_test(options_data->task, (vcf_record_t**) passed_records->items, passed_records->size, individuals, get_num_vcf_samples(vcf_file), factorial_logarithms, output_list); } // Write records that passed and failed filters to separate files, and free them write_filtering_output_files(passed_records, failed_records, passed_file, failed_file); free_filtered_records(passed_records, failed_records, batch->records); // Free batch and its contents vcf_reader_status_free(status); vcf_batch_free(batch); } notify_end_parsing(vcf_file); } double stop = omp_get_wtime(); double total = stop - start; LOG_INFO_F("[%d] Time elapsed = %f s\n", omp_get_thread_num(), total); LOG_INFO_F("[%d] Time elapsed = %e ms\n", omp_get_thread_num(), total*1000); // Free resources for (int i = 0; i < num_filters; i++) { filter_t *filter = filters[i]; filter->free_func(filter); } free(filters); if (sample_ids) { kh_destroy(ids, sample_ids); } if (individuals) { free(individuals); } // Decrease list writers count for (int i = 0; i < shared_options_data->num_threads; i++) { list_decr_writers(output_list); } } #pragma omp section { // Thread that writes the results to the output file LOG_DEBUG_F("Level %d: number of threads in the team - %d\n", 20, omp_get_num_threads()); double start = omp_get_wtime(); // Get the file descriptor char *path; FILE *fd = get_assoc_output_file(options_data->task, shared_options_data, &path); LOG_INFO_F("Association test output filename = %s\n", path); // Write data: header + one line per variant write_output_header(options_data->task, fd); write_output_body(options_data->task, output_list, fd); fclose(fd); // Sort resulting file char *cmd = calloc (40 + strlen(path) * 4, sizeof(char)); sprintf(cmd, "sort -k1,1h -k2,2n %s > %s.tmp && mv %s.tmp %s", path, path, path, path); int sort_ret = system(cmd); if (sort_ret) { LOG_WARN("Association results could not be sorted by chromosome and position, will be shown unsorted\n"); } double stop = omp_get_wtime(); double total = stop - start; LOG_INFO_F("[%dW] Time elapsed = %f s\n", omp_get_thread_num(), total); LOG_INFO_F("[%dW] Time elapsed = %e ms\n", omp_get_thread_num(), total*1000); } } free(output_list); vcf_close(vcf_file); ped_close(ped_file, 1,1); return ret_code; }
int run_merge(shared_options_data_t *shared_options_data, merge_options_data_t *options_data) { if (options_data->num_files == 1) { LOG_INFO("Just one VCF file specified, no need to merge"); return 0; } list_t *read_list[options_data->num_files]; memset(read_list, 0, options_data->num_files * sizeof(list_t*)); list_t *output_header_list = (list_t*) malloc (sizeof(list_t)); list_init("headers", shared_options_data->num_threads, INT_MAX, output_header_list); list_t *output_list = (list_t*) malloc (sizeof(list_t)); list_init("output", shared_options_data->num_threads, shared_options_data->max_batches * shared_options_data->batch_lines, output_list); list_t *merge_tokens = (list_t*) malloc (sizeof(list_t)); list_init("tokens", 1, INT_MAX, merge_tokens); int ret_code = 0; double start, stop, total; vcf_file_t *files[options_data->num_files]; memset(files, 0, options_data->num_files * sizeof(vcf_file_t*)); // Initialize variables related to the different files for (int i = 0; i < options_data->num_files; i++) { files[i] = vcf_open(options_data->input_files[i], shared_options_data->max_batches); if (!files[i]) { LOG_FATAL_F("VCF file %s does not exist!\n", options_data->input_files[i]); } read_list[i] = (list_t*) malloc(sizeof(list_t)); list_init("text", 1, shared_options_data->max_batches, read_list[i]); } ret_code = create_directory(shared_options_data->output_directory); if (ret_code != 0 && errno != EEXIST) { LOG_FATAL_F("Can't create output directory: %s\n", shared_options_data->output_directory); } chromosome_order = get_chromosome_order(shared_options_data->host_url, shared_options_data->species, shared_options_data->version, &num_chromosomes); printf("Number of threads = %d\n", shared_options_data->num_threads); #pragma omp parallel sections private(start, stop, total) { #pragma omp section { LOG_DEBUG_F("Thread %d reads the VCF file\n", omp_get_thread_num()); // Reading start = omp_get_wtime(); ret_code = vcf_multiread_batches(read_list, shared_options_data->batch_lines, files, options_data->num_files); stop = omp_get_wtime(); total = stop - start; if (ret_code) { LOG_ERROR_F("Error %d while reading VCF files\n", ret_code); } LOG_INFO_F("[%dR] Time elapsed = %f s\n", omp_get_thread_num(), total); LOG_INFO_F("[%dR] Time elapsed = %e ms\n", omp_get_thread_num(), total*1000); } #pragma omp section { // Enable nested parallelism omp_set_nested(1); LOG_DEBUG_F("Thread %d processes data\n", omp_get_thread_num()); int num_eof_found = 0; int eof_found[options_data->num_files]; memset(eof_found, 0, options_data->num_files * sizeof(int)); list_item_t *items[options_data->num_files]; memset(items, 0, options_data->num_files * sizeof(list_item_t*)); char *texts[options_data->num_files]; memset(texts, 0, options_data->num_files * sizeof(char*)); khash_t(pos) *positions_read = kh_init(pos); long max_position_merged = LONG_MAX; char *max_chromosome_merged = NULL; int header_merged = 0; int token = 0; double start_parsing, start_insertion, total_parsing = 0, total_insertion = 0; start = omp_get_wtime(); while (num_eof_found < options_data->num_files) { /* Process: * - N threads getting batches of VCF records and inserting them in a data structure. The common minimum * position of each group of batches will also be stored. * - If the data structure reaches certain size or the end of a chromosome, merge positions prior to the * last minimum registered. */ // Getting text elements in a critical region guarantees that each thread gets variants in positions in the same range for (int i = 0; i < options_data->num_files; i++) { if (eof_found[i]) { continue; } items[i] = list_remove_item(read_list[i]); if (items[i] == NULL || !strcmp(items[i]->data_p, "")) { LOG_INFO_F("[%d] EOF found in file %s\n", omp_get_thread_num(), options_data->input_files[i]); eof_found[i] = 1; num_eof_found++; if(items[i] != NULL && !strcmp(items[i]->data_p, "")) { free(items[i]->data_p); list_item_free(items[i]); LOG_DEBUG_F("[%d] Text batch freed\n", omp_get_thread_num()); } else { LOG_DEBUG_F("[%d] No need to free text batch\n", omp_get_thread_num()); } continue; } assert(items[i]->data_p != NULL); texts[i] = items[i]->data_p; // printf("[%d] text batch from file %d\tcontents = '%s'\n", omp_get_thread_num(), i, texts[i]); } for (int i = 0; i < options_data->num_files; i++) { if (eof_found[i]) { continue; } start_parsing = omp_get_wtime(); char *text_begin = texts[i]; char *text_end = text_begin + strlen(text_begin); assert(text_end != NULL); // printf("batch = '%.*s'\n", text_end - text_begin, text_begin); // Get VCF batches from text batches vcf_reader_status *status = vcf_reader_status_new(shared_options_data->batch_lines, 0); ret_code = run_vcf_parser(text_begin, text_end, shared_options_data->batch_lines, files[i], status); if (ret_code) { // TODO stop? LOG_ERROR_F("Error %d while reading the file %s\n", ret_code, files[i]->filename); continue; } // printf("batches = %d\n", files[i]->record_batches->length); vcf_batch_t *batch = fetch_vcf_batch_non_blocking(files[i]); if (!batch) { continue; } total_parsing += omp_get_wtime() - start_parsing; start_insertion = omp_get_wtime(); // Insert records into hashtable for (int j = 0; j < batch->records->size; j++) { vcf_record_t *record = vcf_record_copy(array_list_get(j, batch->records)); vcf_record_file_link *link = vcf_record_file_link_new(record, files[i]); char key[64]; compose_key_value(record->chromosome, record->position, key); int ret = insert_position_read(key, link, positions_read); assert(ret); } total_insertion += omp_get_wtime() - start_insertion; // Update minimum position being a maximum of these batches vcf_record_t *current_record = (vcf_record_t*) array_list_get(batch->records->size - 1, batch->records); calculate_merge_interval(current_record, &max_chromosome_merged, &max_position_merged, chromosome_order, num_chromosomes); // Free batch and its contents vcf_reader_status_free(status); vcf_batch_free(batch); list_item_free(items[i]); } if (num_eof_found == options_data->num_files) { max_chromosome_merged = chromosome_order[num_chromosomes-1]; max_position_merged = LONG_MAX; } // Merge headers, if not previously done if (!header_merged) { merge_vcf_headers(files, options_data->num_files, options_data, output_header_list); header_merged = 1; // Decrease list writers count for (int i = 0; i < shared_options_data->num_threads; i++) { list_decr_writers(output_header_list); } } // If the data structure reaches certain size or the end of a chromosome, // merge positions prior to the last minimum registered if (num_eof_found < options_data->num_files && kh_size(positions_read) > TREE_LIMIT) { LOG_INFO_F("Merging until position %s:%ld\n", max_chromosome_merged, max_position_merged); token = merge_interval(positions_read, max_chromosome_merged, max_position_merged, chromosome_order, num_chromosomes, files, shared_options_data, options_data, output_list); } // When reaching EOF for all files, merge the remaining entries else if (num_eof_found == options_data->num_files && kh_size(positions_read) > 0) { LOG_INFO_F("Merging remaining positions (last = %s:%ld)\n", chromosome_order[num_chromosomes - 1], LONG_MAX); token = merge_remaining_interval(positions_read, files, shared_options_data, options_data, output_list); } if (token) { int *token_ptr = malloc (sizeof(int)); *token_ptr = token; list_item_t *item = list_item_new(1, 0, token_ptr); list_insert_item(item, merge_tokens); } // Set variables ready for next iteration of the algorithm if (max_chromosome_merged) { free(max_chromosome_merged); } token = 0; max_chromosome_merged = NULL; max_position_merged = LONG_MAX; } kh_destroy(pos, positions_read); stop = omp_get_wtime(); total = stop - start; LOG_INFO_F("[%d] Time elapsed = %f s\n", omp_get_thread_num(), total); LOG_INFO_F("[%d] Time elapsed = %e ms\n", omp_get_thread_num(), total*1000); LOG_DEBUG_F("** Time in parsing = %f s\n", total_parsing); LOG_DEBUG_F("** Time in insertion = %f s\n", total_insertion); // for (int i = 0; i < shared_options_data->num_threads; i++) { // printf("[%d] Time in searching = %f s\n", i, total_search[i]); // printf("[%d] Time in merging = %f s\n", i, total_merge[i]); // } // Decrease list writers count for (int i = 0; i < shared_options_data->num_threads; i++) { list_decr_writers(output_list); } list_decr_writers(merge_tokens); } #pragma omp section { LOG_DEBUG_F("Thread %d writes the output\n", omp_get_thread_num()); start = omp_get_wtime(); // Create file streams for results char aux_filename[32]; memset(aux_filename, 0, 32 * sizeof(char)); sprintf(aux_filename, "merge_from_%d_files.vcf", options_data->num_files); char *merge_filename; FILE *merge_fd = get_output_file(shared_options_data, aux_filename, &merge_filename); LOG_INFO_F("Output filename = %s\n", merge_filename); free(merge_filename); list_item_t *item1 = NULL, *item2 = NULL; vcf_header_entry_t *entry; vcf_record_t *record; int *num_records; // Write headers while ((item1 = list_remove_item(output_header_list)) != NULL) { entry = item1->data_p; write_vcf_header_entry(entry, merge_fd); } // Write delimiter array_list_t *sample_names = merge_vcf_sample_names(files, options_data->num_files); write_vcf_delimiter_from_samples((char**) sample_names->items, sample_names->size, merge_fd); // Write records // When a token is present, it means a set of batches has been merged. The token contains the number of records merged. // In this case, the records must be sorted by chromosome and position, and written afterwards. while ((item1 = list_remove_item(merge_tokens)) != NULL) { num_records = item1->data_p; vcf_record_t *records[*num_records]; for (int i = 0; i < *num_records; i++) { item2 = list_remove_item(output_list); if (!item2) { break; } records[i] = item2->data_p; list_item_free(item2); } // Sort records qsort(records, *num_records, sizeof(vcf_record_t*), record_cmp); // Write and free sorted records for (int i = 0; i < *num_records; i++) { record = records[i]; write_vcf_record(record, merge_fd); vcf_record_free_deep(record); } free(num_records); list_item_free(item1); } // Close file if (merge_fd != NULL) { fclose(merge_fd); } stop = omp_get_wtime(); total = stop - start; LOG_INFO_F("[%dW] Time elapsed = %f s\n", omp_get_thread_num(), total); LOG_INFO_F("[%dW] Time elapsed = %e ms\n", omp_get_thread_num(), total*1000); } } // Free variables related to the different files for (int i = 0; i < options_data->num_files; i++) { if(files[i]) { vcf_close(files[i]); } if(read_list[i]) { free(read_list[i]); } } free(output_list); return ret_code; }