static FILE *get_assoc_output_file(enum ASSOC_task task, shared_options_data_t *global_options_data, char **path) { if (task == CHI_SQUARE) { return get_output_file(global_options_data, "hpg-variant.chisq", path); } else if (task == FISHER) { return get_output_file(global_options_data, "hpg-variant.fisher", path); } else { LOG_FATAL("Requested association test is not recognized as a valid test.\n"); } }
static void save_image (cairo_surface_t *surface, const char *test_name, const char *extension) { char *filename = get_output_file (test_name, ".node", extension); g_test_message ("Storing test result image at %s", filename); g_assert (cairo_surface_write_to_png (surface, filename) == CAIRO_STATUS_SUCCESS); g_free (filename); }
int main(int argc, char *argv[]) { if(argc != 2) { printf("Usage: %s input-file\n", argv[0]); exit(1); } struct img input; if(!ppmb_read(argv[1], &input.xsize, &input.ysize, &input.maxrgb, &input.r, &input.g, &input.b)) { if(input.maxrgb > 255) { printf("Maxrgb %d not supported\n", input.maxrgb); exit(1); } int *hist_r, *hist_g, *hist_b; hist_r = (int *) calloc(input.maxrgb+1, sizeof(int)); hist_g = (int *) calloc(input.maxrgb+1, sizeof(int)); hist_b = (int *) calloc(input.maxrgb+1, sizeof(int)); ggc::Timer t("histogram"); t.start(); histogram(&input, hist_r, hist_g, hist_b); t.stop(); char *output = get_output_file(argv[1]); FILE *out = fopen(output, "w"); if(out) { print_histogram(out, hist_r, input.maxrgb); print_histogram(out, hist_g, input.maxrgb); print_histogram(out, hist_b, input.maxrgb); fclose(out); } else { fprintf(stderr, "Unable to output!\n"); } printf("Time: %llu ns\n", t.duration()); } }
/**************** * Decrypt the data, specified by ED with the key DEK. */ int decrypt_data (ctrl_t ctrl, void *procctx, PKT_encrypted *ed, DEK *dek) { decode_filter_ctx_t dfx; byte *p; int rc=0, c, i; byte temp[32]; unsigned blocksize; unsigned nprefix; dfx = xtrycalloc (1, sizeof *dfx); if (!dfx) return gpg_error_from_syserror (); dfx->refcount = 1; if ( opt.verbose && !dek->algo_info_printed ) { if (!openpgp_cipher_test_algo (dek->algo)) log_info (_("%s encrypted data\n"), openpgp_cipher_algo_name (dek->algo)); else log_info (_("encrypted with unknown algorithm %d\n"), dek->algo ); dek->algo_info_printed = 1; } { char buf[20]; snprintf (buf, sizeof buf, "%d %d", ed->mdc_method, dek->algo); write_status_text (STATUS_DECRYPTION_INFO, buf); } if (opt.show_session_key) { char numbuf[25]; char *hexbuf; snprintf (numbuf, sizeof numbuf, "%d:", dek->algo); hexbuf = bin2hex (dek->key, dek->keylen, NULL); if (!hexbuf) { rc = gpg_error_from_syserror (); goto leave; } log_info ("session key: '%s%s'\n", numbuf, hexbuf); write_status_strings (STATUS_SESSION_KEY, numbuf, hexbuf, NULL); xfree (hexbuf); } rc = openpgp_cipher_test_algo (dek->algo); if (rc) goto leave; blocksize = openpgp_cipher_get_algo_blklen (dek->algo); if ( !blocksize || blocksize > 16 ) log_fatal ("unsupported blocksize %u\n", blocksize ); nprefix = blocksize; if ( ed->len && ed->len < (nprefix+2) ) { /* An invalid message. We can't check that during parsing because we may not know the used cipher then. */ rc = gpg_error (GPG_ERR_INV_PACKET); goto leave; } if ( ed->mdc_method ) { if (gcry_md_open (&dfx->mdc_hash, ed->mdc_method, 0 )) BUG (); if ( DBG_HASHING ) gcry_md_debug (dfx->mdc_hash, "checkmdc"); } rc = openpgp_cipher_open (&dfx->cipher_hd, dek->algo, GCRY_CIPHER_MODE_CFB, (GCRY_CIPHER_SECURE | ((ed->mdc_method || dek->algo >= 100)? 0 : GCRY_CIPHER_ENABLE_SYNC))); if (rc) { /* We should never get an error here cause we already checked * that the algorithm is available. */ BUG(); } /* log_hexdump( "thekey", dek->key, dek->keylen );*/ rc = gcry_cipher_setkey (dfx->cipher_hd, dek->key, dek->keylen); if ( gpg_err_code (rc) == GPG_ERR_WEAK_KEY ) { log_info(_("WARNING: message was encrypted with" " a weak key in the symmetric cipher.\n")); rc=0; } else if( rc ) { log_error("key setup failed: %s\n", gpg_strerror (rc) ); goto leave; } if (!ed->buf) { log_error(_("problem handling encrypted packet\n")); goto leave; } gcry_cipher_setiv (dfx->cipher_hd, NULL, 0); if ( ed->len ) { for (i=0; i < (nprefix+2) && ed->len; i++, ed->len-- ) { if ( (c=iobuf_get(ed->buf)) == -1 ) break; else temp[i] = c; } } else { for (i=0; i < (nprefix+2); i++ ) if ( (c=iobuf_get(ed->buf)) == -1 ) break; else temp[i] = c; } gcry_cipher_decrypt (dfx->cipher_hd, temp, nprefix+2, NULL, 0); gcry_cipher_sync (dfx->cipher_hd); p = temp; /* log_hexdump( "prefix", temp, nprefix+2 ); */ if (dek->symmetric && (p[nprefix-2] != p[nprefix] || p[nprefix-1] != p[nprefix+1]) ) { rc = gpg_error (GPG_ERR_BAD_KEY); goto leave; } if ( dfx->mdc_hash ) gcry_md_write (dfx->mdc_hash, temp, nprefix+2); dfx->refcount++; dfx->partial = ed->is_partial; dfx->length = ed->len; if ( ed->mdc_method ) iobuf_push_filter ( ed->buf, mdc_decode_filter, dfx ); else iobuf_push_filter ( ed->buf, decode_filter, dfx ); if (opt.unwrap_encryption) { char *filename; estream_t fp; rc = get_output_file ("", 0, ed->buf, &filename, &fp); if (! rc) { iobuf_t output = iobuf_esopen (fp, "w", 0); armor_filter_context_t *afx = NULL; if (opt.armor) { afx = new_armor_context (); push_armor_filter (afx, output); } iobuf_copy (output, ed->buf); if ((rc = iobuf_error (ed->buf))) log_error (_("error reading '%s': %s\n"), filename, gpg_strerror (rc)); else if ((rc = iobuf_error (output))) log_error (_("error writing '%s': %s\n"), filename, gpg_strerror (rc)); iobuf_close (output); if (afx) release_armor_context (afx); } } else proc_packets (ctrl, procctx, ed->buf ); ed->buf = NULL; if (dfx->eof_seen > 1 ) rc = gpg_error (GPG_ERR_INV_PACKET); else if ( ed->mdc_method ) { /* We used to let parse-packet.c handle the MDC packet but this turned out to be a problem with compressed packets: With old style packets there is no length information available and the decompressor uses an implicit end. However we can't know this implicit end beforehand (:-) and thus may feed the decompressor with more bytes than actually needed. It would be possible to unread the extra bytes but due to our weird iobuf system any unread is non reliable due to filters already popped off. The easy and sane solution is to care about the MDC packet only here and never pass it to the packet parser. Fortunatley the OpenPGP spec requires a strict format for the MDC packet so that we know that 22 bytes are appended. */ int datalen = gcry_md_get_algo_dlen (ed->mdc_method); log_assert (dfx->cipher_hd); log_assert (dfx->mdc_hash); gcry_cipher_decrypt (dfx->cipher_hd, dfx->defer, 22, NULL, 0); gcry_md_write (dfx->mdc_hash, dfx->defer, 2); gcry_md_final (dfx->mdc_hash); if ( dfx->defer[0] != '\xd3' || dfx->defer[1] != '\x14' || datalen != 20 || memcmp (gcry_md_read (dfx->mdc_hash, 0), dfx->defer+2, datalen)) rc = gpg_error (GPG_ERR_BAD_SIGNATURE); /* log_printhex("MDC message:", dfx->defer, 22); */ /* log_printhex("MDC calc:", gcry_md_read (dfx->mdc_hash,0), datalen); */ } leave: release_dfx_context (dfx); return rc; }
int run_merge(shared_options_data_t *shared_options_data, merge_options_data_t *options_data) { if (options_data->num_files == 1) { LOG_INFO("Just one VCF file specified, no need to merge"); return 0; } list_t *read_list[options_data->num_files]; memset(read_list, 0, options_data->num_files * sizeof(list_t*)); list_t *output_header_list = (list_t*) malloc (sizeof(list_t)); list_init("headers", shared_options_data->num_threads, INT_MAX, output_header_list); list_t *output_list = (list_t*) malloc (sizeof(list_t)); list_init("output", shared_options_data->num_threads, shared_options_data->max_batches * shared_options_data->batch_lines, output_list); list_t *merge_tokens = (list_t*) malloc (sizeof(list_t)); list_init("tokens", 1, INT_MAX, merge_tokens); int ret_code = 0; double start, stop, total; vcf_file_t *files[options_data->num_files]; memset(files, 0, options_data->num_files * sizeof(vcf_file_t*)); // Initialize variables related to the different files for (int i = 0; i < options_data->num_files; i++) { files[i] = vcf_open(options_data->input_files[i], shared_options_data->max_batches); if (!files[i]) { LOG_FATAL_F("VCF file %s does not exist!\n", options_data->input_files[i]); } read_list[i] = (list_t*) malloc(sizeof(list_t)); list_init("text", 1, shared_options_data->max_batches, read_list[i]); } ret_code = create_directory(shared_options_data->output_directory); if (ret_code != 0 && errno != EEXIST) { LOG_FATAL_F("Can't create output directory: %s\n", shared_options_data->output_directory); } chromosome_order = get_chromosome_order(shared_options_data->host_url, shared_options_data->species, shared_options_data->version, &num_chromosomes); printf("Number of threads = %d\n", shared_options_data->num_threads); #pragma omp parallel sections private(start, stop, total) { #pragma omp section { LOG_DEBUG_F("Thread %d reads the VCF file\n", omp_get_thread_num()); // Reading start = omp_get_wtime(); ret_code = vcf_multiread_batches(read_list, shared_options_data->batch_lines, files, options_data->num_files); stop = omp_get_wtime(); total = stop - start; if (ret_code) { LOG_ERROR_F("Error %d while reading VCF files\n", ret_code); } LOG_INFO_F("[%dR] Time elapsed = %f s\n", omp_get_thread_num(), total); LOG_INFO_F("[%dR] Time elapsed = %e ms\n", omp_get_thread_num(), total*1000); } #pragma omp section { // Enable nested parallelism omp_set_nested(1); LOG_DEBUG_F("Thread %d processes data\n", omp_get_thread_num()); int num_eof_found = 0; int eof_found[options_data->num_files]; memset(eof_found, 0, options_data->num_files * sizeof(int)); list_item_t *items[options_data->num_files]; memset(items, 0, options_data->num_files * sizeof(list_item_t*)); char *texts[options_data->num_files]; memset(texts, 0, options_data->num_files * sizeof(char*)); khash_t(pos) *positions_read = kh_init(pos); long max_position_merged = LONG_MAX; char *max_chromosome_merged = NULL; int header_merged = 0; int token = 0; double start_parsing, start_insertion, total_parsing = 0, total_insertion = 0; start = omp_get_wtime(); while (num_eof_found < options_data->num_files) { /* Process: * - N threads getting batches of VCF records and inserting them in a data structure. The common minimum * position of each group of batches will also be stored. * - If the data structure reaches certain size or the end of a chromosome, merge positions prior to the * last minimum registered. */ // Getting text elements in a critical region guarantees that each thread gets variants in positions in the same range for (int i = 0; i < options_data->num_files; i++) { if (eof_found[i]) { continue; } items[i] = list_remove_item(read_list[i]); if (items[i] == NULL || !strcmp(items[i]->data_p, "")) { LOG_INFO_F("[%d] EOF found in file %s\n", omp_get_thread_num(), options_data->input_files[i]); eof_found[i] = 1; num_eof_found++; if(items[i] != NULL && !strcmp(items[i]->data_p, "")) { free(items[i]->data_p); list_item_free(items[i]); LOG_DEBUG_F("[%d] Text batch freed\n", omp_get_thread_num()); } else { LOG_DEBUG_F("[%d] No need to free text batch\n", omp_get_thread_num()); } continue; } assert(items[i]->data_p != NULL); texts[i] = items[i]->data_p; // printf("[%d] text batch from file %d\tcontents = '%s'\n", omp_get_thread_num(), i, texts[i]); } for (int i = 0; i < options_data->num_files; i++) { if (eof_found[i]) { continue; } start_parsing = omp_get_wtime(); char *text_begin = texts[i]; char *text_end = text_begin + strlen(text_begin); assert(text_end != NULL); // printf("batch = '%.*s'\n", text_end - text_begin, text_begin); // Get VCF batches from text batches vcf_reader_status *status = vcf_reader_status_new(shared_options_data->batch_lines, 0); ret_code = run_vcf_parser(text_begin, text_end, shared_options_data->batch_lines, files[i], status); if (ret_code) { // TODO stop? LOG_ERROR_F("Error %d while reading the file %s\n", ret_code, files[i]->filename); continue; } // printf("batches = %d\n", files[i]->record_batches->length); vcf_batch_t *batch = fetch_vcf_batch_non_blocking(files[i]); if (!batch) { continue; } total_parsing += omp_get_wtime() - start_parsing; start_insertion = omp_get_wtime(); // Insert records into hashtable for (int j = 0; j < batch->records->size; j++) { vcf_record_t *record = vcf_record_copy(array_list_get(j, batch->records)); vcf_record_file_link *link = vcf_record_file_link_new(record, files[i]); char key[64]; compose_key_value(record->chromosome, record->position, key); int ret = insert_position_read(key, link, positions_read); assert(ret); } total_insertion += omp_get_wtime() - start_insertion; // Update minimum position being a maximum of these batches vcf_record_t *current_record = (vcf_record_t*) array_list_get(batch->records->size - 1, batch->records); calculate_merge_interval(current_record, &max_chromosome_merged, &max_position_merged, chromosome_order, num_chromosomes); // Free batch and its contents vcf_reader_status_free(status); vcf_batch_free(batch); list_item_free(items[i]); } if (num_eof_found == options_data->num_files) { max_chromosome_merged = chromosome_order[num_chromosomes-1]; max_position_merged = LONG_MAX; } // Merge headers, if not previously done if (!header_merged) { merge_vcf_headers(files, options_data->num_files, options_data, output_header_list); header_merged = 1; // Decrease list writers count for (int i = 0; i < shared_options_data->num_threads; i++) { list_decr_writers(output_header_list); } } // If the data structure reaches certain size or the end of a chromosome, // merge positions prior to the last minimum registered if (num_eof_found < options_data->num_files && kh_size(positions_read) > TREE_LIMIT) { LOG_INFO_F("Merging until position %s:%ld\n", max_chromosome_merged, max_position_merged); token = merge_interval(positions_read, max_chromosome_merged, max_position_merged, chromosome_order, num_chromosomes, files, shared_options_data, options_data, output_list); } // When reaching EOF for all files, merge the remaining entries else if (num_eof_found == options_data->num_files && kh_size(positions_read) > 0) { LOG_INFO_F("Merging remaining positions (last = %s:%ld)\n", chromosome_order[num_chromosomes - 1], LONG_MAX); token = merge_remaining_interval(positions_read, files, shared_options_data, options_data, output_list); } if (token) { int *token_ptr = malloc (sizeof(int)); *token_ptr = token; list_item_t *item = list_item_new(1, 0, token_ptr); list_insert_item(item, merge_tokens); } // Set variables ready for next iteration of the algorithm if (max_chromosome_merged) { free(max_chromosome_merged); } token = 0; max_chromosome_merged = NULL; max_position_merged = LONG_MAX; } kh_destroy(pos, positions_read); stop = omp_get_wtime(); total = stop - start; LOG_INFO_F("[%d] Time elapsed = %f s\n", omp_get_thread_num(), total); LOG_INFO_F("[%d] Time elapsed = %e ms\n", omp_get_thread_num(), total*1000); LOG_DEBUG_F("** Time in parsing = %f s\n", total_parsing); LOG_DEBUG_F("** Time in insertion = %f s\n", total_insertion); // for (int i = 0; i < shared_options_data->num_threads; i++) { // printf("[%d] Time in searching = %f s\n", i, total_search[i]); // printf("[%d] Time in merging = %f s\n", i, total_merge[i]); // } // Decrease list writers count for (int i = 0; i < shared_options_data->num_threads; i++) { list_decr_writers(output_list); } list_decr_writers(merge_tokens); } #pragma omp section { LOG_DEBUG_F("Thread %d writes the output\n", omp_get_thread_num()); start = omp_get_wtime(); // Create file streams for results char aux_filename[32]; memset(aux_filename, 0, 32 * sizeof(char)); sprintf(aux_filename, "merge_from_%d_files.vcf", options_data->num_files); char *merge_filename; FILE *merge_fd = get_output_file(shared_options_data, aux_filename, &merge_filename); LOG_INFO_F("Output filename = %s\n", merge_filename); free(merge_filename); list_item_t *item1 = NULL, *item2 = NULL; vcf_header_entry_t *entry; vcf_record_t *record; int *num_records; // Write headers while ((item1 = list_remove_item(output_header_list)) != NULL) { entry = item1->data_p; write_vcf_header_entry(entry, merge_fd); } // Write delimiter array_list_t *sample_names = merge_vcf_sample_names(files, options_data->num_files); write_vcf_delimiter_from_samples((char**) sample_names->items, sample_names->size, merge_fd); // Write records // When a token is present, it means a set of batches has been merged. The token contains the number of records merged. // In this case, the records must be sorted by chromosome and position, and written afterwards. while ((item1 = list_remove_item(merge_tokens)) != NULL) { num_records = item1->data_p; vcf_record_t *records[*num_records]; for (int i = 0; i < *num_records; i++) { item2 = list_remove_item(output_list); if (!item2) { break; } records[i] = item2->data_p; list_item_free(item2); } // Sort records qsort(records, *num_records, sizeof(vcf_record_t*), record_cmp); // Write and free sorted records for (int i = 0; i < *num_records; i++) { record = records[i]; write_vcf_record(record, merge_fd); vcf_record_free_deep(record); } free(num_records); list_item_free(item1); } // Close file if (merge_fd != NULL) { fclose(merge_fd); } stop = omp_get_wtime(); total = stop - start; LOG_INFO_F("[%dW] Time elapsed = %f s\n", omp_get_thread_num(), total); LOG_INFO_F("[%dW] Time elapsed = %e ms\n", omp_get_thread_num(), total*1000); } } // Free variables related to the different files for (int i = 0; i < options_data->num_files; i++) { if(files[i]) { vcf_close(files[i]); } if(read_list[i]) { free(read_list[i]); } } free(output_list); return ret_code; }