static FILE *get_assoc_output_file(enum ASSOC_task task, shared_options_data_t *global_options_data, char **path) {
if (task == CHI_SQUARE) {
return get_output_file(global_options_data, "hpg-variant.chisq", path);
} else if (task == FISHER) {
return get_output_file(global_options_data, "hpg-variant.fisher", path);
} else {
LOG_FATAL("Requested association test is not recognized as a valid test.\n");
}
}
static void
save_image (cairo_surface_t *surface,
const char *test_name,
const char *extension)
{
char *filename = get_output_file (test_name, ".node", extension);
g_test_message ("Storing test result image at %s", filename);
g_assert (cairo_surface_write_to_png (surface, filename) == CAIRO_STATUS_SUCCESS);
g_free (filename);
}
int main(int argc, char *argv[]) {
if(argc != 2) {
printf("Usage: %s input-file\n", argv[0]);
exit(1);
}
struct img input;
if(!ppmb_read(argv[1], &input.xsize, &input.ysize, &input.maxrgb,
&input.r, &input.g, &input.b)) {
if(input.maxrgb > 255) {
printf("Maxrgb %d not supported\n", input.maxrgb);
exit(1);
}
int *hist_r, *hist_g, *hist_b;
hist_r = (int *) calloc(input.maxrgb+1, sizeof(int));
hist_g = (int *) calloc(input.maxrgb+1, sizeof(int));
hist_b = (int *) calloc(input.maxrgb+1, sizeof(int));
ggc::Timer t("histogram");
t.start();
histogram(&input, hist_r, hist_g, hist_b);
t.stop();
char *output = get_output_file(argv[1]);
FILE *out = fopen(output, "w");
if(out) {
print_histogram(out, hist_r, input.maxrgb);
print_histogram(out, hist_g, input.maxrgb);
print_histogram(out, hist_b, input.maxrgb);
fclose(out);
} else {
fprintf(stderr, "Unable to output!\n");
}
printf("Time: %llu ns\n", t.duration());
}
}
/****************
* Decrypt the data, specified by ED with the key DEK.
*/
int
decrypt_data (ctrl_t ctrl, void *procctx, PKT_encrypted *ed, DEK *dek)
{
decode_filter_ctx_t dfx;
byte *p;
int rc=0, c, i;
byte temp[32];
unsigned blocksize;
unsigned nprefix;
dfx = xtrycalloc (1, sizeof *dfx);
if (!dfx)
return gpg_error_from_syserror ();
dfx->refcount = 1;
if ( opt.verbose && !dek->algo_info_printed )
{
if (!openpgp_cipher_test_algo (dek->algo))
log_info (_("%s encrypted data\n"),
openpgp_cipher_algo_name (dek->algo));
else
log_info (_("encrypted with unknown algorithm %d\n"), dek->algo );
dek->algo_info_printed = 1;
}
{
char buf[20];
snprintf (buf, sizeof buf, "%d %d", ed->mdc_method, dek->algo);
write_status_text (STATUS_DECRYPTION_INFO, buf);
}
if (opt.show_session_key)
{
char numbuf[25];
char *hexbuf;
snprintf (numbuf, sizeof numbuf, "%d:", dek->algo);
hexbuf = bin2hex (dek->key, dek->keylen, NULL);
if (!hexbuf)
{
rc = gpg_error_from_syserror ();
goto leave;
}
log_info ("session key: '%s%s'\n", numbuf, hexbuf);
write_status_strings (STATUS_SESSION_KEY, numbuf, hexbuf, NULL);
xfree (hexbuf);
}
rc = openpgp_cipher_test_algo (dek->algo);
if (rc)
goto leave;
blocksize = openpgp_cipher_get_algo_blklen (dek->algo);
if ( !blocksize || blocksize > 16 )
log_fatal ("unsupported blocksize %u\n", blocksize );
nprefix = blocksize;
if ( ed->len && ed->len < (nprefix+2) )
{
/* An invalid message. We can't check that during parsing
because we may not know the used cipher then. */
rc = gpg_error (GPG_ERR_INV_PACKET);
goto leave;
}
if ( ed->mdc_method )
{
if (gcry_md_open (&dfx->mdc_hash, ed->mdc_method, 0 ))
BUG ();
if ( DBG_HASHING )
gcry_md_debug (dfx->mdc_hash, "checkmdc");
}
rc = openpgp_cipher_open (&dfx->cipher_hd, dek->algo,
GCRY_CIPHER_MODE_CFB,
(GCRY_CIPHER_SECURE
| ((ed->mdc_method || dek->algo >= 100)?
0 : GCRY_CIPHER_ENABLE_SYNC)));
if (rc)
{
/* We should never get an error here cause we already checked
* that the algorithm is available. */
BUG();
}
/* log_hexdump( "thekey", dek->key, dek->keylen );*/
rc = gcry_cipher_setkey (dfx->cipher_hd, dek->key, dek->keylen);
if ( gpg_err_code (rc) == GPG_ERR_WEAK_KEY )
{
log_info(_("WARNING: message was encrypted with"
" a weak key in the symmetric cipher.\n"));
rc=0;
}
else if( rc )
{
log_error("key setup failed: %s\n", gpg_strerror (rc) );
goto leave;
}
if (!ed->buf)
{
log_error(_("problem handling encrypted packet\n"));
goto leave;
}
gcry_cipher_setiv (dfx->cipher_hd, NULL, 0);
if ( ed->len )
{
for (i=0; i < (nprefix+2) && ed->len; i++, ed->len-- )
{
if ( (c=iobuf_get(ed->buf)) == -1 )
break;
else
temp[i] = c;
}
}
else
{
for (i=0; i < (nprefix+2); i++ )
if ( (c=iobuf_get(ed->buf)) == -1 )
break;
else
temp[i] = c;
}
gcry_cipher_decrypt (dfx->cipher_hd, temp, nprefix+2, NULL, 0);
gcry_cipher_sync (dfx->cipher_hd);
p = temp;
/* log_hexdump( "prefix", temp, nprefix+2 ); */
if (dek->symmetric
&& (p[nprefix-2] != p[nprefix] || p[nprefix-1] != p[nprefix+1]) )
{
rc = gpg_error (GPG_ERR_BAD_KEY);
goto leave;
}
if ( dfx->mdc_hash )
gcry_md_write (dfx->mdc_hash, temp, nprefix+2);
dfx->refcount++;
dfx->partial = ed->is_partial;
dfx->length = ed->len;
if ( ed->mdc_method )
iobuf_push_filter ( ed->buf, mdc_decode_filter, dfx );
else
iobuf_push_filter ( ed->buf, decode_filter, dfx );
if (opt.unwrap_encryption)
{
char *filename;
estream_t fp;
rc = get_output_file ("", 0, ed->buf, &filename, &fp);
if (! rc)
{
iobuf_t output = iobuf_esopen (fp, "w", 0);
armor_filter_context_t *afx = NULL;
if (opt.armor)
{
afx = new_armor_context ();
push_armor_filter (afx, output);
}
iobuf_copy (output, ed->buf);
if ((rc = iobuf_error (ed->buf)))
log_error (_("error reading '%s': %s\n"),
filename, gpg_strerror (rc));
else if ((rc = iobuf_error (output)))
log_error (_("error writing '%s': %s\n"),
filename, gpg_strerror (rc));
iobuf_close (output);
if (afx)
release_armor_context (afx);
}
}
else
proc_packets (ctrl, procctx, ed->buf );
ed->buf = NULL;
if (dfx->eof_seen > 1 )
rc = gpg_error (GPG_ERR_INV_PACKET);
else if ( ed->mdc_method )
{
/* We used to let parse-packet.c handle the MDC packet but this
turned out to be a problem with compressed packets: With old
style packets there is no length information available and
the decompressor uses an implicit end. However we can't know
this implicit end beforehand (:-) and thus may feed the
decompressor with more bytes than actually needed. It would
be possible to unread the extra bytes but due to our weird
iobuf system any unread is non reliable due to filters
already popped off. The easy and sane solution is to care
about the MDC packet only here and never pass it to the
packet parser. Fortunatley the OpenPGP spec requires a
strict format for the MDC packet so that we know that 22
bytes are appended. */
int datalen = gcry_md_get_algo_dlen (ed->mdc_method);
log_assert (dfx->cipher_hd);
log_assert (dfx->mdc_hash);
gcry_cipher_decrypt (dfx->cipher_hd, dfx->defer, 22, NULL, 0);
gcry_md_write (dfx->mdc_hash, dfx->defer, 2);
gcry_md_final (dfx->mdc_hash);
if ( dfx->defer[0] != '\xd3'
|| dfx->defer[1] != '\x14'
|| datalen != 20
|| memcmp (gcry_md_read (dfx->mdc_hash, 0), dfx->defer+2, datalen))
rc = gpg_error (GPG_ERR_BAD_SIGNATURE);
/* log_printhex("MDC message:", dfx->defer, 22); */
/* log_printhex("MDC calc:", gcry_md_read (dfx->mdc_hash,0), datalen); */
}
leave:
release_dfx_context (dfx);
return rc;
}
int run_merge(shared_options_data_t *shared_options_data, merge_options_data_t *options_data) {
if (options_data->num_files == 1) {
LOG_INFO("Just one VCF file specified, no need to merge");
return 0;
}
list_t *read_list[options_data->num_files];
memset(read_list, 0, options_data->num_files * sizeof(list_t*));
list_t *output_header_list = (list_t*) malloc (sizeof(list_t));
list_init("headers", shared_options_data->num_threads, INT_MAX, output_header_list);
list_t *output_list = (list_t*) malloc (sizeof(list_t));
list_init("output", shared_options_data->num_threads, shared_options_data->max_batches * shared_options_data->batch_lines, output_list);
list_t *merge_tokens = (list_t*) malloc (sizeof(list_t));
list_init("tokens", 1, INT_MAX, merge_tokens);
int ret_code = 0;
double start, stop, total;
vcf_file_t *files[options_data->num_files];
memset(files, 0, options_data->num_files * sizeof(vcf_file_t*));
// Initialize variables related to the different files
for (int i = 0; i < options_data->num_files; i++) {
files[i] = vcf_open(options_data->input_files[i], shared_options_data->max_batches);
if (!files[i]) {
LOG_FATAL_F("VCF file %s does not exist!\n", options_data->input_files[i]);
}
read_list[i] = (list_t*) malloc(sizeof(list_t));
list_init("text", 1, shared_options_data->max_batches, read_list[i]);
}
ret_code = create_directory(shared_options_data->output_directory);
if (ret_code != 0 && errno != EEXIST) {
LOG_FATAL_F("Can't create output directory: %s\n", shared_options_data->output_directory);
}
chromosome_order = get_chromosome_order(shared_options_data->host_url, shared_options_data->species,
shared_options_data->version, &num_chromosomes);
printf("Number of threads = %d\n", shared_options_data->num_threads);
#pragma omp parallel sections private(start, stop, total)
{
#pragma omp section
{
LOG_DEBUG_F("Thread %d reads the VCF file\n", omp_get_thread_num());
// Reading
start = omp_get_wtime();
ret_code = vcf_multiread_batches(read_list, shared_options_data->batch_lines, files, options_data->num_files);
stop = omp_get_wtime();
total = stop - start;
if (ret_code) {
LOG_ERROR_F("Error %d while reading VCF files\n", ret_code);
}
LOG_INFO_F("[%dR] Time elapsed = %f s\n", omp_get_thread_num(), total);
LOG_INFO_F("[%dR] Time elapsed = %e ms\n", omp_get_thread_num(), total*1000);
}
#pragma omp section
{
// Enable nested parallelism
omp_set_nested(1);
LOG_DEBUG_F("Thread %d processes data\n", omp_get_thread_num());
int num_eof_found = 0;
int eof_found[options_data->num_files];
memset(eof_found, 0, options_data->num_files * sizeof(int));
list_item_t *items[options_data->num_files];
memset(items, 0, options_data->num_files * sizeof(list_item_t*));
char *texts[options_data->num_files];
memset(texts, 0, options_data->num_files * sizeof(char*));
khash_t(pos) *positions_read = kh_init(pos);
long max_position_merged = LONG_MAX;
char *max_chromosome_merged = NULL;
int header_merged = 0;
int token = 0;
double start_parsing, start_insertion, total_parsing = 0, total_insertion = 0;
start = omp_get_wtime();
while (num_eof_found < options_data->num_files) {
/* Process:
* - N threads getting batches of VCF records and inserting them in a data structure. The common minimum
* position of each group of batches will also be stored.
* - If the data structure reaches certain size or the end of a chromosome, merge positions prior to the
* last minimum registered.
*/
// Getting text elements in a critical region guarantees that each thread gets variants in positions in the same range
for (int i = 0; i < options_data->num_files; i++) {
if (eof_found[i]) {
continue;
}
items[i] = list_remove_item(read_list[i]);
if (items[i] == NULL || !strcmp(items[i]->data_p, "")) {
LOG_INFO_F("[%d] EOF found in file %s\n", omp_get_thread_num(), options_data->input_files[i]);
eof_found[i] = 1;
num_eof_found++;
if(items[i] != NULL && !strcmp(items[i]->data_p, "")) {
free(items[i]->data_p);
list_item_free(items[i]);
LOG_DEBUG_F("[%d] Text batch freed\n", omp_get_thread_num());
} else {
LOG_DEBUG_F("[%d] No need to free text batch\n", omp_get_thread_num());
}
continue;
}
assert(items[i]->data_p != NULL);
texts[i] = items[i]->data_p;
// printf("[%d] text batch from file %d\tcontents = '%s'\n", omp_get_thread_num(), i, texts[i]);
}
for (int i = 0; i < options_data->num_files; i++) {
if (eof_found[i]) {
continue;
}
start_parsing = omp_get_wtime();
char *text_begin = texts[i];
char *text_end = text_begin + strlen(text_begin);
assert(text_end != NULL);
// printf("batch = '%.*s'\n", text_end - text_begin, text_begin);
// Get VCF batches from text batches
vcf_reader_status *status = vcf_reader_status_new(shared_options_data->batch_lines, 0);
ret_code = run_vcf_parser(text_begin, text_end, shared_options_data->batch_lines, files[i], status);
if (ret_code) {
// TODO stop?
LOG_ERROR_F("Error %d while reading the file %s\n", ret_code, files[i]->filename);
continue;
}
// printf("batches = %d\n", files[i]->record_batches->length);
vcf_batch_t *batch = fetch_vcf_batch_non_blocking(files[i]);
if (!batch) {
continue;
}
total_parsing += omp_get_wtime() - start_parsing;
start_insertion = omp_get_wtime();
// Insert records into hashtable
for (int j = 0; j < batch->records->size; j++) {
vcf_record_t *record = vcf_record_copy(array_list_get(j, batch->records));
vcf_record_file_link *link = vcf_record_file_link_new(record, files[i]);
char key[64];
compose_key_value(record->chromosome, record->position, key);
int ret = insert_position_read(key, link, positions_read);
assert(ret);
}
total_insertion += omp_get_wtime() - start_insertion;
// Update minimum position being a maximum of these batches
vcf_record_t *current_record = (vcf_record_t*) array_list_get(batch->records->size - 1, batch->records);
calculate_merge_interval(current_record, &max_chromosome_merged, &max_position_merged, chromosome_order, num_chromosomes);
// Free batch and its contents
vcf_reader_status_free(status);
vcf_batch_free(batch);
list_item_free(items[i]);
}
if (num_eof_found == options_data->num_files) {
max_chromosome_merged = chromosome_order[num_chromosomes-1];
max_position_merged = LONG_MAX;
}
// Merge headers, if not previously done
if (!header_merged) {
merge_vcf_headers(files, options_data->num_files, options_data, output_header_list);
header_merged = 1;
// Decrease list writers count
for (int i = 0; i < shared_options_data->num_threads; i++) {
list_decr_writers(output_header_list);
}
}
// If the data structure reaches certain size or the end of a chromosome,
// merge positions prior to the last minimum registered
if (num_eof_found < options_data->num_files && kh_size(positions_read) > TREE_LIMIT) {
LOG_INFO_F("Merging until position %s:%ld\n", max_chromosome_merged, max_position_merged);
token = merge_interval(positions_read, max_chromosome_merged, max_position_merged, chromosome_order, num_chromosomes,
files, shared_options_data, options_data, output_list);
}
// When reaching EOF for all files, merge the remaining entries
else if (num_eof_found == options_data->num_files && kh_size(positions_read) > 0) {
LOG_INFO_F("Merging remaining positions (last = %s:%ld)\n", chromosome_order[num_chromosomes - 1], LONG_MAX);
token = merge_remaining_interval(positions_read, files, shared_options_data, options_data, output_list);
}
if (token) {
int *token_ptr = malloc (sizeof(int)); *token_ptr = token;
list_item_t *item = list_item_new(1, 0, token_ptr);
list_insert_item(item, merge_tokens);
}
// Set variables ready for next iteration of the algorithm
if (max_chromosome_merged) {
free(max_chromosome_merged);
}
token = 0;
max_chromosome_merged = NULL;
max_position_merged = LONG_MAX;
}
kh_destroy(pos, positions_read);
stop = omp_get_wtime();
total = stop - start;
LOG_INFO_F("[%d] Time elapsed = %f s\n", omp_get_thread_num(), total);
LOG_INFO_F("[%d] Time elapsed = %e ms\n", omp_get_thread_num(), total*1000);
LOG_DEBUG_F("** Time in parsing = %f s\n", total_parsing);
LOG_DEBUG_F("** Time in insertion = %f s\n", total_insertion);
// for (int i = 0; i < shared_options_data->num_threads; i++) {
// printf("[%d] Time in searching = %f s\n", i, total_search[i]);
// printf("[%d] Time in merging = %f s\n", i, total_merge[i]);
// }
// Decrease list writers count
for (int i = 0; i < shared_options_data->num_threads; i++) {
list_decr_writers(output_list);
}
list_decr_writers(merge_tokens);
}
#pragma omp section
{
LOG_DEBUG_F("Thread %d writes the output\n", omp_get_thread_num());
start = omp_get_wtime();
// Create file streams for results
char aux_filename[32]; memset(aux_filename, 0, 32 * sizeof(char));
sprintf(aux_filename, "merge_from_%d_files.vcf", options_data->num_files);
char *merge_filename;
FILE *merge_fd = get_output_file(shared_options_data, aux_filename, &merge_filename);
LOG_INFO_F("Output filename = %s\n", merge_filename);
free(merge_filename);
list_item_t *item1 = NULL, *item2 = NULL;
vcf_header_entry_t *entry;
vcf_record_t *record;
int *num_records;
// Write headers
while ((item1 = list_remove_item(output_header_list)) != NULL) {
entry = item1->data_p;
write_vcf_header_entry(entry, merge_fd);
}
// Write delimiter
array_list_t *sample_names = merge_vcf_sample_names(files, options_data->num_files);
write_vcf_delimiter_from_samples((char**) sample_names->items, sample_names->size, merge_fd);
// Write records
// When a token is present, it means a set of batches has been merged. The token contains the number of records merged.
// In this case, the records must be sorted by chromosome and position, and written afterwards.
while ((item1 = list_remove_item(merge_tokens)) != NULL) {
num_records = item1->data_p;
vcf_record_t *records[*num_records];
for (int i = 0; i < *num_records; i++) {
item2 = list_remove_item(output_list);
if (!item2) {
break;
}
records[i] = item2->data_p;
list_item_free(item2);
}
// Sort records
qsort(records, *num_records, sizeof(vcf_record_t*), record_cmp);
// Write and free sorted records
for (int i = 0; i < *num_records; i++) {
record = records[i];
write_vcf_record(record, merge_fd);
vcf_record_free_deep(record);
}
free(num_records);
list_item_free(item1);
}
// Close file
if (merge_fd != NULL) { fclose(merge_fd); }
stop = omp_get_wtime();
total = stop - start;
LOG_INFO_F("[%dW] Time elapsed = %f s\n", omp_get_thread_num(), total);
LOG_INFO_F("[%dW] Time elapsed = %e ms\n", omp_get_thread_num(), total*1000);
}
}
// Free variables related to the different files
for (int i = 0; i < options_data->num_files; i++) {
if(files[i]) { vcf_close(files[i]); }
if(read_list[i]) { free(read_list[i]); }
}
free(output_list);
return ret_code;
}
