//URL Encoding not emplemented RFC1738
int connection::check_method_uri_protocol()
{
if (m_method.empty() || m_uri.empty() || m_protocol.empty())
{
return httpcodes::forbidden;
}
char const bad_sym[] = "<>\"%{}|^~[]`\\;:/?@#=&";
std::string::iterator it = std::search(m_uri.begin(), m_uri.end(), bad_sym, bad_sym+sizeof(bad_sym)-1);
if (it != m_uri.end())
{
return httpcodes::forbidden;
}
if (m_uri.find("..") != std::string::npos)
{
//URI contains ..
return httpcodes::forbidden;
}
if (!is_get() && !is_head())
{
//Unsupported method
return httpcodes::not_implemented;
}
if (m_uri.size() > MAX_PATH)
{
return httpcodes::uri_to_long;
}
if ('/' == *m_uri.begin())
{
m_full_file_name.assign(m_uri.begin()+1, m_uri.end());
}
if (m_full_file_name.empty())
{
m_full_file_name = DEFAULT_FILE_NAME;
}
int res = ::stat(m_full_file_name.c_str(), &m_file_stat);
if (res < 0)
{
int last_errno = errno;
return get_last_http_error(last_errno);
}
m_file.open(m_full_file_name.c_str(), std::ifstream::in);
if (!m_file)
{
return httpcodes::not_found;
}
if (m_file.fail())
{
return httpcodes::forbidden;
}
if (m_file.bad())
{
return httpcodes::service_unavailable;
}
return httpcodes::ok;
}
int run_effect(char **urls, shared_options_data_t *shared_options_data, effect_options_data_t *options_data) {
int ret_code = 0;
double start, stop, total;
vcf_file_t *vcf_file = vcf_open(shared_options_data->vcf_filename, shared_options_data->max_batches);
if (!vcf_file) {
LOG_FATAL("VCF file does not exist!\n");
}
ped_file_t *ped_file = NULL;
if (shared_options_data->ped_filename) {
ped_file = ped_open(shared_options_data->ped_filename);
if (!ped_file) {
LOG_FATAL("PED file does not exist!\n");
}
LOG_INFO("About to read PED file...\n");
// Read PED file before doing any processing
ret_code = ped_read(ped_file);
if (ret_code != 0) {
LOG_FATAL_F("Can't read PED file: %s\n", ped_file->filename);
}
}
char *output_directory = shared_options_data->output_directory;
size_t output_directory_len = strlen(output_directory);
ret_code = create_directory(output_directory);
if (ret_code != 0 && errno != EEXIST) {
LOG_FATAL_F("Can't create output directory: %s\n", output_directory);
}
// Remove all .txt files in folder
ret_code = delete_files_by_extension(output_directory, "txt");
if (ret_code != 0) {
return ret_code;
}
// Initialize environment for connecting to the web service
ret_code = init_http_environment(0);
if (ret_code != 0) {
return ret_code;
}
// Output file descriptors
static cp_hashtable *output_files = NULL;
// Lines of the output data in the main .txt files
static list_t *output_list = NULL;
// Consequence type counters (for summary, must be kept between web service calls)
static cp_hashtable *summary_count = NULL;
// Gene list (for genes-with-variants, must be kept between web service calls)
static cp_hashtable *gene_list = NULL;
// Initialize collections of file descriptors and summary counters
ret_code = initialize_output_files(output_directory, output_directory_len, &output_files);
if (ret_code != 0) {
return ret_code;
}
initialize_output_data_structures(shared_options_data, &output_list, &summary_count, &gene_list);
initialize_ws_buffers(shared_options_data->num_threads);
// Create job.status file
char job_status_filename[output_directory_len + 10];
sprintf(job_status_filename, "%s/job.status", output_directory);
FILE *job_status = new_job_status_file(job_status_filename);
if (!job_status) {
LOG_FATAL("Can't create job status file\n");
} else {
update_job_status_file(0, job_status);
}
#pragma omp parallel sections private(start, stop, total)
{
#pragma omp section
{
LOG_DEBUG_F("Thread %d reads the VCF file\n", omp_get_thread_num());
start = omp_get_wtime();
ret_code = vcf_read(vcf_file, 1,
(shared_options_data->batch_bytes > 0) ? shared_options_data->batch_bytes : shared_options_data->batch_lines,
shared_options_data->batch_bytes <= 0);
stop = omp_get_wtime();
total = stop - start;
if (ret_code) {
LOG_ERROR_F("Error %d while reading the file %s\n", ret_code, vcf_file->filename);
}
LOG_INFO_F("[%dR] Time elapsed = %f s\n", omp_get_thread_num(), total);
LOG_INFO_F("[%dR] Time elapsed = %e ms\n", omp_get_thread_num(), total*1000);
notify_end_parsing(vcf_file);
}
#pragma omp section
{
// Enable nested parallelism and set the number of threads the user has chosen
omp_set_nested(1);
LOG_DEBUG_F("Thread %d processes data\n", omp_get_thread_num());
// Filters and files for filtering output
filter_t **filters = NULL;
int num_filters = 0;
if (shared_options_data->chain != NULL) {
filters = sort_filter_chain(shared_options_data->chain, &num_filters);
}
FILE *passed_file = NULL, *failed_file = NULL, *non_processed_file = NULL;
get_filtering_output_files(shared_options_data, &passed_file, &failed_file);
// Pedigree information (used in some filters)
individual_t **individuals = NULL;
khash_t(ids) *sample_ids = NULL;
// Filename structure outdir/vcfname.errors
char *prefix_filename = calloc(strlen(shared_options_data->vcf_filename), sizeof(char));
get_filename_from_path(shared_options_data->vcf_filename, prefix_filename);
char *non_processed_filename = malloc((strlen(shared_options_data->output_directory) + strlen(prefix_filename) + 9) * sizeof(char));
sprintf(non_processed_filename, "%s/%s.errors", shared_options_data->output_directory, prefix_filename);
non_processed_file = fopen(non_processed_filename, "w");
free(non_processed_filename);
// Maximum size processed by each thread (never allow more than 1000 variants per query)
if (shared_options_data->batch_lines > 0) {
shared_options_data->entries_per_thread = MIN(MAX_VARIANTS_PER_QUERY,
ceil((float) shared_options_data->batch_lines / shared_options_data->num_threads));
} else {
shared_options_data->entries_per_thread = MAX_VARIANTS_PER_QUERY;
}
LOG_DEBUG_F("entries-per-thread = %d\n", shared_options_data->entries_per_thread);
int i = 0;
vcf_batch_t *batch = NULL;
int ret_ws_0 = 0, ret_ws_1 = 0, ret_ws_2 = 0;
start = omp_get_wtime();
while (batch = fetch_vcf_batch(vcf_file)) {
if (i == 0) {
// Add headers associated to the defined filters
vcf_header_entry_t **filter_headers = get_filters_as_vcf_headers(filters, num_filters);
for (int j = 0; j < num_filters; j++) {
add_vcf_header_entry(filter_headers[j], vcf_file);
}
// Write file format, header entries and delimiter
if (passed_file != NULL) { write_vcf_header(vcf_file, passed_file); }
if (failed_file != NULL) { write_vcf_header(vcf_file, failed_file); }
if (non_processed_file != NULL) { write_vcf_header(vcf_file, non_processed_file); }
LOG_DEBUG("VCF header written\n");
if (ped_file) {
// Create map to associate the position of individuals in the list of samples defined in the VCF file
sample_ids = associate_samples_and_positions(vcf_file);
// Sort individuals in PED as defined in the VCF file
individuals = sort_individuals(vcf_file, ped_file);
}
}
// printf("batch loaded = '%.*s'\n", 50, batch->text);
// printf("batch text len = %zu\n", strlen(batch->text));
// if (i % 10 == 0) {
LOG_INFO_F("Batch %d reached by thread %d - %zu/%zu records \n",
i, omp_get_thread_num(),
batch->records->size, batch->records->capacity);
// }
int reconnections = 0;
int max_reconnections = 3; // TODO allow to configure?
// Write records that passed to a separate file, and query the WS with them as args
array_list_t *failed_records = NULL;
int num_variables = ped_file? get_num_variables(ped_file): 0;
array_list_t *passed_records = filter_records(filters, num_filters, individuals, sample_ids, num_variables, batch->records, &failed_records);
if (passed_records->size > 0) {
// Divide the list of passed records in ranges of size defined in config file
int num_chunks;
int *chunk_sizes;
int *chunk_starts = create_chunks(passed_records->size, shared_options_data->entries_per_thread, &num_chunks, &chunk_sizes);
do {
// OpenMP: Launch a thread for each range
#pragma omp parallel for num_threads(shared_options_data->num_threads)
for (int j = 0; j < num_chunks; j++) {
int tid = omp_get_thread_num();
LOG_DEBUG_F("[%d] WS invocation\n", tid);
LOG_DEBUG_F("[%d] -- effect WS\n", tid);
if (!reconnections || ret_ws_0) {
ret_ws_0 = invoke_effect_ws(urls[0], (vcf_record_t**) (passed_records->items + chunk_starts[j]),
chunk_sizes[j], options_data->excludes);
parse_effect_response(tid, output_directory, output_directory_len, output_files, output_list, summary_count, gene_list);
free(effect_line[tid]);
effect_line[tid] = (char*) calloc (max_line_size[tid], sizeof(char));
}
if (!options_data->no_phenotypes) {
if (!reconnections || ret_ws_1) {
LOG_DEBUG_F("[%d] -- snp WS\n", omp_get_thread_num());
ret_ws_1 = invoke_snp_phenotype_ws(urls[1], (vcf_record_t**) (passed_records->items + chunk_starts[j]), chunk_sizes[j]);
parse_snp_phenotype_response(tid, output_list);
free(snp_line[tid]);
snp_line[tid] = (char*) calloc (snp_max_line_size[tid], sizeof(char));
}
if (!reconnections || ret_ws_2) {
LOG_DEBUG_F("[%d] -- mutation WS\n", omp_get_thread_num());
ret_ws_2 = invoke_mutation_phenotype_ws(urls[2], (vcf_record_t**) (passed_records->items + chunk_starts[j]), chunk_sizes[j]);
parse_mutation_phenotype_response(tid, output_list);
free(mutation_line[tid]);
mutation_line[tid] = (char*) calloc (mutation_max_line_size[tid], sizeof(char));
}
}
}
LOG_DEBUG_F("*** %dth web services invocation finished\n", i);
if (ret_ws_0 || ret_ws_1 || ret_ws_2) {
if (ret_ws_0) {
LOG_ERROR_F("Effect web service error: %s\n", get_last_http_error(ret_ws_0));
}
if (ret_ws_1) {
LOG_ERROR_F("SNP phenotype web service error: %s\n", get_last_http_error(ret_ws_1));
}
if (ret_ws_2) {
LOG_ERROR_F("Mutations phenotype web service error: %s\n", get_last_http_error(ret_ws_2));
}
// In presence of errors, wait 4 seconds before retrying
reconnections++;
LOG_ERROR_F("Some errors ocurred, reconnection #%d\n", reconnections);
sleep(4);
} else {
free(chunk_starts);
free(chunk_sizes);
}
} while (reconnections < max_reconnections && (ret_ws_0 || ret_ws_1 || ret_ws_2));
}
// If the maximum number of reconnections was reached still with errors,
// write the non-processed batch to the corresponding file
if (reconnections == max_reconnections && (ret_ws_0 || ret_ws_1 || ret_ws_2)) {
#pragma omp critical
{
write_vcf_batch(batch, non_processed_file);
}
}
// Write records that passed and failed filters to separate files, and free them
write_filtering_output_files(passed_records, failed_records, passed_file, failed_file);
free_filtered_records(passed_records, failed_records, batch->records);
// Free batch and its contents
vcf_batch_free(batch);
i++;
}
stop = omp_get_wtime();
total = stop - start;
LOG_INFO_F("[%d] Time elapsed = %f s\n", omp_get_thread_num(), total);
LOG_INFO_F("[%d] Time elapsed = %e ms\n", omp_get_thread_num(), total*1000);
// Free resources
if (passed_file) { fclose(passed_file); }
if (failed_file) { fclose(failed_file); }
if (non_processed_file) { fclose(non_processed_file); }
// Free filters
for (i = 0; i < num_filters; i++) {
filter_t *filter = filters[i];
filter->free_func(filter);
}
free(filters);
// Decrease list writers count
for (i = 0; i < shared_options_data->num_threads; i++) {
list_decr_writers(output_list);
}
}
#pragma omp section
{
// Thread which writes the results to all_variants, summary and one file per consequence type
int ret = 0;
char *line;
list_item_t* item = NULL;
FILE *fd = NULL;
FILE *all_variants_file = cp_hashtable_get(output_files, "all_variants");
FILE *snp_phenotype_file = cp_hashtable_get(output_files, "snp_phenotypes");
FILE *mutation_phenotype_file = cp_hashtable_get(output_files, "mutation_phenotypes");
while ((item = list_remove_item(output_list)) != NULL) {
line = item->data_p;
// Type greater than 0: consequence type identified by its SO code
// Type equals to -1: SNP phenotype
// Type equals to -2: mutation phenotype
if (item->type > 0) {
// Write entry in the consequence type file
fd = cp_hashtable_get(output_files, &(item->type));
int ret = fprintf(fd, "%s\n", line);
if (ret < 0) {
LOG_ERROR_F("Error writing to file: '%s'\n", line);
}
// Write in all_variants
ret = fprintf(all_variants_file, "%s\n", line);
if (ret < 0) {
LOG_ERROR_F("Error writing to all_variants: '%s'\n", line);
}
} else if (item->type == SNP_PHENOTYPE) {
ret = fprintf(snp_phenotype_file, "%s\n", line);
if (ret < 0) {
LOG_ERROR_F("Error writing to snp_phenotypes: '%s'\n", line);
}
} else if (item->type == MUTATION_PHENOTYPE) {
ret = fprintf(mutation_phenotype_file, "%s\n", line);
if (ret < 0) {
LOG_ERROR_F("Error writing to mutation_phenotypes: '%s'\n", line);
}
}
free(line);
list_item_free(item);
}
}
}
write_summary_file(summary_count, cp_hashtable_get(output_files, "summary"));
write_genes_with_variants_file(gene_list, output_directory);
write_result_file(shared_options_data, options_data, summary_count, output_directory);
free_output_data_structures(output_files, summary_count, gene_list);
free_ws_buffers(shared_options_data->num_threads);
free(output_list);
vcf_close(vcf_file);
update_job_status_file(100, job_status);
close_job_status_file(job_status);
return ret_code;
}