void workflow_schedule(workflow_t *wf) { work_item_t *item = NULL; pthread_mutex_lock(&wf->main_mutex); //printf("Workflow schedule mutex lock\n"); while (wf->num_pending_items <= 0 && !wf->completed_producer) { //printf("Waitign in workflow...\n"); pthread_cond_wait(&wf->workers_cond, &wf->main_mutex); } // for (int i = wf->num_stages - 1; i >= 0; i--) { for (int i = 0 ; i <= wf->num_stages - 1; i++) { item = array_list_remove_at(0, wf->pending_items[i]); if (item) { break; } } pthread_mutex_unlock(&wf->main_mutex); if (item) { workflow_stage_function_t stage_function = wf->stage_functions[item->stage_id]; // Extrae_event(6000019, item->stage_id + 1); struct timeval start_time, end_time; double total_time = 0.0; start_timer(start_time); int next_stage = stage_function(item->data); stop_timer(start_time, end_time, total_time); pthread_mutex_lock(&wf->stage_times_mutex[item->stage_id]); wf->stage_times[item->stage_id] += (total_time / 1000000.0f); pthread_mutex_unlock(&wf->stage_times_mutex[item->stage_id]); // Extrae_event(6000019, 0); item->stage_id = next_stage; if (next_stage >= 0 && next_stage < wf->num_stages) { // moving item to the next stage to process pthread_mutex_lock(&wf->main_mutex); array_list_insert(item, wf->pending_items[item->stage_id]); pthread_mutex_unlock(&wf->main_mutex); } else if (next_stage == -1) { // item fully processed !! pthread_mutex_lock(&wf->main_mutex); wf->num_pending_items--; array_list_insert(item, wf->completed_items); pthread_cond_broadcast(&wf->consumer_cond); pthread_mutex_unlock(&wf->main_mutex); } else { // error !! pthread_mutex_lock(&wf->main_mutex); wf->num_pending_items--; pthread_mutex_unlock(&wf->main_mutex); } } }
int array_list_remove(array_list* list, void* data) { int index = array_list_index(list, data); if(index != -1) { array_list_remove_at(list, index); return index; } return -1; }
void *workflow_remove_item(workflow_t *wf) { void *ret = NULL; work_item_t *item; pthread_mutex_lock(&wf->main_mutex); //printf("Writer mutex lock\n"); while (workflow_get_num_completed_items_(wf) <= 0) { //printf("Cond wait completed items...\n"); pthread_cond_wait(&wf->consumer_cond, &wf->main_mutex); } //printf("writer continue\n"); item = array_list_remove_at(0, wf->completed_items); pthread_cond_broadcast(&wf->producer_cond); pthread_mutex_unlock(&wf->main_mutex); if (item) { ret = item->data; work_item_free(item); } return ret; }
size_t bwt_search_pair_anchors(array_list_t *list, unsigned int read_length) { bwt_anchor_t *bwt_anchor; int max_anchor_length = 0; bwt_anchor_t *bwt_anchor_back, *bwt_anchor_forw; int anchor_length_tmp, anchor_back, anchor_forw; int strand = 0, type = 0; int found_anchor = 0, found_double_anchor = 0; const int MIN_ANCHOR = 25; const int MIN_SINGLE_ANCHOR = 40; //const int MIN_DOUBLE_ANCHOR = MIN_ANCHOR*2; const int MAX_BWT_REGIONS = 50; const int MAX_BWT_ANCHOR_DISTANCE = 500000; array_list_t *anchor_list_tmp, *forward_anchor_list, *backward_anchor_list; cal_t *cal; int seed_size, gap_read, gap_genome; array_list_t *backward_anchor_list_0 = array_list_new(MAX_BWT_REGIONS, 1.25f, COLLECTION_MODE_ASYNCHRONIZED); array_list_t *forward_anchor_list_0 = array_list_new(MAX_BWT_REGIONS, 1.25f , COLLECTION_MODE_ASYNCHRONIZED); array_list_t *backward_anchor_list_1 = array_list_new(MAX_BWT_REGIONS, 1.25f, COLLECTION_MODE_ASYNCHRONIZED); array_list_t *forward_anchor_list_1 = array_list_new(MAX_BWT_REGIONS, 1.25f , COLLECTION_MODE_ASYNCHRONIZED); array_list_t *big_anchor_list = array_list_new(MAX_BWT_REGIONS, 1.25f , COLLECTION_MODE_ASYNCHRONIZED); //printf("Tot Anchors %i\n", array_list_size(list)); for (int i = 0; i < array_list_size(list); i++) { bwt_anchor = array_list_get(i, list); if (bwt_anchor->strand == 1) { //printf("(-)bwt anchor %i:%lu-%lu (%i): \n", bwt_anchor->chromosome + 1, bwt_anchor->start, bwt_anchor->end, bwt_anchor->end - bwt_anchor->start + 1); if (bwt_anchor->type == FORWARD_ANCHOR) { array_list_insert(bwt_anchor, forward_anchor_list_1); //printf("FORW\n"); } else { array_list_insert(bwt_anchor, backward_anchor_list_1); //printf("BACK\n"); } } else { //printf("(+)bwt anchor %i:%lu-%lu (%i): \n", bwt_anchor->chromosome + 1, bwt_anchor->start, bwt_anchor->end, bwt_anchor->end - bwt_anchor->start + 1); if (bwt_anchor->type == FORWARD_ANCHOR) { array_list_insert(bwt_anchor, forward_anchor_list_0); //printf("FORW\n"); } else { array_list_insert(bwt_anchor, backward_anchor_list_0); //printf("BACK\n"); } } anchor_length_tmp = bwt_anchor->end - bwt_anchor->start + 1; if (anchor_length_tmp > MIN_SINGLE_ANCHOR && anchor_length_tmp > max_anchor_length) { max_anchor_length = anchor_length_tmp; found_anchor = 1; strand = bwt_anchor->strand; type = bwt_anchor->type; } if (read_length - anchor_length_tmp < 16) { array_list_insert(bwt_anchor, big_anchor_list); } } array_list_clear(list, NULL); if (array_list_size(big_anchor_list) > 0) { for (int i = array_list_size(big_anchor_list) - 1; i >= 0; i--) { //printf("Insert cal %i\n", i); bwt_anchor = array_list_remove_at(i, big_anchor_list); size_t seed_size = bwt_anchor->end - bwt_anchor->start; if (bwt_anchor->type == FORWARD_ANCHOR) { cal = convert_bwt_anchor_to_CAL(bwt_anchor, 0, seed_size); } else { cal = convert_bwt_anchor_to_CAL(bwt_anchor, read_length - seed_size - 1, read_length - 1); } array_list_insert(cal, list); } array_list_set_flag(SINGLE_ANCHORS, list); goto exit; } for (int type = 1; type >= 0; type--) { if (!type) { forward_anchor_list = forward_anchor_list_1; backward_anchor_list = backward_anchor_list_1; //printf("Strand (+): %i-%i\n", array_list_size(forward_anchor_list), array_list_size(backward_anchor_list)); } else { forward_anchor_list = forward_anchor_list_0; backward_anchor_list = backward_anchor_list_0; //printf("Strand (-): %i-%i\n", array_list_size(forward_anchor_list), array_list_size(backward_anchor_list)); } int *set_forward = (int *)calloc(array_list_size(forward_anchor_list), sizeof(int)); int *set_backward = (int *)calloc(array_list_size(backward_anchor_list), sizeof(int)); //Associate Anchors (+)/(-) for (int i = 0; i < array_list_size(forward_anchor_list); i++) { if (set_forward[i]) { continue; } bwt_anchor_forw = array_list_get(i, forward_anchor_list); for (int j = 0; j < array_list_size(backward_anchor_list); j++) { if (set_backward[j]) { continue; } bwt_anchor_back = array_list_get(j, backward_anchor_list); anchor_forw = (bwt_anchor_forw->end - bwt_anchor_forw->start + 1); anchor_back = (bwt_anchor_back->end - bwt_anchor_back->start + 1); anchor_length_tmp = anchor_forw + anchor_back; //printf("\tCommpare %i:%lu-%lu with %i:%lu-%lu\n", bwt_anchor_forw->chromosome + 1, // bwt_anchor_forw->start, bwt_anchor_forw->end, bwt_anchor_back->chromosome + 1, // bwt_anchor_back->start, bwt_anchor_back->end); if (bwt_anchor_forw->chromosome == bwt_anchor_back->chromosome && abs(bwt_anchor_back->start - bwt_anchor_forw->end) <= MAX_BWT_ANCHOR_DISTANCE && anchor_forw >= MIN_ANCHOR && anchor_back >= MIN_ANCHOR) { if (bwt_anchor_back->start < bwt_anchor_forw->end) { continue; } gap_read = read_length - (anchor_forw + anchor_back); gap_genome = bwt_anchor_back->start - bwt_anchor_forw->end; //printf("anchor_forw = %i, anchor_back = %i, gap_read = %i, gap_genome = %i\n", // anchor_forw, anchor_back, gap_read, gap_genome); int apply_flank = 0; if (gap_read < 2 || gap_genome < 2) { int gap; if (gap_read < 0 && gap_genome < 0) { gap = abs(gap_read) > abs(gap_genome) ? abs(gap_read) : abs(gap_genome); } else if (gap_read < 0) { gap = abs(gap_read); } else if (gap_genome < 0) { gap = abs(gap_genome); } else { gap = 2; } int flank = 5; apply_flank = 1; if (abs(gap) >= flank*2) { //Solve read overlap flank = abs(gap)/2 + flank/2; } //printf("\tgap = %i, flank = %i\n", gap, flank); if (flank >= anchor_forw) { bwt_anchor_forw->end -= anchor_forw/2; } else { bwt_anchor_forw->end -= flank; } if (flank >= anchor_back) { bwt_anchor_back->start += anchor_back/2; } else { bwt_anchor_back->start += flank; } } cal = convert_bwt_anchor_to_CAL(bwt_anchor_forw, 0, bwt_anchor_forw->end - bwt_anchor_forw->start); //printf("INSERT-1 (%i)[%i:%lu-%lu]\n", cal->strand, cal->chromosome_id, cal->start, cal->end); array_list_insert(cal, list); seed_size = bwt_anchor_back->end - bwt_anchor_back->start + 1; //if (bwt_anchor_forw->end + read_length >= bwt_anchor_back->start) { //seed_region_t *seed_region = seed_region_new(read_length - seed_size, read_length - 1, //bwt_anchor_back->start, bwt_anchor_back->end, 1); //cal->end = bwt_anchor_back->end; //linked_list_insert_last(seed_region, cal->sr_list); //} else { cal = convert_bwt_anchor_to_CAL(bwt_anchor_back, read_length - seed_size, read_length - 1); //printf("INSERT-2 (%i)[%i:%lu-%lu]\n", cal->strand, cal->chromosome_id, cal->start, cal->end); array_list_insert(cal, list); if (array_list_size(list) > 5) { free(set_backward); free(set_forward); goto exit; } array_list_set_flag(DOUBLE_ANCHORS, list); found_double_anchor = 1; set_forward[i] = 1; set_backward[j] = 1; break; } } } free(set_backward); free(set_forward); } if (!found_double_anchor && found_anchor) { //Not Double anchor found but one Yes!! if (strand == 1) { if (type == FORWARD_ANCHOR) { anchor_list_tmp = forward_anchor_list_1; } else { anchor_list_tmp = backward_anchor_list_1; } } else { if (type == FORWARD_ANCHOR) { anchor_list_tmp = forward_anchor_list_0; } else { anchor_list_tmp = backward_anchor_list_0; } } //printf("LIST SIZE %i\n", array_list_size(anchor_list_tmp)); for (int i = 0; i < array_list_size(anchor_list_tmp); i++) { bwt_anchor = array_list_get(i, anchor_list_tmp); size_t seed_size = bwt_anchor->end - bwt_anchor->start; //array_list_insert(bwt_anchor_new(bwt_anchor->strand, bwt_anchor->chromosome, // bwt_anchor->start, bwt_anchor->end, bwt_anchor->type), anchor_list); if (bwt_anchor->type == FORWARD_ANCHOR) { //printf("------------------------> start %i\n", 0); cal = convert_bwt_anchor_to_CAL(bwt_anchor, 0, seed_size); } else { //printf("------------------------> start %i\n", read_length - seed_size); cal = convert_bwt_anchor_to_CAL(bwt_anchor, read_length - seed_size - 1, read_length - 1); } array_list_insert(cal, list); } array_list_set_flag(SINGLE_ANCHORS, list); } exit: array_list_free(forward_anchor_list_1, (void *)bwt_anchor_free); array_list_free(backward_anchor_list_1, (void *)bwt_anchor_free); array_list_free(forward_anchor_list_0, (void *)bwt_anchor_free); array_list_free(backward_anchor_list_0, (void *)bwt_anchor_free); array_list_free(big_anchor_list, (void *)bwt_anchor_free); return array_list_size(list); }
int apply_caling_rna(cal_seeker_input_t* input, batch_t *batch) { LOG_DEBUG("========= APPLY CALING RNA =========\n"); //if (time_on) { start_timer(start); } bwt_optarg_t *bwt_optarg = input->bwt_optarg; bwt_index_t *bwt_index = input->index; cal_optarg_t *cal_optarg = input->cal_optarg; mapping_batch_t *mapping_batch = batch->mapping_batch; size_t num_cals, select_cals; size_t total_reads = 0; size_t num_targets, target_pos, total_targets, extra_target_pos; fastq_read_t *read; genome_t *genome = input->genome; unsigned int num_chromosomes = genome->num_chromosomes; int min_seeds, max_seeds; int seed_size = input->cal_optarg->seed_size; array_list_t *cal_list, *list; cal_t *cal; //array_list_t *region_list; num_targets = mapping_batch->num_targets; total_targets = 0; extra_target_pos = 0; total_reads += num_targets; target_pos = 0; mapping_batch->extra_stage_do = 1; /* int t, target; for (t = 0; t < num_targets; t++) { target = mapping_batch->targets[t]; mapping_batch->mapping_lists[target]->size = 0; } return RNA_POST_PAIR_STAGE; */ array_list_t *region_list = array_list_new(1000, 1.25f, COLLECTION_MODE_ASYNCHRONIZED); //extern size_t TOTAL_READS_SEEDING, TOTAL_READS_SEEDING2; //pthread_mutex_lock(&mutex_sp); //TOTAL_READS_SEEDING += num_targets; //pthread_mutex_unlock(&mutex_sp); //printf("Num targets = %i\n", num_targets); for (size_t i = 0; i < num_targets; i++) { read = array_list_get(mapping_batch->targets[i], mapping_batch->fq_batch); //printf("From CAL Seeker %s\n", read->id); list = mapping_batch->mapping_lists[mapping_batch->targets[i]]; //if (array_list_get_flag(region_list) == 0 || // array_list_get_flag(region_list) == 2) { //We have normal and extend seeds (anchors) max_seeds = (read->length / 15)*2 + 10; //printf("%i\n", input->cal_optarg->min_cal_size); num_cals = bwt_generate_cals(read->sequence, seed_size, bwt_optarg, cal_optarg, bwt_index, list, num_chromosomes); // if we want to seed with 24-length seeds, if (num_cals == 0) { //printf("No Cals seeding...\n"); //pthread_mutex_lock(&mutex_sp); //extern size_t seeds_1err; //seeds_1err++; //pthread_mutex_unlock(&mutex_sp); int seed_size = 24; //First, Delete old regions array_list_clear(region_list, (void *)region_bwt_free); //Second, Create new regions with seed_size 24 and 1 Mismatch bwt_map_inexact_seeds_seq(read->sequence, seed_size, seed_size/2, bwt_optarg, bwt_index, region_list); max_seeds = (read->length / 15)*2 + 10; //int prev_min_cal = input->cal_optarg->min_cal_size; //input->cal_optarg->min_cal_size = seed_size + seed_size / 2; //printf("NO CALS, new seeds %lu\n", array_list_size(region_list)); num_cals = bwt_generate_cal_list_linked_list(region_list, input->cal_optarg, &min_seeds, &max_seeds, genome->num_chromosomes + 1, list, read->length, cal_optarg->min_cal_size, 0); //input->cal_optarg->min_cal_size = prev_min_cal; //pthread_mutex_lock(&mutex_sp); //TOTAL_READS_SEEDING2++; //pthread_mutex_unlock(&mutex_sp); } array_list_clear(region_list, (void *)region_bwt_free); //filter-incoherent CALs int founds[num_cals], found = 0; for (size_t j = 0; j < num_cals; j++) { founds[j] = 0; cal = array_list_get(j, list); LOG_DEBUG_F("\tcal %i of %i: sr_list size = %i (cal->num_seeds = %i) %i:%lu-%lu\n", j, num_cals, cal->sr_list->size, cal->num_seeds, cal->chromosome_id, cal->start, cal->end); if (cal->sr_list->size > 0) { int start = 0; size_t genome_start = 0; int first = 1; for (linked_list_item_t *list_item = cal->sr_list->first; list_item != NULL; list_item = list_item->next) { seed_region_t *s = list_item->item; LOG_DEBUG_F("\t\t:: star %lu > %lu s->read_start\n", start, s->read_start); LOG_DEBUG_F("\t\t:: read_star %lu > read_end %lu \n", s->read_start, s->read_end); if (start > s->read_start || s->read_start >= s->read_end) { LOG_DEBUG("\t\t\t:: remove\n"); found++; founds[j] = 1; } if (!first && ((s->genome_start < genome_start) || (s->genome_start - genome_start) > 2*read->length)) { //printf("Remove (genome_start = %i s->genome_start = %i)\n", genome_start, s->genome_start); //cal_print(cal); found++; founds[j] = 1; } first = 0; start = s->read_end + 1; genome_start = s->genome_end + 1; } } else { found++; founds[j] = 1; } } if (found) { min_seeds = 100000; max_seeds = 0; cal_list = array_list_new(MAX_CALS, 1.25f, COLLECTION_MODE_ASYNCHRONIZED); for (size_t j = 0; j < num_cals; j++) { if (!founds[j]) { cal = array_list_get(j, list); cal->num_seeds = cal->sr_list->size; if (cal->num_seeds > max_seeds) max_seeds = cal->num_seeds; if (cal->num_seeds < min_seeds) min_seeds = cal->num_seeds; array_list_insert(cal, cal_list); array_list_set(j, NULL, list); } } array_list_free(list, (void *) cal_free); num_cals = array_list_size(cal_list); list = cal_list; } mapping_batch->mapping_lists[mapping_batch->targets[i]] = list; num_cals = array_list_size(list); int max = 100; if (num_cals > max) { select_cals = num_cals - max; for(int j = num_cals - 1; j >= max; j--) { cal_free(array_list_remove_at(j, mapping_batch->mapping_lists[mapping_batch->targets[i]])); } } //mapping_batch->targets[target_pos++] = mapping_batch->targets[i]; //} //else if (num_cals > 0) { mapping_batch->targets[target_pos++] = mapping_batch->targets[i]; /* printf("<<<<<===== CAL SERVER =====>>>>>\n"); */ /* for (int c = 0; c < array_list_size(mapping_batch->mapping_lists[mapping_batch->targets[i]]); c++) { */ /* cal_t *cal_aux = array_list_get(c, mapping_batch->mapping_lists[mapping_batch->targets[i]]); */ /* cal_print(cal_aux); */ /* } */ /* printf("<<<<<===== CAL SERVER END =====>>>>>\n"); */ //printf("Total CALs %i\n", num_cals); } mapping_batch->num_targets = target_pos; array_list_free(region_list, NULL); //if (time_on) { stop_timer(start, end, time); timing_add(time, CAL_SEEKER, timing); } LOG_DEBUG("========= APPLY CALING RNA END =========\n"); // return RNA_STAGE; if (batch->mapping_mode == RNA_MODE) { return RNA_STAGE; } if (batch->pair_input->pair_mng->pair_mode != SINGLE_END_MODE) { return PRE_PAIR_STAGE; } else if (batch->mapping_batch->num_targets > 0) { return SW_STAGE; } return DNA_POST_PAIR_STAGE; }
array_list_t *filter_cals(size_t num_cals, size_t read_length, array_list_t *list) { cal_t *cal; int min_seeds, max_seeds; array_list_t *cal_list; size_t select_cals; //filter-incoherent CALs int founds[num_cals], found = 0; for (size_t j = 0; j < num_cals; j++) { founds[j] = 0; cal = array_list_get(j, list); LOG_DEBUG_F("\tcal %i of %i: sr_list size = %i (cal->num_seeds = %i) %i:%lu-%lu\n", j, num_cals, cal->sr_list->size, cal->num_seeds, cal->chromosome_id, cal->start, cal->end); if (cal->sr_list->size > 0) { int start = 0; size_t genome_start = 0; int first = 1; for (linked_list_item_t *list_item = cal->sr_list->first; list_item != NULL; list_item = list_item->next) { seed_region_t *s = list_item->item; LOG_DEBUG_F("\t\t:: star %lu > %lu s->read_start\n", start, s->read_start); LOG_DEBUG_F("\t\t:: read_star %lu > read_end %lu \n", s->read_start, s->read_end); if (start > s->read_start || s->read_start >= s->read_end) { LOG_DEBUG("\t\t\t:: remove\n"); found++; founds[j] = 1; } if (!first && ((s->genome_start < genome_start) || (s->genome_start - genome_start) > 2 * read_length)) { //printf("Remove (genome_start = %i s->genome_start = %i)\n", genome_start, s->genome_start); //cal_print(cal); found++; founds[j] = 1; } first = 0; start = s->read_end + 1; genome_start = s->genome_end + 1; } } else { found++; founds[j] = 1; } } if (found) { min_seeds = 100000; max_seeds = 0; cal_list = array_list_new(MAX_CALS, 1.25f, COLLECTION_MODE_ASYNCHRONIZED); for (size_t j = 0; j < num_cals; j++) { if (!founds[j]) { cal = array_list_get(j, list); cal->num_seeds = cal->sr_list->size; if (cal->num_seeds > max_seeds) max_seeds = cal->num_seeds; if (cal->num_seeds < min_seeds) min_seeds = cal->num_seeds; array_list_insert(cal, cal_list); array_list_set(j, NULL, list); } } array_list_free(list, (void *) cal_free); num_cals = array_list_size(cal_list); list = cal_list; } num_cals = array_list_size(list); int max = 100; if (num_cals > max) { select_cals = num_cals - max; for(int j = num_cals - 1; j >= max; j--) { cal_free(array_list_remove_at(j, list)); } } return list; }
//==================================================================================== // apply_caling //==================================================================================== int apply_caling(cal_seeker_input_t* input, batch_t *batch) { mapping_batch_t *mapping_batch = batch->mapping_batch; array_list_t *list = NULL; size_t read_index, num_cals; int min_seeds, max_seeds; cal_t *cal; array_list_t *cal_list; fastq_read_t *read; size_t num_chromosomes = input->genome->num_chromosomes + 1; size_t num_targets = mapping_batch->num_targets; size_t *targets = mapping_batch->targets; size_t new_num_targets = 0; array_list_t *region_list; bwt_anchor_t *bwt_anchor_back, *bwt_anchor_forw; linked_list_t *linked_list; int anchor_nt, gap_nt; seed_region_t *seed_region_start, *seed_region_end; //max_seeds = input->cal_optarg->num_seeds; // size_t *new_targets = (size_t *) calloc(num_targets, sizeof(size_t)); // set to zero mapping_batch->num_to_do = 0; for (size_t i = 0; i < num_targets; i++) { read_index = targets[i]; read = array_list_get(read_index, mapping_batch->fq_batch); region_list = mapping_batch->mapping_lists[read_index]; // for debugging // LOG_DEBUG_F("%s\n", ((fastq_read_t *) array_list_get(read_index, mapping_batch->fq_batch))->id); if (!list) { list = array_list_new(1000, 1.25f, COLLECTION_MODE_ASYNCHRONIZED); } if (array_list_get_flag(region_list) == 0 || array_list_get_flag(region_list) == 2) { //We have normal and extend seeds (anchors) max_seeds = (read->length / 15)*2 + 10; num_cals = bwt_generate_cal_list_linked_list(region_list, input->cal_optarg, &min_seeds, &max_seeds, num_chromosomes, list, read->length, input->cal_optarg->min_cal_size, 0); } else { //We have double anchors with smaller distance between they //printf("Easy case... Two anchors and same distance between read gap and genome distance\n"); num_cals = 0; for (int a = array_list_size(region_list) - 1; a >= 0; a -= 2) { max_seeds = 2; min_seeds = 2; bwt_anchor_back = array_list_remove_at(a, region_list); bwt_anchor_forw = array_list_remove_at(a - 1, region_list); linked_list = linked_list_new(COLLECTION_MODE_ASYNCHRONIZED); //Seed for the first anchor anchor_nt = bwt_anchor_forw->end - bwt_anchor_forw->start; //printf("\t seed0[%i-%i][%lu-%lu]\n", 0, anchor_nt - 1, // bwt_anchor_forw->start, bwt_anchor_forw->end); seed_region_start = seed_region_new(0, anchor_nt - 1, bwt_anchor_forw->start, bwt_anchor_forw->end, 0, 0, 0); //Seed for the first anchor gap_nt = read->length - (anchor_nt + (bwt_anchor_back->end - bwt_anchor_back->start)); //printf("\t gap_nt = %i, anchor_nt = %i\n", gap_nt, anchor_nt); //printf("\t seed1[%i-%i][%lu-%lu]\n", anchor_nt + gap_nt, read->length - 1, // bwt_anchor_back->start + 1, bwt_anchor_back->end); seed_region_end = seed_region_new(anchor_nt + gap_nt, read->length - 1, bwt_anchor_back->start + 1, bwt_anchor_back->end, 1, 0, 0); //The reference distance is 0 and the read distance not //The read distance is 0 and the reference distance not //if (seed_region_start->genome_end > seed_region_end->genome_start || // seed_region_start->read_end > seed_region_end->read_start) { //array_list_clear(region_list, NULL); //continue; if (seed_region_end->genome_start - seed_region_start->genome_end < 5 || seed_region_end->read_start - seed_region_start->read_end < 5) { seed_region_start->genome_end -= 5; seed_region_start->read_end -= 5; seed_region_end->genome_start += 5; seed_region_end->read_start += 5; } linked_list_insert(seed_region_start, linked_list); linked_list_insert_last(seed_region_end, linked_list); cal = cal_new(bwt_anchor_forw->chromosome + 1, bwt_anchor_forw->strand, bwt_anchor_forw->start, bwt_anchor_back->end + 1, 2, linked_list, linked_list_new(COLLECTION_MODE_ASYNCHRONIZED)); array_list_insert(cal, list); num_cals++; } } // for debugging LOG_DEBUG_F("read %s : num. cals = %i, min. seeds = %i, max. seeds = %i\n", read->id, num_cals, min_seeds, max_seeds); /* if (num_cals == 0) { int seed_size = 24; //First, Delete old regions array_list_clear(mapping_batch->mapping_lists[read_index], region_bwt_free); //Second, Create new regions with seed_size 24 and 1 Mismatch bwt_map_inexact_seeds_seq(read->sequence, seed_size, seed_size/2, bwt_optarg, bwt_index, mapping_batch->mapping_lists[read_index]); num_cals = bwt_generate_cal_list_linked_list(mapping_batch->mapping_lists[mapping_batch->targets[i]], input->cal_optarg, &min_seeds, &max_seeds, num_chromosomes, list, read->length); }*/ /* for (size_t j = 0; j < num_cals; j++) { cal = array_list_get(j, list); LOG_DEBUG_F("\tchr: %i, strand: %i, start: %lu, end: %lu, num_seeds = %i, num. regions = %lu\n", cal->chromosome_id, cal->strand, cal->start, cal->end, cal->num_seeds, cal->sr_list->size); } */ // printf("min_seeds = %i, max_seeds = %i, min_limit = %i, num_cals = %i\n", // min_seeds, max_seeds, min_limit, array_list_size(list)); // filter incoherent CALs int founds[num_cals], found = 0; for (size_t j = 0; j < num_cals; j++) { founds[j] = 0; cal = array_list_get(j, list); LOG_DEBUG_F("\tcal %i of %i: sr_list size = %i (cal->num_seeds = %i) %i:%lu-%lu\n", j, num_cals, cal->sr_list->size, cal->num_seeds, cal->chromosome_id, cal->start, cal->end); if (cal->sr_list->size > 0) { int start = 0; for (linked_list_item_t *list_item = cal->sr_list->first; list_item != NULL; list_item = list_item->next) { seed_region_t *s = list_item->item; LOG_DEBUG_F("\t\t:: star %lu > %lu s->read_start\n", start, s->read_start); if (start > s->read_start) { LOG_DEBUG("\t\t\t:: remove\n"); found++; founds[j] = 1; } start = s->read_end + 1; } } else { found++; founds[j] = 1; } } if (found) { min_seeds = 100000; max_seeds = 0; cal_list = array_list_new(MAX_CALS, 1.25f, COLLECTION_MODE_ASYNCHRONIZED); for (size_t j = 0; j < num_cals; j++) { if (!founds[j]) { cal = array_list_get(j, list); cal->num_seeds = cal->sr_list->size; if (cal->num_seeds > max_seeds) max_seeds = cal->num_seeds; if (cal->num_seeds < min_seeds) min_seeds = cal->num_seeds; array_list_insert(cal, cal_list); array_list_set(j, NULL, list); } } array_list_free(list, (void *) cal_free); num_cals = array_list_size(cal_list); list = cal_list; } // LOG_FATAL_F("num. cals = %i, min. seeds = %i, max. seeds = %i\n", num_cals, min_seeds, max_seeds); // filter CALs by the number of seeds cal_list = list; list = NULL; /* int min_limit = input->cal_optarg->min_num_seeds_in_cal; if (min_limit < 0) min_limit = max_seeds; // min_limit -= 3; if (min_seeds == max_seeds || min_limit <= min_seeds) { cal_list = list; list = NULL; } else { cal_list = array_list_new(MAX_CALS, 1.25f, COLLECTION_MODE_ASYNCHRONIZED); for (size_t j = 0; j < num_cals; j++) { cal = array_list_get(j, list); if (cal->num_seeds >= min_limit) { array_list_insert(cal, cal_list); array_list_set(j, NULL, list); } } array_list_clear(list, (void *) cal_free); num_cals = array_list_size(cal_list); } */ if (num_cals > MAX_CALS) { for (size_t j = num_cals - 1; j >= MAX_CALS; j--) { cal = (cal_t *) array_list_remove_at(j, cal_list); cal_free(cal); } num_cals = array_list_size(cal_list); } // LOG_DEBUG_F("num. cals = %i, MAX_CALS = %i\n", num_cals, MAX_CALS); if (num_cals > 0 && num_cals <= MAX_CALS) { array_list_set_flag(2, cal_list); targets[new_num_targets++] = read_index; /* int count1 = 0, count2 = 0; // count number of sw to do // method #1 // printf("method #1\n"); seed_region_t *s, *prev_s; linked_list_iterator_t* itr; for (size_t j = 0; j < num_cals; j++) { prev_s = NULL; cal = array_list_get(j, cal_list); itr = linked_list_iterator_new(cal->sr_list); s = (seed_region_t *) linked_list_iterator_curr(itr); while (s != NULL) { if ((prev_s == NULL && s->read_start != 0) || (prev_s != NULL)) { // printf("\t\t\tcase 1\n"); count1++; } prev_s = s; linked_list_iterator_next(itr); s = linked_list_iterator_curr(itr); } if (prev_s != NULL && prev_s->read_end < read->length - 1) { count1++; // printf("\t\t\tcase 2 (%i < %i)\n", prev_s->read_end, read->length - 1); } linked_list_iterator_free(itr); } // method #2 printf("method #2\n"); for (size_t j = 0; j < num_cals; j++) { cal = array_list_get(j, cal_list); printf("\t: %i\n", j); if (cal->sr_list->size > 0) { int start = 0; for (linked_list_item_t *list_item = cal->sr_list->first; list_item != NULL; list_item = list_item->next) { seed_region_t *s = list_item->item; printf("\t\t[%i|%i - %i|%i]\n", s->genome_start, s->read_start, s->read_end, s->genome_end); if (s->read_start != start) { count2++; } start = s->read_end + 1; } if (start < read->length) { count2++; } } } printf("count #1 = %i, count #2 = %i\n", count1, count2); assert(count1 == count2); mapping_batch->num_to_do += count1; */ // we have to free the region list array_list_free(mapping_batch->mapping_lists[read_index], (void *) region_bwt_free); mapping_batch->mapping_lists[read_index] = cal_list; } else { array_list_set_flag(0, mapping_batch->mapping_lists[read_index]); // we have to free the region list array_list_clear(mapping_batch->mapping_lists[read_index], (void *) region_bwt_free); if (cal_list) array_list_free(cal_list, (void *) cal_free); if (list) array_list_clear(list, (void *) cal_free); } /* cal_list = list; list = NULL; array_list_set_flag(2, cal_list); // mapping_batch->num_to_do += num_cals; targets[new_num_targets++] = read_index; // we have to free the region list array_list_free(mapping_batch->mapping_lists[read_index], (void *) region_bwt_free); mapping_batch->mapping_lists[read_index] = cal_list; */ /* // filter CALs by the number of seeds int min_limit = input->cal_optarg->min_num_seeds_in_cal; if (min_limit < 0) min_limit = max_seeds; printf("min_seeds = %i, max_seeds = %i, min_limit = %i, num_cals = %i\n", min_seeds, max_seeds, min_limit, array_list_size(list)); if (min_seeds == max_seeds || min_limit <= min_seeds) { cal_list = list; list = NULL; } else { cal_list = array_list_new(MAX_CALS, 1.25f, COLLECTION_MODE_ASYNCHRONIZED); for (size_t j = 0; j < num_cals; j++) { cal = array_list_get(j, list); if (cal->num_seeds >= min_limit) { array_list_insert(cal, cal_list); array_list_set(j, NULL, list); } } array_list_clear(list, (void *) cal_free); num_cals = array_list_size(cal_list); printf("************, num_cals = %i\n", num_cals); } if (num_cals > MAX_CALS) { for (size_t j = num_cals - 1; j >= MAX_CALS; j--) { cal = (cal_t *) array_list_remove_at(j, cal_list); cal_free(cal); } num_cals = array_list_size(cal_list); } if (num_cals > 0 && num_cals <= MAX_CALS) { array_list_set_flag(2, cal_list); mapping_batch->num_to_do += num_cals; targets[new_num_targets++] = read_index; // we have to free the region list array_list_free(mapping_batch->mapping_lists[read_index], (void *) region_bwt_free); mapping_batch->mapping_lists[read_index] = cal_list; } else { array_list_set_flag(0, mapping_batch->mapping_lists[read_index]); // we have to free the region list array_list_clear(mapping_batch->mapping_lists[read_index], (void *) region_bwt_free); if (cal_list) array_list_free(cal_list, (void *) cal_free); if (list) array_list_clear(list, (void *) cal_free); } */ } // end for 0 ... num_targets // update batch mapping_batch->num_targets = new_num_targets; // LOG_DEBUG_F("num. SW to do: %i\n", mapping_batch->num_to_do); // exit(-1); // free memory if (list) array_list_free(list, NULL); if (batch->mapping_mode == RNA_MODE) { return RNA_STAGE; } if (batch->pair_input->pair_mng->pair_mode != SINGLE_END_MODE) { return PRE_PAIR_STAGE; } else if (batch->mapping_batch->num_targets > 0) { return SW_STAGE; } return DNA_POST_PAIR_STAGE; }
int apply_seeding(region_seeker_input_t* input, batch_t *batch) { //printf("APPLY SEEDING...\n"); //if (time_on) { start_timer(start); } mapping_batch_t *mapping_batch = batch->mapping_batch; size_t num_mappings; int seed_size = input->cal_optarg_p->seed_size; size_t min_seed_size = input->cal_optarg_p->min_seed_size; size_t num_targets = mapping_batch->num_targets; size_t *targets = mapping_batch->targets; size_t new_num_targets = 0; fastq_read_t *read; int min_intron_size = 40; int target; bwt_anchor_t *bwt_anchor = NULL; region_t *region; int gap_nt; int start_search; int end_search; // set to zero mapping_batch->num_to_do = 0; //TODO: omp parallel for !! /*if (batch->mapping_mode == 1000) { for (size_t i = 0; i < num_targets; i++) { //printf("Seq (i=%i)(target=%i): %s\n", i, targets[i], read->sequence); read = array_list_get(targets[i], mapping_batch->fq_batch); num_mappings = bwt_map_exact_seeds_seq(padding_left, padding_right, read->sequence, seed_size, min_seed_size, input->bwt_optarg_p, input->bwt_index_p, mapping_batch->mapping_lists[targets[i]], mapping_batch->extra_stage_id[targets[i]]); //printf("Num mappings %i\n", num_mappings); if (num_mappings > 0) { array_list_set_flag(2, mapping_batch->mapping_lists[targets[i]]); targets[new_num_targets++] = targets[i]; mapping_batch->num_to_do += num_mappings; } } } else {*/ //size_t new_num_targets = 0; //size_t *new_targets = (size_t *)malloc(array_list_size(fq_batch)*sizeof(size_t)); array_list_t *array_list_aux = array_list_new(256, 1.25f, COLLECTION_MODE_ASYNCHRONIZED); //Flag 0: The read has simple anchor or any, and need seeds and normal Cal_Seeker //Flag 1: The read has double anchor and the gap is smaller than MIN_INTRON_SIZE. Cal_Seeker will be make one CAL //Flag 2: The read has double anchor but the gap is bigger than MIN_INTRON_SIZE. for (size_t i = 0; i < num_targets; i++) { read = array_list_get(targets[i], mapping_batch->fq_batch); //printf("Read Region %s: \n", read->id); /* if (array_list_get_flag(mapping_batch->mapping_lists[targets[i]]) == 0 || array_list_get_flag(mapping_batch->mapping_lists[targets[i]]) == 1) { array_list_clear(mapping_batch->mapping_lists[targets[i]], bwt_anchor_free); continue; } */ if (array_list_get_flag(mapping_batch->mapping_lists[targets[i]]) == 0 || array_list_get_flag(mapping_batch->mapping_lists[targets[i]]) == 1) { //Flag 0 Case, Not anchors found, Make normal seeds // printf("***** Normal Case 0. Not anchors found!\n"); for (int j = array_list_size(mapping_batch->mapping_lists[targets[i]]) - 1; j >= 0; j--) { bwt_anchor = array_list_remove_at(j, mapping_batch->mapping_lists[targets[i]]); array_list_insert(bwt_anchor, array_list_aux); } num_mappings = 0; num_mappings = bwt_map_exact_seeds_seq(0, 0, read->sequence, seed_size, min_seed_size, input->bwt_optarg_p, input->bwt_index_p, mapping_batch->mapping_lists[targets[i]], 0); if (num_mappings > 0) { array_list_set_flag(0, mapping_batch->mapping_lists[targets[i]]); targets[new_num_targets++] = targets[i]; //mapping_batch->num_to_do += num_mappings; } } else if (array_list_get_flag(mapping_batch->mapping_lists[targets[i]]) == 1) { //Flag 1 Case, One anchor found, Make displacements seeds printf("***** Case 1. One anchor found!\n"); for (int j = array_list_size(mapping_batch->mapping_lists[targets[i]]) - 1; j >= 0; j--) { bwt_anchor = array_list_remove_at(j, mapping_batch->mapping_lists[targets[i]]); array_list_insert(bwt_anchor, array_list_aux); } int anchor_nt = bwt_anchor->end - bwt_anchor->start; int seed_id = 0; int seed_start, seed_end; int extra_seed; if ((bwt_anchor->type == FORWARD_ANCHOR && bwt_anchor->strand == 0) || (bwt_anchor->type == BACKWARD_ANCHOR && bwt_anchor->strand == 1 )) { start_search = anchor_nt + 1; end_search = read->length - 1; extra_seed = EXTRA_SEED_END; } else { start_search = 0; end_search = read->length - anchor_nt - 2; extra_seed = EXTRA_SEED_START; } printf("end_start %i - start_search %i = %i >= seed_size %i\n", end_search, start_search, end_search - start_search, seed_size); if (end_search - start_search >= seed_size) { printf("00 bwt_map_exact_seeds_between_coords --> searching from %i to %i\n", start_search, end_search); /* num_mappings = bwt_map_exact_seeds_between_coords(start_search, end_search, read->sequence, seed_size, min_seed_size, input->bwt_optarg_p, input->bwt_index_p, mapping_batch->mapping_lists[targets[i]], extra_seed, &seed_id); */ } if (bwt_anchor->type == FORWARD_ANCHOR) { seed_id = 0; seed_start = 0; seed_end = anchor_nt; } else { seed_id += 1; seed_start = read->length - anchor_nt - 1; seed_end = read->length - 1; } for (int j = 0; j < array_list_size(array_list_aux); j++) { bwt_anchor_t *bwt_anchor = array_list_get(j, array_list_aux); // printf("\tCreate seed Anchor [%i:%lu|%i-%i|%lu]\n", bwt_anchor->chromosome + 1, bwt_anchor->start, // seed_start,seed_end,bwt_anchor->end); region = region_bwt_new(bwt_anchor->chromosome + 1, bwt_anchor->strand, bwt_anchor->start, bwt_anchor->end, seed_start, seed_end, read->length, seed_id); array_list_insert(region, mapping_batch->mapping_lists[targets[i]]); } array_list_clear(array_list_aux, (void *)bwt_anchor_free); array_list_set_flag(0, mapping_batch->mapping_lists[targets[i]]); targets[new_num_targets++] = targets[i]; } else { //Flag 2 Case, Pair of anchors found printf("***** Case 2. Double anchor found!\n"); bwt_anchor_t *bwt_anchor; bwt_anchor_t *bwt_anchor_forw, *bwt_anchor_back; int read_nt, genome_nt; int distance; int found = 0; region_t *region; int seed_id = 0; //if (array_list_size(mapping_batch->mapping_lists[targets[i]]) > 2) { int *anchors_targets = (int *)calloc(array_list_size(mapping_batch->mapping_lists[targets[i]]), sizeof(int)); int num = 0; //min_intron_size = 0; //Search if one anchor is at the same distance from the reference and the read for (int b = 0; b < array_list_size(mapping_batch->mapping_lists[targets[i]]); b += 2) { bwt_anchor_forw = array_list_get(b, mapping_batch->mapping_lists[targets[i]]); bwt_anchor_back = array_list_get(b + 1, mapping_batch->mapping_lists[targets[i]]); //printf("FORW=%i:%lu-%lu BACK=%i:%lu-%lu\n", bwt_anchor_forw->chromosome, bwt_anchor_forw->start, bwt_anchor_forw->end, // bwt_anchor_back->chromosome, bwt_anchor_back->start, bwt_anchor_back->end); read_nt = read->length - ((bwt_anchor_forw->end - bwt_anchor_forw->start) + (bwt_anchor_back->end - bwt_anchor_back->start)); genome_nt = bwt_anchor_back->start - bwt_anchor_forw->end; distance = abs(genome_nt - read_nt); //printf("\t%i:Distance %i\n", b, distance); if (distance < min_intron_size) { found = 1; } else { anchors_targets[num++] = b; } } if (found) { //printf("\tFound Exact Case... Delete other anchors\n"); for (int t = num - 1; t >= 0; t--) { target = anchors_targets[t]; //printf("\tDelete %i, %i-->\n", target, target + 1); bwt_anchor = array_list_remove_at(target + 1, mapping_batch->mapping_lists[targets[i]]); bwt_anchor_free(bwt_anchor); bwt_anchor = array_list_remove_at(target, mapping_batch->mapping_lists[targets[i]]); bwt_anchor_free(bwt_anchor); } array_list_set_flag(1, mapping_batch->mapping_lists[targets[i]]); } else { //Seeding between anchors //printf("\tFound gap between anchors \n"); array_list_t *anchors_forward = array_list_new(array_list_size(mapping_batch->mapping_lists[targets[i]]), 1.25f, COLLECTION_MODE_ASYNCHRONIZED); array_list_t *anchors_backward = array_list_new(array_list_size(mapping_batch->mapping_lists[targets[i]]), 1.25f, COLLECTION_MODE_ASYNCHRONIZED); int big_gap = 0; int final_anchor_nt = 0; int anchor_nt; int anchor_type; int anchor_strand; for (int j = array_list_size(mapping_batch->mapping_lists[targets[i]]) - 1; j >= 0; j -= 2) { bwt_anchor_back = array_list_remove_at(j, mapping_batch->mapping_lists[targets[i]]); array_list_insert(bwt_anchor_back, anchors_backward); bwt_anchor_forw = array_list_remove_at(j - 1, mapping_batch->mapping_lists[targets[i]]); array_list_insert(bwt_anchor_forw, anchors_forward); if (bwt_anchor_forw->strand == 0) { anchor_nt = bwt_anchor_forw->end - bwt_anchor_forw->start; gap_nt = read->length - (anchor_nt + (bwt_anchor_back->end - bwt_anchor_back->start)); } else { anchor_nt = bwt_anchor_back->end - bwt_anchor_back->start; gap_nt = read->length - (anchor_nt + (bwt_anchor_forw->end - bwt_anchor_forw->start)); } if (gap_nt < 0) { gap_nt = 0; } //printf("Gap nt (%i - %i): %i\n", anchor_nt, bwt_anchor_back->end - bwt_anchor_back->start, gap_nt); if (gap_nt > big_gap) { big_gap = gap_nt; final_anchor_nt = anchor_nt; anchor_type = bwt_anchor_back->type; anchor_strand = bwt_anchor_back->strand; } } printf("%i, %i\n", big_gap - 2, seed_size); if (big_gap - 2 > seed_size) { //if (anchor_type == FORWARD_ANCHOR && anchor_strand == 0 || // anchor_type == BACKWARD_ANCHOR && anchor_strand == 1 ) { start_search = final_anchor_nt + 1; end_search = final_anchor_nt + big_gap - 1; //} else { // start_search = final_anchor_nt + big_gap - 1; //end_search = final_anchor_nt + 1; //} //printf("Seeding between anchors... gap=%i\n", big_gap); printf("11 bwt_map_exact_seeds_between_coords --> searching from %i to %i\n", start_search, end_search); /* num_mappings = bwt_map_exact_seeds_between_coords(start_search, end_search, read->sequence, seed_size, min_seed_size, input->bwt_optarg_p, input->bwt_index_p, mapping_batch->mapping_lists[targets[i]], EXTRA_SEED_NONE, &seed_id); */ } //printf("Making seeds anchors...\n"); for (int a = 0; a < array_list_size(anchors_forward); a++) { //Insert the last anchor. (Create new seed) bwt_anchor_forw = array_list_get(a, anchors_forward); bwt_anchor_back = array_list_get(a, anchors_backward); anchor_nt = bwt_anchor_forw->end - bwt_anchor_forw->start; gap_nt = read->length - (anchor_nt + (bwt_anchor_back->end - bwt_anchor_back->start)); //printf("\t --> Big Seed: %i, gap_nt: %i, anchor_nt = %i\n", a, gap_nt, anchor_nt); if (gap_nt < 0) { //gap_nt = 0; bwt_anchor_forw->end += gap_nt; bwt_anchor_back->start -= gap_nt; anchor_nt += gap_nt; gap_nt = 0; } else if (gap_nt == 0) { bwt_anchor_forw->end -= 1; bwt_anchor_back->start += 1; anchor_nt -= 1; gap_nt = 1; } region = region_bwt_new(bwt_anchor_forw->chromosome + 1, bwt_anchor_forw->strand, bwt_anchor_forw->start, bwt_anchor_forw->end, 0, anchor_nt, read->length, 0); //printf("Region: %i-%i\n", region->seq_start, region->seq_end); array_list_insert(region, mapping_batch->mapping_lists[targets[i]]); region = region_bwt_new(bwt_anchor_back->chromosome + 1, bwt_anchor_back->strand, bwt_anchor_back->start, bwt_anchor_back->end, anchor_nt + gap_nt, read->length - 1, read->length, seed_id + 1); //printf("Region: %i-%i\n", region->seq_start, region->seq_end); array_list_insert(region, mapping_batch->mapping_lists[targets[i]]); //printf("\tMaking seeds anchors end, %i seeds\n", array_list_size(mapping_batch->mapping_lists[targets[i]])); bwt_anchor_free(bwt_anchor_back); bwt_anchor_free(bwt_anchor_forw); } array_list_free(anchors_forward, NULL); array_list_free(anchors_backward, NULL); //printf("Making seeds anchors end, %i seeds\n", array_list_size(mapping_batch->mapping_lists[targets[i]])); array_list_set_flag(2, mapping_batch->mapping_lists[targets[i]]); } free(anchors_targets); targets[new_num_targets++] = targets[i]; } } mapping_batch->num_targets = new_num_targets; array_list_free(array_list_aux, NULL); //if (time_on) { stop_timer(start, end, time); timing_add(time, REGION_SEEKER, timing); } //printf("APPLY SEEDING DONE!\n"); return CAL_STAGE; }