void test_array_list_clear ()
{
array_list *list = create_array_list_simple ();
TEST_ASSERT_NOT_NULL (list);
char *strs1[] = {"test", "content", "random", "word", "code"};
array_list_append_all (list, (void **) strs1, 5);
// try to clear the list
array_list_clear (list);
TEST_ASSERT_EQUAL_INT (list->item_count, 0);
delete_array_list (list);
}
void array_clear(Array * const list) {
array_list_clear((ArrayList * const ) list);
}
size_t bwt_search_pair_anchors(array_list_t *list, unsigned int read_length) {
bwt_anchor_t *bwt_anchor;
int max_anchor_length = 0;
bwt_anchor_t *bwt_anchor_back, *bwt_anchor_forw;
int anchor_length_tmp, anchor_back, anchor_forw;
int strand = 0, type = 0;
int found_anchor = 0, found_double_anchor = 0;
const int MIN_ANCHOR = 25;
const int MIN_SINGLE_ANCHOR = 40;
//const int MIN_DOUBLE_ANCHOR = MIN_ANCHOR*2;
const int MAX_BWT_REGIONS = 50;
const int MAX_BWT_ANCHOR_DISTANCE = 500000;
array_list_t *anchor_list_tmp, *forward_anchor_list, *backward_anchor_list;
cal_t *cal;
int seed_size, gap_read, gap_genome;
array_list_t *backward_anchor_list_0 = array_list_new(MAX_BWT_REGIONS, 1.25f, COLLECTION_MODE_ASYNCHRONIZED);
array_list_t *forward_anchor_list_0 = array_list_new(MAX_BWT_REGIONS, 1.25f , COLLECTION_MODE_ASYNCHRONIZED);
array_list_t *backward_anchor_list_1 = array_list_new(MAX_BWT_REGIONS, 1.25f, COLLECTION_MODE_ASYNCHRONIZED);
array_list_t *forward_anchor_list_1 = array_list_new(MAX_BWT_REGIONS, 1.25f , COLLECTION_MODE_ASYNCHRONIZED);
array_list_t *big_anchor_list = array_list_new(MAX_BWT_REGIONS, 1.25f , COLLECTION_MODE_ASYNCHRONIZED);
//printf("Tot Anchors %i\n", array_list_size(list));
for (int i = 0; i < array_list_size(list); i++) {
bwt_anchor = array_list_get(i, list);
if (bwt_anchor->strand == 1) {
//printf("(-)bwt anchor %i:%lu-%lu (%i): \n", bwt_anchor->chromosome + 1, bwt_anchor->start, bwt_anchor->end, bwt_anchor->end - bwt_anchor->start + 1);
if (bwt_anchor->type == FORWARD_ANCHOR) {
array_list_insert(bwt_anchor, forward_anchor_list_1);
//printf("FORW\n");
} else {
array_list_insert(bwt_anchor, backward_anchor_list_1);
//printf("BACK\n");
}
} else {
//printf("(+)bwt anchor %i:%lu-%lu (%i): \n", bwt_anchor->chromosome + 1, bwt_anchor->start, bwt_anchor->end, bwt_anchor->end - bwt_anchor->start + 1);
if (bwt_anchor->type == FORWARD_ANCHOR) {
array_list_insert(bwt_anchor, forward_anchor_list_0);
//printf("FORW\n");
} else {
array_list_insert(bwt_anchor, backward_anchor_list_0);
//printf("BACK\n");
}
}
anchor_length_tmp = bwt_anchor->end - bwt_anchor->start + 1;
if (anchor_length_tmp > MIN_SINGLE_ANCHOR && anchor_length_tmp > max_anchor_length) {
max_anchor_length = anchor_length_tmp;
found_anchor = 1;
strand = bwt_anchor->strand;
type = bwt_anchor->type;
}
if (read_length - anchor_length_tmp < 16) {
array_list_insert(bwt_anchor, big_anchor_list);
}
}
array_list_clear(list, NULL);
if (array_list_size(big_anchor_list) > 0) {
for (int i = array_list_size(big_anchor_list) - 1; i >= 0; i--) {
//printf("Insert cal %i\n", i);
bwt_anchor = array_list_remove_at(i, big_anchor_list);
size_t seed_size = bwt_anchor->end - bwt_anchor->start;
if (bwt_anchor->type == FORWARD_ANCHOR) {
cal = convert_bwt_anchor_to_CAL(bwt_anchor, 0, seed_size);
} else {
cal = convert_bwt_anchor_to_CAL(bwt_anchor, read_length - seed_size - 1, read_length - 1);
}
array_list_insert(cal, list);
}
array_list_set_flag(SINGLE_ANCHORS, list);
goto exit;
}
for (int type = 1; type >= 0; type--) {
if (!type) {
forward_anchor_list = forward_anchor_list_1;
backward_anchor_list = backward_anchor_list_1;
//printf("Strand (+): %i-%i\n", array_list_size(forward_anchor_list), array_list_size(backward_anchor_list));
} else {
forward_anchor_list = forward_anchor_list_0;
backward_anchor_list = backward_anchor_list_0;
//printf("Strand (-): %i-%i\n", array_list_size(forward_anchor_list), array_list_size(backward_anchor_list));
}
int *set_forward = (int *)calloc(array_list_size(forward_anchor_list), sizeof(int));
int *set_backward = (int *)calloc(array_list_size(backward_anchor_list), sizeof(int));
//Associate Anchors (+)/(-)
for (int i = 0; i < array_list_size(forward_anchor_list); i++) {
if (set_forward[i]) { continue; }
bwt_anchor_forw = array_list_get(i, forward_anchor_list);
for (int j = 0; j < array_list_size(backward_anchor_list); j++) {
if (set_backward[j]) { continue; }
bwt_anchor_back = array_list_get(j, backward_anchor_list);
anchor_forw = (bwt_anchor_forw->end - bwt_anchor_forw->start + 1);
anchor_back = (bwt_anchor_back->end - bwt_anchor_back->start + 1);
anchor_length_tmp = anchor_forw + anchor_back;
//printf("\tCommpare %i:%lu-%lu with %i:%lu-%lu\n", bwt_anchor_forw->chromosome + 1,
// bwt_anchor_forw->start, bwt_anchor_forw->end, bwt_anchor_back->chromosome + 1,
// bwt_anchor_back->start, bwt_anchor_back->end);
if (bwt_anchor_forw->chromosome == bwt_anchor_back->chromosome &&
abs(bwt_anchor_back->start - bwt_anchor_forw->end) <= MAX_BWT_ANCHOR_DISTANCE &&
anchor_forw >= MIN_ANCHOR && anchor_back >= MIN_ANCHOR) {
if (bwt_anchor_back->start < bwt_anchor_forw->end) { continue; }
gap_read = read_length - (anchor_forw + anchor_back);
gap_genome = bwt_anchor_back->start - bwt_anchor_forw->end;
//printf("anchor_forw = %i, anchor_back = %i, gap_read = %i, gap_genome = %i\n",
// anchor_forw, anchor_back, gap_read, gap_genome);
int apply_flank = 0;
if (gap_read < 2 || gap_genome < 2) {
int gap;
if (gap_read < 0 && gap_genome < 0) {
gap = abs(gap_read) > abs(gap_genome) ? abs(gap_read) : abs(gap_genome);
} else if (gap_read < 0) {
gap = abs(gap_read);
} else if (gap_genome < 0) {
gap = abs(gap_genome);
} else {
gap = 2;
}
int flank = 5;
apply_flank = 1;
if (abs(gap) >= flank*2) {
//Solve read overlap
flank = abs(gap)/2 + flank/2;
}
//printf("\tgap = %i, flank = %i\n", gap, flank);
if (flank >= anchor_forw) {
bwt_anchor_forw->end -= anchor_forw/2;
} else {
bwt_anchor_forw->end -= flank;
}
if (flank >= anchor_back) {
bwt_anchor_back->start += anchor_back/2;
} else {
bwt_anchor_back->start += flank;
}
}
cal = convert_bwt_anchor_to_CAL(bwt_anchor_forw, 0, bwt_anchor_forw->end - bwt_anchor_forw->start);
//printf("INSERT-1 (%i)[%i:%lu-%lu]\n", cal->strand, cal->chromosome_id, cal->start, cal->end);
array_list_insert(cal, list);
seed_size = bwt_anchor_back->end - bwt_anchor_back->start + 1;
//if (bwt_anchor_forw->end + read_length >= bwt_anchor_back->start) {
//seed_region_t *seed_region = seed_region_new(read_length - seed_size, read_length - 1,
//bwt_anchor_back->start, bwt_anchor_back->end, 1);
//cal->end = bwt_anchor_back->end;
//linked_list_insert_last(seed_region, cal->sr_list);
//} else {
cal = convert_bwt_anchor_to_CAL(bwt_anchor_back, read_length - seed_size, read_length - 1);
//printf("INSERT-2 (%i)[%i:%lu-%lu]\n", cal->strand, cal->chromosome_id, cal->start, cal->end);
array_list_insert(cal, list);
if (array_list_size(list) > 5) {
free(set_backward);
free(set_forward);
goto exit;
}
array_list_set_flag(DOUBLE_ANCHORS, list);
found_double_anchor = 1;
set_forward[i] = 1;
set_backward[j] = 1;
break;
}
}
}
free(set_backward);
free(set_forward);
}
if (!found_double_anchor && found_anchor) {
//Not Double anchor found but one Yes!!
if (strand == 1) {
if (type == FORWARD_ANCHOR) {
anchor_list_tmp = forward_anchor_list_1;
} else {
anchor_list_tmp = backward_anchor_list_1;
}
} else {
if (type == FORWARD_ANCHOR) {
anchor_list_tmp = forward_anchor_list_0;
} else {
anchor_list_tmp = backward_anchor_list_0;
}
}
//printf("LIST SIZE %i\n", array_list_size(anchor_list_tmp));
for (int i = 0; i < array_list_size(anchor_list_tmp); i++) {
bwt_anchor = array_list_get(i, anchor_list_tmp);
size_t seed_size = bwt_anchor->end - bwt_anchor->start;
//array_list_insert(bwt_anchor_new(bwt_anchor->strand, bwt_anchor->chromosome,
// bwt_anchor->start, bwt_anchor->end, bwt_anchor->type), anchor_list);
if (bwt_anchor->type == FORWARD_ANCHOR) {
//printf("------------------------> start %i\n", 0);
cal = convert_bwt_anchor_to_CAL(bwt_anchor, 0, seed_size);
} else {
//printf("------------------------> start %i\n", read_length - seed_size);
cal = convert_bwt_anchor_to_CAL(bwt_anchor, read_length - seed_size - 1, read_length - 1);
}
array_list_insert(cal, list);
}
array_list_set_flag(SINGLE_ANCHORS, list);
}
exit:
array_list_free(forward_anchor_list_1, (void *)bwt_anchor_free);
array_list_free(backward_anchor_list_1, (void *)bwt_anchor_free);
array_list_free(forward_anchor_list_0, (void *)bwt_anchor_free);
array_list_free(backward_anchor_list_0, (void *)bwt_anchor_free);
array_list_free(big_anchor_list, (void *)bwt_anchor_free);
return array_list_size(list);
}
int apply_sw_bs(sw_server_input_t* input, batch_t *batch) {
int sw_3_nucleotides = 0;
/*
sw_optarg_t *sw_optarg2 = &input->sw_optarg;
printf("Matrix Table\n\tA\tC\tG\tT\tN\nA\t%+.0f\t%+.0f\t%+.0f\t%+.0f\t%+.0f\nC\t%+.0f\t%+.0f\t%+.0f\t%+.0f\t%+.0f\nG\t%+.0f\t%+.0f\t%+.0f\t%+.0f\t%+.0f\nT\t%+.0f\t%+.0f\t%+.0f\t%+.0f\t%+.0f\nN\t%+.0f\t%+.0f\t%+.0f\t%+.0f\t%+.0f\n\n",
sw_optarg2->subst_matrix['A']['A'],
sw_optarg2->subst_matrix['C']['A'],
sw_optarg2->subst_matrix['G']['A'],
sw_optarg2->subst_matrix['T']['A'],
sw_optarg2->subst_matrix['N']['A'],
sw_optarg2->subst_matrix['A']['C'],
sw_optarg2->subst_matrix['C']['C'],
sw_optarg2->subst_matrix['G']['C'],
sw_optarg2->subst_matrix['T']['C'],
sw_optarg2->subst_matrix['N']['C'],
sw_optarg2->subst_matrix['A']['G'],
sw_optarg2->subst_matrix['C']['G'],
sw_optarg2->subst_matrix['G']['G'],
sw_optarg2->subst_matrix['T']['G'],
sw_optarg2->subst_matrix['N']['G'],
sw_optarg2->subst_matrix['A']['T'],
sw_optarg2->subst_matrix['C']['T'],
sw_optarg2->subst_matrix['G']['T'],
sw_optarg2->subst_matrix['T']['T'],
sw_optarg2->subst_matrix['N']['T'],
sw_optarg2->subst_matrix['A']['N'],
sw_optarg2->subst_matrix['C']['N'],
sw_optarg2->subst_matrix['G']['N'],
sw_optarg2->subst_matrix['T']['N'],
sw_optarg2->subst_matrix['N']['N']
);
*/
if (sw_3_nucleotides == 0) {
apply_sw_bs_4nt(input, batch);
} else {
//printf("START: apply_sw\n");
int tid = omp_get_thread_num();
mapping_batch_t *mapping_batch = batch->mapping_batch;
cal_t *cal = NULL;
array_list_t *cal_list = NULL, *mapping_list = NULL;
array_list_t *fq_batch = mapping_batch->fq_batch;
fastq_read_t *fq_read;
// added by PP for bisulfite
array_list_t *CT_fq_batch = mapping_batch->CT_fq_batch;
array_list_t *GA_fq_batch = mapping_batch->GA_fq_batch;
array_list_t *CT_rev_fq_batch = mapping_batch->CT_rev_fq_batch;
array_list_t *GA_rev_fq_batch = mapping_batch->GA_rev_fq_batch;
fastq_read_t *fq_read2;
// end added by PP for bisulfite
size_t start, end;
size_t start2, end2;
/*
genome_t *genome = input->genome_p;
*/
// added by PP for bisulfite
genome_t *genome1 = input->genome1_p;
genome_t *genome2 = input->genome2_p;
// end added by PP for bisulfite
size_t flank_length = input->flank_length;
// SIMD support for Smith-Waterman
float score, min_score = input->min_score;
sw_output_t *sw_output;
size_t read_index, num_cals;
size_t num_targets = mapping_batch->num_targets;
size_t new_num_targets = 0;
// added by PP for bisulfite
size_t num_targets2 = mapping_batch->num_targets2;
size_t new_num_targets2 = 0;
// added by PP for bisulfite
// added by PP for bisulfite
size_t sw_total1 = mapping_batch->num_to_do;
size_t sw_total2 = mapping_batch->num_to_do2;
size_t sw_total = sw_total1 + sw_total2;
// end added by PP for bisulfite
// set to zero
mapping_batch->num_to_do = 0;
// added by PP for bisulfite
mapping_batch->num_to_do2 = 0;
int g[sw_total];
// end added by PP for bisulfite
sw_optarg_t *sw_optarg = &input->sw_optarg;
sw_multi_output_t *output = sw_multi_output_new(sw_total);
char *q[sw_total], *r[sw_total];
uint8_t strands[sw_total], chromosomes[sw_total];
size_t starts[sw_total];
size_t sw_count = 0, read_indices[sw_total], sw_count2 = 0;
int read_len, ref_len, max_ref_len;
//printf("num of sw to do: %i\n", sw_total);
// initialize query and reference sequences to Smith-Waterman
for (size_t i = 0; i < num_targets; i++) {
// printf("sw_server: target #%i of %i\n", i, num_seqs);
read_index = mapping_batch->targets[i];
// to use with the three nucleotides searches
fq_read = (fastq_read_t *) array_list_get(read_index, GA_fq_batch);
fq_read2 = (fastq_read_t *) array_list_get(read_index, GA_rev_fq_batch);
//printf("read %lu = %s\n", read_index, fq_read->sequence);
//printf("read %lu = %s\n", read_index, fq_read2->sequence);
// printf("sw_server: read #%i\n", read_index);
cal_list = mapping_batch->mapping_lists[read_index];
num_cals = array_list_size(cal_list);
read_len = fq_read->length;
// max_ref_len = read_len + (read_len / 2);
//printf("sw_server: num_cals = %i cals\n", num_cals);
// processing each CAL from this read
for(size_t j = 0; j < num_cals; j++) {
// get cal and read index
cal = array_list_get(j, cal_list);
read_indices[sw_count] = read_index;
if (flank_length >= cal->start) {
start = 0;
} else {
start = cal->start - flank_length;
}
end = cal->end + flank_length;
if (end >= genome1->chr_size[cal->chromosome_id - 1]) {
end = genome1->chr_size[cal->chromosome_id - 1] - 1;
}
ref_len = end - start + 2;
// if (ref_len < max_ref_len) {
// query sequence, revcomp if necessary
q[sw_count] = (char *) calloc((read_len + 1), sizeof(char));
// to use with the three nucleotides searches
if (cal->strand == 0) {
memcpy(q[sw_count], fq_read->sequence, read_len);
//seq_reverse_complementary(q[sw_count], read_len);
} else {
memcpy(q[sw_count], fq_read2->sequence, read_len);
}
//q[sw_count] = &(fq_batch->seq[fq_batch->data_indices[index]]);
// reference sequence
//printf("\tSW: %d.[chromosome:%d]-[strand:%d]-[start:%d, end:%d]\n", j, cal->chromosome_id, cal->strand, cal->start, cal->end);
r[sw_count] = calloc(1, end - start + 2);
// to use with the three nucleotides searches
if (cal->strand == 0) {
genome_read_sequence_by_chr_index(r[sw_count], 0,
cal->chromosome_id - 1, &start, &end, genome1);
} else {
genome_read_sequence_by_chr_index(r[sw_count], 0,
cal->chromosome_id - 1, &start, &end, genome2);
/*
start2 = genome1->chr_size[cal->chromosome_id - 1] - 1 - end;
end2 = genome1->chr_size[cal->chromosome_id - 1] - 1 - start;
genome_read_sequence_by_chr_index(r[sw_count], 0,
cal->chromosome_id - 1, &start2, &end2, genome2);
*/
}
/*
genome_read_sequence_by_chr_index(r[sw_count], cal->strand,
cal->chromosome_id - 1, &start, &end, genome1);
*/
// save some stuff, we'll use them after...
strands[sw_count] = cal->strand;
chromosomes[sw_count] = cal->chromosome_id;
starts[sw_count] = start;
/*
printf("st = %lu\tend = %lu\n", cal->start, cal->end);
printf("1\nseq %s\ngen %s\nstrand %2lu chromo %lu start %lu end %lu\n",
q[sw_count], r[sw_count], cal->strand, cal->chromosome_id, start, end);
*/
// increase counter
sw_count++;
}
// free cal_list
array_list_clear(cal_list, (void *) cal_free);
// batch->mapping_lists[index] = NULL;
}
////////////////
sw_count2 = sw_count;
for (size_t i = 0; i < num_targets2; i++) {
// printf("sw_server: target #%i of %i\n", i, num_seqs);
read_index = mapping_batch->targets2[i];
// to use with the three nucleotides searches
fq_read = (fastq_read_t *) array_list_get(read_index, CT_fq_batch);
fq_read2 = (fastq_read_t *) array_list_get(read_index, CT_rev_fq_batch);
//printf("read %lu = %s\n", read_index, fq_read->sequence);
//printf("read %lu = %s\n", read_index, fq_read2->sequence);
// printf("sw_server: read #%i\n", read_index);
cal_list = mapping_batch->mapping_lists2[read_index];
num_cals = array_list_size(cal_list);
read_len = fq_read->length;
// max_ref_len = read_len + (read_len / 2);
//printf("sw_server: num_cals = %i cals\n", num_cals);
// processing each CAL from this read
for(size_t j = 0; j < num_cals; j++) {
// get cal and read index
cal = array_list_get(j, cal_list);
read_indices[sw_count] = read_index;
if (flank_length >= cal->start) {
start = 0;
} else {
start = cal->start - flank_length;
}
end = cal->end + flank_length;
if (end >= genome1->chr_size[cal->chromosome_id - 1]) {
end = genome1->chr_size[cal->chromosome_id - 1] - 1;
}
ref_len = end - start + 2;
// if (ref_len < max_ref_len) {
// query sequence, revcomp if necessary
q[sw_count] = (char *) calloc((read_len + 1), sizeof(char));
// to use with the three nucleotides searches
if (cal->strand == 0) {
memcpy(q[sw_count], fq_read->sequence, read_len);
//seq_reverse_complementary(q[sw_count], read_len);
} else {
memcpy(q[sw_count], fq_read2->sequence, read_len);
}
//q[sw_count] = &(fq_batch->seq[fq_batch->data_indices[index]]);
// reference sequence
//printf("\tSW: %d.[chromosome:%d]-[strand:%d]-[start:%d, end:%d]\n", j, cal->chromosome_id, cal->strand, cal->start, cal->end);
r[sw_count] = calloc(1, end - start + 2);
// to use with the three nucleotides searches
if (cal->strand == 0) {
genome_read_sequence_by_chr_index(r[sw_count], 0,
cal->chromosome_id - 1, &start, &end, genome2);
} else {
genome_read_sequence_by_chr_index(r[sw_count], 0,
cal->chromosome_id - 1, &start, &end, genome1);
/*
start2 = genome1->chr_size[cal->chromosome_id - 1] - 1 - end;
end2 = genome1->chr_size[cal->chromosome_id - 1] - 1 - start;
genome_read_sequence_by_chr_index(r[sw_count], 0,
cal->chromosome_id - 1, &start2, &end2, genome1);
*/
}
/*
genome_read_sequence_by_chr_index(r[sw_count], cal->strand,
cal->chromosome_id - 1, &start, &end, genome2);
*/
// save some stuff, we'll use them after...
strands[sw_count] = cal->strand;
chromosomes[sw_count] = cal->chromosome_id;
starts[sw_count] = start;
//printf("2\nseq %s\ngen %s\nstrand %2lu chromo %lu start %lu end %lu\n",
// q[sw_count], r[sw_count], cal->strand, cal->chromosome_id, start, end);
// increase counter
sw_count++;
}
// free cal_list
array_list_clear(cal_list, (void *) cal_free);
// batch->mapping_lists[index] = NULL;
}
//printf("before smith_waterman: sw_total = %i, sw_count = %i, sw_count2 = %i\n", sw_total, sw_count, sw_count2);
// run Smith-Waterman
// printf("before smith_waterman: sw_total = %i, sw_count = %i\n", sw_total, sw_count);
smith_waterman_mqmr(q, r, sw_count, sw_optarg, 1, output);
// printf("after smith_waterman\n");
for (size_t i = 0; i < sw_count; i++) {
LOG_DEBUG_F("cal: start = %lu, strand = %i\n", starts[i], strands[i]);
LOG_DEBUG_F("\tquery : %s\n", q[i]);
LOG_DEBUG_F("\tref. : %s\n", r[i]);
LOG_DEBUG_F("\tquery map: %s (start: %i)\n",
output->query_map_p[i], output->query_start_p[i]);
LOG_DEBUG_F("\tref. map : %s (start: %i)\n",
output->ref_map_p[i], output->ref_start_p[i]);
LOG_DEBUG("\n");
}
//size_t mapp = 0, mapp2 = 0;
double norm_score;
// filter alignments by min_score
for (size_t i = 0; i < sw_count2; i++) {
read_index = read_indices[i];
fq_read = (fastq_read_t *) array_list_get(read_index, GA_fq_batch);
fq_read2 = (fastq_read_t *) array_list_get(read_index, GA_rev_fq_batch);
read_len = fq_read->length;
norm_score = NORM_SCORE(output->score_p[i], read_len, input->match);
if (norm_score >= min_score) {
// valid mappings,
//insert in the list for further processing
mapping_list = mapping_batch->mapping_lists[read_index];
array_list_set_flag(0, mapping_list);
if (array_list_size(mapping_list) == 0) {
mapping_batch->targets[new_num_targets++] = read_index;
//mapp++;
}
sw_output = sw_output_new(strands[i],
chromosomes[i],
starts[i],
strlen(r[i]),
strlen(output->query_map_p[i]),
output->query_start_p[i],
output->ref_start_p[i],
output->score_p[i],
norm_score,
output->query_map_p[i],
output->ref_map_p[i]);
array_list_insert(sw_output, mapping_list);
mapping_batch->num_to_do++;
}
// free query and reference
free(q[i]);
free(r[i]);
}
mapping_batch->num_targets = new_num_targets;
for (size_t i = sw_count2; i < sw_count; i++) {
read_index = read_indices[i];
fq_read = (fastq_read_t *) array_list_get(read_index, CT_fq_batch);
fq_read2 = (fastq_read_t *) array_list_get(read_index, CT_rev_fq_batch);
read_len = fq_read->length;
norm_score = NORM_SCORE(output->score_p[i], read_len, input->match);
if (norm_score >= min_score) {
// valid mappings,
//insert in the list for further processing
mapping_list = mapping_batch->mapping_lists2[read_index];
array_list_set_flag(0, mapping_list);
if (array_list_size(mapping_list) == 0) {
mapping_batch->targets2[new_num_targets2++] = read_index;
//mapp2++;
}
sw_output = sw_output_new(strands[i],
chromosomes[i],
starts[i],
strlen(r[i]),
strlen(output->query_map_p[i]),
output->query_start_p[i],
output->ref_start_p[i],
output->score_p[i],
norm_score,
output->query_map_p[i],
output->ref_map_p[i]);
array_list_insert(sw_output, mapping_list);
mapping_batch->num_to_do2++;
}
// free query and reference
free(q[i]);
free(r[i]);
}
mapping_batch->num_targets2 = new_num_targets2;
// update counter
// thr_sw_items[tid] += sw_count;
// free
sw_multi_output_free(output);
// go to the next stage
/*
printf("3 SW1 \t%3lu\tmapp \t%3lu\tno map (discard) \t%3lu\n",
num_targets, mapp, num_targets - mapp);
printf("3 SW2 \t%3lu\tmapp \t%3lu\tno map (discard) \t%3lu\n",
num_targets2, mapp2, num_targets2 - mapp2);
*/
//printf("END: apply_sw, (%d Smith-Waterman)\n", sw_total);
}
//return CONSUMER_STAGE;
return BS_POST_PAIR_STAGE;
// printf("END: apply_sw, (%d Smith-Waterman, %d valids)\n", total, valids);
}
int main (int argc, char *argv[]) {
if(!strcmp("count-lines", argv[1])) {
fastq_file_t *file = fastq_fopen(argv[2]);
array_list_t *reads = array_list_new(2000000, 1.8, COLLECTION_MODE_SYNCHRONIZED);
size_t nread = 1;
int count = 0;
while((nread = fastq_fread_se(reads, 100000, file)) != 0) {
count += nread;
for(int i=0; i<reads->size; i++) {
fastq_read_print(array_list_get(i, reads));
}
// printf("Size: %i, Capacity: %i\n", reads->size, reads->capacity);
array_list_clear(reads, fastq_read_free);
}
// printf("Total num reads: %i\n", reads->size);
// fastq_read_print(array_list_get(0, reads));
// fastq_read_print(array_list_get(reads->size-1, reads));
array_list_free(reads, fastq_read_free);
fastq_fclose(file);
}
if(!strcmp("count-lines-gz", argv[1])) {
fastq_gzfile_t *file = fastq_gzopen(argv[2]);
// printf("=>%i\n", file->ret);
array_list_t *reads = array_list_new(1000000, 1.8, COLLECTION_MODE_SYNCHRONIZED);
size_t nread = 1;
int count = 0;
while((nread = fastq_gzread_se(reads, 100000, file)) != 0) {
// nread = fastq_gzread_se(reads, 1000000, file);
count += nread;
// printf("Size: %i, Capacity: %i, count = %i, nread: %i\n", reads->size, reads->capacity, count, nread);
for(int i=0; i<reads->size; i++) {
fastq_read_print(array_list_get(i, reads));
}
// fastq_read_print((fastq_read_t*)array_list_get(reads->size-1, reads));
array_list_clear(reads, fastq_read_free);
}
// printf("Total num reads: %i\n", count);
// fastq_read_print(array_list_get(0, reads));
array_list_free(reads, fastq_read_free);
fastq_gzclose(file);
}
if(!strcmp("count-bytes-gz", argv[1])) {
fastq_gzfile_t *file = fastq_gzopen(argv[2]);
// printf("=>%i\n", file->ret);
array_list_t *reads = array_list_new(1000000, 1.8, COLLECTION_MODE_SYNCHRONIZED);
size_t nread = 1;
int count = 0;
while((nread = fastq_gzread_bytes_se(reads, 10000000, file)) != 0) {
// nread = fastq_gzread_bytes_se(reads, 100000, file);
count += reads->size;
// printf("Size: %i, Capacity: %i, count = %i, nread: %i\n", reads->size, reads->capacity, count, nread);
for(int i=0; i<reads->size; i++) {
fastq_read_print(array_list_get(i, reads));
}
// fastq_read_print(array_list_get(reads->size-1, reads));
array_list_clear(reads, fastq_read_free);
}
// printf("Total num reads: %i\n", count);
// fastq_read_print(array_list_get(0, reads));
array_list_free(reads, fastq_read_free);
fastq_gzclose(file);
}
if(!strcmp("filter", argv[1])) {
fastq_file_t *file = fastq_fopen(argv[2]);
fastq_filter_options_t *fastq_filter_options = fastq_filter_options_new(50,150, 30, 80, 2, 100);
array_list_t *reads = array_list_new(200000, 1.8, COLLECTION_MODE_SYNCHRONIZED);
array_list_t *passed_reads ;//= array_list_new(200000, 1.8, COLLECTION_MODE_SYNCHRONIZED);
array_list_t *failed_reads ;//= array_list_new(200000, 1.8, COLLECTION_MODE_SYNCHRONIZED);
size_t nread = 1;
int count = 0;
while((nread = fastq_fread_se(reads, 1000000, file)) != 0) {
count += reads->size;
passed_reads = array_list_new(200000, 1.8, COLLECTION_MODE_SYNCHRONIZED);
failed_reads = array_list_new(200000, 1.8, COLLECTION_MODE_SYNCHRONIZED);
// for(int i=0; i<reads->size; i++) {
// fastq_read_print(array_list_get(i, reads));
// }
fastq_filter(reads, passed_reads, failed_reads, fastq_filter_options);
fastq_read_print(array_list_get(0, passed_reads));
fastq_read_print(array_list_get(0, failed_reads));
printf("Total Reads: %lu, Passed Reads: %lu, Reads failed: %lu\n", reads->size, passed_reads->size, failed_reads->size);
array_list_clear(reads, fastq_read_free);
array_list_free(passed_reads, NULL);
array_list_free(failed_reads, NULL);
// fastq_read_print(array_list_get(0, passed_reads));
// fastq_read_print(array_list_get(0, failed_reads));
// printf("Total Reads: %lu, Passed Reads: %lu, Reads filter: %lu\n", reads->size, passed_reads->size, failed_reads->size);
}
// fastq_read_print(array_list_get(0, passed_reads));
// fastq_read_print(array_list_get(0, failed_reads));
// printf("Total Reads: %lu, Passed Reads: %lu, Reads filter: %lu\n", reads->size, passed_reads->size, failed_reads->size);
fastq_filter_options_free(fastq_filter_options);
array_list_free(reads, NULL);
// array_list_free(passed_reads, fastq_read_free);
// array_list_free(failed_reads, fastq_read_free);
fastq_fclose(file);
}
return 0;
}
int apply_caling_rna(cal_seeker_input_t* input, batch_t *batch) {
LOG_DEBUG("========= APPLY CALING RNA =========\n");
//if (time_on) { start_timer(start); }
bwt_optarg_t *bwt_optarg = input->bwt_optarg;
bwt_index_t *bwt_index = input->index;
cal_optarg_t *cal_optarg = input->cal_optarg;
mapping_batch_t *mapping_batch = batch->mapping_batch;
size_t num_cals, select_cals;
size_t total_reads = 0;
size_t num_targets, target_pos, total_targets, extra_target_pos;
fastq_read_t *read;
genome_t *genome = input->genome;
unsigned int num_chromosomes = genome->num_chromosomes;
int min_seeds, max_seeds;
int seed_size = input->cal_optarg->seed_size;
array_list_t *cal_list, *list;
cal_t *cal;
//array_list_t *region_list;
num_targets = mapping_batch->num_targets;
total_targets = 0;
extra_target_pos = 0;
total_reads += num_targets;
target_pos = 0;
mapping_batch->extra_stage_do = 1;
/* int t, target;
for (t = 0; t < num_targets; t++) {
target = mapping_batch->targets[t];
mapping_batch->mapping_lists[target]->size = 0;
}
return RNA_POST_PAIR_STAGE;
*/
array_list_t *region_list = array_list_new(1000,
1.25f,
COLLECTION_MODE_ASYNCHRONIZED);
//extern size_t TOTAL_READS_SEEDING, TOTAL_READS_SEEDING2;
//pthread_mutex_lock(&mutex_sp);
//TOTAL_READS_SEEDING += num_targets;
//pthread_mutex_unlock(&mutex_sp);
//printf("Num targets = %i\n", num_targets);
for (size_t i = 0; i < num_targets; i++) {
read = array_list_get(mapping_batch->targets[i], mapping_batch->fq_batch);
//printf("From CAL Seeker %s\n", read->id);
list = mapping_batch->mapping_lists[mapping_batch->targets[i]];
//if (array_list_get_flag(region_list) == 0 ||
// array_list_get_flag(region_list) == 2) {
//We have normal and extend seeds (anchors)
max_seeds = (read->length / 15)*2 + 10;
//printf("%i\n", input->cal_optarg->min_cal_size);
num_cals = bwt_generate_cals(read->sequence,
seed_size,
bwt_optarg,
cal_optarg,
bwt_index,
list,
num_chromosomes);
// if we want to seed with 24-length seeds,
if (num_cals == 0) {
//printf("No Cals seeding...\n");
//pthread_mutex_lock(&mutex_sp);
//extern size_t seeds_1err;
//seeds_1err++;
//pthread_mutex_unlock(&mutex_sp);
int seed_size = 24;
//First, Delete old regions
array_list_clear(region_list, (void *)region_bwt_free);
//Second, Create new regions with seed_size 24 and 1 Mismatch
bwt_map_inexact_seeds_seq(read->sequence, seed_size, seed_size/2,
bwt_optarg, bwt_index,
region_list);
max_seeds = (read->length / 15)*2 + 10;
//int prev_min_cal = input->cal_optarg->min_cal_size;
//input->cal_optarg->min_cal_size = seed_size + seed_size / 2;
//printf("NO CALS, new seeds %lu\n", array_list_size(region_list));
num_cals = bwt_generate_cal_list_linked_list(region_list,
input->cal_optarg,
&min_seeds, &max_seeds,
genome->num_chromosomes + 1,
list, read->length,
cal_optarg->min_cal_size,
0);
//input->cal_optarg->min_cal_size = prev_min_cal;
//pthread_mutex_lock(&mutex_sp);
//TOTAL_READS_SEEDING2++;
//pthread_mutex_unlock(&mutex_sp);
}
array_list_clear(region_list, (void *)region_bwt_free);
//filter-incoherent CALs
int founds[num_cals], found = 0;
for (size_t j = 0; j < num_cals; j++) {
founds[j] = 0;
cal = array_list_get(j, list);
LOG_DEBUG_F("\tcal %i of %i: sr_list size = %i (cal->num_seeds = %i) %i:%lu-%lu\n",
j, num_cals, cal->sr_list->size, cal->num_seeds,
cal->chromosome_id, cal->start, cal->end);
if (cal->sr_list->size > 0) {
int start = 0;
size_t genome_start = 0;
int first = 1;
for (linked_list_item_t *list_item = cal->sr_list->first; list_item != NULL; list_item = list_item->next) {
seed_region_t *s = list_item->item;
LOG_DEBUG_F("\t\t:: star %lu > %lu s->read_start\n", start, s->read_start);
LOG_DEBUG_F("\t\t:: read_star %lu > read_end %lu \n", s->read_start, s->read_end);
if (start > s->read_start || s->read_start >= s->read_end) {
LOG_DEBUG("\t\t\t:: remove\n");
found++;
founds[j] = 1;
}
if (!first &&
((s->genome_start < genome_start) ||
(s->genome_start - genome_start) > 2*read->length)) {
//printf("Remove (genome_start = %i s->genome_start = %i)\n", genome_start, s->genome_start);
//cal_print(cal);
found++;
founds[j] = 1;
}
first = 0;
start = s->read_end + 1;
genome_start = s->genome_end + 1;
}
} else {
found++;
founds[j] = 1;
}
}
if (found) {
min_seeds = 100000;
max_seeds = 0;
cal_list = array_list_new(MAX_CALS, 1.25f, COLLECTION_MODE_ASYNCHRONIZED);
for (size_t j = 0; j < num_cals; j++) {
if (!founds[j]) {
cal = array_list_get(j, list);
cal->num_seeds = cal->sr_list->size;
if (cal->num_seeds > max_seeds) max_seeds = cal->num_seeds;
if (cal->num_seeds < min_seeds) min_seeds = cal->num_seeds;
array_list_insert(cal, cal_list);
array_list_set(j, NULL, list);
}
}
array_list_free(list, (void *) cal_free);
num_cals = array_list_size(cal_list);
list = cal_list;
}
mapping_batch->mapping_lists[mapping_batch->targets[i]] = list;
num_cals = array_list_size(list);
int max = 100;
if (num_cals > max) {
select_cals = num_cals - max;
for(int j = num_cals - 1; j >= max; j--) {
cal_free(array_list_remove_at(j, mapping_batch->mapping_lists[mapping_batch->targets[i]]));
}
}
//mapping_batch->targets[target_pos++] = mapping_batch->targets[i];
//} //else if (num_cals > 0) {
mapping_batch->targets[target_pos++] = mapping_batch->targets[i];
/* printf("<<<<<===== CAL SERVER =====>>>>>\n"); */
/* for (int c = 0; c < array_list_size(mapping_batch->mapping_lists[mapping_batch->targets[i]]); c++) { */
/* cal_t *cal_aux = array_list_get(c, mapping_batch->mapping_lists[mapping_batch->targets[i]]); */
/* cal_print(cal_aux); */
/* } */
/* printf("<<<<<===== CAL SERVER END =====>>>>>\n"); */
//printf("Total CALs %i\n", num_cals);
}
mapping_batch->num_targets = target_pos;
array_list_free(region_list, NULL);
//if (time_on) { stop_timer(start, end, time); timing_add(time, CAL_SEEKER, timing); }
LOG_DEBUG("========= APPLY CALING RNA END =========\n");
// return RNA_STAGE;
if (batch->mapping_mode == RNA_MODE) {
return RNA_STAGE;
}
if (batch->pair_input->pair_mng->pair_mode != SINGLE_END_MODE) {
return PRE_PAIR_STAGE;
} else if (batch->mapping_batch->num_targets > 0) {
return SW_STAGE;
}
return DNA_POST_PAIR_STAGE;
}
//====================================================================================
// apply_caling
//====================================================================================
int apply_caling(cal_seeker_input_t* input, batch_t *batch) {
mapping_batch_t *mapping_batch = batch->mapping_batch;
array_list_t *list = NULL;
size_t read_index, num_cals;
int min_seeds, max_seeds;
cal_t *cal;
array_list_t *cal_list;
fastq_read_t *read;
size_t num_chromosomes = input->genome->num_chromosomes + 1;
size_t num_targets = mapping_batch->num_targets;
size_t *targets = mapping_batch->targets;
size_t new_num_targets = 0;
array_list_t *region_list;
bwt_anchor_t *bwt_anchor_back, *bwt_anchor_forw;
linked_list_t *linked_list;
int anchor_nt, gap_nt;
seed_region_t *seed_region_start, *seed_region_end;
//max_seeds = input->cal_optarg->num_seeds;
// size_t *new_targets = (size_t *) calloc(num_targets, sizeof(size_t));
// set to zero
mapping_batch->num_to_do = 0;
for (size_t i = 0; i < num_targets; i++) {
read_index = targets[i];
read = array_list_get(read_index, mapping_batch->fq_batch);
region_list = mapping_batch->mapping_lists[read_index];
// for debugging
// LOG_DEBUG_F("%s\n", ((fastq_read_t *) array_list_get(read_index, mapping_batch->fq_batch))->id);
if (!list) {
list = array_list_new(1000,
1.25f,
COLLECTION_MODE_ASYNCHRONIZED);
}
if (array_list_get_flag(region_list) == 0 ||
array_list_get_flag(region_list) == 2) {
//We have normal and extend seeds (anchors)
max_seeds = (read->length / 15)*2 + 10;
num_cals = bwt_generate_cal_list_linked_list(region_list,
input->cal_optarg,
&min_seeds, &max_seeds,
num_chromosomes,
list, read->length,
input->cal_optarg->min_cal_size, 0);
} else {
//We have double anchors with smaller distance between they
//printf("Easy case... Two anchors and same distance between read gap and genome distance\n");
num_cals = 0;
for (int a = array_list_size(region_list) - 1; a >= 0; a -= 2) {
max_seeds = 2;
min_seeds = 2;
bwt_anchor_back = array_list_remove_at(a, region_list);
bwt_anchor_forw = array_list_remove_at(a - 1, region_list);
linked_list = linked_list_new(COLLECTION_MODE_ASYNCHRONIZED);
//Seed for the first anchor
anchor_nt = bwt_anchor_forw->end - bwt_anchor_forw->start;
//printf("\t seed0[%i-%i][%lu-%lu]\n", 0, anchor_nt - 1,
// bwt_anchor_forw->start, bwt_anchor_forw->end);
seed_region_start = seed_region_new(0, anchor_nt - 1,
bwt_anchor_forw->start, bwt_anchor_forw->end, 0, 0, 0);
//Seed for the first anchor
gap_nt = read->length - (anchor_nt + (bwt_anchor_back->end - bwt_anchor_back->start));
//printf("\t gap_nt = %i, anchor_nt = %i\n", gap_nt, anchor_nt);
//printf("\t seed1[%i-%i][%lu-%lu]\n", anchor_nt + gap_nt, read->length - 1,
// bwt_anchor_back->start + 1, bwt_anchor_back->end);
seed_region_end = seed_region_new(anchor_nt + gap_nt, read->length - 1,
bwt_anchor_back->start + 1, bwt_anchor_back->end, 1, 0, 0);
//The reference distance is 0 and the read distance not
//The read distance is 0 and the reference distance not
//if (seed_region_start->genome_end > seed_region_end->genome_start ||
// seed_region_start->read_end > seed_region_end->read_start) {
//array_list_clear(region_list, NULL);
//continue;
if (seed_region_end->genome_start - seed_region_start->genome_end < 5 ||
seed_region_end->read_start - seed_region_start->read_end < 5) {
seed_region_start->genome_end -= 5;
seed_region_start->read_end -= 5;
seed_region_end->genome_start += 5;
seed_region_end->read_start += 5;
}
linked_list_insert(seed_region_start, linked_list);
linked_list_insert_last(seed_region_end, linked_list);
cal = cal_new(bwt_anchor_forw->chromosome + 1,
bwt_anchor_forw->strand,
bwt_anchor_forw->start,
bwt_anchor_back->end + 1,
2,
linked_list,
linked_list_new(COLLECTION_MODE_ASYNCHRONIZED));
array_list_insert(cal, list);
num_cals++;
}
}
// for debugging
LOG_DEBUG_F("read %s : num. cals = %i, min. seeds = %i, max. seeds = %i\n",
read->id, num_cals, min_seeds, max_seeds);
/* if (num_cals == 0) {
int seed_size = 24;
//First, Delete old regions
array_list_clear(mapping_batch->mapping_lists[read_index], region_bwt_free);
//Second, Create new regions with seed_size 24 and 1 Mismatch
bwt_map_inexact_seeds_seq(read->sequence, seed_size, seed_size/2,
bwt_optarg, bwt_index,
mapping_batch->mapping_lists[read_index]);
num_cals = bwt_generate_cal_list_linked_list(mapping_batch->mapping_lists[mapping_batch->targets[i]],
input->cal_optarg,
&min_seeds, &max_seeds,
num_chromosomes,
list, read->length);
}*/
/*
for (size_t j = 0; j < num_cals; j++) {
cal = array_list_get(j, list);
LOG_DEBUG_F("\tchr: %i, strand: %i, start: %lu, end: %lu, num_seeds = %i, num. regions = %lu\n",
cal->chromosome_id, cal->strand, cal->start, cal->end, cal->num_seeds, cal->sr_list->size);
}
*/
// printf("min_seeds = %i, max_seeds = %i, min_limit = %i, num_cals = %i\n",
// min_seeds, max_seeds, min_limit, array_list_size(list));
// filter incoherent CALs
int founds[num_cals], found = 0;
for (size_t j = 0; j < num_cals; j++) {
founds[j] = 0;
cal = array_list_get(j, list);
LOG_DEBUG_F("\tcal %i of %i: sr_list size = %i (cal->num_seeds = %i) %i:%lu-%lu\n",
j, num_cals, cal->sr_list->size, cal->num_seeds,
cal->chromosome_id, cal->start, cal->end);
if (cal->sr_list->size > 0) {
int start = 0;
for (linked_list_item_t *list_item = cal->sr_list->first; list_item != NULL; list_item = list_item->next) {
seed_region_t *s = list_item->item;
LOG_DEBUG_F("\t\t:: star %lu > %lu s->read_start\n", start, s->read_start);
if (start > s->read_start) {
LOG_DEBUG("\t\t\t:: remove\n");
found++;
founds[j] = 1;
}
start = s->read_end + 1;
}
} else {
found++;
founds[j] = 1;
}
}
if (found) {
min_seeds = 100000;
max_seeds = 0;
cal_list = array_list_new(MAX_CALS, 1.25f, COLLECTION_MODE_ASYNCHRONIZED);
for (size_t j = 0; j < num_cals; j++) {
if (!founds[j]) {
cal = array_list_get(j, list);
cal->num_seeds = cal->sr_list->size;
if (cal->num_seeds > max_seeds) max_seeds = cal->num_seeds;
if (cal->num_seeds < min_seeds) min_seeds = cal->num_seeds;
array_list_insert(cal, cal_list);
array_list_set(j, NULL, list);
}
}
array_list_free(list, (void *) cal_free);
num_cals = array_list_size(cal_list);
list = cal_list;
}
// LOG_FATAL_F("num. cals = %i, min. seeds = %i, max. seeds = %i\n", num_cals, min_seeds, max_seeds);
// filter CALs by the number of seeds
cal_list = list;
list = NULL;
/*
int min_limit = input->cal_optarg->min_num_seeds_in_cal;
if (min_limit < 0) min_limit = max_seeds;
// min_limit -= 3;
if (min_seeds == max_seeds || min_limit <= min_seeds) {
cal_list = list;
list = NULL;
} else {
cal_list = array_list_new(MAX_CALS, 1.25f, COLLECTION_MODE_ASYNCHRONIZED);
for (size_t j = 0; j < num_cals; j++) {
cal = array_list_get(j, list);
if (cal->num_seeds >= min_limit) {
array_list_insert(cal, cal_list);
array_list_set(j, NULL, list);
}
}
array_list_clear(list, (void *) cal_free);
num_cals = array_list_size(cal_list);
}
*/
if (num_cals > MAX_CALS) {
for (size_t j = num_cals - 1; j >= MAX_CALS; j--) {
cal = (cal_t *) array_list_remove_at(j, cal_list);
cal_free(cal);
}
num_cals = array_list_size(cal_list);
}
// LOG_DEBUG_F("num. cals = %i, MAX_CALS = %i\n", num_cals, MAX_CALS);
if (num_cals > 0 && num_cals <= MAX_CALS) {
array_list_set_flag(2, cal_list);
targets[new_num_targets++] = read_index;
/*
int count1 = 0, count2 = 0;
// count number of sw to do
// method #1
// printf("method #1\n");
seed_region_t *s, *prev_s;
linked_list_iterator_t* itr;
for (size_t j = 0; j < num_cals; j++) {
prev_s = NULL;
cal = array_list_get(j, cal_list);
itr = linked_list_iterator_new(cal->sr_list);
s = (seed_region_t *) linked_list_iterator_curr(itr);
while (s != NULL) {
if ((prev_s == NULL && s->read_start != 0) || (prev_s != NULL)) {
// printf("\t\t\tcase 1\n");
count1++;
}
prev_s = s;
linked_list_iterator_next(itr);
s = linked_list_iterator_curr(itr);
}
if (prev_s != NULL && prev_s->read_end < read->length - 1) {
count1++;
// printf("\t\t\tcase 2 (%i < %i)\n", prev_s->read_end, read->length - 1);
}
linked_list_iterator_free(itr);
}
// method #2
printf("method #2\n");
for (size_t j = 0; j < num_cals; j++) {
cal = array_list_get(j, cal_list);
printf("\t: %i\n", j);
if (cal->sr_list->size > 0) {
int start = 0;
for (linked_list_item_t *list_item = cal->sr_list->first; list_item != NULL; list_item = list_item->next) {
seed_region_t *s = list_item->item;
printf("\t\t[%i|%i - %i|%i]\n", s->genome_start, s->read_start, s->read_end, s->genome_end);
if (s->read_start != start) {
count2++;
}
start = s->read_end + 1;
}
if (start < read->length) {
count2++;
}
}
}
printf("count #1 = %i, count #2 = %i\n", count1, count2);
assert(count1 == count2);
mapping_batch->num_to_do += count1;
*/
// we have to free the region list
array_list_free(mapping_batch->mapping_lists[read_index], (void *) region_bwt_free);
mapping_batch->mapping_lists[read_index] = cal_list;
} else {
array_list_set_flag(0, mapping_batch->mapping_lists[read_index]);
// we have to free the region list
array_list_clear(mapping_batch->mapping_lists[read_index], (void *) region_bwt_free);
if (cal_list) array_list_free(cal_list, (void *) cal_free);
if (list) array_list_clear(list, (void *) cal_free);
}
/*
cal_list = list;
list = NULL;
array_list_set_flag(2, cal_list);
// mapping_batch->num_to_do += num_cals;
targets[new_num_targets++] = read_index;
// we have to free the region list
array_list_free(mapping_batch->mapping_lists[read_index], (void *) region_bwt_free);
mapping_batch->mapping_lists[read_index] = cal_list;
*/
/*
// filter CALs by the number of seeds
int min_limit = input->cal_optarg->min_num_seeds_in_cal;
if (min_limit < 0) min_limit = max_seeds;
printf("min_seeds = %i, max_seeds = %i, min_limit = %i, num_cals = %i\n",
min_seeds, max_seeds, min_limit, array_list_size(list));
if (min_seeds == max_seeds || min_limit <= min_seeds) {
cal_list = list;
list = NULL;
} else {
cal_list = array_list_new(MAX_CALS, 1.25f, COLLECTION_MODE_ASYNCHRONIZED);
for (size_t j = 0; j < num_cals; j++) {
cal = array_list_get(j, list);
if (cal->num_seeds >= min_limit) {
array_list_insert(cal, cal_list);
array_list_set(j, NULL, list);
}
}
array_list_clear(list, (void *) cal_free);
num_cals = array_list_size(cal_list);
printf("************, num_cals = %i\n", num_cals);
}
if (num_cals > MAX_CALS) {
for (size_t j = num_cals - 1; j >= MAX_CALS; j--) {
cal = (cal_t *) array_list_remove_at(j, cal_list);
cal_free(cal);
}
num_cals = array_list_size(cal_list);
}
if (num_cals > 0 && num_cals <= MAX_CALS) {
array_list_set_flag(2, cal_list);
mapping_batch->num_to_do += num_cals;
targets[new_num_targets++] = read_index;
// we have to free the region list
array_list_free(mapping_batch->mapping_lists[read_index], (void *) region_bwt_free);
mapping_batch->mapping_lists[read_index] = cal_list;
} else {
array_list_set_flag(0, mapping_batch->mapping_lists[read_index]);
// we have to free the region list
array_list_clear(mapping_batch->mapping_lists[read_index], (void *) region_bwt_free);
if (cal_list) array_list_free(cal_list, (void *) cal_free);
if (list) array_list_clear(list, (void *) cal_free);
}
*/
} // end for 0 ... num_targets
// update batch
mapping_batch->num_targets = new_num_targets;
// LOG_DEBUG_F("num. SW to do: %i\n", mapping_batch->num_to_do);
// exit(-1);
// free memory
if (list) array_list_free(list, NULL);
if (batch->mapping_mode == RNA_MODE) {
return RNA_STAGE;
}
if (batch->pair_input->pair_mng->pair_mode != SINGLE_END_MODE) {
return PRE_PAIR_STAGE;
} else if (batch->mapping_batch->num_targets > 0) {
return SW_STAGE;
}
return DNA_POST_PAIR_STAGE;
}
/*
* here,
* we use four rules to filter all the chunks to get the best chunk.
* and this is the core of the mmseg alogrithm.
* 1. maximum match word length.
* 2. larget average word length.
* 3. smallest word length variance.
* 4. largest single word morpheme degrees of freedom.
*/
__STATIC_API__ friso_chunk_t mmseg_core_invoke( friso_array_t chunks ) {
register uint_t t/*, j*/;
float max;
friso_chunk_t e;
friso_array_t __res__, __tmp__;
__res__ = new_array_list_with_opacity( chunks->length );
//1.get the maximum matched chunks.
//count the maximum length
max = ( float ) ( ( friso_chunk_t ) chunks->items[0] )->length;
for ( t = 1; t < chunks->length; t++ ) {
e = ( friso_chunk_t ) chunks->items[t];
if ( e->length > max )
max = ( float ) e->length;
}
//get the chunk items that owns the maximum length.
for ( t = 0; t < chunks->length; t++ ) {
e = ( friso_chunk_t ) chunks->items[t];
if ( e->length >= max ) {
array_list_add( __res__, e );
} else {
free_array_list( e->words );
free_chunk( e );
}
}
//check the left chunks
if ( __res__->length == 1 ) {
e = ( friso_chunk_t ) __res__->items[0];
free_array_list( __res__ );
free_array_list( chunks );
return e;
} else {
__tmp__ = array_list_clear( chunks );
chunks = __res__;
__res__ = __tmp__;
}
//2.get the largest average word length chunks.
//count the maximum average word length.
max = count_chunk_avl( ( friso_chunk_t ) chunks->items[0] );
for ( t = 1; t < chunks->length; t++ ) {
e = ( friso_chunk_t ) chunks->items[t];
if ( count_chunk_avl( e ) > max ) {
max = e->average_word_length;
}
}
//get the chunks items that own the largest average word length.
for ( t = 0; t < chunks->length; t++ ) {
e = ( friso_chunk_t ) chunks->items[t];
if ( e->average_word_length >= max ) {
array_list_add( __res__, e );
} else {
free_array_list( e->words );
free_chunk( e );
}
}
//check the left chunks
if ( __res__->length == 1 ) {
e = ( friso_chunk_t ) __res__->items[0];
free_array_list( chunks );
free_array_list( __res__ );
return e;
} else {
__tmp__ = array_list_clear( chunks );
chunks = __res__;
__res__ = __tmp__;
}
//3.get the smallest word length variance chunks
//count the smallest word length variance
max = count_chunk_var( ( friso_chunk_t ) chunks->items[0] );
for ( t = 1; t < chunks->length; t++ ) {
e = ( friso_chunk_t ) chunks->items[t];
if ( count_chunk_var( e ) < max ) {
max = e->word_length_variance;
}
}
//get the chunks that own the smallest word length variance.
for ( t = 0; t < chunks->length; t++ ) {
e = ( friso_chunk_t ) chunks->items[t];
if ( e->word_length_variance <= max ) {
array_list_add( __res__, e );
} else {
free_array_list( e->words );
free_chunk( e );
}
}
//check the left chunks
if ( __res__->length == 1 ) {
e = ( friso_chunk_t ) __res__->items[0];
free_array_list( chunks );
free_array_list( __res__ );
return e;
} else {
__tmp__ = array_list_clear( chunks );
chunks = __res__;
__res__ = __tmp__;
}
//4.get the largest single word morpheme degrees of freedom.
//count the maximum single word morpheme degreees of freedom
max = count_chunk_mdf( ( friso_chunk_t ) chunks->items[0] );
for ( t = 1; t < chunks->length; t++ ) {
e = ( friso_chunk_t ) chunks->items[t];
if ( count_chunk_mdf( e ) > max ) {
max = e->single_word_dmf;
}
}
//get the chunks that own the largest single word word morpheme degrees of freedom.
for ( t = 0; t < chunks->length; t++ ) {
e = ( friso_chunk_t ) chunks->items[t];
if ( e->single_word_dmf >= max ) {
array_list_add( __res__, e );
} else {
free_array_list( e->words );
free_chunk( e );
}
}
/*
* there is still more than one chunks?
* well, this rarely happen but still got a chance.
* here we simple return the first chunk as the final result.
*/
for ( t = 1; t < __res__->length; t++ ) {
e = ( friso_chunk_t ) __res__->items[t];
free_array_list( e->words );
free_chunk( e );
}
e = ( friso_chunk_t ) __res__->items[0];
free_array_list( chunks );
free_array_list( __res__ );
return e;
}
int apply_seeding(region_seeker_input_t* input, batch_t *batch) {
//printf("APPLY SEEDING...\n");
//if (time_on) { start_timer(start); }
mapping_batch_t *mapping_batch = batch->mapping_batch;
size_t num_mappings;
int seed_size = input->cal_optarg_p->seed_size;
size_t min_seed_size = input->cal_optarg_p->min_seed_size;
size_t num_targets = mapping_batch->num_targets;
size_t *targets = mapping_batch->targets;
size_t new_num_targets = 0;
fastq_read_t *read;
int min_intron_size = 40;
int target;
bwt_anchor_t *bwt_anchor = NULL;
region_t *region;
int gap_nt;
int start_search;
int end_search;
// set to zero
mapping_batch->num_to_do = 0;
//TODO: omp parallel for !!
/*if (batch->mapping_mode == 1000) {
for (size_t i = 0; i < num_targets; i++) {
//printf("Seq (i=%i)(target=%i): %s\n", i, targets[i], read->sequence);
read = array_list_get(targets[i], mapping_batch->fq_batch);
num_mappings = bwt_map_exact_seeds_seq(padding_left,
padding_right,
read->sequence,
seed_size,
min_seed_size,
input->bwt_optarg_p,
input->bwt_index_p,
mapping_batch->mapping_lists[targets[i]],
mapping_batch->extra_stage_id[targets[i]]);
//printf("Num mappings %i\n", num_mappings);
if (num_mappings > 0) {
array_list_set_flag(2, mapping_batch->mapping_lists[targets[i]]);
targets[new_num_targets++] = targets[i];
mapping_batch->num_to_do += num_mappings;
}
}
} else {*/
//size_t new_num_targets = 0;
//size_t *new_targets = (size_t *)malloc(array_list_size(fq_batch)*sizeof(size_t));
array_list_t *array_list_aux = array_list_new(256, 1.25f, COLLECTION_MODE_ASYNCHRONIZED);
//Flag 0: The read has simple anchor or any, and need seeds and normal Cal_Seeker
//Flag 1: The read has double anchor and the gap is smaller than MIN_INTRON_SIZE. Cal_Seeker will be make one CAL
//Flag 2: The read has double anchor but the gap is bigger than MIN_INTRON_SIZE.
for (size_t i = 0; i < num_targets; i++) {
read = array_list_get(targets[i], mapping_batch->fq_batch);
//printf("Read Region %s: \n", read->id);
/* if (array_list_get_flag(mapping_batch->mapping_lists[targets[i]]) == 0 ||
array_list_get_flag(mapping_batch->mapping_lists[targets[i]]) == 1) {
array_list_clear(mapping_batch->mapping_lists[targets[i]], bwt_anchor_free);
continue;
}
*/
if (array_list_get_flag(mapping_batch->mapping_lists[targets[i]]) == 0 ||
array_list_get_flag(mapping_batch->mapping_lists[targets[i]]) == 1) {
//Flag 0 Case, Not anchors found, Make normal seeds
// printf("***** Normal Case 0. Not anchors found!\n");
for (int j = array_list_size(mapping_batch->mapping_lists[targets[i]]) - 1; j >= 0; j--) {
bwt_anchor = array_list_remove_at(j, mapping_batch->mapping_lists[targets[i]]);
array_list_insert(bwt_anchor, array_list_aux);
}
num_mappings = 0;
num_mappings = bwt_map_exact_seeds_seq(0,
0,
read->sequence,
seed_size,
min_seed_size,
input->bwt_optarg_p,
input->bwt_index_p,
mapping_batch->mapping_lists[targets[i]],
0);
if (num_mappings > 0) {
array_list_set_flag(0, mapping_batch->mapping_lists[targets[i]]);
targets[new_num_targets++] = targets[i];
//mapping_batch->num_to_do += num_mappings;
}
} else if (array_list_get_flag(mapping_batch->mapping_lists[targets[i]]) == 1) {
//Flag 1 Case, One anchor found, Make displacements seeds
printf("***** Case 1. One anchor found!\n");
for (int j = array_list_size(mapping_batch->mapping_lists[targets[i]]) - 1; j >= 0; j--) {
bwt_anchor = array_list_remove_at(j, mapping_batch->mapping_lists[targets[i]]);
array_list_insert(bwt_anchor, array_list_aux);
}
int anchor_nt = bwt_anchor->end - bwt_anchor->start;
int seed_id = 0;
int seed_start, seed_end;
int extra_seed;
if ((bwt_anchor->type == FORWARD_ANCHOR && bwt_anchor->strand == 0) ||
(bwt_anchor->type == BACKWARD_ANCHOR && bwt_anchor->strand == 1 )) {
start_search = anchor_nt + 1;
end_search = read->length - 1;
extra_seed = EXTRA_SEED_END;
} else {
start_search = 0;
end_search = read->length - anchor_nt - 2;
extra_seed = EXTRA_SEED_START;
}
printf("end_start %i - start_search %i = %i >= seed_size %i\n", end_search, start_search, end_search - start_search, seed_size);
if (end_search - start_search >= seed_size) {
printf("00 bwt_map_exact_seeds_between_coords --> searching from %i to %i\n", start_search, end_search);
/*
num_mappings = bwt_map_exact_seeds_between_coords(start_search,
end_search,
read->sequence,
seed_size, min_seed_size,
input->bwt_optarg_p,
input->bwt_index_p,
mapping_batch->mapping_lists[targets[i]],
extra_seed, &seed_id);
*/
}
if (bwt_anchor->type == FORWARD_ANCHOR) {
seed_id = 0;
seed_start = 0;
seed_end = anchor_nt;
} else {
seed_id += 1;
seed_start = read->length - anchor_nt - 1;
seed_end = read->length - 1;
}
for (int j = 0; j < array_list_size(array_list_aux); j++) {
bwt_anchor_t *bwt_anchor = array_list_get(j, array_list_aux);
// printf("\tCreate seed Anchor [%i:%lu|%i-%i|%lu]\n", bwt_anchor->chromosome + 1, bwt_anchor->start,
// seed_start,seed_end,bwt_anchor->end);
region = region_bwt_new(bwt_anchor->chromosome + 1,
bwt_anchor->strand,
bwt_anchor->start,
bwt_anchor->end,
seed_start,
seed_end,
read->length,
seed_id);
array_list_insert(region, mapping_batch->mapping_lists[targets[i]]);
}
array_list_clear(array_list_aux, (void *)bwt_anchor_free);
array_list_set_flag(0, mapping_batch->mapping_lists[targets[i]]);
targets[new_num_targets++] = targets[i];
} else {
//Flag 2 Case, Pair of anchors found
printf("***** Case 2. Double anchor found!\n");
bwt_anchor_t *bwt_anchor;
bwt_anchor_t *bwt_anchor_forw, *bwt_anchor_back;
int read_nt, genome_nt;
int distance;
int found = 0;
region_t *region;
int seed_id = 0;
//if (array_list_size(mapping_batch->mapping_lists[targets[i]]) > 2) {
int *anchors_targets = (int *)calloc(array_list_size(mapping_batch->mapping_lists[targets[i]]), sizeof(int));
int num = 0;
//min_intron_size = 0;
//Search if one anchor is at the same distance from the reference and the read
for (int b = 0; b < array_list_size(mapping_batch->mapping_lists[targets[i]]); b += 2) {
bwt_anchor_forw = array_list_get(b, mapping_batch->mapping_lists[targets[i]]);
bwt_anchor_back = array_list_get(b + 1, mapping_batch->mapping_lists[targets[i]]);
//printf("FORW=%i:%lu-%lu BACK=%i:%lu-%lu\n", bwt_anchor_forw->chromosome, bwt_anchor_forw->start, bwt_anchor_forw->end,
// bwt_anchor_back->chromosome, bwt_anchor_back->start, bwt_anchor_back->end);
read_nt = read->length - ((bwt_anchor_forw->end - bwt_anchor_forw->start) + (bwt_anchor_back->end - bwt_anchor_back->start));
genome_nt = bwt_anchor_back->start - bwt_anchor_forw->end;
distance = abs(genome_nt - read_nt);
//printf("\t%i:Distance %i\n", b, distance);
if (distance < min_intron_size) {
found = 1;
} else {
anchors_targets[num++] = b;
}
}
if (found) {
//printf("\tFound Exact Case... Delete other anchors\n");
for (int t = num - 1; t >= 0; t--) {
target = anchors_targets[t];
//printf("\tDelete %i, %i-->\n", target, target + 1);
bwt_anchor = array_list_remove_at(target + 1, mapping_batch->mapping_lists[targets[i]]);
bwt_anchor_free(bwt_anchor);
bwt_anchor = array_list_remove_at(target, mapping_batch->mapping_lists[targets[i]]);
bwt_anchor_free(bwt_anchor);
}
array_list_set_flag(1, mapping_batch->mapping_lists[targets[i]]);
} else {
//Seeding between anchors
//printf("\tFound gap between anchors \n");
array_list_t *anchors_forward = array_list_new(array_list_size(mapping_batch->mapping_lists[targets[i]]),
1.25f, COLLECTION_MODE_ASYNCHRONIZED);
array_list_t *anchors_backward = array_list_new(array_list_size(mapping_batch->mapping_lists[targets[i]]),
1.25f, COLLECTION_MODE_ASYNCHRONIZED);
int big_gap = 0;
int final_anchor_nt = 0;
int anchor_nt;
int anchor_type;
int anchor_strand;
for (int j = array_list_size(mapping_batch->mapping_lists[targets[i]]) - 1; j >= 0; j -= 2) {
bwt_anchor_back = array_list_remove_at(j, mapping_batch->mapping_lists[targets[i]]);
array_list_insert(bwt_anchor_back, anchors_backward);
bwt_anchor_forw = array_list_remove_at(j - 1, mapping_batch->mapping_lists[targets[i]]);
array_list_insert(bwt_anchor_forw, anchors_forward);
if (bwt_anchor_forw->strand == 0) {
anchor_nt = bwt_anchor_forw->end - bwt_anchor_forw->start;
gap_nt = read->length - (anchor_nt + (bwt_anchor_back->end - bwt_anchor_back->start));
} else {
anchor_nt = bwt_anchor_back->end - bwt_anchor_back->start;
gap_nt = read->length - (anchor_nt + (bwt_anchor_forw->end - bwt_anchor_forw->start));
}
if (gap_nt < 0) { gap_nt = 0; }
//printf("Gap nt (%i - %i): %i\n", anchor_nt, bwt_anchor_back->end - bwt_anchor_back->start, gap_nt);
if (gap_nt > big_gap) {
big_gap = gap_nt;
final_anchor_nt = anchor_nt;
anchor_type = bwt_anchor_back->type;
anchor_strand = bwt_anchor_back->strand;
}
}
printf("%i, %i\n", big_gap - 2, seed_size);
if (big_gap - 2 > seed_size) {
//if (anchor_type == FORWARD_ANCHOR && anchor_strand == 0 ||
// anchor_type == BACKWARD_ANCHOR && anchor_strand == 1 ) {
start_search = final_anchor_nt + 1;
end_search = final_anchor_nt + big_gap - 1;
//} else {
// start_search = final_anchor_nt + big_gap - 1;
//end_search = final_anchor_nt + 1;
//}
//printf("Seeding between anchors... gap=%i\n", big_gap);
printf("11 bwt_map_exact_seeds_between_coords --> searching from %i to %i\n", start_search, end_search);
/*
num_mappings = bwt_map_exact_seeds_between_coords(start_search,
end_search,
read->sequence, seed_size, min_seed_size,
input->bwt_optarg_p,
input->bwt_index_p,
mapping_batch->mapping_lists[targets[i]],
EXTRA_SEED_NONE,
&seed_id);
*/
}
//printf("Making seeds anchors...\n");
for (int a = 0; a < array_list_size(anchors_forward); a++) {
//Insert the last anchor. (Create new seed)
bwt_anchor_forw = array_list_get(a, anchors_forward);
bwt_anchor_back = array_list_get(a, anchors_backward);
anchor_nt = bwt_anchor_forw->end - bwt_anchor_forw->start;
gap_nt = read->length - (anchor_nt + (bwt_anchor_back->end - bwt_anchor_back->start));
//printf("\t --> Big Seed: %i, gap_nt: %i, anchor_nt = %i\n", a, gap_nt, anchor_nt);
if (gap_nt < 0) {
//gap_nt = 0;
bwt_anchor_forw->end += gap_nt;
bwt_anchor_back->start -= gap_nt;
anchor_nt += gap_nt;
gap_nt = 0;
} else if (gap_nt == 0) {
bwt_anchor_forw->end -= 1;
bwt_anchor_back->start += 1;
anchor_nt -= 1;
gap_nt = 1;
}
region = region_bwt_new(bwt_anchor_forw->chromosome + 1,
bwt_anchor_forw->strand,
bwt_anchor_forw->start,
bwt_anchor_forw->end,
0,
anchor_nt,
read->length,
0);
//printf("Region: %i-%i\n", region->seq_start, region->seq_end);
array_list_insert(region, mapping_batch->mapping_lists[targets[i]]);
region = region_bwt_new(bwt_anchor_back->chromosome + 1,
bwt_anchor_back->strand,
bwt_anchor_back->start,
bwt_anchor_back->end,
anchor_nt + gap_nt,
read->length - 1,
read->length,
seed_id + 1);
//printf("Region: %i-%i\n", region->seq_start, region->seq_end);
array_list_insert(region, mapping_batch->mapping_lists[targets[i]]);
//printf("\tMaking seeds anchors end, %i seeds\n", array_list_size(mapping_batch->mapping_lists[targets[i]]));
bwt_anchor_free(bwt_anchor_back);
bwt_anchor_free(bwt_anchor_forw);
}
array_list_free(anchors_forward, NULL);
array_list_free(anchors_backward, NULL);
//printf("Making seeds anchors end, %i seeds\n", array_list_size(mapping_batch->mapping_lists[targets[i]]));
array_list_set_flag(2, mapping_batch->mapping_lists[targets[i]]);
}
free(anchors_targets);
targets[new_num_targets++] = targets[i];
}
}
mapping_batch->num_targets = new_num_targets;
array_list_free(array_list_aux, NULL);
//if (time_on) { stop_timer(start, end, time); timing_add(time, REGION_SEEKER, timing); }
//printf("APPLY SEEDING DONE!\n");
return CAL_STAGE;
}
