void BlockQuantify::count()
{
_impl->fasta_to_use.reset(new FastaFile(_impl->ref_fasta));
#ifdef DEBUG_BLOCKQUANTIFY
int lastpos = 0;
std::cerr << "starting block." << "\n";
#endif
auto current_bs_start = _impl->variants.begin();
std::string current_chr;
int current_bs = -1;
bool current_bs_valid = false;
// function to compute the QQ values for truth variants in the current
// benchmarking superlocus
const auto update_bs_filters = [this, ¤t_bs_start](BlockQuantifyImpl::variantlist_t::iterator to)
{
std::set<int> bs_filters;
for(auto cur = current_bs_start; cur != to; ++cur)
{
for(int nf = 0; nf < (*cur)->d.n_flt; ++nf)
{
const int f = (*cur)->d.flt[nf];
if(f != bcf_hdr_id2int(_impl->hdr, BCF_DT_ID, "PASS"))
{
bs_filters.insert(f);
}
}
}
if(bs_filters.empty())
{
return;
}
for(auto cur = current_bs_start; cur != to; ++cur)
{
const std::string bdt = bcfhelpers::getFormatString(_impl->hdr, *cur, "BD", 0);
const std::string bvq = bcfhelpers::getFormatString(_impl->hdr, *cur, "BVT", 1);
// filter TPs where the query call in NOCALL
if(bdt == "TP" && bvq == "NOCALL")
{
for(auto f : bs_filters)
{
bcf_add_filter(_impl->hdr, *cur, f);
}
}
}
};
// function to compute the QQ values for truth variants in the current
// benchmarking superlocus
const auto update_bs_qq = [this, ¤t_bs_start](BlockQuantifyImpl::variantlist_t::iterator to)
{
std::vector<float> tp_qqs;
for(auto cur = current_bs_start; cur != to; ++cur)
{
const float qqq = bcfhelpers::getFormatFloat(_impl->hdr, *cur, "QQ", 1);
if(std::isnan(qqq))
{
continue;
}
const std::string bd = bcfhelpers::getFormatString(_impl->hdr, *cur, "BD", 1);
// we want the scores of all TPs in this BS
if(bd == "TP")
{
tp_qqs.push_back(qqq);
}
}
float t_qq = bcfhelpers::missing_float();
if(!tp_qqs.empty())
{
t_qq = *(std::min_element(tp_qqs.begin(), tp_qqs.end()));
}
/** compute the median over all variants */
int fsize = bcf_hdr_nsamples(_impl->hdr);
float * fmt = (float*)calloc((size_t) fsize, sizeof(float));
for(auto cur = current_bs_start; cur != to; ++cur)
{
const std::string bd = bcfhelpers::getFormatString(_impl->hdr, *cur, "BD", 0);
bcf_get_format_float(_impl->hdr, *cur, "QQ", &fmt, &fsize);
if(bd != "TP")
{
fmt[0] = bcfhelpers::missing_float();
}
else
{
const float qqq = bcfhelpers::getFormatFloat(_impl->hdr, *cur, "QQ", 1);
const std::string bd = bcfhelpers::getFormatString(_impl->hdr, *cur, "BD", 1);
if(bd == "TP" && !std::isnan(qqq))
{
fmt[0] = qqq;
}
else
{
fmt[0] = t_qq;
}
}
bcf_update_format_float(_impl->hdr, *cur, "QQ", fmt, fsize);
}
free(fmt);
#ifdef DEBUG_BLOCKQUANTIFY
const int bs = bcfhelpers::getInfoInt(_impl->hdr, *current_bs_start, "BS", -1);
std::string values;
for(float x : tp_qqs)
{
values += std::to_string(x) + ",";
}
std::cerr << "BS: " << bs << " T_QQ = " << t_qq << " [" << values << "]" << "\n";
#endif
};
const auto update_bs_conf_boundary_flag = [this, ¤t_bs_start](BlockQuantifyImpl::variantlist_t::iterator to)
{
static const int has_conf = 1;
static const int has_non_conf = 2;
int conf_non_conf = 0;
for(auto cur = current_bs_start; cur != to; ++cur)
{
const std::string regions = bcfhelpers::getInfoString(_impl->hdr, *cur, "Regions", "");
if(regions.find("CONF") == std::string::npos)
{
conf_non_conf |= has_non_conf;
}
else
{
conf_non_conf |= has_conf;
}
if(regions.find("TS_boundary") != std::string::npos)
{
conf_non_conf |= has_non_conf | has_conf;
}
}
for(auto cur = current_bs_start; cur != to; ++cur)
{
const std::string regions = bcfhelpers::getInfoString(_impl->hdr, *cur, "Regions", "");
if(conf_non_conf == (has_conf | has_non_conf))
{
if(regions.find("TS_boundary") == std::string::npos)
{
bcf_update_info_string(_impl->hdr,
*cur, "Regions",
(regions.empty() ? "TS_boundary" :
regions + ",TS_boundary").c_str());
}
}
else if(conf_non_conf == has_conf)
{
if(regions.find("TS_contained") == std::string::npos)
{
// also flag fully confident superloci
bcf_update_info_string(_impl->hdr,
*cur, "Regions",
(regions.empty() ? "TS_contained" :
regions + ",TS_contained").c_str());
}
}
}
};
for(auto v_it = _impl->variants.begin(); v_it != _impl->variants.end(); ++v_it)
{
// update fields, must output GA4GH-compliant fields
countVariants(*v_it);
// determine benchmarking superlocus
const std::string vchr = bcfhelpers::getChrom(_impl->hdr, *v_it);
const int vbs = bcfhelpers::getInfoInt(_impl->hdr, *v_it, "BS");
if(!current_bs_valid)
{
current_bs = vbs;
current_chr = vchr;
current_bs_valid = true;
}
#ifdef DEBUG_BLOCKQUANTIFY
std::cerr << "current BS = " << current_bs << " vbs = " << vbs << "\n";
#endif
if( current_bs_start != v_it
&& (vbs != current_bs || vbs < 0 || vchr != current_chr))
{
#ifdef DEBUG_BLOCKQUANTIFY
std::cerr << "finishing BS = " << current_bs << " vbs = " << vbs << "\n";
#endif
update_bs_qq(v_it);
update_bs_filters(v_it);
update_bs_conf_boundary_flag(v_it);
current_bs = vbs;
current_chr = vchr;
current_bs_start = v_it;
}
}
// do final superlocus (if any)
update_bs_qq(_impl->variants.end());
update_bs_filters(_impl->variants.end());
update_bs_conf_boundary_flag(_impl->variants.end());
for(auto & v : _impl->variants)
{
#ifdef DEBUG_BLOCKQUANTIFY
lastpos = v->pos;
#endif
// use BD and BVT to make ROCs
rocEvaluate(v);
}
#ifdef DEBUG_BLOCKQUANTIFY
std::cerr << "finished block " << lastpos << " - " << _impl->variants.size() << " records on thread " << std::this_thread::get_id() << "\n";
#endif
_impl->fasta_to_use.reset(nullptr);
}
static void buffered_filters(args_t *args, bcf1_t *line)
{
/**
* The logic of SnpGap=3. The SNPs at positions 1 and 7 are filtered,
* positions 0 and 8 are not:
* 0123456789
* ref .G.GT..G..
* del .A.G-..A..
* Here the positions 1 and 6 are filtered, 0 and 7 are not:
* 0123-456789
* ref .G.G-..G..
* ins .A.GT..A..
*
* The logic of IndelGap=2. The second indel is filtered:
* 012345678901
* ref .GT.GT..GT..
* del .G-.G-..G-..
* And similarly here, the second is filtered:
* 01 23 456 78
* ref .A-.A-..A-..
* ins .AT.AT..AT..
*/
// To avoid additional data structure, we abuse bcf1_t's var and var_type records.
const int SnpGap_set = VCF_OTHER<<1;
const int IndelGap_set = VCF_OTHER<<2;
const int IndelGap_flush = VCF_OTHER<<3;
int var_type = 0, i;
if ( line )
{
// Still on the same chromosome?
int ilast = rbuf_last(&args->rbuf);
if ( ilast>=0 && line->rid != args->rbuf_lines[ilast]->rid )
flush_buffer(args, args->rbuf.n); // new chromosome, flush everything
rbuf_expand0(&args->rbuf,bcf1_t*,args->rbuf.n,args->rbuf_lines);
// Insert the new record in the buffer. The line would be overwritten in
// the next bcf_sr_next_line call, therefore we need to swap it with an
// unused one
ilast = rbuf_append(&args->rbuf);
if ( !args->rbuf_lines[ilast] ) args->rbuf_lines[ilast] = bcf_init1();
SWAP(bcf1_t*, args->files->readers[0].buffer[0], args->rbuf_lines[ilast]);
var_type = bcf_get_variant_types(line);
// Find out the size of an indel. The indel boundaries are based on REF
// (POS+1,POS+rlen-1). This is not entirely correct: mpileup likes to
// output REF=CAGAGAGAGA, ALT=CAGAGAGAGAGA where REF=C,ALT=CGA could be
// used. This filter is therefore more strict and may remove some valid
// SNPs.
int len = 1;
if ( var_type & VCF_INDEL )
{
for (i=1; i<line->n_allele; i++)
if ( len < 1-line->d.var[i].n ) len = 1-line->d.var[i].n;
}
// Set the REF allele's length to max deletion length or to 1 if a SNP or an insertion.
line->d.var[0].n = len;
}
int k_flush = 1;
if ( args->indel_gap )
{
k_flush = 0;
// Find indels which are too close to each other
int last_to = -1;
for (i=-1; rbuf_next(&args->rbuf,&i); )
{
bcf1_t *rec = args->rbuf_lines[i];
int rec_from = rec->pos;
if ( last_to!=-1 && last_to < rec_from ) break;
k_flush++;
if ( !(rec->d.var_type & VCF_INDEL) ) continue;
rec->d.var_type |= IndelGap_set;
last_to = args->indel_gap + rec->pos + rec->d.var[0].n - 1;
}
if ( i==args->rbuf.f && line && last_to!=-1 ) k_flush = 0;
if ( k_flush || !line )
{
// Select the best indel from the cluster of k_flush indels
int k = 0, max_ac = -1, imax_ac = -1;
for (i=-1; rbuf_next(&args->rbuf,&i) && k<k_flush; )
{
k++;
bcf1_t *rec = args->rbuf_lines[i];
if ( !(rec->d.var_type & IndelGap_set) ) continue;
hts_expand(int, rec->n_allele, args->ntmpi, args->tmpi);
int ret = bcf_calc_ac(args->hdr, rec, args->tmpi, BCF_UN_ALL);
if ( imax_ac==-1 || (ret && max_ac < args->tmpi[1]) ) { max_ac = args->tmpi[1]; imax_ac = i; }
}
// Filter all but the best indel (with max AF or first if AF not available)
k = 0;
for (i=-1; rbuf_next(&args->rbuf,&i) && k<k_flush; )
{
k++;
bcf1_t *rec = args->rbuf_lines[i];
if ( !(rec->d.var_type & IndelGap_set) ) continue;
rec->d.var_type |= IndelGap_flush;
if ( i!=imax_ac ) bcf_add_filter(args->hdr, args->rbuf_lines[i], args->IndelGap_id);
}
}
