/**
* Gets records for the most recent position and fills up the buffer from file i.
* returns true if buffer is filled or it is not necessary to fill buffer.
* returns false if no more records are found to fill buffer
*/
void BCFSyncedStreamReader::fill_buffer(int32_t i)
{
//not necessary to fill buffer
if (buffer[i].size()>=2)
return;
int32_t pos1 = buffer[i].size()==0 ? 0 : bcf_get_pos1(buffer[i].front());
if (ftypes[i]==FT_BCF_GZ)
{
bcf1_t *v = get_bcf1_from_pool();
bool populated = false;
while (itrs[i] && bcf_itr_next(vcfs[i], itrs[i], v) >= 0)
{
populated = true;
bcf_unpack(v, BCF_UN_STR);
buffer[i].push_back(v);
insert_into_pq(i, v);
if (pos1==0)
{
pos1 = bcf_get_pos1(v);
}
if (bcf_get_pos1(v)!=pos1)
{
break;
}
v = get_bcf1_from_pool();
populated = false;
}
if (!populated)
store_bcf1_into_pool(v);
}
else if (ftypes[i]==FT_VCF_GZ)
{
while (itrs[i] && tbx_itr_next(vcfs[i], tbxs[i], itrs[i], &s) >= 0)
{
bcf1_t *v = get_bcf1_from_pool();
vcf_parse(&s, hdrs[i], v);
bcf_unpack(v, BCF_UN_STR);
buffer[i].push_back(v);
insert_into_pq(i, v);
if (pos1==0)
{
pos1 = bcf_get_pos1(v);
}
if (bcf_get_pos1(v)!=pos1)
{
break;
}
}
}
}
int ingest1(const char *input,const char *output,char *ref,bool exit_on_mismatch=true) {
cerr << "Input: " << input << "\tOutput: "<<output<<endl;
kstream_t *ks;
kstring_t str = {0,0,0};
gzFile fp = gzopen(input, "r");
VarBuffer vbuf(1000);
int prev_rid = -1;
if(fp==NULL) {
fprintf(stderr,"problem opening %s\n",input);
exit(1);
}
char *out_fname = (char *)malloc(strlen(output)+5);
strcpy(out_fname,output);
strcat(out_fname,".tmp");
if(fileexists(out_fname)) {
fprintf(stderr,"%s file already exists. will not overwrite\n",out_fname);
exit(1);
}
printf("depth: %s\n",out_fname);
gzFile depth_fp = gzopen(out_fname, "wb1");
strcpy(out_fname,output);
strcat(out_fname,".bcf");
if(fileexists(out_fname)) {
fprintf(stderr,"%s file already exists. will not overwrite\n",out_fname);
exit(1);
}
printf("variants: %s\n",out_fname);
htsFile *variant_fp=hts_open(out_fname,"wb1");
if(variant_fp==NULL) {
fprintf(stderr,"problem opening %s\n",input);
exit(1);
}
ks = ks_init(fp);
htsFile *hfp=hts_open(input, "r");
bcf_hdr_t *hdr_in = bcf_hdr_read(hfp);
hts_close(hfp);
//this is a hack to fix gvcfs where AD is incorrectly defined in the header. (vcf4.2 does not technically allow Number=R)
bcf_hdr_remove(hdr_in,BCF_HL_FMT,"AD");
assert( bcf_hdr_append(hdr_in,"##FORMAT=<ID=AD,Number=R,Type=Integer,Description=\"Allelic depths for the ref and alt alleles in the order listed. For indels this value only includes reads which confidently support each allele (posterior prob 0.999 or higher that read contains indicated allele vs all other intersecting indel alleles)\">") == 0);
//this is a hack to fix broken gvcfs where GQ is incorrectly labelled as float (v4.3 spec says it should be integer)
bcf_hdr_remove(hdr_in,BCF_HL_FMT,"GQ");
assert( bcf_hdr_append(hdr_in,"##FORMAT=<ID=GQ,Number=1,Type=Integer,Description=\"Genotype Quality\">") == 0);
// bcf_hdr_t *hdr_out=hdr_in;
bcf_hdr_t *hdr_out = bcf_hdr_dup(hdr_in);
remove_hdr_lines(hdr_out,BCF_HL_INFO);
remove_hdr_lines(hdr_out,BCF_HL_FLT);
bcf_hdr_sync(hdr_out);
//here we add FORMAT/PF. which is the pass filter flag for alts.
assert( bcf_hdr_append(hdr_out,"##FORMAT=<ID=PF,Number=A,Type=Integer,Description=\"variant was PASS filter in original sample gvcf\">") == 0);
args_t *norm_args = init_vcfnorm(hdr_out,ref);
norm_args->check_ref |= CHECK_REF_WARN;
bcf1_t *bcf_rec = bcf_init();
bcf_hdr_write(variant_fp, hdr_out);
kstring_t work1 = {0,0,0};
int buf[5];
ks_tokaux_t aux;
int ndec=0;
int ref_len,alt_len;
while( ks_getuntil(ks, '\n', &str, 0) >=0) {
// fprintf(stderr,"%s\n",str.s);
if(str.s[0]!='#') {
char *ptr = kstrtok(str.s,"\t",&aux);//chrom
ptr = kstrtok(NULL,NULL,&aux);//pos
work1.l=0;
kputsn(str.s,ptr-str.s-1, &work1);
buf[0] = bcf_hdr_name2id(hdr_in, work1.s);
assert( buf[0]>=0);
buf[1]=atoi(ptr)-1;
ptr = kstrtok(NULL,NULL,&aux);//ID
ptr = kstrtok(NULL,NULL,&aux);//REF
ref_len=0;
while(ptr[ref_len]!='\t') ref_len++;
ptr = kstrtok(NULL,NULL,&aux);//ALT
bool is_variant=false;
alt_len=0;
while(ptr[alt_len]!='\t') alt_len++;
if(ptr[0]!='.')
is_variant=true;
char * QUAL_ptr = kstrtok(NULL, NULL, &aux);
assert (QUAL_ptr != NULL);
for(int i=0;i<2;i++) ptr = kstrtok(NULL,NULL,&aux);// gets us to INFO
//find END if it is there
char *end_ptr=strstr(ptr,"END=") ;
if(end_ptr!=NULL)
buf[2]=atoi(end_ptr+4)-1;
else
buf[2]=buf[1]+alt_len-1;
ptr = kstrtok(NULL,NULL,&aux);//FORMAT
//find index of DP (if present)
//if not present, dont output anything (indels ignored)
char *DP_ptr = find_format(ptr,"DP");
int GQX = 0;
int QUAL = 0;
// AH: change code to use the minimum of GQ and QUAL fields if
// GQX is not defined. See here:
// https://support.basespace.illumina.com/knowledgebase/articles/144844-vcf-file
// "GQXGenotype quality. GQX is the minimum of the GQ value
// and the QUAL column. In general, these are similar values;
// taking the minimum makes GQX the more conservative measure of
// genotype quality."
if(DP_ptr!=NULL) {
buf[3]=atoi(DP_ptr);
char *GQX_ptr = find_format(ptr,"GQX");
if (GQX_ptr == NULL)
{
GQX_ptr = find_format(ptr,"GQ");
GQX = atoi(GQX_ptr);
if (QUAL_ptr[0] != '.')
{
QUAL = atoi(QUAL_ptr);
if (QUAL < GQX)
GQX = QUAL;
}
}
else
{
GQX = atoi(GQX_ptr);
}
//trying to reduce entropy on GQ to get better compression performance.
//1. rounds down to nearest 10.
//2. sets gq to min(gq,100).
buf[4]=GQX/10;
buf[4]*=10;
if(buf[4]>100) buf[4]=100;
// printf("%d\t%d\t%d\t%d\t%d\n",buf[0],buf[1],buf[2],buf[3],buf[4]);
if(gzwrite(depth_fp,buf,5*sizeof(int))!=(5*sizeof(int)))
die("ERROR: problem writing "+(string)out_fname+".tmp");
}
if(is_variant) {//wass this a variant? if so write it out to the bcf
norm_args->ntotal++;
vcf_parse(&str,hdr_in,bcf_rec);
// cerr<<bcf_rec->rid<<":"<<bcf_rec->pos<<endl;
if(prev_rid!=bcf_rec->rid)
vbuf.flush(variant_fp,hdr_out);
else
vbuf.flush(bcf_rec->pos,variant_fp,hdr_out);
prev_rid=bcf_rec->rid;
int32_t pass = bcf_has_filter(hdr_in, bcf_rec, ".");
bcf_update_format_int32(hdr_out,bcf_rec,"PF",&pass,1);
bcf_update_filter(hdr_out,bcf_rec,NULL,0);
if(bcf_rec->n_allele>2) {//split multi-allelics (using vcfnorm.c from bcftools1.3
norm_args->nsplit++;
split_multiallelic_to_biallelics(norm_args,bcf_rec );
for(int i=0;i<norm_args->ntmp_lines;i++){
remove_info(norm_args->tmp_lines[i]);
if(realign(norm_args,norm_args->tmp_lines[i]) != ERR_REF_MISMATCH)
ndec+=decompose(norm_args->tmp_lines[i],hdr_out,vbuf);
else
if(exit_on_mismatch)
die("vcf did not match the reference");
else
norm_args->nskipped++;
}
}
else {
remove_info(bcf_rec);
if( realign(norm_args,bcf_rec) != ERR_REF_MISMATCH)
ndec+=decompose(bcf_rec,hdr_out,vbuf);
else
if(exit_on_mismatch)
die("vcf did not match the reference");
else
norm_args->nskipped++;
}
vbuf.flush(bcf_rec->pos,variant_fp,hdr_out);
}
}
}
vbuf.flush(variant_fp,hdr_out);
bcf_hdr_destroy(hdr_in);
bcf_hdr_destroy(hdr_out);
bcf_destroy1(bcf_rec);
ks_destroy(ks);
gzclose(fp);
gzclose(depth_fp);
free(str.s);
free(work1.s);
hts_close(variant_fp);
destroy_data(norm_args);
fprintf(stderr,"Variant lines total/split/realigned/skipped:\t%d/%d/%d/%d\n", norm_args->ntotal,norm_args->nsplit,norm_args->nchanged,norm_args->nskipped);
fprintf(stderr,"Decomposed %d MNPs\n", ndec);
fprintf(stderr,"Indexing %s\n",out_fname);
bcf_index_build(out_fname, BCF_LIDX_SHIFT);
free(out_fname);
return 0;
}
// @param vcf_pos is 0-based
// @param prev_base is -1 if SNP otherwise previous base
// @param next_base is -1 unless indel at position 0
static void print_vcf_entry(size_t vcf_pos, int8_t prev_base, int8_t next_base,
const char *ref, const char *alt, size_t len,
const uint8_t *gts, size_t nsamples,
CallDecomp *dc, const AlignedCall *call,
size_t max_allele_len)
{
dc->stats.nvars++;
StrBuf *sbuf = &dc->sbuf;
strbuf_reset(sbuf);
// Check actual allele length
size_t i, alt_bases = 0;
for(i = 0; i < len; i++) alt_bases += (alt[i] != '-');
if(alt_bases > max_allele_len) { dc->stats.nallele_too_long++; return; }
// CHROM POS ID REF ALT QUAL FILTER INFO
strbuf_append_str(sbuf, call->chrom->name.b);
strbuf_append_char(sbuf, '\t');
strbuf_append_ulong(sbuf, vcf_pos+1);
strbuf_append_str(sbuf, "\t.\t");
print_vcf_allele(ref, len, prev_base, next_base, sbuf);
strbuf_append_char(sbuf, '\t');
print_vcf_allele(alt, len, prev_base, next_base, sbuf);
strbuf_append_str(sbuf, "\t.\tPASS\t");
strbuf_append_str(sbuf, call->info.b ? call->info.b : ".");
strbuf_append_str(sbuf, "\tGT");
// Print genotypes
for(i = 0; i < nsamples; i++) {
strbuf_append_char(sbuf, '\t');
strbuf_append_char(sbuf, gts[i] ? '1' : '.');
}
strbuf_append_char(sbuf, '\n');
// fprintf(stderr, " prev_base:%i next_base:%i info:%s\n", prev_base, next_base, call->info.b);
// fprintf(stderr, "%s [%zu vs %zu]\n", sbuf->b, sbuf->end, strlen(sbuf->b));
kstring_t ks = {.l = sbuf->end, .m = sbuf->size, .s = sbuf->b};
if(vcf_parse(&ks, dc->vcfhdr, dc->v) != 0)
die("Cannot construct VCF entry: %s", sbuf->b);
if(bcf_write(dc->vcffh, dc->vcfhdr, dc->v) != 0)
die("Cannot write VCF entry [nsamples: %zu vs %zu]", nsamples, (size_t)bcf_hdr_nsamples(dc->vcfhdr));
// Move back into our string buffer
sbuf->b = ks.s;
sbuf->size = ks.m;
dc->stats.nvars_printed++;
}
// `ref` and `alt` are aligned alleles - should both be same length strings
// of 'ACGT-'
// return first mismatch position or -1
static int align_get_start(const char *ref, const char *alt)
{
const char *start = ref;
while(*ref) {
if(*ref != *alt) return (ref - start);
ref++; alt++;
}
return -1;
}
// `ref` and `alt` are aligned alleles - should both be same length strings
// of 'ACGT-'
// return first matching position
static int align_get_end(const char *ref, const char *alt)
{
int i = 0;
while(ref[i] && ref[i] != alt[i]) i++;
return i;
}
/**
* Gets records for the most recent position and fills up the buffer from file i.
* returns true if buffer is filled or it is not necessary to fill buffer.
* returns false if no more records are found to fill buffer
*/
void BCFSyncedReader::fill_buffer(int32_t i)
{
if (buffer[i].size()>=2)
return;
if (random_access)
{
int32_t pos1 = buffer[i].size()==0 ? 0 : bcf_get_pos1(buffer[i].front());
if (ftypes[i].format==bcf)
{
bcf1_t *v = get_bcf1_from_pool();
bool populated = false;
while (itrs[i] && bcf_itr_next(files[i], itrs[i], v)>=0)
{
populated = true;
bcf_unpack(v, BCF_UN_STR);
//check to ensure order
if (!buffer[i].empty())
{
if (!bcf_is_in_order(buffer[i].back(), v))
{
fprintf(stderr, "[E:%s:%d %s] VCF file not in order: %s\n", __FILE__, __LINE__, __FUNCTION__, file_names[i].c_str());
exit(1);
}
}
buffer[i].push_back(v);
insert_into_pq(i, v);
if (pos1==0)
{
pos1 = bcf_get_pos1(v);
}
if (bcf_get_pos1(v)!=pos1)
{
break;
}
v = get_bcf1_from_pool();
populated = false;
}
if (!populated)
store_bcf1_into_pool(v);
}
else if (ftypes[i].format==vcf)
{
while (itrs[i] && tbx_itr_next(files[i], tbxs[i], itrs[i], &s)>=0)
{
bcf1_t *v = get_bcf1_from_pool();
vcf_parse(&s, hdrs[i], v);
bcf_unpack(v, BCF_UN_STR);
//check to ensure order
if (!buffer[i].empty())
{
if (!bcf_is_in_order(buffer[i].back(), v))
{
fprintf(stderr, "[E:%s:%d %s] VCF file not in order: %s\n", __FILE__, __LINE__, __FUNCTION__, file_names[i].c_str());
exit(1);
}
}
buffer[i].push_back(v);
insert_into_pq(i, v);
if (pos1==0)
{
pos1 = bcf_get_pos1(v);
}
if (bcf_get_pos1(v)!=pos1)
{
break;
}
}
}
}
else
{
int32_t rid = buffer[i].size()==0 ? -1 : bcf_get_rid(buffer[i].front());
int32_t pos1 = buffer[i].size()==0 ? 0 : bcf_get_pos1(buffer[i].front());
bcf1_t *v = get_bcf1_from_pool();
bool populated = false;
while (bcf_read(files[i], hdrs[i], v)>=0)
{
populated = true;
bcf_unpack(v, BCF_UN_STR);
//check to ensure order
if (!buffer[i].empty())
{
if (!bcf_is_in_order(buffer[i].back(), v))
{
fprintf(stderr, "[E:%s:%d %s] VCF file not in order: %s\n", __FILE__, __LINE__, __FUNCTION__, file_names[i].c_str());
exit(1);
}
}
buffer[i].push_back(v);
insert_into_pq(i, v);
if (rid==-1)
{
rid = bcf_get_rid(v);
pos1 = bcf_get_pos1(v);
}
if (bcf_get_rid(v)!=rid || bcf_get_pos1(v)!=pos1)
{
break;
}
v = get_bcf1_from_pool();
populated = false;
}
if (!populated)
store_bcf1_into_pool(v);
}
}
