/**
* Returns true if chrom:start1-end1 overlaps with a region in the file.
*/
bool OrderedBCFOverlapMatcher::overlaps_with(std::string& chrom, int32_t start1, int32_t end1)
{
bool overlaps = false;
//moves to new chromosome
if (current_interval.seq!=chrom)
{
buffer.clear();
current_interval.set(chrom);
odr->jump_to_interval(current_interval);
std::cerr << "Jumped to chromosome " << chrom << "\n";
while (odr->read(v))
{
if (bcf_get_end_pos1(v)<start1) continue;
overlaps = overlaps || (bcf_get_pos1(v)<=end1);
buffer.push_back(v);
if (bcf_get_pos1(v)>end1) break;
v = bcf_init();
}
}
else
{
//scythe preceding bed records
std::list<bcf1_t*>::iterator i = buffer.begin();
while (i!=buffer.end())
{
if (bcf_get_end_pos1(*i)<start1)
{
bcf_destroy(*i);
i = buffer.erase(i);
continue;
}
overlaps = (bcf_get_pos1(*i)<=end1);
break;
}
if (!overlaps)
{
if (buffer.empty())
{
while (odr->read(v))
{
if (bcf_get_end_pos1(*i)<start1) continue;
overlaps = overlaps || (bcf_get_pos1(*i)<=end1);
buffer.push_back(v);
if (bcf_get_pos1(v)>end1) break;
v = bcf_init();
}
}
}
}
return overlaps;
};
/**
* Constructor.
*/
OrderedBCFOverlapMatcher::OrderedBCFOverlapMatcher(std::string& file, std::vector<GenomeInterval>& intervals)
{
odr = new BCFOrderedReader(file, intervals);
v = bcf_init();
no_regions = 0;
current_interval.seq = "";
};
CallDecomp* call_decomp_init(htsFile *vcffh, bcf_hdr_t *vcfhdr)
{
CallDecomp *dc = ctx_calloc(1, sizeof(CallDecomp));
dc->nw_aligner = needleman_wunsch_new();
dc->aln = alignment_create(1024);
dc->scoring = ctx_calloc(1, sizeof(dc->scoring[0]));
scoring_system_default(dc->scoring);
dc->vcffh = vcffh;
dc->vcfhdr = vcfhdr;
dc->v = bcf_init();
strbuf_alloc(&dc->sbuf, 256);
return dc;
}
static int view_vcf(hFILE *hfp, const char *filename)
{
vcfFile *in = hts_hopen(hfp, filename, "r");
if (in == NULL) return 0;
vcfFile *out = dup_stdout("w");
bcf_hdr_t *hdr = bcf_hdr_read(in);
if (show_headers) bcf_hdr_write(out, hdr);
if (mode == view_all) {
bcf1_t *rec = bcf_init();
while (bcf_read(in, hdr, rec) >= 0)
bcf_write(out, hdr, rec);
bcf_destroy(rec);
}
bcf_hdr_destroy(hdr);
hts_close(out);
hts_close(in);
return 1;
}
static void init_data(args_t *args)
{
bcf1_t *line = NULL;
// With phased concat, the chunks overlap and come in the right order. To
// avoid opening all files at once, store start positions to recognise need
// for the next one. This way we can keep only two open chunks at once.
if ( args->phased_concat )
{
args->start_pos = (int*) malloc(sizeof(int)*args->nfnames);
line = bcf_init();
}
kstring_t str = {0,0,0};
int i, prev_chrid = -1;
for (i=0; i<args->nfnames; i++)
{
htsFile *fp = hts_open(args->fnames[i], "r"); if ( !fp ) error("Failed to open: %s\n", args->fnames[i]);
bcf_hdr_t *hdr = bcf_hdr_read(fp); if ( !hdr ) error("Failed to parse header: %s\n", args->fnames[i]);
args->out_hdr = bcf_hdr_merge(args->out_hdr,hdr);
if ( bcf_hdr_nsamples(hdr) != bcf_hdr_nsamples(args->out_hdr) )
error("Different number of samples in %s. Perhaps \"bcftools merge\" is what you are looking for?\n", args->fnames[i]);
int j;
for (j=0; j<bcf_hdr_nsamples(hdr); j++)
if ( strcmp(args->out_hdr->samples[j],hdr->samples[j]) )
error("Different sample names in %s. Perhaps \"bcftools merge\" is what you are looking for?\n", args->fnames[i]);
if ( args->phased_concat )
{
int ret = bcf_read(fp, hdr, line);
if ( ret!=0 ) args->start_pos[i] = -2; // empty file
else
{
int chrid = bcf_hdr_id2int(args->out_hdr,BCF_DT_CTG,bcf_seqname(hdr,line));
args->start_pos[i] = chrid==prev_chrid ? line->pos : -1;
prev_chrid = chrid;
}
}
bcf_hdr_destroy(hdr);
hts_close(fp);
}
free(str.s);
if ( line ) bcf_destroy(line);
args->seen_seq = (int*) calloc(args->out_hdr->n[BCF_DT_CTG],sizeof(int));
if ( args->phased_concat )
{
bcf_hdr_append(args->out_hdr,"##FORMAT=<ID=PQ,Number=1,Type=Integer,Description=\"Phasing Quality (bigger is better)\">");
bcf_hdr_append(args->out_hdr,"##FORMAT=<ID=PS,Number=1,Type=Integer,Description=\"Phase Set\">");
}
if (args->record_cmd_line) bcf_hdr_append_version(args->out_hdr, args->argc, args->argv, "bcftools_concat");
args->out_fh = hts_open(args->output_fname,hts_bcf_wmode(args->output_type));
if ( args->out_fh == NULL ) error("Can't write to \"%s\": %s\n", args->output_fname, strerror(errno));
if ( args->n_threads ) hts_set_threads(args->out_fh, args->n_threads);
bcf_hdr_write(args->out_fh, args->out_hdr);
if ( args->allow_overlaps )
{
args->files = bcf_sr_init();
args->files->require_index = 1;
if ( args->regions_list )
{
if ( bcf_sr_set_regions(args->files, args->regions_list, args->regions_is_file)<0 )
error("Failed to read the regions: %s\n", args->regions_list);
}
if ( args->remove_dups )
{
if ( !strcmp(args->remove_dups,"snps") ) args->files->collapse |= COLLAPSE_SNPS;
else if ( !strcmp(args->remove_dups,"indels") ) args->files->collapse |= COLLAPSE_INDELS;
else if ( !strcmp(args->remove_dups,"both") ) args->files->collapse |= COLLAPSE_SNPS | COLLAPSE_INDELS;
else if ( !strcmp(args->remove_dups,"any") ) args->files->collapse |= COLLAPSE_ANY;
else if ( !strcmp(args->remove_dups,"all") ) args->files->collapse |= COLLAPSE_ANY;
else if ( !strcmp(args->remove_dups,"none") ) args->files->collapse = COLLAPSE_NONE;
else error("The -D string \"%s\" not recognised.\n", args->remove_dups);
}
for (i=0; i<args->nfnames; i++)
if ( !bcf_sr_add_reader(args->files,args->fnames[i]) ) error("Failed to open %s: %s\n", args->fnames[i],bcf_sr_strerror(args->files->errnum));
}
else if ( args->phased_concat )
{
// Remove empty files from the list
int nok = 0;
while (1)
{
while ( nok<args->nfnames && args->start_pos[nok]!=-2 ) nok++;
if ( nok==args->nfnames ) break;
i = nok;
while ( i<args->nfnames && args->start_pos[i]==-2 ) i++;
if ( i==args->nfnames ) break;
int tmp = args->start_pos[nok]; args->start_pos[nok] = args->start_pos[i]; args->start_pos[i] = tmp;
char *str = args->fnames[nok]; args->fnames[nok] = args->fnames[i]; args->fnames[i] = str;
}
for (i=nok; i<args->nfnames; i++) free(args->fnames[i]);
args->nfnames = nok;
for (i=1; i<args->nfnames; i++)
if ( args->start_pos[i-1]!=-1 && args->start_pos[i]!=-1 && args->start_pos[i]<args->start_pos[i-1] )
error("The files not in ascending order: %d in %s, %d in %s\n", args->start_pos[i-1]+1,args->fnames[i-1],args->start_pos[i]+1,args->fnames[i]);
args->prev_chr = -1;
args->swap_phase = (int*) calloc(bcf_hdr_nsamples(args->out_hdr),sizeof(int));
args->nmatch = (int*) calloc(bcf_hdr_nsamples(args->out_hdr),sizeof(int));
args->nmism = (int*) calloc(bcf_hdr_nsamples(args->out_hdr),sizeof(int));
args->phase_qual = (int32_t*) malloc(bcf_hdr_nsamples(args->out_hdr)*sizeof(int32_t));
args->phase_set = (int32_t*) malloc(bcf_hdr_nsamples(args->out_hdr)*sizeof(int32_t));
args->files = bcf_sr_init();
args->files->require_index = 1;
args->ifname = 0;
}
}
static void concat(args_t *args)
{
int i;
if ( args->phased_concat ) // phased concat
{
// keep only two open files at a time
while ( args->ifname < args->nfnames )
{
int new_file = 0;
while ( args->files->nreaders < 2 && args->ifname < args->nfnames )
{
if ( !bcf_sr_add_reader(args->files,args->fnames[args->ifname]) ) error("Failed to open %s: %s\n", args->fnames[args->ifname],bcf_sr_strerror(args->files->errnum));
new_file = 1;
args->ifname++;
if ( args->start_pos[args->ifname-1]==-1 ) break; // new chromosome, start with only one file open
if ( args->ifname < args->nfnames && args->start_pos[args->ifname]==-1 ) break; // next file starts on a different chromosome
}
// is there a line from the previous run? Seek the newly opened reader to that position
int seek_pos = -1;
int seek_chr = -1;
if ( bcf_sr_has_line(args->files,0) )
{
bcf1_t *line = bcf_sr_get_line(args->files,0);
bcf_sr_seek(args->files, bcf_seqname(args->files->readers[0].header,line), line->pos);
seek_pos = line->pos;
seek_chr = bcf_hdr_name2id(args->out_hdr, bcf_seqname(args->files->readers[0].header,line));
}
else if ( new_file )
bcf_sr_seek(args->files,NULL,0); // set to start
int nret;
while ( (nret = bcf_sr_next_line(args->files)) )
{
if ( !bcf_sr_has_line(args->files,0) ) // no input from the first reader
{
// We are assuming that there is a perfect overlap, sites which are not present in both files are dropped
if ( ! bcf_sr_region_done(args->files,0) ) continue;
phased_flush(args);
bcf_sr_remove_reader(args->files, 0);
}
// Get a line to learn about current position
for (i=0; i<args->files->nreaders; i++)
if ( bcf_sr_has_line(args->files,i) ) break;
bcf1_t *line = bcf_sr_get_line(args->files,i);
// This can happen after bcf_sr_seek: indel may start before the coordinate which we seek to.
if ( seek_chr>=0 && seek_pos>line->pos && seek_chr==bcf_hdr_name2id(args->out_hdr, bcf_seqname(args->files->readers[i].header,line)) ) continue;
seek_pos = seek_chr = -1;
// Check if the position overlaps with the next, yet unopened, reader
int must_seek = 0;
while ( args->ifname < args->nfnames && args->start_pos[args->ifname]!=-1 && line->pos >= args->start_pos[args->ifname] )
{
must_seek = 1;
if ( !bcf_sr_add_reader(args->files,args->fnames[args->ifname]) ) error("Failed to open %s: %s\n", args->fnames[args->ifname],bcf_sr_strerror(args->files->errnum));
args->ifname++;
}
if ( must_seek )
{
bcf_sr_seek(args->files, bcf_seqname(args->files->readers[i].header,line), line->pos);
seek_pos = line->pos;
seek_chr = bcf_hdr_name2id(args->out_hdr, bcf_seqname(args->files->readers[i].header,line));
continue;
}
// We are assuming that there is a perfect overlap, sites which are not present in both files are dropped
if ( args->files->nreaders>1 && !bcf_sr_has_line(args->files,1) && !bcf_sr_region_done(args->files,1) ) continue;
phased_push(args, bcf_sr_get_line(args->files,0), args->files->nreaders>1 ? bcf_sr_get_line(args->files,1) : NULL);
}
if ( args->files->nreaders )
{
phased_flush(args);
while ( args->files->nreaders )
bcf_sr_remove_reader(args->files, 0);
}
}
}
else if ( args->files ) // combining overlapping files, using synced reader
{
while ( bcf_sr_next_line(args->files) )
{
for (i=0; i<args->files->nreaders; i++)
{
bcf1_t *line = bcf_sr_get_line(args->files,i);
if ( !line ) continue;
bcf_translate(args->out_hdr, args->files->readers[i].header, line);
bcf_write1(args->out_fh, args->out_hdr, line);
if ( args->remove_dups ) break;
}
}
}
else // concatenating
{
kstring_t tmp = {0,0,0};
int prev_chr_id = -1, prev_pos;
bcf1_t *line = bcf_init();
for (i=0; i<args->nfnames; i++)
{
htsFile *fp = hts_open(args->fnames[i], "r"); if ( !fp ) error("Failed to open: %s\n", args->fnames[i]);
bcf_hdr_t *hdr = bcf_hdr_read(fp); if ( !hdr ) error("Failed to parse header: %s\n", args->fnames[i]);
if ( !fp->is_bin && args->output_type&FT_VCF )
{
line->max_unpack = BCF_UN_STR;
// if VCF is on both input and output, avoid VCF to BCF conversion
while ( hts_getline(fp, KS_SEP_LINE, &fp->line) >=0 )
{
char *str = fp->line.s;
while ( *str && *str!='\t' ) str++;
tmp.l = 0;
kputsn(fp->line.s,str-fp->line.s,&tmp);
int chr_id = bcf_hdr_name2id(args->out_hdr, tmp.s);
if ( chr_id<0 ) error("The sequence \"%s\" not defined in the header: %s\n(Quick workaround: index the file.)\n", tmp.s, args->fnames[i]);
if ( prev_chr_id!=chr_id )
{
prev_pos = -1;
if ( args->seen_seq[chr_id] )
error("\nThe chromosome block %s is not contiguous, consider running with -a.\n", tmp.s);
}
char *end;
int pos = strtol(str+1,&end,10) - 1;
if ( end==str+1 ) error("Could not parse line: %s\n", fp->line.s);
if ( prev_pos > pos )
error("The chromosome block %s is not sorted, consider running with -a.\n", tmp.s);
args->seen_seq[chr_id] = 1;
prev_chr_id = chr_id;
if ( vcf_write_line(args->out_fh, &fp->line)!=0 ) error("Failed to write %d bytes\n", fp->line.l);
}
}
else
{
// BCF conversion is required
line->max_unpack = 0;
while ( bcf_read(fp, hdr, line)==0 )
{
bcf_translate(args->out_hdr, hdr, line);
if ( prev_chr_id!=line->rid )
{
prev_pos = -1;
if ( args->seen_seq[line->rid] )
error("\nThe chromosome block %s is not contiguous, consider running with -a.\n", bcf_seqname(args->out_hdr, line));
}
if ( prev_pos > line->pos )
error("The chromosome block %s is not sorted, consider running with -a.\n", bcf_seqname(args->out_hdr, line));
args->seen_seq[line->rid] = 1;
prev_chr_id = line->rid;
if ( bcf_write(args->out_fh, args->out_hdr, line)!=0 ) error("Failed to write\n");
}
}
bcf_hdr_destroy(hdr);
hts_close(fp);
}
bcf_destroy(line);
free(tmp.s);
}
}
int ingest1(const char *input,const char *output,char *ref,bool exit_on_mismatch=true) {
cerr << "Input: " << input << "\tOutput: "<<output<<endl;
kstream_t *ks;
kstring_t str = {0,0,0};
gzFile fp = gzopen(input, "r");
VarBuffer vbuf(1000);
int prev_rid = -1;
if(fp==NULL) {
fprintf(stderr,"problem opening %s\n",input);
exit(1);
}
char *out_fname = (char *)malloc(strlen(output)+5);
strcpy(out_fname,output);
strcat(out_fname,".tmp");
if(fileexists(out_fname)) {
fprintf(stderr,"%s file already exists. will not overwrite\n",out_fname);
exit(1);
}
printf("depth: %s\n",out_fname);
gzFile depth_fp = gzopen(out_fname, "wb1");
strcpy(out_fname,output);
strcat(out_fname,".bcf");
if(fileexists(out_fname)) {
fprintf(stderr,"%s file already exists. will not overwrite\n",out_fname);
exit(1);
}
printf("variants: %s\n",out_fname);
htsFile *variant_fp=hts_open(out_fname,"wb1");
if(variant_fp==NULL) {
fprintf(stderr,"problem opening %s\n",input);
exit(1);
}
ks = ks_init(fp);
htsFile *hfp=hts_open(input, "r");
bcf_hdr_t *hdr_in = bcf_hdr_read(hfp);
hts_close(hfp);
//this is a hack to fix gvcfs where AD is incorrectly defined in the header. (vcf4.2 does not technically allow Number=R)
bcf_hdr_remove(hdr_in,BCF_HL_FMT,"AD");
assert( bcf_hdr_append(hdr_in,"##FORMAT=<ID=AD,Number=R,Type=Integer,Description=\"Allelic depths for the ref and alt alleles in the order listed. For indels this value only includes reads which confidently support each allele (posterior prob 0.999 or higher that read contains indicated allele vs all other intersecting indel alleles)\">") == 0);
//this is a hack to fix broken gvcfs where GQ is incorrectly labelled as float (v4.3 spec says it should be integer)
bcf_hdr_remove(hdr_in,BCF_HL_FMT,"GQ");
assert( bcf_hdr_append(hdr_in,"##FORMAT=<ID=GQ,Number=1,Type=Integer,Description=\"Genotype Quality\">") == 0);
// bcf_hdr_t *hdr_out=hdr_in;
bcf_hdr_t *hdr_out = bcf_hdr_dup(hdr_in);
remove_hdr_lines(hdr_out,BCF_HL_INFO);
remove_hdr_lines(hdr_out,BCF_HL_FLT);
bcf_hdr_sync(hdr_out);
//here we add FORMAT/PF. which is the pass filter flag for alts.
assert( bcf_hdr_append(hdr_out,"##FORMAT=<ID=PF,Number=A,Type=Integer,Description=\"variant was PASS filter in original sample gvcf\">") == 0);
args_t *norm_args = init_vcfnorm(hdr_out,ref);
norm_args->check_ref |= CHECK_REF_WARN;
bcf1_t *bcf_rec = bcf_init();
bcf_hdr_write(variant_fp, hdr_out);
kstring_t work1 = {0,0,0};
int buf[5];
ks_tokaux_t aux;
int ndec=0;
int ref_len,alt_len;
while( ks_getuntil(ks, '\n', &str, 0) >=0) {
// fprintf(stderr,"%s\n",str.s);
if(str.s[0]!='#') {
char *ptr = kstrtok(str.s,"\t",&aux);//chrom
ptr = kstrtok(NULL,NULL,&aux);//pos
work1.l=0;
kputsn(str.s,ptr-str.s-1, &work1);
buf[0] = bcf_hdr_name2id(hdr_in, work1.s);
assert( buf[0]>=0);
buf[1]=atoi(ptr)-1;
ptr = kstrtok(NULL,NULL,&aux);//ID
ptr = kstrtok(NULL,NULL,&aux);//REF
ref_len=0;
while(ptr[ref_len]!='\t') ref_len++;
ptr = kstrtok(NULL,NULL,&aux);//ALT
bool is_variant=false;
alt_len=0;
while(ptr[alt_len]!='\t') alt_len++;
if(ptr[0]!='.')
is_variant=true;
char * QUAL_ptr = kstrtok(NULL, NULL, &aux);
assert (QUAL_ptr != NULL);
for(int i=0;i<2;i++) ptr = kstrtok(NULL,NULL,&aux);// gets us to INFO
//find END if it is there
char *end_ptr=strstr(ptr,"END=") ;
if(end_ptr!=NULL)
buf[2]=atoi(end_ptr+4)-1;
else
buf[2]=buf[1]+alt_len-1;
ptr = kstrtok(NULL,NULL,&aux);//FORMAT
//find index of DP (if present)
//if not present, dont output anything (indels ignored)
char *DP_ptr = find_format(ptr,"DP");
int GQX = 0;
int QUAL = 0;
// AH: change code to use the minimum of GQ and QUAL fields if
// GQX is not defined. See here:
// https://support.basespace.illumina.com/knowledgebase/articles/144844-vcf-file
// "GQXGenotype quality. GQX is the minimum of the GQ value
// and the QUAL column. In general, these are similar values;
// taking the minimum makes GQX the more conservative measure of
// genotype quality."
if(DP_ptr!=NULL) {
buf[3]=atoi(DP_ptr);
char *GQX_ptr = find_format(ptr,"GQX");
if (GQX_ptr == NULL)
{
GQX_ptr = find_format(ptr,"GQ");
GQX = atoi(GQX_ptr);
if (QUAL_ptr[0] != '.')
{
QUAL = atoi(QUAL_ptr);
if (QUAL < GQX)
GQX = QUAL;
}
}
else
{
GQX = atoi(GQX_ptr);
}
//trying to reduce entropy on GQ to get better compression performance.
//1. rounds down to nearest 10.
//2. sets gq to min(gq,100).
buf[4]=GQX/10;
buf[4]*=10;
if(buf[4]>100) buf[4]=100;
// printf("%d\t%d\t%d\t%d\t%d\n",buf[0],buf[1],buf[2],buf[3],buf[4]);
if(gzwrite(depth_fp,buf,5*sizeof(int))!=(5*sizeof(int)))
die("ERROR: problem writing "+(string)out_fname+".tmp");
}
if(is_variant) {//wass this a variant? if so write it out to the bcf
norm_args->ntotal++;
vcf_parse(&str,hdr_in,bcf_rec);
// cerr<<bcf_rec->rid<<":"<<bcf_rec->pos<<endl;
if(prev_rid!=bcf_rec->rid)
vbuf.flush(variant_fp,hdr_out);
else
vbuf.flush(bcf_rec->pos,variant_fp,hdr_out);
prev_rid=bcf_rec->rid;
int32_t pass = bcf_has_filter(hdr_in, bcf_rec, ".");
bcf_update_format_int32(hdr_out,bcf_rec,"PF",&pass,1);
bcf_update_filter(hdr_out,bcf_rec,NULL,0);
if(bcf_rec->n_allele>2) {//split multi-allelics (using vcfnorm.c from bcftools1.3
norm_args->nsplit++;
split_multiallelic_to_biallelics(norm_args,bcf_rec );
for(int i=0;i<norm_args->ntmp_lines;i++){
remove_info(norm_args->tmp_lines[i]);
if(realign(norm_args,norm_args->tmp_lines[i]) != ERR_REF_MISMATCH)
ndec+=decompose(norm_args->tmp_lines[i],hdr_out,vbuf);
else
if(exit_on_mismatch)
die("vcf did not match the reference");
else
norm_args->nskipped++;
}
}
else {
remove_info(bcf_rec);
if( realign(norm_args,bcf_rec) != ERR_REF_MISMATCH)
ndec+=decompose(bcf_rec,hdr_out,vbuf);
else
if(exit_on_mismatch)
die("vcf did not match the reference");
else
norm_args->nskipped++;
}
vbuf.flush(bcf_rec->pos,variant_fp,hdr_out);
}
}
}
vbuf.flush(variant_fp,hdr_out);
bcf_hdr_destroy(hdr_in);
bcf_hdr_destroy(hdr_out);
bcf_destroy1(bcf_rec);
ks_destroy(ks);
gzclose(fp);
gzclose(depth_fp);
free(str.s);
free(work1.s);
hts_close(variant_fp);
destroy_data(norm_args);
fprintf(stderr,"Variant lines total/split/realigned/skipped:\t%d/%d/%d/%d\n", norm_args->ntotal,norm_args->nsplit,norm_args->nchanged,norm_args->nskipped);
fprintf(stderr,"Decomposed %d MNPs\n", ndec);
fprintf(stderr,"Indexing %s\n",out_fname);
bcf_index_build(out_fname, BCF_LIDX_SHIFT);
free(out_fname);
return 0;
}
static int query_regions(args_t *args, char *fname, char **regs, int nregs)
{
int i;
htsFile *fp = hts_open(fname,"r");
if ( !fp ) error("Could not read %s\n", fname);
enum htsExactFormat format = hts_get_format(fp)->format;
regidx_t *reg_idx = NULL;
if ( args->targets_fname )
{
reg_idx = regidx_init(args->targets_fname, NULL, NULL, 0, NULL);
if ( !reg_idx ) error("Could not read %s\n", args->targets_fname);
}
if ( format == bcf )
{
htsFile *out = hts_open("-","w");
if ( !out ) error("Could not open stdout\n", fname);
hts_idx_t *idx = bcf_index_load(fname);
if ( !idx ) error("Could not load .csi index of %s\n", fname);
bcf_hdr_t *hdr = bcf_hdr_read(fp);
if ( !hdr ) error("Could not read the header: %s\n", fname);
if ( args->print_header )
bcf_hdr_write(out,hdr);
if ( !args->header_only )
{
bcf1_t *rec = bcf_init();
for (i=0; i<nregs; i++)
{
hts_itr_t *itr = bcf_itr_querys(idx,hdr,regs[i]);
while ( bcf_itr_next(fp, itr, rec) >=0 )
{
if ( reg_idx && !regidx_overlap(reg_idx, bcf_seqname(hdr,rec),rec->pos,rec->pos+rec->rlen-1, NULL) ) continue;
bcf_write(out,hdr,rec);
}
tbx_itr_destroy(itr);
}
bcf_destroy(rec);
}
if ( hts_close(out) ) error("hts_close returned non-zero status for stdout\n");
bcf_hdr_destroy(hdr);
hts_idx_destroy(idx);
}
else if ( format==vcf || format==sam || format==unknown_format )
{
tbx_t *tbx = tbx_index_load(fname);
if ( !tbx ) error("Could not load .tbi/.csi index of %s\n", fname);
kstring_t str = {0,0,0};
if ( args->print_header )
{
while ( hts_getline(fp, KS_SEP_LINE, &str) >= 0 )
{
if ( !str.l || str.s[0]!=tbx->conf.meta_char ) break;
puts(str.s);
}
}
if ( !args->header_only )
{
int nseq;
const char **seq = NULL;
if ( reg_idx ) seq = tbx_seqnames(tbx, &nseq);
for (i=0; i<nregs; i++)
{
hts_itr_t *itr = tbx_itr_querys(tbx, regs[i]);
if ( !itr ) continue;
while (tbx_itr_next(fp, tbx, itr, &str) >= 0)
{
if ( reg_idx && !regidx_overlap(reg_idx,seq[itr->curr_tid],itr->curr_beg,itr->curr_end, NULL) ) continue;
puts(str.s);
}
tbx_itr_destroy(itr);
}
free(seq);
}
free(str.s);
tbx_destroy(tbx);
}
else if ( format==bam )
error("Please use \"samtools view\" for querying BAM files.\n");
if ( reg_idx ) regidx_destroy(reg_idx);
if ( hts_close(fp) ) error("hts_close returned non-zero status: %s\n", fname);
for (i=0; i<nregs; i++) free(regs[i]);
free(regs);
return 0;
}
static void reheader_bcf(args_t *args, int is_compressed)
{
htsFile *fp = hts_open(args->fname, "r"); if ( !fp ) error("Failed to open: %s\n", args->fname);
bcf_hdr_t *hdr = bcf_hdr_read(fp); if ( !hdr ) error("Failed to read the header: %s\n", args->fname);
kstring_t htxt = {0,0,0};
int hlen;
htxt.s = bcf_hdr_fmt_text(hdr, 1, &hlen);
htxt.l = hlen;
int i, nsamples = 0;
char **samples = NULL;
if ( args->samples_fname )
samples = hts_readlines(args->samples_fname, &nsamples);
if ( args->header_fname )
{
free(htxt.s); htxt.s = NULL; htxt.l = htxt.m = 0;
read_header_file(args->header_fname, &htxt);
}
if ( samples )
{
set_samples(samples, nsamples, &htxt);
for (i=0; i<nsamples; i++) free(samples[i]);
free(samples);
}
bcf_hdr_t *hdr_out = bcf_hdr_init("r");
bcf_hdr_parse(hdr_out, htxt.s);
if ( args->header_fname ) hdr_out = strip_header(hdr, hdr_out);
// write the header and the body
htsFile *fp_out = hts_open("-",is_compressed ? "wb" : "wbu");
bcf_hdr_write(fp_out, hdr_out);
bcf1_t *rec = bcf_init();
while ( bcf_read(fp, hdr, rec)==0 )
{
// sanity checking, this slows things down. Make it optional?
bcf_unpack(rec, BCF_UN_ALL);
if ( rec->rid >= hdr_out->n[BCF_DT_CTG] || strcmp(bcf_hdr_int2id(hdr,BCF_DT_CTG,rec->rid),bcf_hdr_int2id(hdr_out,BCF_DT_CTG,rec->rid)) )
error("The CHROM is not defined: \"%s\"\n", bcf_hdr_int2id(hdr,BCF_DT_CTG,rec->rid));
for (i=0; i<rec->d.n_flt; i++)
{
int id = rec->d.flt[i];
if ( id >= hdr_out->n[BCF_DT_ID] ) break;
if ( !bcf_hdr_idinfo_exists(hdr_out,BCF_HL_FLT,id) ) break;
if ( strcmp(hdr->id[BCF_DT_ID][id].key,hdr_out->id[BCF_DT_ID][id].key) )
error("FIXME: Broken FILTER ids: %s vs %s\n", hdr->id[BCF_DT_ID][id].key,hdr_out->id[BCF_DT_ID][id].key);
}
if ( i!=rec->d.n_flt )
error("The FILTER is not defined: \"%s\"\n", bcf_hdr_int2id(hdr,BCF_DT_ID,rec->d.flt[i]));
for (i=0; i<rec->n_info; i++)
{
int id = rec->d.info[i].key;
if ( id >= hdr_out->n[BCF_DT_ID] ) break;
if ( !hdr_out->id[BCF_DT_ID][id].key ) break;
if ( !bcf_hdr_idinfo_exists(hdr_out,BCF_HL_INFO,id) ) break;
if ( strcmp(hdr->id[BCF_DT_ID][id].key,hdr_out->id[BCF_DT_ID][id].key) )
error("FIXME: Broken INFO ids: %s vs %s\n", hdr->id[BCF_DT_ID][id].key,hdr_out->id[BCF_DT_ID][id].key);
}
if ( i!=rec->n_info )
error("The INFO tag is not defined: \"%s\"\n", bcf_hdr_int2id(hdr,BCF_DT_ID,rec->d.info[i].key));
for (i=0; i<rec->n_fmt; i++)
{
int id = rec->d.fmt[i].id;
if ( id >= hdr_out->n[BCF_DT_ID] ) break;
if ( !hdr_out->id[BCF_DT_ID][id].key ) break;
if ( !bcf_hdr_idinfo_exists(hdr_out,BCF_HL_FMT,id) ) break;
if ( strcmp(hdr->id[BCF_DT_ID][id].key,hdr_out->id[BCF_DT_ID][id].key) )
error("FIXME: Broken FORMAT ids: %s vs %s\n", hdr->id[BCF_DT_ID][id].key,hdr_out->id[BCF_DT_ID][id].key);
}
if ( i!=rec->n_fmt )
error("The FORMAT tag is not defined: \"%s\"\n", bcf_hdr_int2id(hdr,BCF_DT_ID,rec->d.fmt[i].id));
bcf_write(fp_out,hdr_out,rec);
}
bcf_destroy(rec);
free(htxt.s);
hts_close(fp_out);
hts_close(fp);
bcf_hdr_destroy(hdr_out);
bcf_hdr_destroy(hdr);
}
/**
* Collects sufficient statistics from read for variants to be genotyped.
*
* The VCF records in the buffer must never occur before
*/
void BCFSingleGenotypingBufferedReader::process_read(bam_hdr_t *h, bam1_t *s)
{
//wrap bam1_t in AugmentBAMRecord
as.initialize(h, s);
uint32_t tid = bam_get_tid(s);
uint32_t beg1 = as.beg1;
uint32_t end1 = as.end1;
//collect statistics for variant records that are in the buffer and overlap with the read
GenotypingRecord* g;
for (std::list<GenotypingRecord*>::iterator i=buffer.begin(); i!=buffer.end(); ++i)
{
g = *i;
// std::cerr << g->pos1 << " " << g->beg1 << " " << g->end1 << " ";
//same chromosome
if (tid==g->rid)
{
if (end1 < g->beg1)
{
//can terminate
return;
}
else if (beg1 > g->end1)
{
//this should not occur if the buffer was flushed before invoking process read
continue;
}
//else if (beg1 <= g->beg1 && g->end1 <= end1)
else if (beg1 <= g->pos1 && g->pos1 <= end1)
{
// collect_sufficient_statistics(*i, as);
}
else
{
//other types of overlap, just ignore
}
// std::cerr << "\n";
}
//prior chromosome
else if (tid<g->rid)
{
//this should not occur if the buffer was flushed before invoking process read
return;
}
//latter chromosome
else if (tid>g->rid)
{
//in case if the buffer has a VCF record later in the list which matches it
continue;
}
}
//you will only reach here if a read occurs after or overlaps the last record in the buffer
//adding new VCF records and collecting statistics if necessary
bcf1_t *v = bcf_init();
while (odr->read(v))
{
int32_t vtype = vm->classify_variant(odr->hdr, v, variant);
g = create_genotyping_record(odr->hdr, v, 2, variant);
buffer.push_back(g);
if (tid==g->rid)
{
//if (end1>=g->beg1 && pos1<=g->end1)
if (beg1 <= g->pos1 && g->pos1 <= end1)
{
// collect_sufficient_statistics(g, as);
}
}
//VCF record occurs after the read
if (tid < g->rid || end1 < g->beg1)
{
return;
}
else
{
v = bcf_init();
}
}
//this means end of file
bcf_destroy(v);
}
/**
* Flush records.
*/
void BCFSingleGenotypingBufferedReader::flush(bam_hdr_t *h, bam1_t *s, bool flush_all)
{
if (flush_all)
{
//read all the remaining from the reference genotyping file
bcf1_t *v = bcf_init();
while (odr->read(v))
{
int32_t vtype = vm->classify_variant(odr->hdr, v, variant);
GenotypingRecord* g = create_genotyping_record(odr->hdr, v, 2, variant);
buffer.push_back(g);
v = bcf_init();
}
bcf_destroy(v);
GenotypingRecord* g;
while (!buffer.empty())
{
g = buffer.front();
// genotype_and_print(g);
delete g;
buffer.pop_front();
}
}
else
{
//std::cerr << "partial flush\n";
uint32_t tid = bam_get_tid(s);
GenotypingRecord* g;
while (!buffer.empty())
{
g = buffer.front();
if (tid==g->rid)
{
if (bam_get_pos1(s) > g->end1)
{
// genotype_and_print(g);
delete g;
buffer.pop_front();
}
else
{
return;
}
}
else if (tid>g->rid)
{
// genotype_and_print(g);
delete g;
buffer.pop_front();
}
else
{
return;
}
}
}
}
int main(int argc, char **argv)
{
if ( argc == 1 )
error("Usage : bed_annos -c config.json -O z -o output.vcf.gz input.vcf.gz");
int i;
for ( i = 1; i < argc; ) {
const char *a = argv[i++];
const char **var = 0;
if ( strcmp(a, "-c") == 0 )
var = &json_fname;
else if ( strcmp(a, "-O") == 0 )
var = &output_fname_type;
else if ( strcmp(a, "-o") == 0 )
var = &output_fname;
if ( var != 0 ) {
if ( i == argc )
error("Missing an argument after %s", a);
*var = argv[i++];
continue;
}
if ( input_fname == 0 ) {
input_fname = a;
continue;
}
error("Unknown argument : %s.", a);
}
struct vcfanno_config *con = vcfanno_config_init();
if ( vcfanno_load_config(con, json_fname) != 0 )
error("Failed to load configure file. %s : %s", json_fname, strerror(errno));
vcfanno_config_debug(con);
if ( con->beds.n_beds == 0)
error("No bed database specified.");
if ( input_fname == 0 && (!isatty(fileno(stdin))) )
input_fname = "-";
if ( input_fname == 0 )
error("No input file.");
int out_type = FT_VCF;
if ( output_fname_type != 0 ) {
switch (output_fname_type[0]) {
case 'b':
out_type = FT_BCF_GZ; break;
case 'u':
out_type = FT_BCF; break;
case 'z':
out_type = FT_VCF_GZ; break;
case 'v':
out_type = FT_VCF; break;
default :
error("The output type \"%d\" not recognised\n", out_type);
};
}
htsFile *fp = hts_open(input_fname, "r");
if ( fp == NULL )
error("Failed to open %s : %s.", input_fname, strerror(errno));
htsFormat type = *hts_get_format(fp);
if ( type.format != vcf && type.format != bcf )
error("Unsupported input format. %s", input_fname);
bcf_hdr_t *hdr = bcf_hdr_read(fp);
if ( hdr == NULL )
error("Failed to parse header.");
bcf_hdr_t *hdr_out = bcf_hdr_dup(hdr);
htsFile *fout = output_fname == 0 ? hts_open("-", hts_bcf_wmode(out_type)) : hts_open(output_fname, hts_bcf_wmode(out_type));
struct beds_options opts = { .beds_is_inited = 0,};
beds_options_init(&opts);
opts.hdr_out = hdr_out;
for ( i = 0; i < con->beds.n_beds; ++i ) {
beds_database_add(&opts, con->beds.files[i].fname, con->beds.files[i].columns);
}
bcf_hdr_write(fout, hdr_out);
bcf1_t *line = bcf_init();
while ( bcf_read(fp, hdr, line) == 0 ) {
anno_beds_core(&opts, line);
bcf_write(fout, hdr_out, line);
}
bcf_destroy(line);
bcf_hdr_destroy(hdr);
bcf_hdr_destroy(hdr_out);
beds_options_destroy(&opts);
hts_close(fp);
hts_close(fout);
return 0;
}