static void init_data(args_t *args)
{
args->hdr = args->files->readers[0].header;
args->hdr_out = bcf_hdr_dup(args->hdr);
init_plugin(args);
if ( args->filter_str )
args->filter = filter_init(args->hdr, args->filter_str);
bcf_hdr_append_version(args->hdr_out, args->argc, args->argv, "bcftools_plugin");
if ( !args->drop_header )
{
args->out_fh = hts_open(args->output_fname,hts_bcf_wmode(args->output_type));
if ( args->out_fh == NULL ) error("Can't write to \"%s\": %s\n", args->output_fname, strerror(errno));
bcf_hdr_write(args->out_fh, args->hdr_out);
}
}
int main(int argc, char **argv)
{
if (argc != 3) {
fprintf(stderr,"anno_setter <in.vcf.gz> <columns_string>\n");
return 1;
}
bcf_hdr_t *h = NULL; //bcf_hdr_init();
htsFile *fp = hts_open(argv[1], "r");
if (fp == NULL)
error("%s : %s", argv[1], strerror(errno));
h = bcf_hdr_read(fp);
if (h == NULL)
error("failed to prase header");
bcf_hdr_t *out = bcf_hdr_dup(h);
char *string = strdup(argv[2]);
int ncols = 0;
anno_col_t *cols = init_columns(string, h, out, &ncols, anno_is_vcf);
print_anno_cols(cols, ncols);
hts_close(fp);
return 0;
}
static void init_data(args_t *args)
{
args->aux.srs = bcf_sr_init();
// Open files for input and output, initialize structures
if ( args->targets )
{
if ( bcf_sr_set_targets(args->aux.srs, args->targets, args->targets_is_file, args->aux.flag&CALL_CONSTR_ALLELES ? 3 : 0)<0 )
error("Failed to read the targets: %s\n", args->targets);
if ( args->aux.flag&CALL_CONSTR_ALLELES && args->flag&CF_INS_MISSED )
{
args->aux.srs->targets->missed_reg_handler = print_missed_line;
args->aux.srs->targets->missed_reg_data = args;
}
}
if ( args->regions )
{
if ( bcf_sr_set_regions(args->aux.srs, args->regions, args->regions_is_file)<0 )
error("Failed to read the targets: %s\n", args->regions);
}
int i;
if ( !bcf_sr_add_reader(args->aux.srs, args->bcf_fname) ) error("Failed to open: %s\n", args->bcf_fname);
if ( args->nsamples && args->nsamples != bcf_hdr_nsamples(args->aux.srs->readers[0].header) )
{
args->samples_map = (int *) malloc(sizeof(int)*args->nsamples);
args->aux.hdr = bcf_hdr_subset(args->aux.srs->readers[0].header, args->nsamples, args->samples, args->samples_map);
for (i=0; i<args->nsamples; i++)
if ( args->samples_map[i]<0 ) error("No such sample: %s\n", args->samples[i]);
if ( !bcf_hdr_nsamples(args->aux.hdr) ) error("No matching sample found\n");
}
else
{
args->aux.hdr = bcf_hdr_dup(args->aux.srs->readers[0].header);
for (i=0; i<args->nsamples; i++)
if ( bcf_hdr_id2int(args->aux.hdr,BCF_DT_SAMPLE,args->samples[i])<0 )
error("No such sample: %s\n", args->samples[i]);
}
// Reorder ploidy and family indexes to match mpileup's output and exclude samples which are not available
if ( args->aux.ploidy )
{
for (i=0; i<args->aux.nfams; i++)
{
int j;
for (j=0; j<3; j++)
{
int k = bcf_hdr_id2int(args->aux.hdr, BCF_DT_SAMPLE, args->samples[ args->aux.fams[i].sample[j] ]);
if ( k<0 ) error("No such sample: %s\n", args->samples[ args->aux.fams[i].sample[j] ]);
args->aux.fams[i].sample[j] = k;
}
}
uint8_t *ploidy = (uint8_t*) calloc(bcf_hdr_nsamples(args->aux.hdr), 1);
for (i=0; i<args->nsamples; i++) // i index in -s sample list
{
int j = bcf_hdr_id2int(args->aux.hdr, BCF_DT_SAMPLE, args->samples[i]); // j index in the output VCF / subset VCF
if ( j<0 )
{
fprintf(stderr,"Warning: no such sample: \"%s\"\n", args->samples[i]);
continue;
}
ploidy[j] = args->aux.ploidy[i];
}
args->nsamples = bcf_hdr_nsamples(args->aux.hdr);
for (i=0; i<args->nsamples; i++)
assert( ploidy[i]==0 || ploidy[i]==1 || ploidy[i]==2 );
free(args->aux.ploidy);
args->aux.ploidy = ploidy;
}
args->out_fh = hts_open(args->output_fname, hts_bcf_wmode(args->output_type));
if ( args->out_fh == NULL ) error("Can't write to \"%s\": %s\n", args->output_fname, strerror(errno));
if ( args->flag & CF_QCALL )
return;
if ( args->flag & CF_MCALL )
mcall_init(&args->aux);
if ( args->flag & CF_CCALL )
ccall_init(&args->aux);
if ( args->flag&CF_GVCF )
{
bcf_hdr_append(args->aux.hdr,"##INFO=<ID=END,Number=1,Type=Integer,Description=\"End position of the variant described in this record\">");
args->gvcf.rid = -1;
args->gvcf.line = bcf_init1();
args->gvcf.gt = (int32_t*) malloc(2*sizeof(int32_t)*bcf_hdr_nsamples(args->aux.hdr));
for (i=0; i<bcf_hdr_nsamples(args->aux.hdr); i++)
{
args->gvcf.gt[2*i+0] = bcf_gt_unphased(0);
args->gvcf.gt[2*i+1] = bcf_gt_unphased(0);
}
}
bcf_hdr_remove(args->aux.hdr, BCF_HL_INFO, "QS");
bcf_hdr_remove(args->aux.hdr, BCF_HL_INFO, "I16");
bcf_hdr_append_version(args->aux.hdr, args->argc, args->argv, "bcftools_call");
bcf_hdr_write(args->out_fh, args->aux.hdr);
if ( args->flag&CF_INS_MISSED ) init_missed_line(args);
}
static void init_data(args_t *args)
{
args->sr = bcf_sr_init();
if ( args->region )
{
args->sr->require_index = 1;
if ( bcf_sr_set_regions(args->sr, args->region, args->region_is_file)<0 ) error("Failed to read the regions: %s\n",args->region);
}
if ( args->target && bcf_sr_set_targets(args->sr, args->target, args->target_is_file, 0)<0 ) error("Failed to read the targets: %s\n",args->target);
if ( !bcf_sr_add_reader(args->sr,args->fname) ) error("Error: %s\n", bcf_sr_strerror(args->sr->errnum));
args->hdr_in = bcf_sr_get_header(args->sr,0);
args->hdr_out = bcf_hdr_dup(args->hdr_in);
if ( args->filter_str )
args->filter = filter_init(args->hdr_in, args->filter_str);
mkdir_p("%s/",args->output_dir);
int i, nsmpl = bcf_hdr_nsamples(args->hdr_in);
if ( !nsmpl ) error("No samples to split: %s\n", args->fname);
args->fh = (htsFile**)calloc(nsmpl,sizeof(*args->fh));
args->bnames = set_file_base_names(args);
kstring_t str = {0,0,0};
for (i=0; i<nsmpl; i++)
{
if ( !args->bnames[i] ) continue;
str.l = 0;
kputs(args->output_dir, &str);
if ( str.s[str.l-1] != '/' ) kputc('/', &str);
int k, l = str.l;
kputs(args->bnames[i], &str);
for (k=l; k<str.l; k++) if ( isspace(str.s[k]) ) str.s[k] = '_';
if ( args->output_type & FT_BCF ) kputs(".bcf", &str);
else if ( args->output_type & FT_GZ ) kputs(".vcf.gz", &str);
else kputs(".vcf", &str);
args->fh[i] = hts_open(str.s, hts_bcf_wmode(args->output_type));
if ( args->fh[i] == NULL ) error("Can't write to \"%s\": %s\n", str.s, strerror(errno));
bcf_hdr_nsamples(args->hdr_out) = 1;
args->hdr_out->samples[0] = args->bnames[i];
bcf_hdr_write(args->fh[i], args->hdr_out);
}
free(str.s);
// parse tags
int is_info = 0, is_fmt = 0;
char *beg = args->keep_tags;
while ( beg && *beg )
{
if ( !strncasecmp("INFO/",beg,5) ) { is_info = 1; is_fmt = 0; beg += 5; }
else if ( !strcasecmp("INFO",beg) ) { args->keep_info = 1; break; }
else if ( !strncasecmp("INFO,",beg,5) ) { args->keep_info = 1; beg += 5; continue; }
else if ( !strncasecmp("FMT/",beg,4) ) { is_info = 0; is_fmt = 1; beg += 4; }
else if ( !strncasecmp("FORMAT/",beg,7) ) { is_info = 0; is_fmt = 1; beg += 7; }
else if ( !strcasecmp("FMT",beg) ) { args->keep_fmt = 1; break; }
else if ( !strcasecmp("FORMAT",beg) ) { args->keep_fmt = 1; break; }
else if ( !strncasecmp("FMT,",beg,4) ) { args->keep_fmt = 1; beg += 4; continue; }
else if ( !strncasecmp("FORMAT,",beg,7) ) { args->keep_fmt = 1; beg += 7; continue; }
char *end = beg;
while ( *end && *end!=',' ) end++;
char tmp = *end; *end = 0;
int id = bcf_hdr_id2int(args->hdr_in, BCF_DT_ID, beg);
beg = tmp ? end + 1 : end;
if ( is_info && bcf_hdr_idinfo_exists(args->hdr_in,BCF_HL_INFO,id) )
{
if ( id >= args->ninfo_tags ) args->ninfo_tags = id + 1;
hts_expand0(uint8_t, args->ninfo_tags, args->minfo_tags, args->info_tags);
args->info_tags[id] = 1;
}
if ( is_fmt && bcf_hdr_idinfo_exists(args->hdr_in,BCF_HL_FMT,id) )
{
if ( id >= args->nfmt_tags ) args->nfmt_tags = id + 1;
hts_expand0(uint8_t, args->nfmt_tags, args->mfmt_tags, args->fmt_tags);
args->fmt_tags[id] = 1;
}
}
if ( !args->keep_info && !args->keep_fmt && !args->ninfo_tags && !args->nfmt_tags )
{
args->keep_info = args->keep_fmt = 1;
}
}
int ingest1(const char *input,const char *output,char *ref,bool exit_on_mismatch=true) {
cerr << "Input: " << input << "\tOutput: "<<output<<endl;
kstream_t *ks;
kstring_t str = {0,0,0};
gzFile fp = gzopen(input, "r");
VarBuffer vbuf(1000);
int prev_rid = -1;
if(fp==NULL) {
fprintf(stderr,"problem opening %s\n",input);
exit(1);
}
char *out_fname = (char *)malloc(strlen(output)+5);
strcpy(out_fname,output);
strcat(out_fname,".tmp");
if(fileexists(out_fname)) {
fprintf(stderr,"%s file already exists. will not overwrite\n",out_fname);
exit(1);
}
printf("depth: %s\n",out_fname);
gzFile depth_fp = gzopen(out_fname, "wb1");
strcpy(out_fname,output);
strcat(out_fname,".bcf");
if(fileexists(out_fname)) {
fprintf(stderr,"%s file already exists. will not overwrite\n",out_fname);
exit(1);
}
printf("variants: %s\n",out_fname);
htsFile *variant_fp=hts_open(out_fname,"wb1");
if(variant_fp==NULL) {
fprintf(stderr,"problem opening %s\n",input);
exit(1);
}
ks = ks_init(fp);
htsFile *hfp=hts_open(input, "r");
bcf_hdr_t *hdr_in = bcf_hdr_read(hfp);
hts_close(hfp);
//this is a hack to fix gvcfs where AD is incorrectly defined in the header. (vcf4.2 does not technically allow Number=R)
bcf_hdr_remove(hdr_in,BCF_HL_FMT,"AD");
assert( bcf_hdr_append(hdr_in,"##FORMAT=<ID=AD,Number=R,Type=Integer,Description=\"Allelic depths for the ref and alt alleles in the order listed. For indels this value only includes reads which confidently support each allele (posterior prob 0.999 or higher that read contains indicated allele vs all other intersecting indel alleles)\">") == 0);
//this is a hack to fix broken gvcfs where GQ is incorrectly labelled as float (v4.3 spec says it should be integer)
bcf_hdr_remove(hdr_in,BCF_HL_FMT,"GQ");
assert( bcf_hdr_append(hdr_in,"##FORMAT=<ID=GQ,Number=1,Type=Integer,Description=\"Genotype Quality\">") == 0);
// bcf_hdr_t *hdr_out=hdr_in;
bcf_hdr_t *hdr_out = bcf_hdr_dup(hdr_in);
remove_hdr_lines(hdr_out,BCF_HL_INFO);
remove_hdr_lines(hdr_out,BCF_HL_FLT);
bcf_hdr_sync(hdr_out);
//here we add FORMAT/PF. which is the pass filter flag for alts.
assert( bcf_hdr_append(hdr_out,"##FORMAT=<ID=PF,Number=A,Type=Integer,Description=\"variant was PASS filter in original sample gvcf\">") == 0);
args_t *norm_args = init_vcfnorm(hdr_out,ref);
norm_args->check_ref |= CHECK_REF_WARN;
bcf1_t *bcf_rec = bcf_init();
bcf_hdr_write(variant_fp, hdr_out);
kstring_t work1 = {0,0,0};
int buf[5];
ks_tokaux_t aux;
int ndec=0;
int ref_len,alt_len;
while( ks_getuntil(ks, '\n', &str, 0) >=0) {
// fprintf(stderr,"%s\n",str.s);
if(str.s[0]!='#') {
char *ptr = kstrtok(str.s,"\t",&aux);//chrom
ptr = kstrtok(NULL,NULL,&aux);//pos
work1.l=0;
kputsn(str.s,ptr-str.s-1, &work1);
buf[0] = bcf_hdr_name2id(hdr_in, work1.s);
assert( buf[0]>=0);
buf[1]=atoi(ptr)-1;
ptr = kstrtok(NULL,NULL,&aux);//ID
ptr = kstrtok(NULL,NULL,&aux);//REF
ref_len=0;
while(ptr[ref_len]!='\t') ref_len++;
ptr = kstrtok(NULL,NULL,&aux);//ALT
bool is_variant=false;
alt_len=0;
while(ptr[alt_len]!='\t') alt_len++;
if(ptr[0]!='.')
is_variant=true;
char * QUAL_ptr = kstrtok(NULL, NULL, &aux);
assert (QUAL_ptr != NULL);
for(int i=0;i<2;i++) ptr = kstrtok(NULL,NULL,&aux);// gets us to INFO
//find END if it is there
char *end_ptr=strstr(ptr,"END=") ;
if(end_ptr!=NULL)
buf[2]=atoi(end_ptr+4)-1;
else
buf[2]=buf[1]+alt_len-1;
ptr = kstrtok(NULL,NULL,&aux);//FORMAT
//find index of DP (if present)
//if not present, dont output anything (indels ignored)
char *DP_ptr = find_format(ptr,"DP");
int GQX = 0;
int QUAL = 0;
// AH: change code to use the minimum of GQ and QUAL fields if
// GQX is not defined. See here:
// https://support.basespace.illumina.com/knowledgebase/articles/144844-vcf-file
// "GQXGenotype quality. GQX is the minimum of the GQ value
// and the QUAL column. In general, these are similar values;
// taking the minimum makes GQX the more conservative measure of
// genotype quality."
if(DP_ptr!=NULL) {
buf[3]=atoi(DP_ptr);
char *GQX_ptr = find_format(ptr,"GQX");
if (GQX_ptr == NULL)
{
GQX_ptr = find_format(ptr,"GQ");
GQX = atoi(GQX_ptr);
if (QUAL_ptr[0] != '.')
{
QUAL = atoi(QUAL_ptr);
if (QUAL < GQX)
GQX = QUAL;
}
}
else
{
GQX = atoi(GQX_ptr);
}
//trying to reduce entropy on GQ to get better compression performance.
//1. rounds down to nearest 10.
//2. sets gq to min(gq,100).
buf[4]=GQX/10;
buf[4]*=10;
if(buf[4]>100) buf[4]=100;
// printf("%d\t%d\t%d\t%d\t%d\n",buf[0],buf[1],buf[2],buf[3],buf[4]);
if(gzwrite(depth_fp,buf,5*sizeof(int))!=(5*sizeof(int)))
die("ERROR: problem writing "+(string)out_fname+".tmp");
}
if(is_variant) {//wass this a variant? if so write it out to the bcf
norm_args->ntotal++;
vcf_parse(&str,hdr_in,bcf_rec);
// cerr<<bcf_rec->rid<<":"<<bcf_rec->pos<<endl;
if(prev_rid!=bcf_rec->rid)
vbuf.flush(variant_fp,hdr_out);
else
vbuf.flush(bcf_rec->pos,variant_fp,hdr_out);
prev_rid=bcf_rec->rid;
int32_t pass = bcf_has_filter(hdr_in, bcf_rec, ".");
bcf_update_format_int32(hdr_out,bcf_rec,"PF",&pass,1);
bcf_update_filter(hdr_out,bcf_rec,NULL,0);
if(bcf_rec->n_allele>2) {//split multi-allelics (using vcfnorm.c from bcftools1.3
norm_args->nsplit++;
split_multiallelic_to_biallelics(norm_args,bcf_rec );
for(int i=0;i<norm_args->ntmp_lines;i++){
remove_info(norm_args->tmp_lines[i]);
if(realign(norm_args,norm_args->tmp_lines[i]) != ERR_REF_MISMATCH)
ndec+=decompose(norm_args->tmp_lines[i],hdr_out,vbuf);
else
if(exit_on_mismatch)
die("vcf did not match the reference");
else
norm_args->nskipped++;
}
}
else {
remove_info(bcf_rec);
if( realign(norm_args,bcf_rec) != ERR_REF_MISMATCH)
ndec+=decompose(bcf_rec,hdr_out,vbuf);
else
if(exit_on_mismatch)
die("vcf did not match the reference");
else
norm_args->nskipped++;
}
vbuf.flush(bcf_rec->pos,variant_fp,hdr_out);
}
}
}
vbuf.flush(variant_fp,hdr_out);
bcf_hdr_destroy(hdr_in);
bcf_hdr_destroy(hdr_out);
bcf_destroy1(bcf_rec);
ks_destroy(ks);
gzclose(fp);
gzclose(depth_fp);
free(str.s);
free(work1.s);
hts_close(variant_fp);
destroy_data(norm_args);
fprintf(stderr,"Variant lines total/split/realigned/skipped:\t%d/%d/%d/%d\n", norm_args->ntotal,norm_args->nsplit,norm_args->nchanged,norm_args->nskipped);
fprintf(stderr,"Decomposed %d MNPs\n", ndec);
fprintf(stderr,"Indexing %s\n",out_fname);
bcf_index_build(out_fname, BCF_LIDX_SHIFT);
free(out_fname);
return 0;
}
static void init_data(args_t *args)
{
bcf1_t *line = NULL;
// With phased concat, the chunks overlap and come in the right order. To
// avoid opening all files at once, store start positions to recognise need
// for the next one. This way we can keep only two open chunks at once.
if ( args->phased_concat )
{
args->start_pos = (int*) malloc(sizeof(int)*args->nfnames);
line = bcf_init();
}
kstring_t str = {0,0,0};
int i, prev_chrid = -1;
for (i=0; i<args->nfnames; i++)
{
htsFile *fp = hts_open(args->fnames[i], "r");
if ( !fp ) error("Failed to open: %s\n", args->fnames[i]);
bcf_hdr_t *hdr = bcf_hdr_read(fp);
if ( !hdr ) error("Failed to parse header: %s\n", args->fnames[i]);
if ( !args->out_hdr )
args->out_hdr = bcf_hdr_dup(hdr);
else
{
bcf_hdr_combine(args->out_hdr, hdr);
if ( bcf_hdr_nsamples(hdr) != bcf_hdr_nsamples(args->out_hdr) )
error("Different number of samples in %s. Perhaps \"bcftools merge\" is what you are looking for?\n", args->fnames[i]);
int j;
for (j=0; j<bcf_hdr_nsamples(hdr); j++)
if ( strcmp(args->out_hdr->samples[j],hdr->samples[j]) )
error("Different sample names in %s. Perhaps \"bcftools merge\" is what you are looking for?\n", args->fnames[i]);
}
if ( args->phased_concat )
{
int ret = bcf_read(fp, hdr, line);
if ( ret!=0 ) args->start_pos[i] = -2; // empty file
else
{
int chrid = bcf_hdr_id2int(args->out_hdr,BCF_DT_CTG,bcf_seqname(hdr,line));
args->start_pos[i] = chrid==prev_chrid ? line->pos : -1;
prev_chrid = chrid;
}
}
bcf_hdr_destroy(hdr);
hts_close(fp);
}
free(str.s);
if ( line ) bcf_destroy(line);
args->seen_seq = (int*) calloc(args->out_hdr->n[BCF_DT_CTG],sizeof(int));
if ( args->phased_concat )
{
bcf_hdr_append(args->out_hdr,"##FORMAT=<ID=PQ,Number=1,Type=Integer,Description=\"Phasing Quality (bigger is better)\">");
bcf_hdr_append(args->out_hdr,"##FORMAT=<ID=PS,Number=1,Type=Integer,Description=\"Phase Set\">");
}
bcf_hdr_append_version(args->out_hdr, args->argc, args->argv, "bcftools_concat");
args->out_fh = hts_open(args->output_fname,hts_bcf_wmode(args->output_type));
if ( args->out_fh == NULL ) error("Can't write to \"%s\": %s\n", args->output_fname, strerror(errno));
bcf_hdr_write(args->out_fh, args->out_hdr);
if ( args->allow_overlaps )
{
args->files = bcf_sr_init();
args->files->require_index = 1;
for (i=0; i<args->nfnames; i++)
if ( !bcf_sr_add_reader(args->files,args->fnames[i]) ) error("Failed to open, is the file indexed? %s\n", args->fnames[i]);
}
else if ( args->phased_concat )
{
// Remove empty files from the list
int nok = 0;
while (1)
{
while ( nok<args->nfnames && args->start_pos[nok]!=-2 ) nok++;
if ( nok==args->nfnames ) break;
i = nok;
while ( i<args->nfnames && args->start_pos[i]==-2 ) i++;
if ( i==args->nfnames ) break;
int tmp = args->start_pos[nok];
args->start_pos[nok] = args->start_pos[i];
args->start_pos[i] = tmp;
char *str = args->fnames[nok];
args->fnames[nok] = args->fnames[i];
args->fnames[i] = str;
}
for (i=nok; i<args->nfnames; i++) free(args->fnames[i]);
args->nfnames = nok;
for (i=1; i<args->nfnames; i++)
if ( args->start_pos[i-1]!=-1 && args->start_pos[i]!=-1 && args->start_pos[i]<args->start_pos[i-1] )
error("The files not in ascending order: %d in %s, %d in %s\n", args->start_pos[i-1]+1,args->fnames[i-1],args->start_pos[i]+1,args->fnames[i]);
args->prev_chr = -1;
args->swap_phase = (int*) calloc(bcf_hdr_nsamples(args->out_hdr),sizeof(int));
args->nmatch = (int*) calloc(bcf_hdr_nsamples(args->out_hdr),sizeof(int));
args->nmism = (int*) calloc(bcf_hdr_nsamples(args->out_hdr),sizeof(int));
args->phase_qual = (int32_t*) malloc(bcf_hdr_nsamples(args->out_hdr)*sizeof(int32_t));
args->phase_set = (int32_t*) malloc(bcf_hdr_nsamples(args->out_hdr)*sizeof(int32_t));
args->files = bcf_sr_init();
args->files->require_index = 1;
args->ifname = 0;
}
}
int init(int argc, char **argv, bcf_hdr_t *in, bcf_hdr_t *out)
{
memset(&args,0,sizeof(args_t));
int i;
static struct option loptions[] =
{
{"help", no_argument, 0,'h'},
{"sample-list", required_argument, 0,'s'},
{0,0,0,0}
};
char **smps_strs = NULL;
int c;
while ((c = getopt_long(argc, argv, "?s:h",loptions,NULL)) >= 0)
{
switch (c)
{
case 's': smps_strs = hts_readlist(optarg,0,&(args.n_sel_smps));
if ( args.n_sel_smps == 0 )
{
fprintf(stderr, "Sample specification not valid.\n");
error("%s", usage());
}
break;
case 'h': usage(); break;
case '?':
default: error("%s", usage()); break;
}
}
if ( optind != argc ) usage(); // too many files given
args.hdr = bcf_hdr_dup(in);
// Samples parsing from header and input option
if ( !bcf_hdr_nsamples(args.hdr) )
{
error("No samples in input file.\n");
}
args.nsmp = bcf_hdr_nsamples(args.hdr);
args.selected_smps = (int*) calloc(args.nsmp,sizeof(int));
for ( i = 0; i < args.n_sel_smps; i++ )
{
int ind = bcf_hdr_id2int(args.hdr, BCF_DT_SAMPLE, smps_strs[i]);
if ( ind == -1 )
{
error("Sample '%s' not in input vcf file.\n", smps_strs[i]);
} else {
args.selected_smps[ind] = 1;
}
free(smps_strs[i]);
}
free(smps_strs);
/*
fprintf(stderr, "Selected samples array:[");
for (i=0;i<args.nsmp;i++)
{
fprintf(stderr, " %i", args.selected_smps[i]);
}
fprintf(stderr, " ]\n");
*/
if ( bcf_hdr_id2int(args.hdr, BCF_DT_ID, "GT")<0 ) error("[E::%s] GT not present in the header\n", __func__);
args.gt_arr = NULL;
return 0;
}
int main(int argc, char **argv)
{
if ( argc == 1 )
error("Usage : bed_annos -c config.json -O z -o output.vcf.gz input.vcf.gz");
int i;
for ( i = 1; i < argc; ) {
const char *a = argv[i++];
const char **var = 0;
if ( strcmp(a, "-c") == 0 )
var = &json_fname;
else if ( strcmp(a, "-O") == 0 )
var = &output_fname_type;
else if ( strcmp(a, "-o") == 0 )
var = &output_fname;
if ( var != 0 ) {
if ( i == argc )
error("Missing an argument after %s", a);
*var = argv[i++];
continue;
}
if ( input_fname == 0 ) {
input_fname = a;
continue;
}
error("Unknown argument : %s.", a);
}
struct vcfanno_config *con = vcfanno_config_init();
if ( vcfanno_load_config(con, json_fname) != 0 )
error("Failed to load configure file. %s : %s", json_fname, strerror(errno));
vcfanno_config_debug(con);
if ( con->beds.n_beds == 0)
error("No bed database specified.");
if ( input_fname == 0 && (!isatty(fileno(stdin))) )
input_fname = "-";
if ( input_fname == 0 )
error("No input file.");
int out_type = FT_VCF;
if ( output_fname_type != 0 ) {
switch (output_fname_type[0]) {
case 'b':
out_type = FT_BCF_GZ; break;
case 'u':
out_type = FT_BCF; break;
case 'z':
out_type = FT_VCF_GZ; break;
case 'v':
out_type = FT_VCF; break;
default :
error("The output type \"%d\" not recognised\n", out_type);
};
}
htsFile *fp = hts_open(input_fname, "r");
if ( fp == NULL )
error("Failed to open %s : %s.", input_fname, strerror(errno));
htsFormat type = *hts_get_format(fp);
if ( type.format != vcf && type.format != bcf )
error("Unsupported input format. %s", input_fname);
bcf_hdr_t *hdr = bcf_hdr_read(fp);
if ( hdr == NULL )
error("Failed to parse header.");
bcf_hdr_t *hdr_out = bcf_hdr_dup(hdr);
htsFile *fout = output_fname == 0 ? hts_open("-", hts_bcf_wmode(out_type)) : hts_open(output_fname, hts_bcf_wmode(out_type));
struct beds_options opts = { .beds_is_inited = 0,};
beds_options_init(&opts);
opts.hdr_out = hdr_out;
for ( i = 0; i < con->beds.n_beds; ++i ) {
beds_database_add(&opts, con->beds.files[i].fname, con->beds.files[i].columns);
}
bcf_hdr_write(fout, hdr_out);
bcf1_t *line = bcf_init();
while ( bcf_read(fp, hdr, line) == 0 ) {
anno_beds_core(&opts, line);
bcf_write(fout, hdr_out, line);
}
bcf_destroy(line);
bcf_hdr_destroy(hdr);
bcf_hdr_destroy(hdr_out);
beds_options_destroy(&opts);
hts_close(fp);
hts_close(fout);
return 0;
}
/**
* @brief creates a deep copy of an existing bcf_hdr_t
* @param original an htslib raw bcf header pointer
*/
bcf_hdr_t* variant_header_deep_copy(bcf_hdr_t* original) {
return bcf_hdr_dup(original);
}
int ctx_vcfcov(int argc, char **argv)
{
struct MemArgs memargs = MEM_ARGS_INIT;
const char *out_path = NULL, *out_type = NULL;
uint32_t max_allele_len = 0, max_gt_vars = 0;
char *ref_path = NULL;
bool low_mem = false;
// Arg parsing
char cmd[100];
char shortopts[300];
cmd_long_opts_to_short(longopts, shortopts, sizeof(shortopts));
int c;
size_t i;
// silence error messages from getopt_long
// opterr = 0;
while((c = getopt_long_only(argc, argv, shortopts, longopts, NULL)) != -1) {
cmd_get_longopt_str(longopts, c, cmd, sizeof(cmd));
switch(c) {
case 0: /* flag set */ break;
case 'h': cmd_print_usage(NULL); break;
case 'o': cmd_check(!out_path, cmd); out_path = optarg; break;
case 'O': cmd_check(!out_type, cmd); out_type = optarg; break;
case 'f': cmd_check(!futil_get_force(), cmd); futil_set_force(true); break;
case 'm': cmd_mem_args_set_memory(&memargs, optarg); break;
case 'n': cmd_mem_args_set_nkmers(&memargs, optarg); break;
case 'r': cmd_check(!ref_path, cmd); ref_path = optarg; break;
case 'L': cmd_check(!max_allele_len,cmd); max_allele_len = cmd_uint32(cmd,optarg); break;
case 'N': cmd_check(!max_gt_vars,cmd); max_gt_vars = cmd_uint32(cmd,optarg); break;
case 'M': cmd_check(!low_mem, cmd); low_mem = true; break;
case ':': /* BADARG */
case '?': /* BADCH getopt_long has already printed error */
// cmd_print_usage(NULL);
die("`"CMD" "SUBCMD" -h` for help. Bad option: %s", argv[optind-1]);
default: abort();
}
}
// Defaults for unset values
if(out_path == NULL) out_path = "-";
if(ref_path == NULL) cmd_print_usage("Require a reference (-r,--ref <ref.fa>)");
if(optind+2 > argc) cmd_print_usage("Require VCF and graph files");
if(!max_allele_len) max_allele_len = DEFAULT_MAX_ALLELE_LEN;
if(!max_gt_vars) max_gt_vars = DEFAULT_MAX_GT_VARS;
status("[vcfcov] max allele length: %u; max number of variants: %u",
max_allele_len, max_gt_vars);
// open ref
// index fasta with: samtools faidx ref.fa
faidx_t *fai = fai_load(ref_path);
if(fai == NULL) die("Cannot load ref index: %s / %s.fai", ref_path, ref_path);
// Open input VCF file
const char *vcf_path = argv[optind++];
htsFile *vcffh = hts_open(vcf_path, "r");
if(vcffh == NULL) die("Cannot open VCF file: %s", vcf_path);
bcf_hdr_t *vcfhdr = bcf_hdr_read(vcffh);
if(vcfhdr == NULL) die("Cannot read VCF header: %s", vcf_path);
// Test we can close and reopen files
if(low_mem) {
if((vcffh = hts_open(vcf_path, "r")) == NULL)
die("Cannot re-open VCF file: %s", vcf_path);
if((vcfhdr = bcf_hdr_read(vcffh)) == NULL)
die("Cannot re-read VCF header: %s", vcf_path);
}
//
// Open graph files
//
const size_t num_gfiles = argc - optind;
char **graph_paths = argv + optind;
ctx_assert(num_gfiles > 0);
GraphFileReader *gfiles = ctx_calloc(num_gfiles, sizeof(GraphFileReader));
size_t ncols, ctx_max_kmers = 0, ctx_sum_kmers = 0;
ncols = graph_files_open(graph_paths, gfiles, num_gfiles,
&ctx_max_kmers, &ctx_sum_kmers);
// Check graph + paths are compatible
graphs_gpaths_compatible(gfiles, num_gfiles, NULL, 0, -1);
//
// Decide on memory
//
size_t bits_per_kmer, kmers_in_hash, graph_mem;
bits_per_kmer = sizeof(BinaryKmer)*8 + sizeof(Covg)*8 * ncols;
kmers_in_hash = cmd_get_kmers_in_hash(memargs.mem_to_use,
memargs.mem_to_use_set,
memargs.num_kmers,
memargs.num_kmers_set,
bits_per_kmer,
low_mem ? -1 : (int64_t)ctx_max_kmers,
ctx_sum_kmers,
true, &graph_mem);
cmd_check_mem_limit(memargs.mem_to_use, graph_mem);
//
// Open output file
//
// v=>vcf, z=>compressed vcf, b=>bcf, bu=>uncompressed bcf
int mode = vcf_misc_get_outtype(out_type, out_path);
futil_create_output(out_path);
htsFile *outfh = hts_open(out_path, modes_htslib[mode]);
status("[vcfcov] Output format: %s", hsmodes_htslib[mode]);
// Allocate memory
dBGraph db_graph;
db_graph_alloc(&db_graph, gfiles[0].hdr.kmer_size, ncols, 1, kmers_in_hash,
DBG_ALLOC_COVGS);
//
// Set up tag names
//
// *R => ref, *A => alt
sprintf(kcov_ref_tag, "K%zuR", db_graph.kmer_size); // mean coverage
sprintf(kcov_alt_tag, "K%zuA", db_graph.kmer_size);
// #SAMPLE=<ID=...,K29KCOV=...,K29NK=...,K29RLK>
// - K29_kcov is empirical kmer coverage
// - K29_nkmers is the number of kmers in the sample
// - mean_read_length is the mean read length in bases
char sample_kcov_tag[20], sample_nk_tag[20], sample_rlk_tag[20];
sprintf(sample_kcov_tag, "K%zu_kcov", db_graph.kmer_size); // mean coverage
sprintf(sample_nk_tag, "K%zu_nkmers", db_graph.kmer_size);
sprintf(sample_rlk_tag, "mean_read_length");
//
// Load kmers if we are using --low-mem
//
VcfCovStats st;
memset(&st, 0, sizeof(st));
VcfCovPrefs prefs = {.kcov_ref_tag = kcov_ref_tag,
.kcov_alt_tag = kcov_alt_tag,
.max_allele_len = max_allele_len,
.max_gt_vars = max_gt_vars,
.load_kmers_only = false};
if(low_mem)
{
status("[vcfcov] Loading kmers from VCF+ref");
prefs.load_kmers_only = true;
vcfcov_file(vcffh, vcfhdr, NULL, NULL, vcf_path, fai,
NULL, &prefs, &st, &db_graph);
// Close files
hts_close(vcffh);
bcf_hdr_destroy(vcfhdr);
// Re-open files
if((vcffh = hts_open(vcf_path, "r")) == NULL)
die("Cannot re-open VCF file: %s", vcf_path);
if((vcfhdr = bcf_hdr_read(vcffh)) == NULL)
die("Cannot re-read VCF header: %s", vcf_path);
prefs.load_kmers_only = false;
}
//
// Load graphs
//
GraphLoadingStats gstats;
memset(&gstats, 0, sizeof(gstats));
GraphLoadingPrefs gprefs = graph_loading_prefs(&db_graph);
gprefs.must_exist_in_graph = low_mem;
for(i = 0; i < num_gfiles; i++) {
graph_load(&gfiles[i], gprefs, &gstats);
graph_file_close(&gfiles[i]);
}
ctx_free(gfiles);
hash_table_print_stats(&db_graph.ht);
//
// Set up VCF header / graph matchup
//
size_t *samplehdrids = ctx_malloc(db_graph.num_of_cols * sizeof(size_t));
// Add samples to vcf header
bcf_hdr_t *outhdr = bcf_hdr_dup(vcfhdr);
bcf_hrec_t *hrec;
int sid;
char hdrstr[200];
for(i = 0; i < db_graph.num_of_cols; i++) {
char *sname = db_graph.ginfo[i].sample_name.b;
if((sid = bcf_hdr_id2int(outhdr, BCF_DT_SAMPLE, sname)) < 0) {
bcf_hdr_add_sample(outhdr, sname);
sid = bcf_hdr_id2int(outhdr, BCF_DT_SAMPLE, sname);
}
samplehdrids[i] = sid;
// Add SAMPLE field
hrec = bcf_hdr_get_hrec(outhdr, BCF_HL_STR, "ID", sname, "SAMPLE");
if(hrec == NULL) {
sprintf(hdrstr, "##SAMPLE=<ID=%s,%s=%"PRIu64",%s=%"PRIu64",%s=%zu>", sname,
sample_kcov_tag,
gstats.nkmers[i] ? gstats.sumcov[i] / gstats.nkmers[i] : 0,
sample_nk_tag, gstats.nkmers[i],
sample_rlk_tag, (size_t)db_graph.ginfo[i].mean_read_length);
bcf_hdr_append(outhdr, hdrstr);
}
else {
// mean kcovg
sprintf(hdrstr, "%"PRIu64, gstats.sumcov[i] / gstats.nkmers[i]);
vcf_misc_add_update_hrec(hrec, sample_kcov_tag, hdrstr);
// num kmers
sprintf(hdrstr, "%"PRIu64, gstats.nkmers[i]);
vcf_misc_add_update_hrec(hrec, sample_nk_tag, hdrstr);
// mean read length in kmers
sprintf(hdrstr, "%zu", (size_t)db_graph.ginfo[i].mean_read_length);
vcf_misc_add_update_hrec(hrec, sample_rlk_tag, hdrstr);
}
status("[vcfcov] Colour %zu: %s [VCF column %zu]", i, sname, samplehdrids[i]);
}
// Add genotype format fields
// One field per alternative allele
sprintf(hdrstr, "##FORMAT=<ID=%s,Number=A,Type=Integer,"
"Description=\"Coverage on ref (k=%zu): sum(kmer_covs) / exp_num_kmers\">\n",
kcov_ref_tag, db_graph.kmer_size);
bcf_hdr_append(outhdr, hdrstr);
sprintf(hdrstr, "##FORMAT=<ID=%s,Number=A,Type=Integer,"
"Description=\"Coverage on alt (k=%zu): sum(kmer_covs) / exp_num_kmers\">\n",
kcov_alt_tag, db_graph.kmer_size);
bcf_hdr_append(outhdr, hdrstr);
bcf_hdr_set_version(outhdr, "VCFv4.2");
// Add command string to header
vcf_misc_hdr_add_cmd(outhdr, cmd_get_cmdline(), cmd_get_cwd());
if(bcf_hdr_write(outfh, outhdr) != 0)
die("Cannot write header to: %s", futil_outpath_str(out_path));
status("[vcfcov] Reading %s and adding coverage", vcf_path);
// Reset stats and get coverage
memset(&st, 0, sizeof(st));
vcfcov_file(vcffh, vcfhdr, outfh, outhdr, vcf_path, fai,
samplehdrids, &prefs, &st, &db_graph);
// Print statistics
char ns0[50], ns1[50];
status("[vcfcov] Read %s VCF lines", ulong_to_str(st.nvcf_lines, ns0));
status("[vcfcov] Read %s ALTs", ulong_to_str(st.nalts_read, ns0));
status("[vcfcov] Used %s kmers", ulong_to_str(st.ngt_kmers, ns0));
status("[vcfcov] ALTs used: %s / %s (%.2f%%)",
ulong_to_str(st.nalts_loaded, ns0), ulong_to_str(st.nalts_read, ns1),
st.nalts_read ? (100.0*st.nalts_loaded) / st.nalts_read : 0.0);
status("[vcfcov] ALTs too long (>%ubp): %s / %s (%.2f%%)", max_allele_len,
ulong_to_str(st.nalts_too_long, ns0), ulong_to_str(st.nalts_read, ns1),
st.nalts_read ? (100.0*st.nalts_too_long) / st.nalts_read : 0.0);
status("[vcfcov] ALTs too dense (>%u within %zubp): %s / %s (%.2f%%)",
max_gt_vars, db_graph.kmer_size,
ulong_to_str(st.nalts_no_covg, ns0), ulong_to_str(st.nalts_read, ns1),
st.nalts_read ? (100.0*st.nalts_no_covg) / st.nalts_read : 0.0);
status("[vcfcov] ALTs printed with coverage: %s / %s (%.2f%%)",
ulong_to_str(st.nalts_with_covg, ns0), ulong_to_str(st.nalts_read, ns1),
st.nalts_read ? (100.0*st.nalts_with_covg) / st.nalts_read : 0.0);
status("[vcfcov] Saved to: %s\n", out_path);
ctx_free(samplehdrids);
graph_loading_stats_destroy(&gstats);
bcf_hdr_destroy(vcfhdr);
bcf_hdr_destroy(outhdr);
hts_close(vcffh);
hts_close(outfh);
fai_destroy(fai);
db_graph_dealloc(&db_graph);
return EXIT_SUCCESS;
}
static void init_data(args_t *args)
{
args->aux.srs = bcf_sr_init();
// Open files for input and output, initialize structures
if ( args->targets )
{
if ( bcf_sr_set_targets(args->aux.srs, args->targets, args->targets_is_file, args->aux.flag&CALL_CONSTR_ALLELES ? 3 : 0)<0 )
error("Failed to read the targets: %s\n", args->targets);
if ( args->aux.flag&CALL_CONSTR_ALLELES && args->flag&CF_INS_MISSED )
{
args->aux.srs->targets->missed_reg_handler = print_missed_line;
args->aux.srs->targets->missed_reg_data = args;
}
}
if ( args->regions )
{
if ( bcf_sr_set_regions(args->aux.srs, args->regions, args->regions_is_file)<0 )
error("Failed to read the targets: %s\n", args->regions);
}
if ( !bcf_sr_add_reader(args->aux.srs, args->bcf_fname) ) error("Failed to open %s: %s\n", args->bcf_fname,bcf_sr_strerror(args->aux.srs->errnum));
args->aux.hdr = bcf_sr_get_header(args->aux.srs,0);
int i;
if ( args->samples_fname )
{
set_samples(args, args->samples_fname, args->samples_is_file);
if ( args->aux.flag&CALL_CONSTR_TRIO )
{
if ( 3*args->aux.nfams!=args->nsamples ) error("Expected only trios in %s, sorry!\n", args->samples_fname);
fprintf(stderr,"Detected %d samples in %d trio families\n", args->nsamples,args->aux.nfams);
}
args->nsex = ploidy_nsex(args->ploidy);
args->sex2ploidy = (int*) calloc(args->nsex,sizeof(int));
args->sex2ploidy_prev = (int*) calloc(args->nsex,sizeof(int));
args->aux.ploidy = (uint8_t*) malloc(args->nsamples);
for (i=0; i<args->nsamples; i++) args->aux.ploidy[i] = 2;
for (i=0; i<args->nsex; i++) args->sex2ploidy_prev[i] = 2;
}
if ( args->samples_map )
{
args->aux.hdr = bcf_hdr_subset(bcf_sr_get_header(args->aux.srs,0), args->nsamples, args->samples, args->samples_map);
if ( !args->aux.hdr ) error("Error occurred while subsetting samples\n");
for (i=0; i<args->nsamples; i++)
if ( args->samples_map[i]<0 ) error("No such sample: %s\n", args->samples[i]);
if ( !bcf_hdr_nsamples(args->aux.hdr) ) error("No matching sample found\n");
}
else
{
args->aux.hdr = bcf_hdr_dup(bcf_sr_get_header(args->aux.srs,0));
for (i=0; i<args->nsamples; i++)
if ( bcf_hdr_id2int(args->aux.hdr,BCF_DT_SAMPLE,args->samples[i])<0 )
error("No such sample: %s\n", args->samples[i]);
}
args->out_fh = hts_open(args->output_fname, hts_bcf_wmode(args->output_type));
if ( args->out_fh == NULL ) error("Can't write to \"%s\": %s\n", args->output_fname, strerror(errno));
if ( args->flag & CF_QCALL )
return;
if ( args->flag & CF_MCALL )
mcall_init(&args->aux);
if ( args->flag & CF_CCALL )
ccall_init(&args->aux);
if ( args->flag&CF_GVCF )
{
bcf_hdr_append(args->aux.hdr,"##INFO=<ID=END,Number=1,Type=Integer,Description=\"End position of the variant described in this record\">");
args->gvcf.rid = -1;
args->gvcf.line = bcf_init1();
args->gvcf.gt = (int32_t*) malloc(2*sizeof(int32_t)*bcf_hdr_nsamples(args->aux.hdr));
for (i=0; i<bcf_hdr_nsamples(args->aux.hdr); i++)
{
args->gvcf.gt[2*i+0] = bcf_gt_unphased(0);
args->gvcf.gt[2*i+1] = bcf_gt_unphased(0);
}
}
bcf_hdr_remove(args->aux.hdr, BCF_HL_INFO, "QS");
bcf_hdr_remove(args->aux.hdr, BCF_HL_INFO, "I16");
bcf_hdr_append_version(args->aux.hdr, args->argc, args->argv, "bcftools_call");
bcf_hdr_write(args->out_fh, args->aux.hdr);
if ( args->flag&CF_INS_MISSED ) init_missed_line(args);
}
