static void destroy_data(args_t *args)
{
int i;
if ( args->imap ) {
for (i = 0; i < args->n_samples; ++i)
free(args->samples[i]);
free(args->samples);
free(args->imap);
}
if (args->hnull) bcf_hdr_destroy(args->hnull);
if (args->hsub) bcf_hdr_destroy(args->hsub);
if ( args->filter )
filter_destroy(args->filter);
free(args->ac);
}
/**
* Reads header of a VCF file and returns the bcf header object.
* This wraps around vcf_hdr_read from the original htslib to
* allow for an alternative header file to be read in.
*
* this searches for the alternative header saved as <filename>.hdr
*/
bcf_hdr_t *bcf_alt_hdr_read(htsFile *fp)
{
bcf_hdr_t *h = NULL;
//check for existence of alternative header
kstring_t alt_hdr_fn = {0, 0, 0};
kputs(fp->fn, &alt_hdr_fn);
kputs(".hdr", &alt_hdr_fn);
FILE *file = fopen(alt_hdr_fn.s, "r");
if (!file)
{
h = bcf_hdr_read(fp);
}
else
{
fprintf(stderr, "[I:%s:%d %s] read alternative header for %s\n", __FILE__, __LINE__, __FUNCTION__, fp->fn);
fclose(file);
htsFile *alt_hdr = hts_open(alt_hdr_fn.s, "r");
h = bcf_hdr_read(alt_hdr);
hts_close(alt_hdr);
//helps move the pointer to the right place
bcf_hdr_t *temp_h = bcf_hdr_read(fp);
bcf_hdr_destroy(temp_h);
}
if (alt_hdr_fn.m) free(alt_hdr_fn.s);
return h;
}
void bcf_sr_destroy(readers_t *files)
{
if ( !files->nreaders ) return;
int i;
for (i=0; i<files->nreaders; i++)
{
reader_t *reader = &files->readers[i];
if ( reader->tbx ) tbx_destroy(reader->tbx);
if ( reader->bcf ) hts_idx_destroy(reader->bcf);
bcf_hdr_destroy(reader->header);
hts_close(reader->file);
if ( reader->itr ) tbx_itr_destroy(reader->itr);
int j;
for (j=0; j<reader->mbuffer; j++)
bcf_destroy1(reader->buffer[j]);
free(reader->buffer);
if ( reader->samples ) free(reader->samples);
}
free(files->readers);
free(files->seqs);
for (i=0; i<files->n_smpl; i++) free(files->samples[i]);
free(files->samples);
if (files->targets)
{
if (files->targets->itr) tbx_itr_destroy(files->targets->itr);
tbx_destroy(files->targets->tbx);
if (files->targets->line.m) free(files->targets->line.s);
hts_close(files->targets->file);
free(files->targets->seq_names);
free(files->targets);
}
if ( files->tmps.m ) free(files->tmps.s);
free(files);
}
static void destroy_data(args_t *args)
{
if ( args->flag & CF_CCALL ) ccall_destroy(&args->aux);
else if ( args->flag & CF_MCALL ) mcall_destroy(&args->aux);
else if ( args->flag & CF_QCALL ) qcall_destroy(&args->aux);
int i;
for (i=0; i<args->nsamples; i++) free(args->samples[i]);
if ( args->aux.fams )
{
for (i=0; i<args->aux.nfams; i++) free(args->aux.fams[i].name);
free(args->aux.fams);
}
if ( args->missed_line ) bcf_destroy(args->missed_line);
ploidy_destroy(args->ploidy);
if ( args->gvcf.line ) bcf_destroy(args->gvcf.line);
free(args->gvcf.gt);
free(args->gvcf.dp);
free(args->sex2ploidy);
free(args->sex2ploidy_prev);
free(args->samples);
free(args->samples_map);
free(args->sample2sex);
free(args->aux.ploidy);
bcf_hdr_destroy(args->aux.hdr);
hts_close(args->out_fh);
bcf_sr_destroy(args->aux.srs);
}
void destroy(void)
{
/* freeing up args */
bcf_hdr_destroy(args.hdr);
free(args.gt_arr);
free(args.selected_smps);
}
static int query_chroms(char *fname)
{
const char **seq;
int i, nseq, ftype = file_type(fname);
if ( ftype & IS_TXT || !ftype )
{
tbx_t *tbx = tbx_index_load(fname);
if ( !tbx ) error("Could not load .tbi index of %s\n", fname);
seq = tbx_seqnames(tbx, &nseq);
for (i=0; i<nseq; i++)
printf("%s\n", seq[i]);
free(seq);
tbx_destroy(tbx);
}
else if ( ftype==IS_BCF )
{
htsFile *fp = hts_open(fname,"r");
if ( !fp ) error("Could not read %s\n", fname);
bcf_hdr_t *hdr = bcf_hdr_read(fp);
if ( !hdr ) error("Could not read the header: %s\n", fname);
hts_close(fp);
hts_idx_t *idx = bcf_index_load(fname);
if ( !idx ) error("Could not load .csi index of %s\n", fname);
seq = bcf_index_seqnames(idx, hdr, &nseq);
for (i=0; i<nseq; i++)
printf("%s\n", seq[i]);
free(seq);
bcf_hdr_destroy(hdr);
hts_idx_destroy(idx);
}
else if ( ftype==IS_BAM ) // todo: BAM
error("BAM: todo\n");
return 0;
}
/**
* Closes files.
*/
void BCFSyncedStreamReader::close()
{
for (int32_t i = 0; i<nfiles; ++i)
{
bcf_close(vcfs[i]);
bcf_hdr_destroy(hdrs[i]);
bcf_itr_destroy(itrs[i]);
}
}
int main_vcfview(int argc, char *argv[])
{
int c, clevel = -1, in_type = FT_BCF, out_type = FT_VCF;
char *fname_out = NULL, moder[8], modew[8];
while ((c = getopt(argc, argv, "l:bvo:n:z?hu")) >= 0) {
switch (c) {
case 'o':
switch (optarg[0]) {
case 'b': out_type = FT_BCF_GZ; break;
case 'u': out_type = FT_BCF; break;
case 'z': out_type = FT_VCF_GZ; break;
case 'v': out_type = FT_VCF; break;
default: error("The output type \"%s\" not recognised\n", optarg);
}
break;
case 'l': clevel = atoi(optarg); out_type |= FT_GZ; break;
case 'v': in_type = FT_VCF; break;
case 'b': out_type = FT_BCF_GZ; break;
case 'u': out_type = FT_BCF; break;
case 'z': out_type = FT_VCF_GZ; break;
case 'n': fname_out = optarg; break;
case '?':
case 'h': usage(); return 1; break;
}
}
if (argc!=optind+1) { usage(); return 1; }
// Init reader
strcpy(moder, "r");
if ( (!strcmp("-",argv[optind]) && (in_type & FT_BCF)) || (hts_file_type(argv[optind]) & FT_BCF)) strcat(moder, "b");
htsFile *fp_in = hts_open(argv[optind], moder, NULL);
if ( !fp_in ) error("Fail to open: %s\n", argv[optind]);
bcf_hdr_t *hdr = vcf_hdr_read(fp_in);
if ( !hdr ) error("Fail to read VCF/BCF header: %s\n", argv[optind]);
bcf1_t *rec = bcf_init1();
// Init writer
strcpy(modew, "w");
if (clevel >= 0 && clevel <= 9) sprintf(modew + 1, "%d", clevel);
if (out_type & FT_GZ) strcat(modew,"z");
if (out_type & FT_BCF) strcat(modew, "b");
if (out_type == FT_BCF) strcat(modew, "u"); // uncompressed BCF output
htsFile *fp_out = hts_open(fname_out ? fname_out : "-", modew, NULL);
vcf_hdr_write(fp_out, hdr);
while ( vcf_read1(fp_in, hdr, rec) >= 0) vcf_write1(fp_out, hdr, rec);
bcf_destroy1(rec);
bcf_hdr_destroy(hdr);
hts_close(fp_in);
hts_close(fp_out);
return 0;
}
void bcf_sweep_destroy(bcf_sweep_t *sw)
{
int i;
for (i=0; i<sw->mrec; i++) bcf_empty1(&sw->rec[i]);
free(sw->idx);
free(sw->rec);
free(sw->lals);
bcf_hdr_destroy(sw->hdr);
hts_close(sw->file);
free(sw);
}
static bcf_hdr_t *strip_header(bcf_hdr_t *src, bcf_hdr_t *dst)
{
bcf_hrec_t *src_hrec, *dst_hrec, *tmp;
bcf_hdr_t *out = bcf_hdr_init("r");
int i;
for (i=0; i<dst->nhrec; i++)
{
// first insert lines which do not code BCF ids, their order does not matter
dst_hrec = dst->hrec[i];
if ( dst_hrec->type==BCF_HL_FLT || dst_hrec->type==BCF_HL_INFO || dst_hrec->type==BCF_HL_FMT || dst_hrec->type== BCF_HL_CTG ) continue;
bcf_hdr_add_hrec(out, bcf_hrec_dup(dst_hrec));
}
for (i=0; i<src->nhrec; i++)
{
// now transfer header lines which define BCF ids
src_hrec = src->hrec[i];
if ( src_hrec->type==BCF_HL_FLT || src_hrec->type==BCF_HL_INFO || src_hrec->type==BCF_HL_FMT || src_hrec->type== BCF_HL_CTG )
{
int j = bcf_hrec_find_key(src_hrec, "ID");
dst_hrec = bcf_hdr_get_hrec(dst, src_hrec->type, "ID", src_hrec->vals[j], NULL);
if ( !dst_hrec ) continue;
tmp = bcf_hrec_dup(dst_hrec);
j = bcf_hrec_find_key(src_hrec, "IDX");
if ( j>=0 )
{
j = atoi(src_hrec->vals[j]);
hrec_add_idx(tmp, j);
}
bcf_hdr_add_hrec(out, tmp);
}
}
bcf_hdr_sync(out);
for (i=0; i<dst->nhrec; i++)
{
// finally add new structured fields
dst_hrec = dst->hrec[i];
if ( dst_hrec->type==BCF_HL_FLT || dst_hrec->type==BCF_HL_INFO || dst_hrec->type==BCF_HL_FMT || dst_hrec->type== BCF_HL_CTG )
{
int j = bcf_hrec_find_key(dst_hrec, "ID");
tmp = bcf_hdr_get_hrec(out, dst_hrec->type, "ID", dst_hrec->vals[j], NULL);
if ( !tmp )
bcf_hdr_add_hrec(out, bcf_hrec_dup(dst_hrec));
}
}
for (i=0; i<dst->n[BCF_DT_SAMPLE]; i++) bcf_hdr_add_sample(out, dst->samples[i]);
bcf_hdr_add_sample(out, NULL);
bcf_hdr_destroy(dst);
return out;
}
static void bcf_sr_destroy1(bcf_sr_t *reader)
{
if ( reader->tbx_idx ) tbx_destroy(reader->tbx_idx);
if ( reader->bcf_idx ) hts_idx_destroy(reader->bcf_idx);
bcf_hdr_destroy(reader->header);
hts_close(reader->file);
if ( reader->itr ) tbx_itr_destroy(reader->itr);
int j;
for (j=0; j<reader->mbuffer; j++)
bcf_destroy1(reader->buffer[j]);
free(reader->buffer);
free(reader->samples);
free(reader->filter_ids);
}
static void destroy_data(args_t *args)
{
free(args->plugin.name);
args->plugin.destroy();
dlclose(args->plugin.handle);
if ( args->hdr_out ) bcf_hdr_destroy(args->hdr_out);
if ( args->nplugin_paths>0 )
{
free(args->plugin_paths[0]);
free(args->plugin_paths);
}
if ( args->filter )
filter_destroy(args->filter);
if (args->out_fh) hts_close(args->out_fh);
}
/**
* Closes files.
*/
void BCFSyncedReader::close()
{
for (size_t i=0; i<nfiles; ++i)
{
bcf_close(files[i]);
bcf_hdr_destroy(hdrs[i]);
bcf_itr_destroy(itrs[i]);
}
while (pool.size()!=0)
{
bcf_destroy(pool.front());
pool.pop_front();
}
}
bcf_hdr_t *bcf_hdr_read(BGZF *fp)
{
uint8_t magic[5];
bcf_hdr_t *h;
h = bcf_hdr_init();
bgzf_read(fp, magic, 5);
if (strncmp((char*)magic, "BCF\2\1", 5) != 0) {
if (hts_verbose >= 2)
fprintf(stderr, "[E::%s] invalid BCF2 magic string\n", __func__);
bcf_hdr_destroy(h);
return 0;
}
bgzf_read(fp, &h->l_text, 4);
h->text = (char*)malloc(h->l_text);
bgzf_read(fp, h->text, h->l_text);
bcf_hdr_parse(h);
return h;
}
static void destroy_data(args_t *args)
{
free(args->info_tags);
free(args->fmt_tags);
if ( args->filter )
filter_destroy(args->filter);
int i, nsmpl = bcf_hdr_nsamples(args->hdr_in);
for (i=0; i<nsmpl; i++)
{
if ( args->fh[i] && hts_close(args->fh[i])!=0 ) error("Error: close failed!\n");
free(args->bnames[i]);
}
free(args->bnames);
free(args->fh);
bcf_sr_destroy(args->sr);
bcf_hdr_destroy(args->hdr_out);
free(args);
}
int main_getalt(int argc, char *argv[])
{
int c;
char *fn;
BGZF *fp;
bcf1_t *b;
bcf_hdr_t *h;
kstring_t s = {0,0,0};
while ((c = getopt(argc, argv, "")) >= 0) {
}
if (argc - optind == 0) {
fprintf(stderr, "Usage: bgt getalt <bgt-base>\n");
return 1;
}
fn = (char*)calloc(strlen(argv[optind]) + 5, 1);
sprintf(fn, "%s.bcf", argv[optind]);
fp = bgzf_open(fn, "r");
free(fn);
assert(fp);
h = bcf_hdr_read(fp);
b = bcf_init1();
while (bcf_read1(fp, b) >= 0) {
char *ref, *alt;
int l_ref, l_alt, i, min_l;
bcf_get_ref_alt1(b, &l_ref, &ref, &l_alt, &alt);
min_l = l_ref < l_alt? l_ref : l_alt;
for (i = 0; i < min_l && ref[i] == alt[i]; ++i);
s.l = 0;
kputs(h->id[BCF_DT_CTG][b->rid].key, &s);
kputc(':', &s); kputw(b->pos + 1 + i, &s);
kputc(':', &s); kputw(b->rlen - i, &s);
kputc(':', &s); kputsn(alt + i, l_alt - i, &s);
puts(s.s);
}
bcf_destroy1(b);
bcf_hdr_destroy(h);
bgzf_close(fp);
free(s.s);
return 0;
}
static void destroy_data(args_t *args)
{
int i;
for (i=0; i<args->nfnames; i++) free(args->fnames[i]);
free(args->fnames);
if ( args->files ) bcf_sr_destroy(args->files);
if ( hts_close(args->out_fh)!=0 ) error("hts_close error\n");
bcf_hdr_destroy(args->out_hdr);
free(args->seen_seq);
free(args->start_pos);
free(args->swap_phase);
for (i=0; i<args->mbuf; i++) bcf_destroy(args->buf[i]);
free(args->buf);
free(args->GTa);
free(args->GTb);
free(args->nmatch);
free(args->nmism);
free(args->phase_qual);
free(args->phase_set);
}
static int view_vcf(hFILE *hfp, const char *filename)
{
vcfFile *in = hts_hopen(hfp, filename, "r");
if (in == NULL) return 0;
vcfFile *out = dup_stdout("w");
bcf_hdr_t *hdr = bcf_hdr_read(in);
if (show_headers) bcf_hdr_write(out, hdr);
if (mode == view_all) {
bcf1_t *rec = bcf_init();
while (bcf_read(in, hdr, rec) >= 0)
bcf_write(out, hdr, rec);
bcf_destroy(rec);
}
bcf_hdr_destroy(hdr);
hts_close(out);
hts_close(in);
return 1;
}
static int mpileup(mplp_conf_t *conf, int n, char **fn)
{
extern void *bcf_call_add_rg(void *rghash, const char *hdtext, const char *list);
extern void bcf_call_del_rghash(void *rghash);
mplp_aux_t **data;
int i, tid, pos, *n_plp, tid0 = -1, beg0 = 0, end0 = 1u<<29, ref_len, ref_tid = -1, max_depth, max_indel_depth;
const bam_pileup1_t **plp;
bam_mplp_t iter;
bam_header_t *h = 0;
char *ref;
void *rghash = 0;
bcf_callaux_t *bca = 0;
bcf_callret1_t *bcr = 0;
bcf_call_t bc;
bcf_t *bp = 0;
bcf_hdr_t *bh = 0;
bam_sample_t *sm = 0;
kstring_t buf;
mplp_pileup_t gplp;
memset(&gplp, 0, sizeof(mplp_pileup_t));
memset(&buf, 0, sizeof(kstring_t));
memset(&bc, 0, sizeof(bcf_call_t));
data = calloc(n, sizeof(void*));
plp = calloc(n, sizeof(void*));
n_plp = calloc(n, sizeof(int*));
sm = bam_smpl_init();
// read the header and initialize data
for (i = 0; i < n; ++i) {
bam_header_t *h_tmp;
data[i] = calloc(1, sizeof(mplp_aux_t));
data[i]->fp = strcmp(fn[i], "-") == 0? bam_dopen(fileno(stdin), "r") : bam_open(fn[i], "r");
data[i]->conf = conf;
h_tmp = bam_header_read(data[i]->fp);
data[i]->h = i? h : h_tmp; // for i==0, "h" has not been set yet
bam_smpl_add(sm, fn[i], (conf->flag&MPLP_IGNORE_RG)? 0 : h_tmp->text);
rghash = bcf_call_add_rg(rghash, h_tmp->text, conf->pl_list);
if (conf->reg) {
int beg, end;
bam_index_t *idx;
idx = bam_index_load(fn[i]);
if (idx == 0) {
fprintf(stderr, "[%s] fail to load index for %d-th input.\n", __func__, i+1);
exit(1);
}
if (bam_parse_region(h_tmp, conf->reg, &tid, &beg, &end) < 0) {
fprintf(stderr, "[%s] malformatted region or wrong seqname for %d-th input.\n", __func__, i+1);
exit(1);
}
if (i == 0) tid0 = tid, beg0 = beg, end0 = end;
data[i]->iter = bam_iter_query(idx, tid, beg, end);
bam_index_destroy(idx);
}
if (i == 0) h = h_tmp;
else {
// FIXME: to check consistency
bam_header_destroy(h_tmp);
}
}
gplp.n = sm->n;
gplp.n_plp = calloc(sm->n, sizeof(int));
gplp.m_plp = calloc(sm->n, sizeof(int));
gplp.plp = calloc(sm->n, sizeof(void*));
fprintf(stderr, "[%s] %d samples in %d input files\n", __func__, sm->n, n);
// write the VCF header
if (conf->flag & MPLP_GLF) {
kstring_t s;
bh = calloc(1, sizeof(bcf_hdr_t));
s.l = s.m = 0; s.s = 0;
bp = bcf_open("-", (conf->flag&MPLP_NO_COMP)? "wu" : "w");
for (i = 0; i < h->n_targets; ++i) {
kputs(h->target_name[i], &s);
kputc('\0', &s);
}
bh->l_nm = s.l;
bh->name = malloc(s.l);
memcpy(bh->name, s.s, s.l);
s.l = 0;
for (i = 0; i < sm->n; ++i) {
kputs(sm->smpl[i], &s); kputc('\0', &s);
}
bh->l_smpl = s.l;
bh->sname = malloc(s.l);
memcpy(bh->sname, s.s, s.l);
bh->txt = malloc(strlen(BAM_VERSION) + 64);
bh->l_txt = 1 + sprintf(bh->txt, "##samtoolsVersion=%s\n", BAM_VERSION);
free(s.s);
bcf_hdr_sync(bh);
bcf_hdr_write(bp, bh);
bca = bcf_call_init(-1., conf->min_baseQ);
bcr = calloc(sm->n, sizeof(bcf_callret1_t));
bca->rghash = rghash;
bca->openQ = conf->openQ, bca->extQ = conf->extQ, bca->tandemQ = conf->tandemQ;
bca->min_frac = conf->min_frac;
bca->min_support = conf->min_support;
}
if (tid0 >= 0 && conf->fai) { // region is set
ref = faidx_fetch_seq(conf->fai, h->target_name[tid0], 0, 0x7fffffff, &ref_len);
ref_tid = tid0;
for (i = 0; i < n; ++i) data[i]->ref = ref, data[i]->ref_id = tid0;
} else ref_tid = -1, ref = 0;
iter = bam_mplp_init(n, mplp_func, (void**)data);
max_depth = conf->max_depth;
if (max_depth * sm->n > 1<<20)
fprintf(stderr, "(%s) Max depth is above 1M. Potential memory hog!\n", __func__);
if (max_depth * sm->n < 8000) {
max_depth = 8000 / sm->n;
fprintf(stderr, "<%s> Set max per-file depth to %d\n", __func__, max_depth);
}
max_indel_depth = conf->max_indel_depth * sm->n;
bam_mplp_set_maxcnt(iter, max_depth);
while (bam_mplp_auto(iter, &tid, &pos, n_plp, plp) > 0) {
if (conf->reg && (pos < beg0 || pos >= end0)) continue; // out of the region requested
if (conf->bed && tid >= 0 && !bed_overlap(conf->bed, h->target_name[tid], pos, pos+1)) continue;
if (tid != ref_tid) {
free(ref); ref = 0;
if (conf->fai) ref = faidx_fetch_seq(conf->fai, h->target_name[tid], 0, 0x7fffffff, &ref_len);
for (i = 0; i < n; ++i) data[i]->ref = ref, data[i]->ref_id = tid;
ref_tid = tid;
}
if (conf->flag & MPLP_GLF) {
int total_depth, _ref0, ref16;
bcf1_t *b = calloc(1, sizeof(bcf1_t));
for (i = total_depth = 0; i < n; ++i) total_depth += n_plp[i];
group_smpl(&gplp, sm, &buf, n, fn, n_plp, plp, conf->flag & MPLP_IGNORE_RG);
_ref0 = (ref && pos < ref_len)? ref[pos] : 'N';
ref16 = bam_nt16_table[_ref0];
for (i = 0; i < gplp.n; ++i)
bcf_call_glfgen(gplp.n_plp[i], gplp.plp[i], ref16, bca, bcr + i);
bcf_call_combine(gplp.n, bcr, ref16, &bc);
bcf_call2bcf(tid, pos, &bc, b, (conf->flag&(MPLP_FMT_DP|MPLP_FMT_SP))? bcr : 0,
(conf->flag&MPLP_FMT_SP), 0, 0);
bcf_write(bp, bh, b);
bcf_destroy(b);
// call indels
if (!(conf->flag&MPLP_NO_INDEL) && total_depth < max_indel_depth && bcf_call_gap_prep(gplp.n, gplp.n_plp, gplp.plp, pos, bca, ref, rghash) >= 0) {
for (i = 0; i < gplp.n; ++i)
bcf_call_glfgen(gplp.n_plp[i], gplp.plp[i], -1, bca, bcr + i);
if (bcf_call_combine(gplp.n, bcr, -1, &bc) >= 0) {
b = calloc(1, sizeof(bcf1_t));
bcf_call2bcf(tid, pos, &bc, b, (conf->flag&(MPLP_FMT_DP|MPLP_FMT_SP))? bcr : 0,
(conf->flag&MPLP_FMT_SP), bca, ref);
bcf_write(bp, bh, b);
bcf_destroy(b);
}
}
} else {
printf("%s\t%d\t%c", h->target_name[tid], pos + 1, (ref && pos < ref_len)? ref[pos] : 'N');
for (i = 0; i < n; ++i) {
int j;
printf("\t%d\t", n_plp[i]);
if (n_plp[i] == 0) {
printf("*\t*"); // FIXME: printf() is very slow...
if (conf->flag & MPLP_PRINT_POS) printf("\t*");
} else {
for (j = 0; j < n_plp[i]; ++j)
pileup_seq(plp[i] + j, pos, ref_len, ref);
putchar('\t');
for (j = 0; j < n_plp[i]; ++j) {
const bam_pileup1_t *p = plp[i] + j;
int c = bam1_qual(p->b)[p->qpos] + 33;
if (c > 126) c = 126;
putchar(c);
}
if (conf->flag & MPLP_PRINT_MAPQ) {
putchar('\t');
for (j = 0; j < n_plp[i]; ++j) {
int c = plp[i][j].b->core.qual + 33;
if (c > 126) c = 126;
putchar(c);
}
}
if (conf->flag & MPLP_PRINT_POS) {
putchar('\t');
for (j = 0; j < n_plp[i]; ++j) {
if (j > 0) putchar(',');
printf("%d", plp[i][j].qpos + 1); // FIXME: printf() is very slow...
}
}
}
}
putchar('\n');
}
}
bcf_close(bp);
bam_smpl_destroy(sm); free(buf.s);
for (i = 0; i < gplp.n; ++i) free(gplp.plp[i]);
free(gplp.plp); free(gplp.n_plp); free(gplp.m_plp);
bcf_call_del_rghash(rghash);
bcf_hdr_destroy(bh); bcf_call_destroy(bca); free(bc.PL); free(bcr);
bam_mplp_destroy(iter);
bam_header_destroy(h);
for (i = 0; i < n; ++i) {
bam_close(data[i]->fp);
if (data[i]->iter) bam_iter_destroy(data[i]->iter);
free(data[i]);
}
free(data); free(plp); free(ref); free(n_plp);
return 0;
}
static void init_data(args_t *args)
{
bcf1_t *line = NULL;
// With phased concat, the chunks overlap and come in the right order. To
// avoid opening all files at once, store start positions to recognise need
// for the next one. This way we can keep only two open chunks at once.
if ( args->phased_concat )
{
args->start_pos = (int*) malloc(sizeof(int)*args->nfnames);
line = bcf_init();
}
kstring_t str = {0,0,0};
int i, prev_chrid = -1;
for (i=0; i<args->nfnames; i++)
{
htsFile *fp = hts_open(args->fnames[i], "r"); if ( !fp ) error("Failed to open: %s\n", args->fnames[i]);
bcf_hdr_t *hdr = bcf_hdr_read(fp); if ( !hdr ) error("Failed to parse header: %s\n", args->fnames[i]);
args->out_hdr = bcf_hdr_merge(args->out_hdr,hdr);
if ( bcf_hdr_nsamples(hdr) != bcf_hdr_nsamples(args->out_hdr) )
error("Different number of samples in %s. Perhaps \"bcftools merge\" is what you are looking for?\n", args->fnames[i]);
int j;
for (j=0; j<bcf_hdr_nsamples(hdr); j++)
if ( strcmp(args->out_hdr->samples[j],hdr->samples[j]) )
error("Different sample names in %s. Perhaps \"bcftools merge\" is what you are looking for?\n", args->fnames[i]);
if ( args->phased_concat )
{
int ret = bcf_read(fp, hdr, line);
if ( ret!=0 ) args->start_pos[i] = -2; // empty file
else
{
int chrid = bcf_hdr_id2int(args->out_hdr,BCF_DT_CTG,bcf_seqname(hdr,line));
args->start_pos[i] = chrid==prev_chrid ? line->pos : -1;
prev_chrid = chrid;
}
}
bcf_hdr_destroy(hdr);
hts_close(fp);
}
free(str.s);
if ( line ) bcf_destroy(line);
args->seen_seq = (int*) calloc(args->out_hdr->n[BCF_DT_CTG],sizeof(int));
if ( args->phased_concat )
{
bcf_hdr_append(args->out_hdr,"##FORMAT=<ID=PQ,Number=1,Type=Integer,Description=\"Phasing Quality (bigger is better)\">");
bcf_hdr_append(args->out_hdr,"##FORMAT=<ID=PS,Number=1,Type=Integer,Description=\"Phase Set\">");
}
if (args->record_cmd_line) bcf_hdr_append_version(args->out_hdr, args->argc, args->argv, "bcftools_concat");
args->out_fh = hts_open(args->output_fname,hts_bcf_wmode(args->output_type));
if ( args->out_fh == NULL ) error("Can't write to \"%s\": %s\n", args->output_fname, strerror(errno));
if ( args->n_threads ) hts_set_threads(args->out_fh, args->n_threads);
bcf_hdr_write(args->out_fh, args->out_hdr);
if ( args->allow_overlaps )
{
args->files = bcf_sr_init();
args->files->require_index = 1;
if ( args->regions_list )
{
if ( bcf_sr_set_regions(args->files, args->regions_list, args->regions_is_file)<0 )
error("Failed to read the regions: %s\n", args->regions_list);
}
if ( args->remove_dups )
{
if ( !strcmp(args->remove_dups,"snps") ) args->files->collapse |= COLLAPSE_SNPS;
else if ( !strcmp(args->remove_dups,"indels") ) args->files->collapse |= COLLAPSE_INDELS;
else if ( !strcmp(args->remove_dups,"both") ) args->files->collapse |= COLLAPSE_SNPS | COLLAPSE_INDELS;
else if ( !strcmp(args->remove_dups,"any") ) args->files->collapse |= COLLAPSE_ANY;
else if ( !strcmp(args->remove_dups,"all") ) args->files->collapse |= COLLAPSE_ANY;
else if ( !strcmp(args->remove_dups,"none") ) args->files->collapse = COLLAPSE_NONE;
else error("The -D string \"%s\" not recognised.\n", args->remove_dups);
}
for (i=0; i<args->nfnames; i++)
if ( !bcf_sr_add_reader(args->files,args->fnames[i]) ) error("Failed to open %s: %s\n", args->fnames[i],bcf_sr_strerror(args->files->errnum));
}
else if ( args->phased_concat )
{
// Remove empty files from the list
int nok = 0;
while (1)
{
while ( nok<args->nfnames && args->start_pos[nok]!=-2 ) nok++;
if ( nok==args->nfnames ) break;
i = nok;
while ( i<args->nfnames && args->start_pos[i]==-2 ) i++;
if ( i==args->nfnames ) break;
int tmp = args->start_pos[nok]; args->start_pos[nok] = args->start_pos[i]; args->start_pos[i] = tmp;
char *str = args->fnames[nok]; args->fnames[nok] = args->fnames[i]; args->fnames[i] = str;
}
for (i=nok; i<args->nfnames; i++) free(args->fnames[i]);
args->nfnames = nok;
for (i=1; i<args->nfnames; i++)
if ( args->start_pos[i-1]!=-1 && args->start_pos[i]!=-1 && args->start_pos[i]<args->start_pos[i-1] )
error("The files not in ascending order: %d in %s, %d in %s\n", args->start_pos[i-1]+1,args->fnames[i-1],args->start_pos[i]+1,args->fnames[i]);
args->prev_chr = -1;
args->swap_phase = (int*) calloc(bcf_hdr_nsamples(args->out_hdr),sizeof(int));
args->nmatch = (int*) calloc(bcf_hdr_nsamples(args->out_hdr),sizeof(int));
args->nmism = (int*) calloc(bcf_hdr_nsamples(args->out_hdr),sizeof(int));
args->phase_qual = (int32_t*) malloc(bcf_hdr_nsamples(args->out_hdr)*sizeof(int32_t));
args->phase_set = (int32_t*) malloc(bcf_hdr_nsamples(args->out_hdr)*sizeof(int32_t));
args->files = bcf_sr_init();
args->files->require_index = 1;
args->ifname = 0;
}
}
static void concat(args_t *args)
{
int i;
if ( args->phased_concat ) // phased concat
{
// keep only two open files at a time
while ( args->ifname < args->nfnames )
{
int new_file = 0;
while ( args->files->nreaders < 2 && args->ifname < args->nfnames )
{
if ( !bcf_sr_add_reader(args->files,args->fnames[args->ifname]) ) error("Failed to open %s: %s\n", args->fnames[args->ifname],bcf_sr_strerror(args->files->errnum));
new_file = 1;
args->ifname++;
if ( args->start_pos[args->ifname-1]==-1 ) break; // new chromosome, start with only one file open
if ( args->ifname < args->nfnames && args->start_pos[args->ifname]==-1 ) break; // next file starts on a different chromosome
}
// is there a line from the previous run? Seek the newly opened reader to that position
int seek_pos = -1;
int seek_chr = -1;
if ( bcf_sr_has_line(args->files,0) )
{
bcf1_t *line = bcf_sr_get_line(args->files,0);
bcf_sr_seek(args->files, bcf_seqname(args->files->readers[0].header,line), line->pos);
seek_pos = line->pos;
seek_chr = bcf_hdr_name2id(args->out_hdr, bcf_seqname(args->files->readers[0].header,line));
}
else if ( new_file )
bcf_sr_seek(args->files,NULL,0); // set to start
int nret;
while ( (nret = bcf_sr_next_line(args->files)) )
{
if ( !bcf_sr_has_line(args->files,0) ) // no input from the first reader
{
// We are assuming that there is a perfect overlap, sites which are not present in both files are dropped
if ( ! bcf_sr_region_done(args->files,0) ) continue;
phased_flush(args);
bcf_sr_remove_reader(args->files, 0);
}
// Get a line to learn about current position
for (i=0; i<args->files->nreaders; i++)
if ( bcf_sr_has_line(args->files,i) ) break;
bcf1_t *line = bcf_sr_get_line(args->files,i);
// This can happen after bcf_sr_seek: indel may start before the coordinate which we seek to.
if ( seek_chr>=0 && seek_pos>line->pos && seek_chr==bcf_hdr_name2id(args->out_hdr, bcf_seqname(args->files->readers[i].header,line)) ) continue;
seek_pos = seek_chr = -1;
// Check if the position overlaps with the next, yet unopened, reader
int must_seek = 0;
while ( args->ifname < args->nfnames && args->start_pos[args->ifname]!=-1 && line->pos >= args->start_pos[args->ifname] )
{
must_seek = 1;
if ( !bcf_sr_add_reader(args->files,args->fnames[args->ifname]) ) error("Failed to open %s: %s\n", args->fnames[args->ifname],bcf_sr_strerror(args->files->errnum));
args->ifname++;
}
if ( must_seek )
{
bcf_sr_seek(args->files, bcf_seqname(args->files->readers[i].header,line), line->pos);
seek_pos = line->pos;
seek_chr = bcf_hdr_name2id(args->out_hdr, bcf_seqname(args->files->readers[i].header,line));
continue;
}
// We are assuming that there is a perfect overlap, sites which are not present in both files are dropped
if ( args->files->nreaders>1 && !bcf_sr_has_line(args->files,1) && !bcf_sr_region_done(args->files,1) ) continue;
phased_push(args, bcf_sr_get_line(args->files,0), args->files->nreaders>1 ? bcf_sr_get_line(args->files,1) : NULL);
}
if ( args->files->nreaders )
{
phased_flush(args);
while ( args->files->nreaders )
bcf_sr_remove_reader(args->files, 0);
}
}
}
else if ( args->files ) // combining overlapping files, using synced reader
{
while ( bcf_sr_next_line(args->files) )
{
for (i=0; i<args->files->nreaders; i++)
{
bcf1_t *line = bcf_sr_get_line(args->files,i);
if ( !line ) continue;
bcf_translate(args->out_hdr, args->files->readers[i].header, line);
bcf_write1(args->out_fh, args->out_hdr, line);
if ( args->remove_dups ) break;
}
}
}
else // concatenating
{
kstring_t tmp = {0,0,0};
int prev_chr_id = -1, prev_pos;
bcf1_t *line = bcf_init();
for (i=0; i<args->nfnames; i++)
{
htsFile *fp = hts_open(args->fnames[i], "r"); if ( !fp ) error("Failed to open: %s\n", args->fnames[i]);
bcf_hdr_t *hdr = bcf_hdr_read(fp); if ( !hdr ) error("Failed to parse header: %s\n", args->fnames[i]);
if ( !fp->is_bin && args->output_type&FT_VCF )
{
line->max_unpack = BCF_UN_STR;
// if VCF is on both input and output, avoid VCF to BCF conversion
while ( hts_getline(fp, KS_SEP_LINE, &fp->line) >=0 )
{
char *str = fp->line.s;
while ( *str && *str!='\t' ) str++;
tmp.l = 0;
kputsn(fp->line.s,str-fp->line.s,&tmp);
int chr_id = bcf_hdr_name2id(args->out_hdr, tmp.s);
if ( chr_id<0 ) error("The sequence \"%s\" not defined in the header: %s\n(Quick workaround: index the file.)\n", tmp.s, args->fnames[i]);
if ( prev_chr_id!=chr_id )
{
prev_pos = -1;
if ( args->seen_seq[chr_id] )
error("\nThe chromosome block %s is not contiguous, consider running with -a.\n", tmp.s);
}
char *end;
int pos = strtol(str+1,&end,10) - 1;
if ( end==str+1 ) error("Could not parse line: %s\n", fp->line.s);
if ( prev_pos > pos )
error("The chromosome block %s is not sorted, consider running with -a.\n", tmp.s);
args->seen_seq[chr_id] = 1;
prev_chr_id = chr_id;
if ( vcf_write_line(args->out_fh, &fp->line)!=0 ) error("Failed to write %d bytes\n", fp->line.l);
}
}
else
{
// BCF conversion is required
line->max_unpack = 0;
while ( bcf_read(fp, hdr, line)==0 )
{
bcf_translate(args->out_hdr, hdr, line);
if ( prev_chr_id!=line->rid )
{
prev_pos = -1;
if ( args->seen_seq[line->rid] )
error("\nThe chromosome block %s is not contiguous, consider running with -a.\n", bcf_seqname(args->out_hdr, line));
}
if ( prev_pos > line->pos )
error("The chromosome block %s is not sorted, consider running with -a.\n", bcf_seqname(args->out_hdr, line));
args->seen_seq[line->rid] = 1;
prev_chr_id = line->rid;
if ( bcf_write(args->out_fh, args->out_hdr, line)!=0 ) error("Failed to write\n");
}
}
bcf_hdr_destroy(hdr);
hts_close(fp);
}
bcf_destroy(line);
free(tmp.s);
}
}
static int mpileup(mplp_conf_t *conf)
{
if (conf->nfiles == 0) {
fprintf(stderr,"[%s] no input file/data given\n", __func__);
exit(EXIT_FAILURE);
}
mplp_ref_t mp_ref = MPLP_REF_INIT;
conf->gplp = (mplp_pileup_t *) calloc(1,sizeof(mplp_pileup_t));
conf->mplp_data = (mplp_aux_t**) calloc(conf->nfiles, sizeof(mplp_aux_t*));
conf->plp = (const bam_pileup1_t**) calloc(conf->nfiles, sizeof(bam_pileup1_t*));
conf->n_plp = (int*) calloc(conf->nfiles, sizeof(int));
// Allow to run mpileup on multiple regions in one go. This comes at cost: the bai index
// must be kept in the memory for the whole time which can be a problem with many bams.
// Therefore if none or only one region is requested, we initialize the bam iterator as
// before and free the index. Only when multiple regions are queried, we keep the index.
int nregs = 0;
if ( conf->reg_fname )
{
if ( conf->reg_is_file )
{
conf->reg = regidx_init(conf->reg_fname,NULL,NULL,0,NULL);
if ( !conf->reg ) {
fprintf(stderr,"Could not parse the regions: %s\n", conf->reg_fname);
exit(EXIT_FAILURE);
}
}
else
{
conf->reg = regidx_init(NULL,regidx_parse_reg,NULL,sizeof(char*),NULL);
if ( regidx_insert_list(conf->reg,conf->reg_fname,',') !=0 ) {
fprintf(stderr,"Could not parse the regions: %s\n", conf->reg_fname);
exit(EXIT_FAILURE);
}
}
nregs = regidx_nregs(conf->reg);
conf->reg_itr = regitr_init(conf->reg);
regitr_loop(conf->reg_itr); // region iterator now positioned at the first region
}
// read the header of each file in the list and initialize data
// beware: mpileup has always assumed that tid's are consistent in the headers, add sanity check at least!
bam_hdr_t *hdr = NULL; // header of first file in input list
int i;
for (i = 0; i < conf->nfiles; ++i) {
bam_hdr_t *h_tmp;
conf->mplp_data[i] = (mplp_aux_t*) calloc(1, sizeof(mplp_aux_t));
conf->mplp_data[i]->fp = sam_open(conf->files[i], "rb");
if ( !conf->mplp_data[i]->fp )
{
fprintf(stderr, "[%s] failed to open %s: %s\n", __func__, conf->files[i], strerror(errno));
exit(EXIT_FAILURE);
}
if (hts_set_opt(conf->mplp_data[i]->fp, CRAM_OPT_DECODE_MD, 0)) {
fprintf(stderr, "Failed to set CRAM_OPT_DECODE_MD value\n");
exit(EXIT_FAILURE);
}
if (conf->fai_fname && hts_set_fai_filename(conf->mplp_data[i]->fp, conf->fai_fname) != 0) {
fprintf(stderr, "[%s] failed to process %s: %s\n",
__func__, conf->fai_fname, strerror(errno));
exit(EXIT_FAILURE);
}
conf->mplp_data[i]->conf = conf;
conf->mplp_data[i]->ref = &mp_ref;
h_tmp = sam_hdr_read(conf->mplp_data[i]->fp);
if ( !h_tmp ) {
fprintf(stderr,"[%s] fail to read the header of %s\n", __func__, conf->files[i]);
exit(EXIT_FAILURE);
}
conf->mplp_data[i]->h = i ? hdr : h_tmp; // for j==0, "h" has not been set yet
conf->mplp_data[i]->bam_id = bam_smpl_add_bam(conf->bsmpl,h_tmp->text,conf->files[i]);
if ( conf->mplp_data[i]->bam_id<0 )
{
// no usable readgroups in this bam, it can be skipped
sam_close(conf->mplp_data[i]->fp);
free(conf->mplp_data[i]);
bam_hdr_destroy(h_tmp);
free(conf->files[i]);
if ( i+1<conf->nfiles ) memmove(&conf->files[i],&conf->files[i+1],sizeof(*conf->files)*(conf->nfiles-i-1));
conf->nfiles--;
i--;
continue;
}
if (conf->reg) {
hts_idx_t *idx = sam_index_load(conf->mplp_data[i]->fp, conf->files[i]);
if (idx == NULL) {
fprintf(stderr, "[%s] fail to load index for %s\n", __func__, conf->files[i]);
exit(EXIT_FAILURE);
}
conf->buf.l = 0;
ksprintf(&conf->buf,"%s:%u-%u",conf->reg_itr->seq,conf->reg_itr->beg+1,conf->reg_itr->end+1);
conf->mplp_data[i]->iter = sam_itr_querys(idx, conf->mplp_data[i]->h, conf->buf.s);
if ( !conf->mplp_data[i]->iter )
{
conf->mplp_data[i]->iter = sam_itr_querys(idx, conf->mplp_data[i]->h, conf->reg_itr->seq);
if ( conf->mplp_data[i]->iter ) {
fprintf(stderr,"[E::%s] fail to parse region '%s'\n", __func__, conf->buf.s);
exit(EXIT_FAILURE);
}
fprintf(stderr,"[E::%s] the sequence \"%s\" not found: %s\n",__func__,conf->reg_itr->seq,conf->files[i]);
exit(EXIT_FAILURE);
}
if ( nregs==1 ) // no need to keep the index in memory
hts_idx_destroy(idx);
else
conf->mplp_data[i]->idx = idx;
}
if ( !hdr ) hdr = h_tmp; /* save the header of first file in list */
else {
// FIXME: check consistency between h and h_tmp
bam_hdr_destroy(h_tmp);
// we store only the first file's header; it's (alleged to be)
// compatible with the i-th file's target_name lookup needs
conf->mplp_data[i]->h = hdr;
}
}
// allocate data storage proportionate to number of samples being studied sm->n
bam_smpl_get_samples(conf->bsmpl, &conf->gplp->n);
conf->gplp->n_plp = (int*) calloc(conf->gplp->n, sizeof(int));
conf->gplp->m_plp = (int*) calloc(conf->gplp->n, sizeof(int));
conf->gplp->plp = (bam_pileup1_t**) calloc(conf->gplp->n, sizeof(bam_pileup1_t*));
fprintf(stderr, "[%s] %d samples in %d input files\n", __func__, conf->gplp->n, conf->nfiles);
// write the VCF header
conf->bcf_fp = hts_open(conf->output_fname?conf->output_fname:"-", hts_bcf_wmode(conf->output_type));
if (conf->bcf_fp == NULL) {
fprintf(stderr, "[%s] failed to write to %s: %s\n", __func__, conf->output_fname? conf->output_fname : "standard output", strerror(errno));
exit(EXIT_FAILURE);
}
if ( conf->n_threads ) hts_set_threads(conf->bcf_fp, conf->n_threads);
// BCF header creation
conf->bcf_hdr = bcf_hdr_init("w");
conf->buf.l = 0;
if (conf->record_cmd_line)
{
ksprintf(&conf->buf, "##bcftoolsVersion=%s+htslib-%s\n",bcftools_version(),hts_version());
bcf_hdr_append(conf->bcf_hdr, conf->buf.s);
conf->buf.l = 0;
ksprintf(&conf->buf, "##bcftoolsCommand=mpileup");
for (i=1; i<conf->argc; i++) ksprintf(&conf->buf, " %s", conf->argv[i]);
kputc('\n', &conf->buf);
bcf_hdr_append(conf->bcf_hdr, conf->buf.s);
}
if (conf->fai_fname)
{
conf->buf.l = 0;
ksprintf(&conf->buf, "##reference=file://%s\n", conf->fai_fname);
bcf_hdr_append(conf->bcf_hdr, conf->buf.s);
}
// Translate BAM @SQ tags to BCF ##contig tags
// todo: use/write new BAM header manipulation routines, fill also UR, M5
for (i=0; i<hdr->n_targets; i++)
{
conf->buf.l = 0;
ksprintf(&conf->buf, "##contig=<ID=%s,length=%d>", hdr->target_name[i], hdr->target_len[i]);
bcf_hdr_append(conf->bcf_hdr, conf->buf.s);
}
conf->buf.l = 0;
bcf_hdr_append(conf->bcf_hdr,"##ALT=<ID=*,Description=\"Represents allele(s) other than observed.\">");
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=INDEL,Number=0,Type=Flag,Description=\"Indicates that the variant is an INDEL.\">");
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=IDV,Number=1,Type=Integer,Description=\"Maximum number of reads supporting an indel\">");
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=IMF,Number=1,Type=Float,Description=\"Maximum fraction of reads supporting an indel\">");
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=DP,Number=1,Type=Integer,Description=\"Raw read depth\">");
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=VDB,Number=1,Type=Float,Description=\"Variant Distance Bias for filtering splice-site artefacts in RNA-seq data (bigger is better)\",Version=\"3\">");
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=RPB,Number=1,Type=Float,Description=\"Mann-Whitney U test of Read Position Bias (bigger is better)\">");
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=MQB,Number=1,Type=Float,Description=\"Mann-Whitney U test of Mapping Quality Bias (bigger is better)\">");
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=BQB,Number=1,Type=Float,Description=\"Mann-Whitney U test of Base Quality Bias (bigger is better)\">");
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=MQSB,Number=1,Type=Float,Description=\"Mann-Whitney U test of Mapping Quality vs Strand Bias (bigger is better)\">");
#if CDF_MWU_TESTS
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=RPB2,Number=1,Type=Float,Description=\"Mann-Whitney U test of Read Position Bias [CDF] (bigger is better)\">");
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=MQB2,Number=1,Type=Float,Description=\"Mann-Whitney U test of Mapping Quality Bias [CDF] (bigger is better)\">");
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=BQB2,Number=1,Type=Float,Description=\"Mann-Whitney U test of Base Quality Bias [CDF] (bigger is better)\">");
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=MQSB2,Number=1,Type=Float,Description=\"Mann-Whitney U test of Mapping Quality vs Strand Bias [CDF] (bigger is better)\">");
#endif
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=SGB,Number=1,Type=Float,Description=\"Segregation based metric.\">");
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=MQ0F,Number=1,Type=Float,Description=\"Fraction of MQ0 reads (smaller is better)\">");
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=I16,Number=16,Type=Float,Description=\"Auxiliary tag used for calling, see description of bcf_callret1_t in bam2bcf.h\">");
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=QS,Number=R,Type=Float,Description=\"Auxiliary tag used for calling\">");
bcf_hdr_append(conf->bcf_hdr,"##FORMAT=<ID=PL,Number=G,Type=Integer,Description=\"List of Phred-scaled genotype likelihoods\">");
if ( conf->fmt_flag&B2B_FMT_DP )
bcf_hdr_append(conf->bcf_hdr,"##FORMAT=<ID=DP,Number=1,Type=Integer,Description=\"Number of high-quality bases\">");
if ( conf->fmt_flag&B2B_FMT_DV )
bcf_hdr_append(conf->bcf_hdr,"##FORMAT=<ID=DV,Number=1,Type=Integer,Description=\"Number of high-quality non-reference bases\">");
if ( conf->fmt_flag&B2B_FMT_DPR )
bcf_hdr_append(conf->bcf_hdr,"##FORMAT=<ID=DPR,Number=R,Type=Integer,Description=\"Number of high-quality bases observed for each allele\">");
if ( conf->fmt_flag&B2B_INFO_DPR )
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=DPR,Number=R,Type=Integer,Description=\"Number of high-quality bases observed for each allele\">");
if ( conf->fmt_flag&B2B_FMT_DP4 )
bcf_hdr_append(conf->bcf_hdr,"##FORMAT=<ID=DP4,Number=4,Type=Integer,Description=\"Number of high-quality ref-fwd, ref-reverse, alt-fwd and alt-reverse bases\">");
if ( conf->fmt_flag&B2B_FMT_SP )
bcf_hdr_append(conf->bcf_hdr,"##FORMAT=<ID=SP,Number=1,Type=Integer,Description=\"Phred-scaled strand bias P-value\">");
if ( conf->fmt_flag&B2B_FMT_AD )
bcf_hdr_append(conf->bcf_hdr,"##FORMAT=<ID=AD,Number=R,Type=Integer,Description=\"Allelic depths\">");
if ( conf->fmt_flag&B2B_FMT_ADF )
bcf_hdr_append(conf->bcf_hdr,"##FORMAT=<ID=ADF,Number=R,Type=Integer,Description=\"Allelic depths on the forward strand\">");
if ( conf->fmt_flag&B2B_FMT_ADR )
bcf_hdr_append(conf->bcf_hdr,"##FORMAT=<ID=ADR,Number=R,Type=Integer,Description=\"Allelic depths on the reverse strand\">");
if ( conf->fmt_flag&B2B_INFO_AD )
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=AD,Number=R,Type=Integer,Description=\"Total allelic depths\">");
if ( conf->fmt_flag&B2B_INFO_ADF )
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=ADF,Number=R,Type=Integer,Description=\"Total allelic depths on the forward strand\">");
if ( conf->fmt_flag&B2B_INFO_ADR )
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=ADR,Number=R,Type=Integer,Description=\"Total allelic depths on the reverse strand\">");
if ( conf->gvcf )
gvcf_update_header(conf->gvcf, conf->bcf_hdr);
int nsmpl;
const char **smpl = bam_smpl_get_samples(conf->bsmpl, &nsmpl);
for (i=0; i<nsmpl; i++)
bcf_hdr_add_sample(conf->bcf_hdr, smpl[i]);
bcf_hdr_write(conf->bcf_fp, conf->bcf_hdr);
conf->bca = bcf_call_init(-1., conf->min_baseQ);
conf->bcr = (bcf_callret1_t*) calloc(nsmpl, sizeof(bcf_callret1_t));
conf->bca->openQ = conf->openQ, conf->bca->extQ = conf->extQ, conf->bca->tandemQ = conf->tandemQ;
conf->bca->min_frac = conf->min_frac;
conf->bca->min_support = conf->min_support;
conf->bca->per_sample_flt = conf->flag & MPLP_PER_SAMPLE;
conf->bc.bcf_hdr = conf->bcf_hdr;
conf->bc.n = nsmpl;
conf->bc.PL = (int32_t*) malloc(15 * nsmpl * sizeof(*conf->bc.PL));
if (conf->fmt_flag)
{
assert( sizeof(float)==sizeof(int32_t) );
conf->bc.DP4 = (int32_t*) malloc(nsmpl * sizeof(int32_t) * 4);
conf->bc.fmt_arr = (uint8_t*) malloc(nsmpl * sizeof(float)); // all fmt_flag fields, float and int32
if ( conf->fmt_flag&(B2B_INFO_DPR|B2B_FMT_DPR|B2B_INFO_AD|B2B_INFO_ADF|B2B_INFO_ADR|B2B_FMT_AD|B2B_FMT_ADF|B2B_FMT_ADR) )
{
// first B2B_MAX_ALLELES fields for total numbers, the rest per-sample
conf->bc.ADR = (int32_t*) malloc((nsmpl+1)*B2B_MAX_ALLELES*sizeof(int32_t));
conf->bc.ADF = (int32_t*) malloc((nsmpl+1)*B2B_MAX_ALLELES*sizeof(int32_t));
for (i=0; i<nsmpl; i++)
{
conf->bcr[i].ADR = conf->bc.ADR + (i+1)*B2B_MAX_ALLELES;
conf->bcr[i].ADF = conf->bc.ADF + (i+1)*B2B_MAX_ALLELES;
}
}
}
// init mpileup
conf->iter = bam_mplp_init(conf->nfiles, mplp_func, (void**)conf->mplp_data);
if ( conf->flag & MPLP_SMART_OVERLAPS ) bam_mplp_init_overlaps(conf->iter);
if ( (double)conf->max_depth * conf->nfiles > 1<<20)
fprintf(stderr, "Warning: Potential memory hog, up to %.0fM reads in the pileup!\n", (double)conf->max_depth*conf->nfiles);
if ( (double)conf->max_depth * conf->nfiles / nsmpl < 250 )
fprintf(stderr, "Note: The maximum per-sample depth with -d %d is %.1fx\n", conf->max_depth,(double)conf->max_depth * conf->nfiles / nsmpl);
bam_mplp_set_maxcnt(conf->iter, conf->max_depth);
conf->max_indel_depth = conf->max_indel_depth * nsmpl;
conf->bcf_rec = bcf_init1();
bam_mplp_constructor(conf->iter, pileup_constructor);
// Run mpileup for multiple regions
if ( nregs )
{
int ireg = 0;
do
{
// first region is already positioned
if ( ireg++ > 0 )
{
conf->buf.l = 0;
ksprintf(&conf->buf,"%s:%u-%u",conf->reg_itr->seq,conf->reg_itr->beg,conf->reg_itr->end);
for (i=0; i<conf->nfiles; i++)
{
hts_itr_destroy(conf->mplp_data[i]->iter);
conf->mplp_data[i]->iter = sam_itr_querys(conf->mplp_data[i]->idx, conf->mplp_data[i]->h, conf->buf.s);
if ( !conf->mplp_data[i]->iter )
{
conf->mplp_data[i]->iter = sam_itr_querys(conf->mplp_data[i]->idx, conf->mplp_data[i]->h, conf->reg_itr->seq);
if ( conf->mplp_data[i]->iter ) {
fprintf(stderr,"[E::%s] fail to parse region '%s'\n", __func__, conf->buf.s);
exit(EXIT_FAILURE);
}
fprintf(stderr,"[E::%s] the sequence \"%s\" not found: %s\n",__func__,conf->reg_itr->seq,conf->files[i]);
exit(EXIT_FAILURE);
}
bam_mplp_reset(conf->iter);
}
}
mpileup_reg(conf,conf->reg_itr->beg,conf->reg_itr->end);
}
while ( regitr_loop(conf->reg_itr) );
}
else
mpileup_reg(conf,0,0);
flush_bcf_records(conf, conf->bcf_fp, conf->bcf_hdr, NULL);
// clean up
free(conf->bc.tmp.s);
bcf_destroy1(conf->bcf_rec);
if (conf->bcf_fp)
{
hts_close(conf->bcf_fp);
bcf_hdr_destroy(conf->bcf_hdr);
bcf_call_destroy(conf->bca);
free(conf->bc.PL);
free(conf->bc.DP4);
free(conf->bc.ADR);
free(conf->bc.ADF);
free(conf->bc.fmt_arr);
free(conf->bcr);
}
if ( conf->gvcf ) gvcf_destroy(conf->gvcf);
free(conf->buf.s);
for (i = 0; i < conf->gplp->n; ++i) free(conf->gplp->plp[i]);
free(conf->gplp->plp); free(conf->gplp->n_plp); free(conf->gplp->m_plp); free(conf->gplp);
bam_mplp_destroy(conf->iter);
bam_hdr_destroy(hdr);
for (i = 0; i < conf->nfiles; ++i) {
if ( nregs>1 ) hts_idx_destroy(conf->mplp_data[i]->idx);
sam_close(conf->mplp_data[i]->fp);
if ( conf->mplp_data[i]->iter) hts_itr_destroy(conf->mplp_data[i]->iter);
free(conf->mplp_data[i]);
}
if ( conf->reg_itr ) regitr_destroy(conf->reg_itr);
free(conf->mplp_data); free(conf->plp); free(conf->n_plp);
free(mp_ref.ref[0]);
free(mp_ref.ref[1]);
return 0;
}
int vcf_index_stats(char *fname, int stats)
{
char *fn_out = NULL;
FILE *out;
out = fn_out ? fopen(fn_out, "w") : stdout;
const char **seq;
int i, nseq;
tbx_t *tbx = NULL;
hts_idx_t *idx = NULL;
htsFile *fp = hts_open(fname,"r");
if ( !fp ) { fprintf(stderr,"Could not read %s\n", fname); return 1; }
bcf_hdr_t *hdr = bcf_hdr_read(fp);
if ( !hdr ) { fprintf(stderr,"Could not read the header: %s\n", fname); return 1; }
if ( hts_get_format(fp)->format==vcf )
{
tbx = tbx_index_load(fname);
if ( !tbx ) { fprintf(stderr,"Could not load TBI index: %s\n", fname); return 1; }
}
else if ( hts_get_format(fp)->format==bcf )
{
idx = bcf_index_load(fname);
if ( !idx ) { fprintf(stderr,"Could not load CSI index: %s\n", fname); return 1; }
}
else
{
fprintf(stderr,"Could not detect the file type as VCF or BCF: %s\n", fname);
return 1;
}
seq = tbx ? tbx_seqnames(tbx, &nseq) : bcf_index_seqnames(idx, hdr, &nseq);
uint64_t sum = 0;
for (i=0; i<nseq; i++)
{
uint64_t records, v;
hts_idx_get_stat(tbx ? tbx->idx : idx, i, &records, &v);
sum+=records;
if (stats&2 || !records) continue;
bcf_hrec_t *hrec = bcf_hdr_get_hrec(hdr, BCF_HL_CTG, "ID", seq[i], NULL);
int hkey = hrec ? bcf_hrec_find_key(hrec, "length") : -1;
fprintf(out,"%s\t%s\t%" PRIu64 "\n", seq[i], hkey<0?".":hrec->vals[hkey], records);
}
if (!sum)
{
// No counts found.
// Is this because index version has no stored count data, or no records?
bcf1_t *rec = bcf_init1();
if (bcf_read1(fp, hdr, rec) >= 0)
{
fprintf(stderr,"%s index of %s does not contain any count metadata. Please re-index with a newer version of bcftools or tabix.\n", tbx ? "TBI" : "CSI", fname);
return 1;
}
bcf_destroy1(rec);
}
if (stats&2) fprintf(out, "%" PRIu64 "\n", sum);
free(seq);
fclose(out);
hts_close(fp);
bcf_hdr_destroy(hdr);
if (tbx)
tbx_destroy(tbx);
if (idx)
hts_idx_destroy(idx);
return 0;
}
int ingest1(const char *input,const char *output,char *ref,bool exit_on_mismatch=true) {
cerr << "Input: " << input << "\tOutput: "<<output<<endl;
kstream_t *ks;
kstring_t str = {0,0,0};
gzFile fp = gzopen(input, "r");
VarBuffer vbuf(1000);
int prev_rid = -1;
if(fp==NULL) {
fprintf(stderr,"problem opening %s\n",input);
exit(1);
}
char *out_fname = (char *)malloc(strlen(output)+5);
strcpy(out_fname,output);
strcat(out_fname,".tmp");
if(fileexists(out_fname)) {
fprintf(stderr,"%s file already exists. will not overwrite\n",out_fname);
exit(1);
}
printf("depth: %s\n",out_fname);
gzFile depth_fp = gzopen(out_fname, "wb1");
strcpy(out_fname,output);
strcat(out_fname,".bcf");
if(fileexists(out_fname)) {
fprintf(stderr,"%s file already exists. will not overwrite\n",out_fname);
exit(1);
}
printf("variants: %s\n",out_fname);
htsFile *variant_fp=hts_open(out_fname,"wb1");
if(variant_fp==NULL) {
fprintf(stderr,"problem opening %s\n",input);
exit(1);
}
ks = ks_init(fp);
htsFile *hfp=hts_open(input, "r");
bcf_hdr_t *hdr_in = bcf_hdr_read(hfp);
hts_close(hfp);
//this is a hack to fix gvcfs where AD is incorrectly defined in the header. (vcf4.2 does not technically allow Number=R)
bcf_hdr_remove(hdr_in,BCF_HL_FMT,"AD");
assert( bcf_hdr_append(hdr_in,"##FORMAT=<ID=AD,Number=R,Type=Integer,Description=\"Allelic depths for the ref and alt alleles in the order listed. For indels this value only includes reads which confidently support each allele (posterior prob 0.999 or higher that read contains indicated allele vs all other intersecting indel alleles)\">") == 0);
//this is a hack to fix broken gvcfs where GQ is incorrectly labelled as float (v4.3 spec says it should be integer)
bcf_hdr_remove(hdr_in,BCF_HL_FMT,"GQ");
assert( bcf_hdr_append(hdr_in,"##FORMAT=<ID=GQ,Number=1,Type=Integer,Description=\"Genotype Quality\">") == 0);
// bcf_hdr_t *hdr_out=hdr_in;
bcf_hdr_t *hdr_out = bcf_hdr_dup(hdr_in);
remove_hdr_lines(hdr_out,BCF_HL_INFO);
remove_hdr_lines(hdr_out,BCF_HL_FLT);
bcf_hdr_sync(hdr_out);
//here we add FORMAT/PF. which is the pass filter flag for alts.
assert( bcf_hdr_append(hdr_out,"##FORMAT=<ID=PF,Number=A,Type=Integer,Description=\"variant was PASS filter in original sample gvcf\">") == 0);
args_t *norm_args = init_vcfnorm(hdr_out,ref);
norm_args->check_ref |= CHECK_REF_WARN;
bcf1_t *bcf_rec = bcf_init();
bcf_hdr_write(variant_fp, hdr_out);
kstring_t work1 = {0,0,0};
int buf[5];
ks_tokaux_t aux;
int ndec=0;
int ref_len,alt_len;
while( ks_getuntil(ks, '\n', &str, 0) >=0) {
// fprintf(stderr,"%s\n",str.s);
if(str.s[0]!='#') {
char *ptr = kstrtok(str.s,"\t",&aux);//chrom
ptr = kstrtok(NULL,NULL,&aux);//pos
work1.l=0;
kputsn(str.s,ptr-str.s-1, &work1);
buf[0] = bcf_hdr_name2id(hdr_in, work1.s);
assert( buf[0]>=0);
buf[1]=atoi(ptr)-1;
ptr = kstrtok(NULL,NULL,&aux);//ID
ptr = kstrtok(NULL,NULL,&aux);//REF
ref_len=0;
while(ptr[ref_len]!='\t') ref_len++;
ptr = kstrtok(NULL,NULL,&aux);//ALT
bool is_variant=false;
alt_len=0;
while(ptr[alt_len]!='\t') alt_len++;
if(ptr[0]!='.')
is_variant=true;
char * QUAL_ptr = kstrtok(NULL, NULL, &aux);
assert (QUAL_ptr != NULL);
for(int i=0;i<2;i++) ptr = kstrtok(NULL,NULL,&aux);// gets us to INFO
//find END if it is there
char *end_ptr=strstr(ptr,"END=") ;
if(end_ptr!=NULL)
buf[2]=atoi(end_ptr+4)-1;
else
buf[2]=buf[1]+alt_len-1;
ptr = kstrtok(NULL,NULL,&aux);//FORMAT
//find index of DP (if present)
//if not present, dont output anything (indels ignored)
char *DP_ptr = find_format(ptr,"DP");
int GQX = 0;
int QUAL = 0;
// AH: change code to use the minimum of GQ and QUAL fields if
// GQX is not defined. See here:
// https://support.basespace.illumina.com/knowledgebase/articles/144844-vcf-file
// "GQXGenotype quality. GQX is the minimum of the GQ value
// and the QUAL column. In general, these are similar values;
// taking the minimum makes GQX the more conservative measure of
// genotype quality."
if(DP_ptr!=NULL) {
buf[3]=atoi(DP_ptr);
char *GQX_ptr = find_format(ptr,"GQX");
if (GQX_ptr == NULL)
{
GQX_ptr = find_format(ptr,"GQ");
GQX = atoi(GQX_ptr);
if (QUAL_ptr[0] != '.')
{
QUAL = atoi(QUAL_ptr);
if (QUAL < GQX)
GQX = QUAL;
}
}
else
{
GQX = atoi(GQX_ptr);
}
//trying to reduce entropy on GQ to get better compression performance.
//1. rounds down to nearest 10.
//2. sets gq to min(gq,100).
buf[4]=GQX/10;
buf[4]*=10;
if(buf[4]>100) buf[4]=100;
// printf("%d\t%d\t%d\t%d\t%d\n",buf[0],buf[1],buf[2],buf[3],buf[4]);
if(gzwrite(depth_fp,buf,5*sizeof(int))!=(5*sizeof(int)))
die("ERROR: problem writing "+(string)out_fname+".tmp");
}
if(is_variant) {//wass this a variant? if so write it out to the bcf
norm_args->ntotal++;
vcf_parse(&str,hdr_in,bcf_rec);
// cerr<<bcf_rec->rid<<":"<<bcf_rec->pos<<endl;
if(prev_rid!=bcf_rec->rid)
vbuf.flush(variant_fp,hdr_out);
else
vbuf.flush(bcf_rec->pos,variant_fp,hdr_out);
prev_rid=bcf_rec->rid;
int32_t pass = bcf_has_filter(hdr_in, bcf_rec, ".");
bcf_update_format_int32(hdr_out,bcf_rec,"PF",&pass,1);
bcf_update_filter(hdr_out,bcf_rec,NULL,0);
if(bcf_rec->n_allele>2) {//split multi-allelics (using vcfnorm.c from bcftools1.3
norm_args->nsplit++;
split_multiallelic_to_biallelics(norm_args,bcf_rec );
for(int i=0;i<norm_args->ntmp_lines;i++){
remove_info(norm_args->tmp_lines[i]);
if(realign(norm_args,norm_args->tmp_lines[i]) != ERR_REF_MISMATCH)
ndec+=decompose(norm_args->tmp_lines[i],hdr_out,vbuf);
else
if(exit_on_mismatch)
die("vcf did not match the reference");
else
norm_args->nskipped++;
}
}
else {
remove_info(bcf_rec);
if( realign(norm_args,bcf_rec) != ERR_REF_MISMATCH)
ndec+=decompose(bcf_rec,hdr_out,vbuf);
else
if(exit_on_mismatch)
die("vcf did not match the reference");
else
norm_args->nskipped++;
}
vbuf.flush(bcf_rec->pos,variant_fp,hdr_out);
}
}
}
vbuf.flush(variant_fp,hdr_out);
bcf_hdr_destroy(hdr_in);
bcf_hdr_destroy(hdr_out);
bcf_destroy1(bcf_rec);
ks_destroy(ks);
gzclose(fp);
gzclose(depth_fp);
free(str.s);
free(work1.s);
hts_close(variant_fp);
destroy_data(norm_args);
fprintf(stderr,"Variant lines total/split/realigned/skipped:\t%d/%d/%d/%d\n", norm_args->ntotal,norm_args->nsplit,norm_args->nchanged,norm_args->nskipped);
fprintf(stderr,"Decomposed %d MNPs\n", ndec);
fprintf(stderr,"Indexing %s\n",out_fname);
bcf_index_build(out_fname, BCF_LIDX_SHIFT);
free(out_fname);
return 0;
}
int ctx_calls2vcf(int argc, char **argv)
{
const char *in_path = NULL, *out_path = NULL, *out_type = NULL;
// Filtering parameters
int32_t min_mapq = -1, max_align_len = -1, max_allele_len = -1;
// Alignment parameters
int nwmatch = 1, nwmismatch = -2, nwgapopen = -4, nwgapextend = -1;
// ref paths
char const*const* ref_paths = NULL;
size_t nref_paths = 0;
// flank file
const char *sam_path = NULL;
//
// Things we figure out by looking at the input
//
bool isbubble = false;
// samples in VCF, (0 for bubble, does not include ref in breakpoint calls)
size_t i, kmer_size, num_samples;
//
// Reference genome
//
// Hash map of chromosome name -> sequence
ChromHash *genome;
ReadBuffer chroms;
// Arg parsing
char cmd[100];
char shortopts[300];
cmd_long_opts_to_short(longopts, shortopts, sizeof(shortopts));
int c;
// silence error messages from getopt_long
// opterr = 0;
while((c = getopt_long_only(argc, argv, shortopts, longopts, NULL)) != -1) {
cmd_get_longopt_str(longopts, c, cmd, sizeof(cmd));
switch(c) {
case 0: /* flag set */ break;
case 'h': cmd_print_usage(NULL); break;
case 'o': cmd_check(!out_path, cmd); out_path = optarg; break;
case 'O': cmd_check(!out_type, cmd); out_type = optarg; break;
case 'f': cmd_check(!futil_get_force(), cmd); futil_set_force(true); break;
case 'F': cmd_check(!sam_path,cmd); sam_path = optarg; break;
case 'Q': cmd_check(min_mapq < 0,cmd); min_mapq = cmd_uint32(cmd, optarg); break;
case 'A': cmd_check(max_align_len < 0,cmd); max_align_len = cmd_uint32(cmd, optarg); break;
case 'L': cmd_check(max_allele_len < 0,cmd); max_allele_len = cmd_uint32(cmd, optarg); break;
case 'm': nwmatch = cmd_int32(cmd, optarg); break;
case 'M': nwmismatch = cmd_int32(cmd, optarg); break;
case 'g': nwgapopen = cmd_int32(cmd, optarg); break;
case 'G': nwgapextend = cmd_int32(cmd, optarg); break;
case ':': /* BADARG */
case '?': /* BADCH getopt_long has already printed error */
die("`"CMD" "SUBCMD" -h` for help. Bad option: %s", argv[optind-1]);
default: ctx_assert2(0, "shouldn't reach here: %c", c);
}
}
// Defaults for unset values
if(out_path == NULL) out_path = "-";
if(max_align_len < 0) max_align_len = DEFAULT_MAX_ALIGN;
if(max_allele_len < 0) max_allele_len = DEFAULT_MAX_ALLELE;
if(optind+2 > argc)
cmd_print_usage("Require <in.txt.gz> and at least one reference");
in_path = argv[optind++];
ref_paths = (char const*const*)argv + optind;
nref_paths = argc - optind;
// These functions call die() on error
gzFile gzin = futil_gzopen(in_path, "r");
// Read call file header
cJSON *json = json_hdr_load(gzin, in_path);
// Check we can handle the kmer size
kmer_size = json_hdr_get_kmer_size(json, in_path);
db_graph_check_kmer_size(kmer_size, in_path);
// Get format (bubble or breakpoint file)
cJSON *json_fmt = json_hdr_get(json, "file_format", cJSON_String, in_path);
if(strcmp(json_fmt->valuestring,"CtxBreakpoints") == 0) isbubble = false;
else if(strcmp(json_fmt->valuestring,"CtxBubbles") == 0) isbubble = true;
else die("Unknown format: '%s'", json_fmt->valuestring);
status("Reading %s in %s format", futil_inpath_str(in_path),
isbubble ? "bubble" : "breakpoint");
if(isbubble) {
// bubble specific
if(sam_path == NULL)
cmd_print_usage("Require -F <flanks.sam> with bubble file");
if(min_mapq < 0) min_mapq = DEFAULT_MIN_MAPQ;
}
else {
// breakpoint specific
if(min_mapq >= 0)
cmd_print_usage("-Q,--min-mapq <Q> only valid with bubble calls");
}
// Open flank file if it exists
htsFile *samfh = NULL;
bam_hdr_t *bam_hdr = NULL;
bam1_t *mflank = NULL;
if(sam_path)
{
if((samfh = hts_open(sam_path, "r")) == NULL)
die("Cannot open SAM/BAM %s", sam_path);
// Load BAM header
bam_hdr = sam_hdr_read(samfh);
if(bam_hdr == NULL) die("Cannot load BAM header: %s", sam_path);
mflank = bam_init1();
}
// Output VCF has 0 samples if bubbles file, otherwise has N where N is
// number of samples/colours in the breakpoint graph
size_t num_graph_samples = json_hdr_get_ncols(json, in_path);
size_t num_graph_nonref = json_hdr_get_nonref_ncols(json, in_path);
num_samples = 0;
if(!isbubble) {
// If last colour has "is_ref", drop number of samples by one
num_samples = num_graph_nonref < num_graph_samples ? num_graph_samples-1
: num_graph_samples;
}
//
// Open output file
//
if(!out_path) out_path = "-";
int mode = vcf_misc_get_outtype(out_type, out_path);
futil_create_output(out_path);
htsFile *vcffh = hts_open(out_path, modes_htslib[mode]);
status("[calls2vcf] Reading %s call file with %zu samples",
isbubble ? "Bubble" : "Breakpoint", num_graph_samples);
status("[calls2vcf] %zu sample output to: %s format: %s",
num_samples, futil_outpath_str(out_path), hsmodes_htslib[mode]);
if(isbubble) status("[calls2vcf] min. MAPQ: %i", min_mapq);
status("[calls2vcf] max alignment length: %i", max_align_len);
status("[calls2vcf] max VCF allele length: %i", max_allele_len);
status("[calls2vcf] alignment match:%i mismatch:%i gap open:%i extend:%i",
nwmatch, nwmismatch, nwgapopen, nwgapextend);
// Load reference genome
read_buf_alloc(&chroms, 1024);
genome = chrom_hash_init();
chrom_hash_load(ref_paths, nref_paths, &chroms, genome);
// convert to upper case
char *s;
for(i = 0; i < chroms.len; i++)
for(s = chroms.b[i].seq.b; *s; s++) *s = toupper(*s);
if(!isbubble) brkpnt_check_refs_match(json, genome, in_path);
bcf_hdr_t *vcfhdr = make_vcf_hdr(json, in_path, !isbubble, kmer_size,
ref_paths, nref_paths,
chroms.b, chroms.len);
if(bcf_hdr_write(vcffh, vcfhdr) != 0) die("Cannot write VCF header");
AlignedCall *call = acall_init();
CallDecomp *aligner = call_decomp_init(vcffh, vcfhdr);
scoring_t *scoring = call_decomp_get_scoring(aligner);
scoring_init(scoring, nwmatch, nwmismatch, nwgapopen, nwgapextend,
false, false, 0, 0, 0, 0);
CallFileEntry centry;
call_file_entry_alloc(¢ry);
char kmer_str[50];
sprintf(kmer_str, ";K%zu", kmer_size);
if(isbubble)
{
// Bubble calls
DecompBubble *bubbles = decomp_bubble_init();
// Set scoring for aligning 3' flank
scoring = decomp_bubble_get_scoring(bubbles);
scoring_init(scoring, nwmatch, nwmismatch, nwgapopen, nwgapextend,
true, true, 0, 0, 0, 0);
while(call_file_read(gzin, in_path, ¢ry)) {
do {
if(sam_read1(samfh, bam_hdr, mflank) < 0)
die("We've run out of SAM entries!");
} while(mflank->core.flag & (BAM_FSECONDARY | BAM_FSUPPLEMENTARY));
// Align call
strbuf_reset(&call->info);
decomp_bubble_call(bubbles, genome, kmer_size, min_mapq,
¢ry, mflank, bam_hdr, call);
strbuf_append_str(&call->info, kmer_str);
acall_decompose(aligner, call, max_align_len, max_allele_len);
}
// print bubble stats
DecompBubbleStats *bub_stats = ctx_calloc(1, sizeof(*bub_stats));
decomp_bubble_cpy_stats(bub_stats, bubbles);
print_bubble_stats(bub_stats);
ctx_free(bub_stats);
decomp_bubble_destroy(bubbles);
}
else
{
// Breakpoint calls
DecompBreakpoint *breakpoints = decomp_brkpt_init();
while(call_file_read(gzin, in_path, ¢ry)) {
strbuf_reset(&call->info);
decomp_brkpt_call(breakpoints, genome, num_samples, ¢ry, call);
strbuf_append_str(&call->info, kmer_str);
acall_decompose(aligner, call, max_align_len, max_allele_len);
}
// print bubble stats
DecompBreakpointStats *brk_stats = ctx_calloc(1, sizeof(*brk_stats));
decomp_brkpt_cpy_stats(brk_stats, breakpoints);
print_breakpoint_stats(brk_stats);
ctx_free(brk_stats);
decomp_brkpt_destroy(breakpoints);
}
// Print stats
DecomposeStats *astats = ctx_calloc(1, sizeof(*astats));
call_decomp_cpy_stats(astats, aligner);
print_acall_stats(astats);
ctx_free(astats);
call_file_entry_dealloc(¢ry);
call_decomp_destroy(aligner);
acall_destroy(call);
// Finished - clean up
cJSON_Delete(json);
gzclose(gzin);
bcf_hdr_destroy(vcfhdr);
hts_close(vcffh);
for(i = 0; i < chroms.len; i++) seq_read_dealloc(&chroms.b[i]);
read_buf_dealloc(&chroms);
chrom_hash_destroy(genome);
if(sam_path) {
hts_close(samfh);
bam_hdr_destroy(bam_hdr);
bam_destroy1(mflank);
}
return EXIT_SUCCESS;
}
int main(int argc, char* argv[]) {
namespace po = boost::program_options;
std::string file;
std::string output;
try
{
// Declare the supported options.
po::options_description desc("Allowed options");
desc.add_options()
("help,h", "produce help message")
("version", "Show version")
("input-file", po::value< std::string >(), "The input files")
("output-file", po::value<std::string>(), "The output file name.")
;
po::positional_options_description popts;
popts.add("input-file", 1);
popts.add("output-file", 1);
po::options_description cmdline_options;
cmdline_options
.add(desc)
;
po::variables_map vm;
po::store(po::command_line_parser(argc, argv).
options(cmdline_options).positional(popts).run(), vm);
po::notify(vm);
if (vm.count("version"))
{
std::cout << "vcfhdr2json version " << HAPLOTYPES_VERSION << "\n";
return 0;
}
if (vm.count("help"))
{
std::cout << desc << "\n";
return 1;
}
if (vm.count("input-file"))
{
file = vm["input-file"].as< std::string > ();
}
if (vm.count("output-file"))
{
output = vm["output-file"].as< std::string >();
}
if(file.size() == 0)
{
std::cerr << "Please specify an input file.\n";
return 1;
}
if (output == "")
{
std::cerr << "Please specify an output file.\n";
return 1;
}
}
catch (po::error & e)
{
std::cerr << e.what() << "\n";
return 1;
}
try
{
Json::StyledWriter writer;
htsFile * fp = bcf_open(file.c_str(), "r");
bcf_hdr_t * hdr = bcf_hdr_read(fp);
Json::Value root;
Json::Value a;
for (int i = 0; i < bcf_hdr_nsamples(hdr); ++i)
{
a.append(hdr->samples[i]);
}
root["samples"] = a;
Json::Value fields;
for (int i = 0; i < hdr->nhrec; i++)
{
Json::Value field;
field["key"] = hdr->hrec[i]->key;
if (!hdr->hrec[i]->value)
{
Json::Value values;
for (int j = 0; j < hdr->hrec[i]->nkeys; j++)
{
values[hdr->hrec[i]->keys[j]] = hdr->hrec[i]->vals[j];
}
field["values"] = values;
}
else
{
field["value"] = hdr->hrec[i]->value;
}
fields.append(field);
}
root["fields"] = fields;
tbx_t * tbx_idx = tbx_index_load(file.c_str());
if ( !tbx_idx )
{
hts_idx_t * csi_idx = bcf_index_load(file.c_str());
if(!csi_idx)
{
root["tabix"] = Json::Value::null;
}
else
{
root["tabix"] = Json::Value();
root["tabix"]["chromosomes"] = Json::Value();
int count = 0;
const char ** tbx_names = bcf_index_seqnames(csi_idx, hdr, &count);
for (int i = 0; i < count; ++i)
{
root["tabix"]["chromosomes"].append(tbx_names[i]);
}
free(tbx_names);
hts_idx_destroy(csi_idx);
}
}
else
{
root["tabix"] = Json::Value();
root["tabix"]["chromosomes"] = Json::Value();
int count = 0;
const char ** tbx_names = tbx_seqnames(tbx_idx, &count);
for (int i = 0; i < count; ++i)
{
root["tabix"]["chromosomes"].append(tbx_names[i]);
}
free(tbx_names);
tbx_destroy(tbx_idx);
}
std::ofstream out(output.c_str());
out << writer.write(root);
bcf_close(fp);
bcf_hdr_destroy(hdr);
}
catch(std::runtime_error & e)
{
std::cerr << e.what() << std::endl;
return 1;
}
catch(std::logic_error & e)
{
std::cerr << e.what() << std::endl;
return 1;
}
return 0;
}
bcf_hdr_t *vcf_hdr_read(htsFile *fp)
{
if (!fp->is_bin) {
kstring_t txt, *s = &fp->line;
bcf_hdr_t *h;
h = bcf_hdr_init();
txt.l = txt.m = 0; txt.s = 0;
while (hts_getline(fp, KS_SEP_LINE, s) >= 0) {
if (s->l == 0) continue;
if (s->s[0] != '#') {
if (hts_verbose >= 2)
fprintf(stderr, "[E::%s] no sample line\n", __func__);
free(txt.s);
bcf_hdr_destroy(h);
return 0;
}
if (s->s[1] != '#' && fp->fn_aux) { // insert contigs here
int dret;
gzFile f;
kstream_t *ks;
kstring_t tmp;
tmp.l = tmp.m = 0; tmp.s = 0;
f = gzopen(fp->fn_aux, "r");
ks = ks_init(f);
while (ks_getuntil(ks, 0, &tmp, &dret) >= 0) {
int c;
kputs("##contig=<ID=", &txt); kputs(tmp.s, &txt);
ks_getuntil(ks, 0, &tmp, &dret);
kputs(",length=", &txt); kputw(atol(tmp.s), &txt);
kputsn(">\n", 2, &txt);
if (dret != '\n')
while ((c = ks_getc(ks)) != '\n' && c != -1); // skip the rest of the line
}
free(tmp.s);
ks_destroy(ks);
gzclose(f);
}
kputsn(s->s, s->l, &txt);
if (s->s[1] != '#') break;
kputc('\n', &txt);
}
h->l_text = txt.l + 1; // including NULL
h->text = txt.s;
bcf_hdr_parse(h);
// check tabix index, are all contigs listed in the header? add the missing ones
tbx_t *idx = tbx_index_load(fp->fn);
if ( idx )
{
int i, n, need_sync = 0;
const char **names = tbx_seqnames(idx, &n);
for (i=0; i<n; i++)
{
bcf_hrec_t *hrec = bcf_hdr_get_hrec(h, BCF_DT_CTG, (char*) names[i]);
if ( hrec ) continue;
hrec = (bcf_hrec_t*) calloc(1,sizeof(bcf_hrec_t));
hrec->key = strdup("contig");
bcf_hrec_add_key(hrec, "ID", strlen("ID"));
bcf_hrec_set_val(hrec, hrec->nkeys-1, (char*) names[i], strlen(names[i]), 0);
bcf_hrec_add_key(hrec, "length", strlen("length"));
bcf_hrec_set_val(hrec, hrec->nkeys-1, "-1", strlen("-1"), 0); // what is a good default value?
bcf_hdr_add_hrec(h, hrec);
need_sync = 1;
}
free(names);
tbx_destroy(idx);
if ( need_sync )
{
bcf_hdr_sync(h);
bcf_hdr_fmt_text(h);
}
}
return h;
} else return bcf_hdr_read((BGZF*)fp->fp);
}
int main(int argc, char **argv)
{
int i, n;
static struct option const long_opts[] =
{
{"out", required_argument, NULL, 1},
{"report", required_argument, NULL, 2},
{"dotasref", no_argument, NULL, 3},
{"help", no_argument, NULL, 0},
{"version", no_argument, NULL, 4},
{"export_uncov", no_argument, NULL, 5}
};
bool help = FALSE;
bool report_version = FALSE;
while ((n = getopt_long(argc, argv, "1:2:304", long_opts, NULL)) >= 0)
{
switch (n)
{
case 1 : outfile = strdup(optarg); break;
case 2 : report = strdup(optarg); break;
case 3 : dotasref = TRUE; break;
case 0 : help = TRUE; break;
case 4 : report_version = TRUE; break;
case 5 : export_uncover = TRUE; break;
default : return 1;
}
if ( help ) return usage();
if ( report_version ) return show_version();
}
n = argc - optind;
if ( n > 1 ) errabort("only accept one input vcf");
if ( export_uncover == TRUE && outfile == FALSE) {
warnings("export uncove region only used with option --out");
export_uncover = FALSE;
}
char * input;
if ( n == 0 ) input = strdup("-");
else input = strdup(argv[optind]);
htsFile * fp = read_vcf_file(input);
enum htsExactFormat fmt = hts_get_format(fp)->format;
if ( fmt != vcf && fmt != bcf ) errabort("This is not a VCF/BCF file : %s", input);
bcf_hdr_t * hdr = bcf_hdr_read(fp);
int n_samples = bcf_hdr_nsamples(hdr);
if ( n_samples != 2 ) errabort("the input VCF/BCF file must contain only two samples! %d", n_samples);
LOG("Using sample %s as ref ...", hdr->samples[0]);
LOG("Using sample %s as test ...", hdr->samples[1]);
uint32_t matrix[4][4] = { {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0} };
bcf1_t * v = bcf_init1();
kstring_t str = { 0, 0, 0 };
uint32_t line = 0;
htsFile *out = NULL;
if ( outfile && !check_filename(outfile) ) out = hts_open(outfile, mode);
if ( out != NULL ) bcf_hdr_write(out, hdr);
while ( bcf_read1(fp, hdr, v) >= 0 )
{
bcf_unpack(v, BCF_UN_STR|BCF_UN_FMT);
int k;
str.l = 0;
int tag_id = bcf_hdr_id2int(hdr, BCF_DT_ID, "GT");
if ( !bcf_hdr_idinfo_exists(hdr, BCF_HL_FMT, tag_id) ) warnings("There is no 'GT' in the header!");
for ( i = 0; i < v->n_fmt; ++i )
if ( v->d.fmt[i].id == tag_id ) break;
if ( i == v->n_fmt ) {
vcf_format1(hdr, v, &str);
LOG("There is no tag GT in this line : %s", str.s);
continue;
}
corr_t xy[2] = { {-1, -2, -2}, {-1, -2, -2} };
bcf_fmt_t * fmt = &v->d.fmt[i];
for ( i = 0; i < 2; ++i )
{
int corr = i;
if ( fmt == NULL ) {
if ( dotasref == TRUE ) xy[corr].alt = ALT_IS_REF;
else xy[corr].alt = ALT_IS_UNC;
continue;
}
int last = -2;
uint8_t *d = (uint8_t*)((char*)fmt->p + fmt->size*i);
for ( k = 0; k < fmt->n && d[k] != (uint8_t)bcf_int8_vector_end; ++k )
{
int curr = d[k]>>1;
if ( last != curr ) {
if ( curr ) {
if ( last == -2 ) xy[corr].alt = curr > 1 ? ALT_IS_HOM : ALT_IS_REF;
else xy[corr].alt = ALT_IS_HET;
} else {
xy[corr].alt = dotasref == TRUE ? ALT_IS_REF : ALT_IS_UNC;
}
} else {
if ( curr ) {
xy[corr].alt = curr > 1 ? ALT_IS_HOM : ALT_IS_REF;
} else {
xy[corr].alt = dotasref == TRUE ? ALT_IS_REF : ALT_IS_UNC;
}
}
if (last == -2 ) {
xy[corr].min = xy[corr].max = curr;
} else {
if ( curr < xy[corr].min ) xy[corr].min = curr;
else if ( curr > xy[corr].max ) xy[corr].max = curr;
}
last = curr;
}
}
matrix[xy[0].alt][xy[1].alt]++;
if ( xy[0].alt != xy[1].alt && out != NULL) {
if ( xy[0].alt == ALT_IS_UNC || xy[1].alt == ALT_IS_UNC ) {
if ( export_uncover == TRUE ) {
str.l = 0;
vcf_format1(hdr, v, &str);
vcf_write(out, hdr, v);
}
} else {
str.l = 0;
vcf_format1(hdr, v, &str);
vcf_write(out, hdr, v);
}
}
if ( xy[0].alt == ALT_IS_HET && xy[1].alt == ALT_IS_HET && (xy[0].min != xy[1].min || xy[0].max != xy[1].max ) ) {
bias++;
matrix[ALT_IS_HET][ALT_IS_HET]--;
if ( out != NULL ) {
str.l = 0;
vcf_format1(hdr, v, &str);
vcf_write(out, hdr, v);
}
}
line++;
}
if ( out ) hts_close(out);
if ( str.m ) free(str.s);
write_report(matrix, hdr);
bcf_hdr_destroy(hdr);
free(input);
bcf_destroy1(v);
if ( outfile ) free(outfile);
if ( report ) free(report);
if ( hts_close(fp) ) warnings("hts_close returned non-zero status: %s", input);
return 0;
}
int main_vcfview(int argc, char *argv[])
{
int i, c, clevel = -1, flag = 0, n_samples = -1, *imap = 0, excl_snp = 0, excl_indel = 0;
char *fn_ref = 0, *fn_out = 0, moder[8], **samples = 0;
bcf_hdr_t *h, *hsub = 0;
htsFile *in;
bcf1_t *b;
while ((c = getopt(argc, argv, "l:bSt:o:T:s:GNI")) >= 0) {
switch (c) {
case 'l': clevel = atoi(optarg); flag |= 2; break;
case 'S': flag |= 1; break;
case 'b': flag |= 2; break;
case 'G': n_samples = 0; break;
case 't': fn_ref = optarg; flag |= 1; break;
case 'o': fn_out = optarg; break;
case 's': samples = hts_readlines(optarg, &n_samples); break;
case 'N': excl_snp = 1; break;
case 'I': excl_indel = 1; break;
}
}
if (argc == optind) {
fprintf(stderr, "\nUsage: vcfview [options] <in.bcf>|<in.vcf>|<in.vcf.gz>\n\n");
fprintf(stderr, "Options: -b output in BCF\n");
fprintf(stderr, " -S input is VCF\n");
fprintf(stderr, " -o FILE output file name [stdout]\n");
fprintf(stderr, " -l INT compression level [%d]\n", clevel);
fprintf(stderr, " -t FILE list of reference names and lengths [null]\n");
fprintf(stderr, " -s FILE/STR list of samples (STR if started with ':'; FILE otherwise) [null]\n");
fprintf(stderr, " -G drop individual genotype information\n");
fprintf(stderr, " -N exclude SNPs\n");
fprintf(stderr, " -I exclude INDELs\n");
fprintf(stderr, "\n");
return 1;
}
strcpy(moder, "r");
if ((flag&1) == 0 && !(file_type(argv[optind])&(IS_VCF|IS_VCF_GZ))) strcat(moder, "b");
in = hts_open(argv[optind], moder, fn_ref);
h = vcf_hdr_read(in);
if (h == 0) {
fprintf(stderr, "[E::%s] fail to read the VCF/BCF2 header\n", __func__);
hts_close(in);
return 1;
}
if (n_samples >= 0) {
if (n_samples) imap = (int*)malloc(n_samples * sizeof(int));
hsub = bcf_hdr_subset(h, n_samples, samples, imap);
}
b = bcf_init1();
if ((flag&4) == 0) { // VCF/BCF output
htsFile *out;
char modew[8];
strcpy(modew, "w");
if (clevel >= 0 && clevel <= 9) sprintf(modew + 1, "%d", clevel);
if (flag&2) strcat(modew, "b");
out = hts_open(fn_out? fn_out : "-", modew, 0);
vcf_hdr_write(out, hsub? hsub : h);
if (optind + 1 < argc && !(flag&1)) { // BAM input and has a region
hts_idx_t *idx;
if ((idx = bcf_index_load(argv[optind])) == 0) {
fprintf(stderr, "[E::%s] fail to load the BCF index\n", __func__);
return 1;
}
for (i = optind + 1; i < argc; ++i) {
hts_itr_t *iter;
if ((iter = bcf_itr_querys(idx, h, argv[i])) == 0) {
fprintf(stderr, "[E::%s] fail to parse region '%s'\n", __func__, argv[i]);
continue;
}
while (bcf_itr_next((BGZF*)in->fp, iter, b) >= 0) {
if (excl_snp && bcf_is_snp(b)) continue;
if (excl_indel && !bcf_is_snp(b)) continue;
if (n_samples >= 0) {
bcf_subset(h, b, n_samples, imap);
vcf_write1(out, hsub, b);
} else vcf_write1(out, h, b);
}
hts_itr_destroy(iter);
}
hts_idx_destroy(idx);
} else {
while (vcf_read1(in, h, b) >= 0) {
if (excl_snp && bcf_is_snp(b)) continue;
if (excl_indel && !bcf_is_snp(b)) continue;
if (n_samples >= 0) {
bcf_subset(h, b, n_samples, imap);
vcf_write1(out, hsub, b);
} else vcf_write1(out, h, b);
}
}
hts_close(out);
}
bcf_destroy1(b);
if (n_samples > 0) {
for (i = 0; i < n_samples; ++i) free(samples[i]);
free(samples);
bcf_hdr_destroy(hsub);
free(imap);
}
bcf_hdr_destroy(h);
hts_close(in);
return 0;
}
static int query_regions(args_t *args, char *fname, char **regs, int nregs)
{
int i;
htsFile *fp = hts_open(fname,"r");
if ( !fp ) error("Could not read %s\n", fname);
enum htsExactFormat format = hts_get_format(fp)->format;
regidx_t *reg_idx = NULL;
if ( args->targets_fname )
{
reg_idx = regidx_init(args->targets_fname, NULL, NULL, 0, NULL);
if ( !reg_idx ) error("Could not read %s\n", args->targets_fname);
}
if ( format == bcf )
{
htsFile *out = hts_open("-","w");
if ( !out ) error("Could not open stdout\n", fname);
hts_idx_t *idx = bcf_index_load(fname);
if ( !idx ) error("Could not load .csi index of %s\n", fname);
bcf_hdr_t *hdr = bcf_hdr_read(fp);
if ( !hdr ) error("Could not read the header: %s\n", fname);
if ( args->print_header )
bcf_hdr_write(out,hdr);
if ( !args->header_only )
{
bcf1_t *rec = bcf_init();
for (i=0; i<nregs; i++)
{
hts_itr_t *itr = bcf_itr_querys(idx,hdr,regs[i]);
while ( bcf_itr_next(fp, itr, rec) >=0 )
{
if ( reg_idx && !regidx_overlap(reg_idx, bcf_seqname(hdr,rec),rec->pos,rec->pos+rec->rlen-1, NULL) ) continue;
bcf_write(out,hdr,rec);
}
tbx_itr_destroy(itr);
}
bcf_destroy(rec);
}
if ( hts_close(out) ) error("hts_close returned non-zero status for stdout\n");
bcf_hdr_destroy(hdr);
hts_idx_destroy(idx);
}
else if ( format==vcf || format==sam || format==unknown_format )
{
tbx_t *tbx = tbx_index_load(fname);
if ( !tbx ) error("Could not load .tbi/.csi index of %s\n", fname);
kstring_t str = {0,0,0};
if ( args->print_header )
{
while ( hts_getline(fp, KS_SEP_LINE, &str) >= 0 )
{
if ( !str.l || str.s[0]!=tbx->conf.meta_char ) break;
puts(str.s);
}
}
if ( !args->header_only )
{
int nseq;
const char **seq = NULL;
if ( reg_idx ) seq = tbx_seqnames(tbx, &nseq);
for (i=0; i<nregs; i++)
{
hts_itr_t *itr = tbx_itr_querys(tbx, regs[i]);
if ( !itr ) continue;
while (tbx_itr_next(fp, tbx, itr, &str) >= 0)
{
if ( reg_idx && !regidx_overlap(reg_idx,seq[itr->curr_tid],itr->curr_beg,itr->curr_end, NULL) ) continue;
puts(str.s);
}
tbx_itr_destroy(itr);
}
free(seq);
}
free(str.s);
tbx_destroy(tbx);
}
else if ( format==bam )
error("Please use \"samtools view\" for querying BAM files.\n");
if ( reg_idx ) regidx_destroy(reg_idx);
if ( hts_close(fp) ) error("hts_close returned non-zero status: %s\n", fname);
for (i=0; i<nregs; i++) free(regs[i]);
free(regs);
return 0;
}
