static void copy_check_alignment(const char *infname, const char *informat,
const char *outfname, const char *outmode, const char *outref)
{
samFile *in = sam_open(infname, "r");
samFile *out = sam_open(outfname, outmode);
bam1_t *aln = bam_init1();
bam_hdr_t *header = NULL;
int res;
if (!in) {
fail("couldn't open %s", infname);
goto err;
}
if (!out) {
fail("couldn't open %s with mode %s", outfname, outmode);
goto err;
}
if (!aln) {
fail("bam_init1() failed");
goto err;
}
if (outref) {
if (hts_set_opt(out, CRAM_OPT_REFERENCE, outref) < 0) {
fail("setting reference %s for %s", outref, outfname);
goto err;
}
}
header = sam_hdr_read(in);
if (!header) {
fail("reading header from %s", infname);
goto err;
}
if (sam_hdr_write(out, header) < 0) fail("writing headers to %s", outfname);
while ((res = sam_read1(in, header, aln)) >= 0) {
int mod4 = ((intptr_t) bam_get_cigar(aln)) % 4;
if (mod4 != 0)
fail("%s CIGAR not 4-byte aligned; offset is 4k+%d for \"%s\"",
informat, mod4, bam_get_qname(aln));
if (sam_write1(out, header, aln) < 0) fail("writing to %s", outfname);
}
if (res < -1) {
fail("failed to read alignment from %s", infname);
}
err:
bam_destroy1(aln);
bam_hdr_destroy(header);
if (in) sam_close(in);
if (out) sam_close(out);
}
int main(int argc, char *argv[])
{
samFile *in;
char *fn_ref = 0;
int flag = 0, c, clevel = -1, ignore_sam_err = 0;
char moder[8];
bam_hdr_t *h;
bam1_t *b;
htsFile *out;
char modew[8];
int r = 0, exit_code = 0;
hts_opt *in_opts = NULL, *out_opts = NULL, *last = NULL;
int nreads = 0;
int benchmark = 0;
while ((c = getopt(argc, argv, "IbDCSl:t:i:o:N:B")) >= 0) {
switch (c) {
case 'S': flag |= 1; break;
case 'b': flag |= 2; break;
case 'D': flag |= 4; break;
case 'C': flag |= 8; break;
case 'B': benchmark = 1; break;
case 'l': clevel = atoi(optarg); flag |= 2; break;
case 't': fn_ref = optarg; break;
case 'I': ignore_sam_err = 1; break;
case 'i': if (add_option(&in_opts, optarg)) return 1; break;
case 'o': if (add_option(&out_opts, optarg)) return 1; break;
case 'N': nreads = atoi(optarg);
}
}
if (argc == optind) {
fprintf(stderr, "Usage: samview [-bSCSIB] [-N num_reads] [-l level] [-o option=value] <in.bam>|<in.sam>|<in.cram> [region]\n");
return 1;
}
strcpy(moder, "r");
if (flag&4) strcat(moder, "c");
else if ((flag&1) == 0) strcat(moder, "b");
in = sam_open(argv[optind], moder);
if (in == NULL) {
fprintf(stderr, "Error opening \"%s\"\n", argv[optind]);
return EXIT_FAILURE;
}
h = sam_hdr_read(in);
h->ignore_sam_err = ignore_sam_err;
b = bam_init1();
strcpy(modew, "w");
if (clevel >= 0 && clevel <= 9) sprintf(modew + 1, "%d", clevel);
if (flag&8) strcat(modew, "c");
else if (flag&2) strcat(modew, "b");
out = hts_open("-", modew);
if (out == NULL) {
fprintf(stderr, "Error opening standard output\n");
return EXIT_FAILURE;
}
/* CRAM output */
if (flag & 8) {
int ret;
// Parse input header and use for CRAM output
out->fp.cram->header = sam_hdr_parse_(h->text, h->l_text);
// Create CRAM references arrays
if (fn_ref)
ret = cram_set_option(out->fp.cram, CRAM_OPT_REFERENCE, fn_ref);
else
// Attempt to fill out a cram->refs[] array from @SQ headers
ret = cram_set_option(out->fp.cram, CRAM_OPT_REFERENCE, NULL);
if (ret != 0)
return EXIT_FAILURE;
}
// Process any options; currently cram only.
for (; in_opts; in_opts = (last=in_opts)->next, free(last)) {
hts_set_opt(in, in_opts->opt, in_opts->val);
if (in_opts->opt == CRAM_OPT_REFERENCE)
if (hts_set_opt(out, in_opts->opt, in_opts->val) != 0)
return EXIT_FAILURE;
}
for (; out_opts; out_opts = (last=out_opts)->next, free(last))
if (hts_set_opt(out, out_opts->opt, out_opts->val) != 0)
return EXIT_FAILURE;
if (!benchmark)
sam_hdr_write(out, h);
if (optind + 1 < argc && !(flag&1)) { // BAM input and has a region
int i;
hts_idx_t *idx;
if ((idx = sam_index_load(in, argv[optind])) == 0) {
fprintf(stderr, "[E::%s] fail to load the BAM index\n", __func__);
return 1;
}
for (i = optind + 1; i < argc; ++i) {
hts_itr_t *iter;
if ((iter = sam_itr_querys(idx, h, argv[i])) == 0) {
fprintf(stderr, "[E::%s] fail to parse region '%s'\n", __func__, argv[i]);
continue;
}
while ((r = sam_itr_next(in, iter, b)) >= 0) {
if (!benchmark && sam_write1(out, h, b) < 0) {
fprintf(stderr, "Error writing output.\n");
exit_code = 1;
break;
}
if (nreads && --nreads == 0)
break;
}
hts_itr_destroy(iter);
}
hts_idx_destroy(idx);
} else while ((r = sam_read1(in, h, b)) >= 0) {
if (!benchmark && sam_write1(out, h, b) < 0) {
fprintf(stderr, "Error writing output.\n");
exit_code = 1;
break;
}
if (nreads && --nreads == 0)
break;
}
if (r < -1) {
fprintf(stderr, "Error parsing input.\n");
exit_code = 1;
}
r = sam_close(out);
if (r < 0) {
fprintf(stderr, "Error closing output.\n");
exit_code = 1;
}
bam_destroy1(b);
bam_hdr_destroy(h);
r = sam_close(in);
if (r < 0) {
fprintf(stderr, "Error closing input.\n");
exit_code = 1;
}
return exit_code;
}
int main_depth(int argc, char *argv[])
{
int i, n, tid, reg_tid, beg, end, pos, *n_plp, baseQ = 0, mapQ = 0, min_len = 0;
int all = 0, status = EXIT_SUCCESS, nfiles, max_depth = -1;
const bam_pileup1_t **plp;
char *reg = 0; // specified region
void *bed = 0; // BED data structure
char *file_list = NULL, **fn = NULL;
bam_hdr_t *h = NULL; // BAM header of the 1st input
aux_t **data;
bam_mplp_t mplp;
int last_pos = -1, last_tid = -1, ret;
sam_global_args ga = SAM_GLOBAL_ARGS_INIT;
static const struct option lopts[] = {
SAM_OPT_GLOBAL_OPTIONS('-', 0, '-', '-', 0),
{ NULL, 0, NULL, 0 }
};
// parse the command line
while ((n = getopt_long(argc, argv, "r:b:q:Q:l:f:am:d:", lopts, NULL)) >= 0) {
switch (n) {
case 'l':
min_len = atoi(optarg);
break; // minimum query length
case 'r':
reg = strdup(optarg);
break; // parsing a region requires a BAM header
case 'b':
bed = bed_read(optarg); // BED or position list file can be parsed now
if (!bed) {
print_error_errno("depth", "Could not read file \"%s\"", optarg);
return 1;
}
break;
case 'q':
baseQ = atoi(optarg);
break; // base quality threshold
case 'Q':
mapQ = atoi(optarg);
break; // mapping quality threshold
case 'f':
file_list = optarg;
break;
case 'a':
all++;
break;
case 'd':
case 'm':
max_depth = atoi(optarg);
break; // maximum coverage depth
default:
if (parse_sam_global_opt(n, optarg, lopts, &ga) == 0) break;
/* else fall-through */
case '?':
return usage();
}
}
if (optind == argc && !file_list)
return usage();
// initialize the auxiliary data structures
if (file_list)
{
if ( read_file_list(file_list,&nfiles,&fn) ) return 1;
n = nfiles;
argv = fn;
optind = 0;
}
else
n = argc - optind; // the number of BAMs on the command line
data = calloc(n, sizeof(aux_t*)); // data[i] for the i-th input
reg_tid = 0;
beg = 0;
end = INT_MAX; // set the default region
for (i = 0; i < n; ++i) {
int rf;
data[i] = calloc(1, sizeof(aux_t));
data[i]->fp = sam_open_format(argv[optind+i], "r", &ga.in); // open BAM
if (data[i]->fp == NULL) {
print_error_errno("depth", "Could not open \"%s\"", argv[optind+i]);
status = EXIT_FAILURE;
goto depth_end;
}
rf = SAM_FLAG | SAM_RNAME | SAM_POS | SAM_MAPQ | SAM_CIGAR | SAM_SEQ;
if (baseQ) rf |= SAM_QUAL;
if (hts_set_opt(data[i]->fp, CRAM_OPT_REQUIRED_FIELDS, rf)) {
fprintf(stderr, "Failed to set CRAM_OPT_REQUIRED_FIELDS value\n");
return 1;
}
if (hts_set_opt(data[i]->fp, CRAM_OPT_DECODE_MD, 0)) {
fprintf(stderr, "Failed to set CRAM_OPT_DECODE_MD value\n");
return 1;
}
data[i]->min_mapQ = mapQ; // set the mapQ filter
data[i]->min_len = min_len; // set the qlen filter
data[i]->hdr = sam_hdr_read(data[i]->fp); // read the BAM header
if (data[i]->hdr == NULL) {
fprintf(stderr, "Couldn't read header for \"%s\"\n",
argv[optind+i]);
status = EXIT_FAILURE;
goto depth_end;
}
if (reg) { // if a region is specified
hts_idx_t *idx = sam_index_load(data[i]->fp, argv[optind+i]); // load the index
if (idx == NULL) {
print_error("depth", "can't load index for \"%s\"", argv[optind+i]);
status = EXIT_FAILURE;
goto depth_end;
}
data[i]->iter = sam_itr_querys(idx, data[i]->hdr, reg); // set the iterator
hts_idx_destroy(idx); // the index is not needed any more; free the memory
if (data[i]->iter == NULL) {
print_error("depth", "can't parse region \"%s\"", reg);
status = EXIT_FAILURE;
goto depth_end;
}
}
}
h = data[0]->hdr; // easy access to the header of the 1st BAM
if (reg) {
beg = data[0]->iter->beg; // and to the parsed region coordinates
end = data[0]->iter->end;
reg_tid = data[0]->iter->tid;
}
// the core multi-pileup loop
mplp = bam_mplp_init(n, read_bam, (void**)data); // initialization
if (0 < max_depth)
bam_mplp_set_maxcnt(mplp,max_depth); // set maximum coverage depth
n_plp = calloc(n, sizeof(int)); // n_plp[i] is the number of covering reads from the i-th BAM
plp = calloc(n, sizeof(bam_pileup1_t*)); // plp[i] points to the array of covering reads (internal in mplp)
while ((ret=bam_mplp_auto(mplp, &tid, &pos, n_plp, plp)) > 0) { // come to the next covered position
if (pos < beg || pos >= end) continue; // out of range; skip
if (tid >= h->n_targets) continue; // diff number of @SQ lines per file?
if (all) {
while (tid > last_tid) {
if (last_tid >= 0 && !reg) {
// Deal with remainder or entirety of last tid.
while (++last_pos < h->target_len[last_tid]) {
// Horribly inefficient, but the bed API is an obfuscated black box.
if (bed && bed_overlap(bed, h->target_name[last_tid], last_pos, last_pos + 1) == 0)
continue;
fputs(h->target_name[last_tid], stdout);
printf("\t%d", last_pos+1);
for (i = 0; i < n; i++)
putchar('\t'), putchar('0');
putchar('\n');
}
}
last_tid++;
last_pos = -1;
if (all < 2)
break;
}
// Deal with missing portion of current tid
while (++last_pos < pos) {
if (last_pos < beg) continue; // out of range; skip
if (bed && bed_overlap(bed, h->target_name[tid], last_pos, last_pos + 1) == 0)
continue;
fputs(h->target_name[tid], stdout);
printf("\t%d", last_pos+1);
for (i = 0; i < n; i++)
putchar('\t'), putchar('0');
putchar('\n');
}
last_tid = tid;
last_pos = pos;
}
if (bed && bed_overlap(bed, h->target_name[tid], pos, pos + 1) == 0) continue;
fputs(h->target_name[tid], stdout);
printf("\t%d", pos+1); // a customized printf() would be faster
for (i = 0; i < n; ++i) { // base level filters have to go here
int j, m = 0;
for (j = 0; j < n_plp[i]; ++j) {
const bam_pileup1_t *p = plp[i] + j; // DON'T modfity plp[][] unless you really know
if (p->is_del || p->is_refskip) ++m; // having dels or refskips at tid:pos
else if (bam_get_qual(p->b)[p->qpos] < baseQ) ++m; // low base quality
}
printf("\t%d", n_plp[i] - m); // this the depth to output
}
putchar('\n');
}
if (ret < 0) status = EXIT_FAILURE;
free(n_plp);
free(plp);
bam_mplp_destroy(mplp);
if (all) {
// Handle terminating region
if (last_tid < 0 && reg && all > 1) {
last_tid = reg_tid;
last_pos = beg-1;
}
while (last_tid >= 0 && last_tid < h->n_targets) {
while (++last_pos < h->target_len[last_tid]) {
if (last_pos >= end) break;
if (bed && bed_overlap(bed, h->target_name[last_tid], last_pos, last_pos + 1) == 0)
continue;
fputs(h->target_name[last_tid], stdout);
printf("\t%d", last_pos+1);
for (i = 0; i < n; i++)
putchar('\t'), putchar('0');
putchar('\n');
}
last_tid++;
last_pos = -1;
if (all < 2 || reg)
break;
}
}
depth_end:
for (i = 0; i < n && data[i]; ++i) {
bam_hdr_destroy(data[i]->hdr);
if (data[i]->fp) sam_close(data[i]->fp);
hts_itr_destroy(data[i]->iter);
free(data[i]);
}
free(data);
free(reg);
if (bed) bed_destroy(bed);
if ( file_list )
{
for (i=0; i<n; i++) free(fn[i]);
free(fn);
}
sam_global_args_free(&ga);
return status;
}
static int mpileup(mplp_conf_t *conf)
{
if (conf->nfiles == 0) {
fprintf(stderr,"[%s] no input file/data given\n", __func__);
exit(EXIT_FAILURE);
}
mplp_ref_t mp_ref = MPLP_REF_INIT;
conf->gplp = (mplp_pileup_t *) calloc(1,sizeof(mplp_pileup_t));
conf->mplp_data = (mplp_aux_t**) calloc(conf->nfiles, sizeof(mplp_aux_t*));
conf->plp = (const bam_pileup1_t**) calloc(conf->nfiles, sizeof(bam_pileup1_t*));
conf->n_plp = (int*) calloc(conf->nfiles, sizeof(int));
// Allow to run mpileup on multiple regions in one go. This comes at cost: the bai index
// must be kept in the memory for the whole time which can be a problem with many bams.
// Therefore if none or only one region is requested, we initialize the bam iterator as
// before and free the index. Only when multiple regions are queried, we keep the index.
int nregs = 0;
if ( conf->reg_fname )
{
if ( conf->reg_is_file )
{
conf->reg = regidx_init(conf->reg_fname,NULL,NULL,0,NULL);
if ( !conf->reg ) {
fprintf(stderr,"Could not parse the regions: %s\n", conf->reg_fname);
exit(EXIT_FAILURE);
}
}
else
{
conf->reg = regidx_init(NULL,regidx_parse_reg,NULL,sizeof(char*),NULL);
if ( regidx_insert_list(conf->reg,conf->reg_fname,',') !=0 ) {
fprintf(stderr,"Could not parse the regions: %s\n", conf->reg_fname);
exit(EXIT_FAILURE);
}
}
nregs = regidx_nregs(conf->reg);
conf->reg_itr = regitr_init(conf->reg);
regitr_loop(conf->reg_itr); // region iterator now positioned at the first region
}
// read the header of each file in the list and initialize data
// beware: mpileup has always assumed that tid's are consistent in the headers, add sanity check at least!
bam_hdr_t *hdr = NULL; // header of first file in input list
int i;
for (i = 0; i < conf->nfiles; ++i) {
bam_hdr_t *h_tmp;
conf->mplp_data[i] = (mplp_aux_t*) calloc(1, sizeof(mplp_aux_t));
conf->mplp_data[i]->fp = sam_open(conf->files[i], "rb");
if ( !conf->mplp_data[i]->fp )
{
fprintf(stderr, "[%s] failed to open %s: %s\n", __func__, conf->files[i], strerror(errno));
exit(EXIT_FAILURE);
}
if (hts_set_opt(conf->mplp_data[i]->fp, CRAM_OPT_DECODE_MD, 0)) {
fprintf(stderr, "Failed to set CRAM_OPT_DECODE_MD value\n");
exit(EXIT_FAILURE);
}
if (conf->fai_fname && hts_set_fai_filename(conf->mplp_data[i]->fp, conf->fai_fname) != 0) {
fprintf(stderr, "[%s] failed to process %s: %s\n",
__func__, conf->fai_fname, strerror(errno));
exit(EXIT_FAILURE);
}
conf->mplp_data[i]->conf = conf;
conf->mplp_data[i]->ref = &mp_ref;
h_tmp = sam_hdr_read(conf->mplp_data[i]->fp);
if ( !h_tmp ) {
fprintf(stderr,"[%s] fail to read the header of %s\n", __func__, conf->files[i]);
exit(EXIT_FAILURE);
}
conf->mplp_data[i]->h = i ? hdr : h_tmp; // for j==0, "h" has not been set yet
conf->mplp_data[i]->bam_id = bam_smpl_add_bam(conf->bsmpl,h_tmp->text,conf->files[i]);
if ( conf->mplp_data[i]->bam_id<0 )
{
// no usable readgroups in this bam, it can be skipped
sam_close(conf->mplp_data[i]->fp);
free(conf->mplp_data[i]);
bam_hdr_destroy(h_tmp);
free(conf->files[i]);
if ( i+1<conf->nfiles ) memmove(&conf->files[i],&conf->files[i+1],sizeof(*conf->files)*(conf->nfiles-i-1));
conf->nfiles--;
i--;
continue;
}
if (conf->reg) {
hts_idx_t *idx = sam_index_load(conf->mplp_data[i]->fp, conf->files[i]);
if (idx == NULL) {
fprintf(stderr, "[%s] fail to load index for %s\n", __func__, conf->files[i]);
exit(EXIT_FAILURE);
}
conf->buf.l = 0;
ksprintf(&conf->buf,"%s:%u-%u",conf->reg_itr->seq,conf->reg_itr->beg+1,conf->reg_itr->end+1);
conf->mplp_data[i]->iter = sam_itr_querys(idx, conf->mplp_data[i]->h, conf->buf.s);
if ( !conf->mplp_data[i]->iter )
{
conf->mplp_data[i]->iter = sam_itr_querys(idx, conf->mplp_data[i]->h, conf->reg_itr->seq);
if ( conf->mplp_data[i]->iter ) {
fprintf(stderr,"[E::%s] fail to parse region '%s'\n", __func__, conf->buf.s);
exit(EXIT_FAILURE);
}
fprintf(stderr,"[E::%s] the sequence \"%s\" not found: %s\n",__func__,conf->reg_itr->seq,conf->files[i]);
exit(EXIT_FAILURE);
}
if ( nregs==1 ) // no need to keep the index in memory
hts_idx_destroy(idx);
else
conf->mplp_data[i]->idx = idx;
}
if ( !hdr ) hdr = h_tmp; /* save the header of first file in list */
else {
// FIXME: check consistency between h and h_tmp
bam_hdr_destroy(h_tmp);
// we store only the first file's header; it's (alleged to be)
// compatible with the i-th file's target_name lookup needs
conf->mplp_data[i]->h = hdr;
}
}
// allocate data storage proportionate to number of samples being studied sm->n
bam_smpl_get_samples(conf->bsmpl, &conf->gplp->n);
conf->gplp->n_plp = (int*) calloc(conf->gplp->n, sizeof(int));
conf->gplp->m_plp = (int*) calloc(conf->gplp->n, sizeof(int));
conf->gplp->plp = (bam_pileup1_t**) calloc(conf->gplp->n, sizeof(bam_pileup1_t*));
fprintf(stderr, "[%s] %d samples in %d input files\n", __func__, conf->gplp->n, conf->nfiles);
// write the VCF header
conf->bcf_fp = hts_open(conf->output_fname?conf->output_fname:"-", hts_bcf_wmode(conf->output_type));
if (conf->bcf_fp == NULL) {
fprintf(stderr, "[%s] failed to write to %s: %s\n", __func__, conf->output_fname? conf->output_fname : "standard output", strerror(errno));
exit(EXIT_FAILURE);
}
if ( conf->n_threads ) hts_set_threads(conf->bcf_fp, conf->n_threads);
// BCF header creation
conf->bcf_hdr = bcf_hdr_init("w");
conf->buf.l = 0;
if (conf->record_cmd_line)
{
ksprintf(&conf->buf, "##bcftoolsVersion=%s+htslib-%s\n",bcftools_version(),hts_version());
bcf_hdr_append(conf->bcf_hdr, conf->buf.s);
conf->buf.l = 0;
ksprintf(&conf->buf, "##bcftoolsCommand=mpileup");
for (i=1; i<conf->argc; i++) ksprintf(&conf->buf, " %s", conf->argv[i]);
kputc('\n', &conf->buf);
bcf_hdr_append(conf->bcf_hdr, conf->buf.s);
}
if (conf->fai_fname)
{
conf->buf.l = 0;
ksprintf(&conf->buf, "##reference=file://%s\n", conf->fai_fname);
bcf_hdr_append(conf->bcf_hdr, conf->buf.s);
}
// Translate BAM @SQ tags to BCF ##contig tags
// todo: use/write new BAM header manipulation routines, fill also UR, M5
for (i=0; i<hdr->n_targets; i++)
{
conf->buf.l = 0;
ksprintf(&conf->buf, "##contig=<ID=%s,length=%d>", hdr->target_name[i], hdr->target_len[i]);
bcf_hdr_append(conf->bcf_hdr, conf->buf.s);
}
conf->buf.l = 0;
bcf_hdr_append(conf->bcf_hdr,"##ALT=<ID=*,Description=\"Represents allele(s) other than observed.\">");
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=INDEL,Number=0,Type=Flag,Description=\"Indicates that the variant is an INDEL.\">");
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=IDV,Number=1,Type=Integer,Description=\"Maximum number of reads supporting an indel\">");
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=IMF,Number=1,Type=Float,Description=\"Maximum fraction of reads supporting an indel\">");
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=DP,Number=1,Type=Integer,Description=\"Raw read depth\">");
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=VDB,Number=1,Type=Float,Description=\"Variant Distance Bias for filtering splice-site artefacts in RNA-seq data (bigger is better)\",Version=\"3\">");
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=RPB,Number=1,Type=Float,Description=\"Mann-Whitney U test of Read Position Bias (bigger is better)\">");
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=MQB,Number=1,Type=Float,Description=\"Mann-Whitney U test of Mapping Quality Bias (bigger is better)\">");
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=BQB,Number=1,Type=Float,Description=\"Mann-Whitney U test of Base Quality Bias (bigger is better)\">");
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=MQSB,Number=1,Type=Float,Description=\"Mann-Whitney U test of Mapping Quality vs Strand Bias (bigger is better)\">");
#if CDF_MWU_TESTS
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=RPB2,Number=1,Type=Float,Description=\"Mann-Whitney U test of Read Position Bias [CDF] (bigger is better)\">");
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=MQB2,Number=1,Type=Float,Description=\"Mann-Whitney U test of Mapping Quality Bias [CDF] (bigger is better)\">");
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=BQB2,Number=1,Type=Float,Description=\"Mann-Whitney U test of Base Quality Bias [CDF] (bigger is better)\">");
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=MQSB2,Number=1,Type=Float,Description=\"Mann-Whitney U test of Mapping Quality vs Strand Bias [CDF] (bigger is better)\">");
#endif
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=SGB,Number=1,Type=Float,Description=\"Segregation based metric.\">");
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=MQ0F,Number=1,Type=Float,Description=\"Fraction of MQ0 reads (smaller is better)\">");
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=I16,Number=16,Type=Float,Description=\"Auxiliary tag used for calling, see description of bcf_callret1_t in bam2bcf.h\">");
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=QS,Number=R,Type=Float,Description=\"Auxiliary tag used for calling\">");
bcf_hdr_append(conf->bcf_hdr,"##FORMAT=<ID=PL,Number=G,Type=Integer,Description=\"List of Phred-scaled genotype likelihoods\">");
if ( conf->fmt_flag&B2B_FMT_DP )
bcf_hdr_append(conf->bcf_hdr,"##FORMAT=<ID=DP,Number=1,Type=Integer,Description=\"Number of high-quality bases\">");
if ( conf->fmt_flag&B2B_FMT_DV )
bcf_hdr_append(conf->bcf_hdr,"##FORMAT=<ID=DV,Number=1,Type=Integer,Description=\"Number of high-quality non-reference bases\">");
if ( conf->fmt_flag&B2B_FMT_DPR )
bcf_hdr_append(conf->bcf_hdr,"##FORMAT=<ID=DPR,Number=R,Type=Integer,Description=\"Number of high-quality bases observed for each allele\">");
if ( conf->fmt_flag&B2B_INFO_DPR )
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=DPR,Number=R,Type=Integer,Description=\"Number of high-quality bases observed for each allele\">");
if ( conf->fmt_flag&B2B_FMT_DP4 )
bcf_hdr_append(conf->bcf_hdr,"##FORMAT=<ID=DP4,Number=4,Type=Integer,Description=\"Number of high-quality ref-fwd, ref-reverse, alt-fwd and alt-reverse bases\">");
if ( conf->fmt_flag&B2B_FMT_SP )
bcf_hdr_append(conf->bcf_hdr,"##FORMAT=<ID=SP,Number=1,Type=Integer,Description=\"Phred-scaled strand bias P-value\">");
if ( conf->fmt_flag&B2B_FMT_AD )
bcf_hdr_append(conf->bcf_hdr,"##FORMAT=<ID=AD,Number=R,Type=Integer,Description=\"Allelic depths\">");
if ( conf->fmt_flag&B2B_FMT_ADF )
bcf_hdr_append(conf->bcf_hdr,"##FORMAT=<ID=ADF,Number=R,Type=Integer,Description=\"Allelic depths on the forward strand\">");
if ( conf->fmt_flag&B2B_FMT_ADR )
bcf_hdr_append(conf->bcf_hdr,"##FORMAT=<ID=ADR,Number=R,Type=Integer,Description=\"Allelic depths on the reverse strand\">");
if ( conf->fmt_flag&B2B_INFO_AD )
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=AD,Number=R,Type=Integer,Description=\"Total allelic depths\">");
if ( conf->fmt_flag&B2B_INFO_ADF )
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=ADF,Number=R,Type=Integer,Description=\"Total allelic depths on the forward strand\">");
if ( conf->fmt_flag&B2B_INFO_ADR )
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=ADR,Number=R,Type=Integer,Description=\"Total allelic depths on the reverse strand\">");
if ( conf->gvcf )
gvcf_update_header(conf->gvcf, conf->bcf_hdr);
int nsmpl;
const char **smpl = bam_smpl_get_samples(conf->bsmpl, &nsmpl);
for (i=0; i<nsmpl; i++)
bcf_hdr_add_sample(conf->bcf_hdr, smpl[i]);
bcf_hdr_write(conf->bcf_fp, conf->bcf_hdr);
conf->bca = bcf_call_init(-1., conf->min_baseQ);
conf->bcr = (bcf_callret1_t*) calloc(nsmpl, sizeof(bcf_callret1_t));
conf->bca->openQ = conf->openQ, conf->bca->extQ = conf->extQ, conf->bca->tandemQ = conf->tandemQ;
conf->bca->min_frac = conf->min_frac;
conf->bca->min_support = conf->min_support;
conf->bca->per_sample_flt = conf->flag & MPLP_PER_SAMPLE;
conf->bc.bcf_hdr = conf->bcf_hdr;
conf->bc.n = nsmpl;
conf->bc.PL = (int32_t*) malloc(15 * nsmpl * sizeof(*conf->bc.PL));
if (conf->fmt_flag)
{
assert( sizeof(float)==sizeof(int32_t) );
conf->bc.DP4 = (int32_t*) malloc(nsmpl * sizeof(int32_t) * 4);
conf->bc.fmt_arr = (uint8_t*) malloc(nsmpl * sizeof(float)); // all fmt_flag fields, float and int32
if ( conf->fmt_flag&(B2B_INFO_DPR|B2B_FMT_DPR|B2B_INFO_AD|B2B_INFO_ADF|B2B_INFO_ADR|B2B_FMT_AD|B2B_FMT_ADF|B2B_FMT_ADR) )
{
// first B2B_MAX_ALLELES fields for total numbers, the rest per-sample
conf->bc.ADR = (int32_t*) malloc((nsmpl+1)*B2B_MAX_ALLELES*sizeof(int32_t));
conf->bc.ADF = (int32_t*) malloc((nsmpl+1)*B2B_MAX_ALLELES*sizeof(int32_t));
for (i=0; i<nsmpl; i++)
{
conf->bcr[i].ADR = conf->bc.ADR + (i+1)*B2B_MAX_ALLELES;
conf->bcr[i].ADF = conf->bc.ADF + (i+1)*B2B_MAX_ALLELES;
}
}
}
// init mpileup
conf->iter = bam_mplp_init(conf->nfiles, mplp_func, (void**)conf->mplp_data);
if ( conf->flag & MPLP_SMART_OVERLAPS ) bam_mplp_init_overlaps(conf->iter);
if ( (double)conf->max_depth * conf->nfiles > 1<<20)
fprintf(stderr, "Warning: Potential memory hog, up to %.0fM reads in the pileup!\n", (double)conf->max_depth*conf->nfiles);
if ( (double)conf->max_depth * conf->nfiles / nsmpl < 250 )
fprintf(stderr, "Note: The maximum per-sample depth with -d %d is %.1fx\n", conf->max_depth,(double)conf->max_depth * conf->nfiles / nsmpl);
bam_mplp_set_maxcnt(conf->iter, conf->max_depth);
conf->max_indel_depth = conf->max_indel_depth * nsmpl;
conf->bcf_rec = bcf_init1();
bam_mplp_constructor(conf->iter, pileup_constructor);
// Run mpileup for multiple regions
if ( nregs )
{
int ireg = 0;
do
{
// first region is already positioned
if ( ireg++ > 0 )
{
conf->buf.l = 0;
ksprintf(&conf->buf,"%s:%u-%u",conf->reg_itr->seq,conf->reg_itr->beg,conf->reg_itr->end);
for (i=0; i<conf->nfiles; i++)
{
hts_itr_destroy(conf->mplp_data[i]->iter);
conf->mplp_data[i]->iter = sam_itr_querys(conf->mplp_data[i]->idx, conf->mplp_data[i]->h, conf->buf.s);
if ( !conf->mplp_data[i]->iter )
{
conf->mplp_data[i]->iter = sam_itr_querys(conf->mplp_data[i]->idx, conf->mplp_data[i]->h, conf->reg_itr->seq);
if ( conf->mplp_data[i]->iter ) {
fprintf(stderr,"[E::%s] fail to parse region '%s'\n", __func__, conf->buf.s);
exit(EXIT_FAILURE);
}
fprintf(stderr,"[E::%s] the sequence \"%s\" not found: %s\n",__func__,conf->reg_itr->seq,conf->files[i]);
exit(EXIT_FAILURE);
}
bam_mplp_reset(conf->iter);
}
}
mpileup_reg(conf,conf->reg_itr->beg,conf->reg_itr->end);
}
while ( regitr_loop(conf->reg_itr) );
}
else
mpileup_reg(conf,0,0);
flush_bcf_records(conf, conf->bcf_fp, conf->bcf_hdr, NULL);
// clean up
free(conf->bc.tmp.s);
bcf_destroy1(conf->bcf_rec);
if (conf->bcf_fp)
{
hts_close(conf->bcf_fp);
bcf_hdr_destroy(conf->bcf_hdr);
bcf_call_destroy(conf->bca);
free(conf->bc.PL);
free(conf->bc.DP4);
free(conf->bc.ADR);
free(conf->bc.ADF);
free(conf->bc.fmt_arr);
free(conf->bcr);
}
if ( conf->gvcf ) gvcf_destroy(conf->gvcf);
free(conf->buf.s);
for (i = 0; i < conf->gplp->n; ++i) free(conf->gplp->plp[i]);
free(conf->gplp->plp); free(conf->gplp->n_plp); free(conf->gplp->m_plp); free(conf->gplp);
bam_mplp_destroy(conf->iter);
bam_hdr_destroy(hdr);
for (i = 0; i < conf->nfiles; ++i) {
if ( nregs>1 ) hts_idx_destroy(conf->mplp_data[i]->idx);
sam_close(conf->mplp_data[i]->fp);
if ( conf->mplp_data[i]->iter) hts_itr_destroy(conf->mplp_data[i]->iter);
free(conf->mplp_data[i]);
}
if ( conf->reg_itr ) regitr_destroy(conf->reg_itr);
free(conf->mplp_data); free(conf->plp); free(conf->n_plp);
free(mp_ref.ref[0]);
free(mp_ref.ref[1]);
return 0;
}
static bool init_state(const bam2fq_opts_t* opts, bam2fq_state_t** state_out)
{
bam2fq_state_t* state = calloc(1, sizeof(bam2fq_state_t));
state->flag_on = opts->flag_on;
state->flag_off = opts->flag_off;
state->has12 = opts->has12;
state->use_oq = opts->use_oq;
state->copy_tags = opts->copy_tags;
state->filetype = opts->filetype;
state->def_qual = opts->def_qual;
state->fp = sam_open(opts->fn_input, "r");
if (state->fp == NULL) {
print_error_errno("bam2fq","Cannot read file \"%s\"", opts->fn_input);
free(state);
return false;
}
uint32_t rf = SAM_QNAME | SAM_FLAG | SAM_SEQ | SAM_QUAL;
if (opts->use_oq) rf |= SAM_AUX;
if (hts_set_opt(state->fp, CRAM_OPT_REQUIRED_FIELDS, rf)) {
fprintf(stderr, "Failed to set CRAM_OPT_REQUIRED_FIELDS value\n");
free(state);
return false;
}
if (hts_set_opt(state->fp, CRAM_OPT_DECODE_MD, 0)) {
fprintf(stderr, "Failed to set CRAM_OPT_DECODE_MD value\n");
free(state);
return false;
}
if (opts->fnse) {
state->fpse = fopen(opts->fnse,"w");
if (state->fpse == NULL) {
print_error_errno("bam2fq", "Cannot write to singleton file \"%s\"", opts->fnse);
free(state);
return false;
}
}
int i;
for (i = 0; i < 3; ++i) {
if (opts->fnr[i]) {
state->fpr[i] = fopen(opts->fnr[i], "w");
if (state->fpr[i] == NULL) {
print_error_errno("bam2fq", "Cannot write to r%d file \"%s\"", i, opts->fnr[i]);
free(state);
return false;
}
} else {
state->fpr[i] = stdout;
}
}
state->h = sam_hdr_read(state->fp);
if (state->h == NULL) {
fprintf(stderr, "Failed to read header for \"%s\"\n", opts->fn_input);
free(state);
return false;
}
*state_out = state;
return true;
}
int main_depth(int argc, char *argv[])
{
int i, n, tid, beg, end, pos, *n_plp, baseQ = 0, mapQ = 0, min_len = 0, status = EXIT_SUCCESS, nfiles;
const bam_pileup1_t **plp;
char *reg = 0; // specified region
void *bed = 0; // BED data structure
char *file_list = NULL, **fn = NULL;
bam_hdr_t *h = NULL; // BAM header of the 1st input
aux_t **data;
bam_mplp_t mplp;
// parse the command line
while ((n = getopt(argc, argv, "r:b:q:Q:l:f:")) >= 0) {
switch (n) {
case 'l': min_len = atoi(optarg); break; // minimum query length
case 'r': reg = strdup(optarg); break; // parsing a region requires a BAM header
case 'b':
bed = bed_read(optarg); // BED or position list file can be parsed now
if (!bed) { print_error_errno("Could not read file \"%s\"", optarg); return 1; }
break;
case 'q': baseQ = atoi(optarg); break; // base quality threshold
case 'Q': mapQ = atoi(optarg); break; // mapping quality threshold
case 'f': file_list = optarg; break;
}
}
if (optind == argc && !file_list) {
fprintf(pysamerr, "\n");
fprintf(pysamerr, "Usage: samtools depth [options] in1.bam [in2.bam [...]]\n");
fprintf(pysamerr, "Options:\n");
fprintf(pysamerr, " -b <bed> list of positions or regions\n");
fprintf(pysamerr, " -f <list> list of input BAM filenames, one per line [null]\n");
fprintf(pysamerr, " -l <int> read length threshold (ignore reads shorter than <int>)\n");
fprintf(pysamerr, " -q <int> base quality threshold\n");
fprintf(pysamerr, " -Q <int> mapping quality threshold\n");
fprintf(pysamerr, " -r <chr:from-to> region\n");
fprintf(pysamerr, "\n");
return 1;
}
// initialize the auxiliary data structures
if (file_list)
{
if ( read_file_list(file_list,&nfiles,&fn) ) return 1;
n = nfiles;
argv = fn;
optind = 0;
}
else
n = argc - optind; // the number of BAMs on the command line
data = calloc(n, sizeof(aux_t*)); // data[i] for the i-th input
beg = 0; end = 1<<30; // set the default region
for (i = 0; i < n; ++i) {
data[i] = calloc(1, sizeof(aux_t));
data[i]->fp = sam_open(argv[optind+i], "r"); // open BAM
if (data[i]->fp == NULL) {
print_error_errno("Could not open \"%s\"", argv[optind+i]);
status = EXIT_FAILURE;
goto depth_end;
}
if (hts_set_opt(data[i]->fp, CRAM_OPT_REQUIRED_FIELDS,
SAM_FLAG | SAM_RNAME | SAM_POS | SAM_MAPQ | SAM_CIGAR |
SAM_SEQ)) {
fprintf(pysamerr, "Failed to set CRAM_OPT_REQUIRED_FIELDS value\n");
return 1;
}
if (hts_set_opt(data[i]->fp, CRAM_OPT_DECODE_MD, 0)) {
fprintf(pysamerr, "Failed to set CRAM_OPT_DECODE_MD value\n");
return 1;
}
data[i]->min_mapQ = mapQ; // set the mapQ filter
data[i]->min_len = min_len; // set the qlen filter
data[i]->hdr = sam_hdr_read(data[i]->fp); // read the BAM header
if (reg) { // if a region is specified
hts_idx_t *idx = sam_index_load(data[i]->fp, argv[optind+i]); // load the index
if (idx == NULL) {
print_error("can't load index for \"%s\"", argv[optind+i]);
status = EXIT_FAILURE;
goto depth_end;
}
data[i]->iter = sam_itr_querys(idx, data[i]->hdr, reg); // set the iterator
hts_idx_destroy(idx); // the index is not needed any more; free the memory
if (data[i]->iter == NULL) {
print_error("can't parse region \"%s\"", reg);
status = EXIT_FAILURE;
goto depth_end;
}
}
}
h = data[0]->hdr; // easy access to the header of the 1st BAM
if (reg) {
beg = data[0]->iter->beg; // and to the parsed region coordinates
end = data[0]->iter->end;
}
// the core multi-pileup loop
mplp = bam_mplp_init(n, read_bam, (void**)data); // initialization
n_plp = calloc(n, sizeof(int)); // n_plp[i] is the number of covering reads from the i-th BAM
plp = calloc(n, sizeof(bam_pileup1_t*)); // plp[i] points to the array of covering reads (internal in mplp)
while (bam_mplp_auto(mplp, &tid, &pos, n_plp, plp) > 0) { // come to the next covered position
if (pos < beg || pos >= end) continue; // out of range; skip
if (bed && bed_overlap(bed, h->target_name[tid], pos, pos + 1) == 0) continue; // not in BED; skip
fputs(h->target_name[tid], stdout); printf("\t%d", pos+1); // a customized printf() would be faster
for (i = 0; i < n; ++i) { // base level filters have to go here
int j, m = 0;
for (j = 0; j < n_plp[i]; ++j) {
const bam_pileup1_t *p = plp[i] + j; // DON'T modfity plp[][] unless you really know
if (p->is_del || p->is_refskip) ++m; // having dels or refskips at tid:pos
else if (bam_get_qual(p->b)[p->qpos] < baseQ) ++m; // low base quality
}
printf("\t%d", n_plp[i] - m); // this the depth to output
}
putchar('\n');
}
free(n_plp); free(plp);
bam_mplp_destroy(mplp);
depth_end:
for (i = 0; i < n && data[i]; ++i) {
bam_hdr_destroy(data[i]->hdr);
if (data[i]->fp) sam_close(data[i]->fp);
hts_itr_destroy(data[i]->iter);
free(data[i]);
}
free(data); free(reg);
if (bed) bed_destroy(bed);
if ( file_list )
{
for (i=0; i<n; i++) free(fn[i]);
free(fn);
}
return status;
}
/*
* Performs pileup
* @param conf configuration for this pileup
* @param n number of files specified in fn
* @param fn filenames
*/
static int mpileup(mplp_conf_t *conf, int n, char **fn)
{
extern void *bcf_call_add_rg(void *rghash, const char *hdtext, const char *list);
extern void bcf_call_del_rghash(void *rghash);
mplp_aux_t **data;
int i, tid, pos, *n_plp, beg0 = 0, end0 = INT_MAX, ref_len, max_depth, max_indel_depth;
const bam_pileup1_t **plp;
mplp_ref_t mp_ref = MPLP_REF_INIT;
bam_mplp_t iter;
bam_hdr_t *h = NULL; /* header of first file in input list */
char *ref;
void *rghash = NULL;
FILE *pileup_fp = NULL;
bcf_callaux_t *bca = NULL;
bcf_callret1_t *bcr = NULL;
bcf_call_t bc;
htsFile *bcf_fp = NULL;
bcf_hdr_t *bcf_hdr = NULL;
bam_sample_t *sm = NULL;
kstring_t buf;
mplp_pileup_t gplp;
memset(&gplp, 0, sizeof(mplp_pileup_t));
memset(&buf, 0, sizeof(kstring_t));
memset(&bc, 0, sizeof(bcf_call_t));
data = calloc(n, sizeof(mplp_aux_t*));
plp = calloc(n, sizeof(bam_pileup1_t*));
n_plp = calloc(n, sizeof(int));
sm = bam_smpl_init();
if (n == 0) {
fprintf(stderr,"[%s] no input file/data given\n", __func__);
exit(EXIT_FAILURE);
}
// read the header of each file in the list and initialize data
for (i = 0; i < n; ++i) {
bam_hdr_t *h_tmp;
data[i] = calloc(1, sizeof(mplp_aux_t));
data[i]->fp = sam_open(fn[i], "rb");
if ( !data[i]->fp )
{
fprintf(stderr, "[%s] failed to open %s: %s\n", __func__, fn[i], strerror(errno));
exit(EXIT_FAILURE);
}
if (hts_set_opt(data[i]->fp, CRAM_OPT_DECODE_MD, 0)) {
fprintf(stderr, "Failed to set CRAM_OPT_DECODE_MD value\n");
exit(EXIT_FAILURE);
}
if (conf->fai_fname && hts_set_fai_filename(data[i]->fp, conf->fai_fname) != 0) {
fprintf(stderr, "[%s] failed to process %s: %s\n",
__func__, conf->fai_fname, strerror(errno));
exit(EXIT_FAILURE);
}
data[i]->conf = conf;
data[i]->ref = &mp_ref;
h_tmp = sam_hdr_read(data[i]->fp);
if ( !h_tmp ) {
fprintf(stderr,"[%s] fail to read the header of %s\n", __func__, fn[i]);
exit(EXIT_FAILURE);
}
bam_smpl_add(sm, fn[i], (conf->flag&MPLP_IGNORE_RG)? 0 : h_tmp->text);
// Collect read group IDs with PL (platform) listed in pl_list (note: fragile, strstr search)
rghash = bcf_call_add_rg(rghash, h_tmp->text, conf->pl_list);
if (conf->reg) {
hts_idx_t *idx = sam_index_load(data[i]->fp, fn[i]);
if (idx == NULL) {
fprintf(stderr, "[%s] fail to load index for %s\n", __func__, fn[i]);
exit(EXIT_FAILURE);
}
if ( (data[i]->iter=sam_itr_querys(idx, h_tmp, conf->reg)) == 0) {
fprintf(stderr, "[E::%s] fail to parse region '%s' with %s\n", __func__, conf->reg, fn[i]);
exit(EXIT_FAILURE);
}
if (i == 0) beg0 = data[i]->iter->beg, end0 = data[i]->iter->end;
hts_idx_destroy(idx);
}
else
data[i]->iter = NULL;
if (i == 0) h = data[i]->h = h_tmp; // save the header of the first file
else {
// FIXME: check consistency between h and h_tmp
bam_hdr_destroy(h_tmp);
// we store only the first file's header; it's (alleged to be)
// compatible with the i-th file's target_name lookup needs
data[i]->h = h;
}
}
// allocate data storage proportionate to number of samples being studied sm->n
gplp.n = sm->n;
gplp.n_plp = calloc(sm->n, sizeof(int));
gplp.m_plp = calloc(sm->n, sizeof(int));
gplp.plp = calloc(sm->n, sizeof(bam_pileup1_t*));
fprintf(stderr, "[%s] %d samples in %d input files\n", __func__, sm->n, n);
// write the VCF header
if (conf->flag & MPLP_BCF)
{
const char *mode;
if ( conf->flag & MPLP_VCF )
mode = (conf->flag&MPLP_NO_COMP)? "wu" : "wz"; // uncompressed VCF or compressed VCF
else
mode = (conf->flag&MPLP_NO_COMP)? "wub" : "wb"; // uncompressed BCF or compressed BCF
bcf_fp = bcf_open(conf->output_fname? conf->output_fname : "-", mode);
if (bcf_fp == NULL) {
fprintf(stderr, "[%s] failed to write to %s: %s\n", __func__, conf->output_fname? conf->output_fname : "standard output", strerror(errno));
exit(EXIT_FAILURE);
}
// BCF header creation
bcf_hdr = bcf_hdr_init("w");
kstring_t str = {0,0,NULL};
ksprintf(&str, "##samtoolsVersion=%s+htslib-%s\n",samtools_version(),hts_version());
bcf_hdr_append(bcf_hdr, str.s);
str.l = 0;
ksprintf(&str, "##samtoolsCommand=samtools mpileup");
for (i=1; i<conf->argc; i++) ksprintf(&str, " %s", conf->argv[i]);
kputc('\n', &str);
bcf_hdr_append(bcf_hdr, str.s);
if (conf->fai_fname)
{
str.l = 0;
ksprintf(&str, "##reference=file://%s\n", conf->fai_fname);
bcf_hdr_append(bcf_hdr, str.s);
}
// Translate BAM @SQ tags to BCF ##contig tags
// todo: use/write new BAM header manipulation routines, fill also UR, M5
for (i=0; i<h->n_targets; i++)
{
str.l = 0;
ksprintf(&str, "##contig=<ID=%s,length=%d>", h->target_name[i], h->target_len[i]);
bcf_hdr_append(bcf_hdr, str.s);
}
free(str.s);
bcf_hdr_append(bcf_hdr,"##ALT=<ID=X,Description=\"Represents allele(s) other than observed.\">");
bcf_hdr_append(bcf_hdr,"##INFO=<ID=INDEL,Number=0,Type=Flag,Description=\"Indicates that the variant is an INDEL.\">");
bcf_hdr_append(bcf_hdr,"##INFO=<ID=IDV,Number=1,Type=Integer,Description=\"Maximum number of reads supporting an indel\">");
bcf_hdr_append(bcf_hdr,"##INFO=<ID=IMF,Number=1,Type=Float,Description=\"Maximum fraction of reads supporting an indel\">");
bcf_hdr_append(bcf_hdr,"##INFO=<ID=DP,Number=1,Type=Integer,Description=\"Raw read depth\">");
bcf_hdr_append(bcf_hdr,"##INFO=<ID=VDB,Number=1,Type=Float,Description=\"Variant Distance Bias for filtering splice-site artefacts in RNA-seq data (bigger is better)\",Version=\"3\">");
bcf_hdr_append(bcf_hdr,"##INFO=<ID=RPB,Number=1,Type=Float,Description=\"Mann-Whitney U test of Read Position Bias (bigger is better)\">");
bcf_hdr_append(bcf_hdr,"##INFO=<ID=MQB,Number=1,Type=Float,Description=\"Mann-Whitney U test of Mapping Quality Bias (bigger is better)\">");
bcf_hdr_append(bcf_hdr,"##INFO=<ID=BQB,Number=1,Type=Float,Description=\"Mann-Whitney U test of Base Quality Bias (bigger is better)\">");
bcf_hdr_append(bcf_hdr,"##INFO=<ID=MQSB,Number=1,Type=Float,Description=\"Mann-Whitney U test of Mapping Quality vs Strand Bias (bigger is better)\">");
#if CDF_MWU_TESTS
bcf_hdr_append(bcf_hdr,"##INFO=<ID=RPB2,Number=1,Type=Float,Description=\"Mann-Whitney U test of Read Position Bias [CDF] (bigger is better)\">");
bcf_hdr_append(bcf_hdr,"##INFO=<ID=MQB2,Number=1,Type=Float,Description=\"Mann-Whitney U test of Mapping Quality Bias [CDF] (bigger is better)\">");
bcf_hdr_append(bcf_hdr,"##INFO=<ID=BQB2,Number=1,Type=Float,Description=\"Mann-Whitney U test of Base Quality Bias [CDF] (bigger is better)\">");
bcf_hdr_append(bcf_hdr,"##INFO=<ID=MQSB2,Number=1,Type=Float,Description=\"Mann-Whitney U test of Mapping Quality vs Strand Bias [CDF] (bigger is better)\">");
#endif
bcf_hdr_append(bcf_hdr,"##INFO=<ID=SGB,Number=1,Type=Float,Description=\"Segregation based metric.\">");
bcf_hdr_append(bcf_hdr,"##INFO=<ID=MQ0F,Number=1,Type=Float,Description=\"Fraction of MQ0 reads (smaller is better)\">");
bcf_hdr_append(bcf_hdr,"##INFO=<ID=I16,Number=16,Type=Float,Description=\"Auxiliary tag used for calling, see description of bcf_callret1_t in bam2bcf.h\">");
bcf_hdr_append(bcf_hdr,"##INFO=<ID=QS,Number=R,Type=Float,Description=\"Auxiliary tag used for calling\">");
bcf_hdr_append(bcf_hdr,"##FORMAT=<ID=PL,Number=G,Type=Integer,Description=\"List of Phred-scaled genotype likelihoods\">");
if ( conf->fmt_flag&B2B_FMT_DP )
bcf_hdr_append(bcf_hdr,"##FORMAT=<ID=DP,Number=1,Type=Integer,Description=\"Number of high-quality bases\">");
if ( conf->fmt_flag&B2B_FMT_DV )
bcf_hdr_append(bcf_hdr,"##FORMAT=<ID=DV,Number=1,Type=Integer,Description=\"Number of high-quality non-reference bases\">");
if ( conf->fmt_flag&B2B_FMT_DPR )
bcf_hdr_append(bcf_hdr,"##FORMAT=<ID=DPR,Number=R,Type=Integer,Description=\"Number of high-quality bases observed for each allele\">");
if ( conf->fmt_flag&B2B_INFO_DPR )
bcf_hdr_append(bcf_hdr,"##INFO=<ID=DPR,Number=R,Type=Integer,Description=\"Number of high-quality bases observed for each allele\">");
if ( conf->fmt_flag&B2B_FMT_DP4 )
bcf_hdr_append(bcf_hdr,"##FORMAT=<ID=DP4,Number=4,Type=Integer,Description=\"Number of high-quality ref-fwd, ref-reverse, alt-fwd and alt-reverse bases\">");
if ( conf->fmt_flag&B2B_FMT_SP )
bcf_hdr_append(bcf_hdr,"##FORMAT=<ID=SP,Number=1,Type=Integer,Description=\"Phred-scaled strand bias P-value\">");
for (i=0; i<sm->n; i++)
bcf_hdr_add_sample(bcf_hdr, sm->smpl[i]);
bcf_hdr_add_sample(bcf_hdr, NULL);
bcf_hdr_write(bcf_fp, bcf_hdr);
// End of BCF header creation
// Initialise the calling algorithm
bca = bcf_call_init(-1., conf->min_baseQ);
bcr = calloc(sm->n, sizeof(bcf_callret1_t));
bca->rghash = rghash;
bca->openQ = conf->openQ, bca->extQ = conf->extQ, bca->tandemQ = conf->tandemQ;
bca->min_frac = conf->min_frac;
bca->min_support = conf->min_support;
bca->per_sample_flt = conf->flag & MPLP_PER_SAMPLE;
bc.bcf_hdr = bcf_hdr;
bc.n = sm->n;
bc.PL = malloc(15 * sm->n * sizeof(*bc.PL));
if (conf->fmt_flag)
{
assert( sizeof(float)==sizeof(int32_t) );
bc.DP4 = malloc(sm->n * sizeof(int32_t) * 4);
bc.fmt_arr = malloc(sm->n * sizeof(float)); // all fmt_flag fields
if ( conf->fmt_flag&(B2B_INFO_DPR|B2B_FMT_DPR) )
{
// first B2B_MAX_ALLELES fields for total numbers, the rest per-sample
bc.DPR = malloc((sm->n+1)*B2B_MAX_ALLELES*sizeof(int32_t));
for (i=0; i<sm->n; i++)
bcr[i].DPR = bc.DPR + (i+1)*B2B_MAX_ALLELES;
}
}
}
else {
pileup_fp = conf->output_fname? fopen(conf->output_fname, "w") : stdout;
if (pileup_fp == NULL) {
fprintf(stderr, "[%s] failed to write to %s: %s\n", __func__, conf->output_fname, strerror(errno));
exit(EXIT_FAILURE);
}
}
// init pileup
iter = bam_mplp_init(n, mplp_func, (void**)data);
if ( conf->flag & MPLP_SMART_OVERLAPS ) bam_mplp_init_overlaps(iter);
max_depth = conf->max_depth;
if (max_depth * sm->n > 1<<20)
fprintf(stderr, "(%s) Max depth is above 1M. Potential memory hog!\n", __func__);
if (max_depth * sm->n < 8000) {
max_depth = 8000 / sm->n;
fprintf(stderr, "<%s> Set max per-file depth to %d\n", __func__, max_depth);
}
max_indel_depth = conf->max_indel_depth * sm->n;
bam_mplp_set_maxcnt(iter, max_depth);
bcf1_t *bcf_rec = bcf_init1();
int ret;
// begin pileup
while ( (ret=bam_mplp_auto(iter, &tid, &pos, n_plp, plp)) > 0) {
if (conf->reg && (pos < beg0 || pos >= end0)) continue; // out of the region requested
if (conf->bed && tid >= 0 && !bed_overlap(conf->bed, h->target_name[tid], pos, pos+1)) continue;
mplp_get_ref(data[0], tid, &ref, &ref_len);
//printf("tid=%d len=%d ref=%p/%s\n", tid, ref_len, ref, ref);
if (conf->flag & MPLP_BCF) {
int total_depth, _ref0, ref16;
for (i = total_depth = 0; i < n; ++i) total_depth += n_plp[i];
group_smpl(&gplp, sm, &buf, n, fn, n_plp, plp, conf->flag & MPLP_IGNORE_RG);
_ref0 = (ref && pos < ref_len)? ref[pos] : 'N';
ref16 = seq_nt16_table[_ref0];
bcf_callaux_clean(bca, &bc);
for (i = 0; i < gplp.n; ++i)
bcf_call_glfgen(gplp.n_plp[i], gplp.plp[i], ref16, bca, bcr + i);
bc.tid = tid; bc.pos = pos;
bcf_call_combine(gplp.n, bcr, bca, ref16, &bc);
bcf_clear1(bcf_rec);
bcf_call2bcf(&bc, bcf_rec, bcr, conf->fmt_flag, 0, 0);
bcf_write1(bcf_fp, bcf_hdr, bcf_rec);
// call indels; todo: subsampling with total_depth>max_indel_depth instead of ignoring?
if (!(conf->flag&MPLP_NO_INDEL) && total_depth < max_indel_depth && bcf_call_gap_prep(gplp.n, gplp.n_plp, gplp.plp, pos, bca, ref, rghash) >= 0)
{
bcf_callaux_clean(bca, &bc);
for (i = 0; i < gplp.n; ++i)
bcf_call_glfgen(gplp.n_plp[i], gplp.plp[i], -1, bca, bcr + i);
if (bcf_call_combine(gplp.n, bcr, bca, -1, &bc) >= 0) {
bcf_clear1(bcf_rec);
bcf_call2bcf(&bc, bcf_rec, bcr, conf->fmt_flag, bca, ref);
bcf_write1(bcf_fp, bcf_hdr, bcf_rec);
}
}
} else {
fprintf(pileup_fp, "%s\t%d\t%c", h->target_name[tid], pos + 1, (ref && pos < ref_len)? ref[pos] : 'N');
for (i = 0; i < n; ++i) {
int j, cnt;
for (j = cnt = 0; j < n_plp[i]; ++j) {
const bam_pileup1_t *p = plp[i] + j;
int c = p->qpos < p->b->core.l_qseq
? bam_get_qual(p->b)[p->qpos]
: 0;
if (c >= conf->min_baseQ) ++cnt;
}
fprintf(pileup_fp, "\t%d\t", cnt);
if (n_plp[i] == 0) {
fputs("*\t*", pileup_fp);
if (conf->flag & MPLP_PRINT_MAPQ) fputs("\t*", pileup_fp);
if (conf->flag & MPLP_PRINT_POS) fputs("\t*", pileup_fp);
} else {
for (j = 0; j < n_plp[i]; ++j) {
const bam_pileup1_t *p = plp[i] + j;
int c = p->qpos < p->b->core.l_qseq
? bam_get_qual(p->b)[p->qpos]
: 0;
if (c >= conf->min_baseQ)
pileup_seq(pileup_fp, plp[i] + j, pos, ref_len, ref);
}
putc('\t', pileup_fp);
for (j = 0; j < n_plp[i]; ++j) {
const bam_pileup1_t *p = plp[i] + j;
int c = p->qpos < p->b->core.l_qseq
? bam_get_qual(p->b)[p->qpos]
: 0;
if (c >= conf->min_baseQ) {
c = c + 33 < 126? c + 33 : 126;
putc(c, pileup_fp);
}
}
if (conf->flag & MPLP_PRINT_MAPQ) {
putc('\t', pileup_fp);
for (j = 0; j < n_plp[i]; ++j) {
const bam_pileup1_t *p = plp[i] + j;
int c = bam_get_qual(p->b)[p->qpos];
if ( c < conf->min_baseQ ) continue;
c = plp[i][j].b->core.qual + 33;
if (c > 126) c = 126;
putc(c, pileup_fp);
}
}
if (conf->flag & MPLP_PRINT_POS) {
putc('\t', pileup_fp);
for (j = 0; j < n_plp[i]; ++j) {
if (j > 0) putc(',', pileup_fp);
fprintf(pileup_fp, "%d", plp[i][j].qpos + 1); // FIXME: printf() is very slow...
}
}
}
}
putc('\n', pileup_fp);
}
}
// clean up
free(bc.tmp.s);
bcf_destroy1(bcf_rec);
if (bcf_fp)
{
hts_close(bcf_fp);
bcf_hdr_destroy(bcf_hdr);
bcf_call_destroy(bca);
free(bc.PL);
free(bc.DP4);
free(bc.DPR);
free(bc.fmt_arr);
free(bcr);
}
if (pileup_fp && conf->output_fname) fclose(pileup_fp);
bam_smpl_destroy(sm); free(buf.s);
for (i = 0; i < gplp.n; ++i) free(gplp.plp[i]);
free(gplp.plp); free(gplp.n_plp); free(gplp.m_plp);
bcf_call_del_rghash(rghash);
bam_mplp_destroy(iter);
bam_hdr_destroy(h);
for (i = 0; i < n; ++i) {
sam_close(data[i]->fp);
if (data[i]->iter) hts_itr_destroy(data[i]->iter);
free(data[i]);
}
free(data); free(plp); free(n_plp);
if (mp_ref.ref[0]) free(mp_ref.ref[0]);
if (mp_ref.ref[1]) free(mp_ref.ref[1]);
return ret;
}
int main_pad2unpad(int argc, char *argv[])
{
samFile *in = 0, *out = 0;
bam_hdr_t *h = 0, *h_fix = 0;
faidx_t *fai = 0;
int c, compress_level = -1, is_long_help = 0;
char in_mode[5], out_mode[6], *fn_out = 0, *fn_list = 0;
int ret=0;
sam_global_args ga = SAM_GLOBAL_ARGS_INIT;
static const struct option lopts[] = {
SAM_OPT_GLOBAL_OPTIONS('-', 0, 0, 0, 'T'),
{ NULL, 0, NULL, 0 }
};
/* parse command-line options */
strcpy(in_mode, "r"); strcpy(out_mode, "w");
while ((c = getopt_long(argc, argv, "SCso:u1T:?", lopts, NULL)) >= 0) {
switch (c) {
case 'S': break;
case 'C': hts_parse_format(&ga.out, "cram"); break;
case 's': assert(compress_level == -1); hts_parse_format(&ga.out, "sam"); break;
case 'o': fn_out = strdup(optarg); break;
case 'u':
compress_level = 0;
if (ga.out.format == unknown_format)
hts_parse_format(&ga.out, "bam");
break;
case '1':
compress_level = 1;
if (ga.out.format == unknown_format)
hts_parse_format(&ga.out, "bam");
break;
case '?': is_long_help = 1; break;
default: if (parse_sam_global_opt(c, optarg, lopts, &ga) == 0) break;
fprintf(stderr, "[bam_fillmd] unrecognized option '-%c'\n\n", c);
return usage(is_long_help);
}
}
if (argc == optind) return usage(is_long_help);
strcat(out_mode, "h");
if (compress_level >= 0) {
char tmp[2];
tmp[0] = compress_level + '0'; tmp[1] = '\0';
strcat(out_mode, tmp);
}
// Load FASTA reference (also needed for SAM -> BAM if missing header)
if (ga.reference) {
fn_list = samfaipath(ga.reference);
fai = fai_load(ga.reference);
}
// open file handlers
if ((in = sam_open_format(argv[optind], in_mode, &ga.in)) == 0) {
fprintf(stderr, "[depad] failed to open \"%s\" for reading.\n", argv[optind]);
ret = 1;
goto depad_end;
}
if (fn_list && hts_set_fai_filename(in, fn_list) != 0) {
fprintf(stderr, "[depad] failed to load reference file \"%s\".\n", fn_list);
ret = 1;
goto depad_end;
}
if ((h = sam_hdr_read(in)) == 0) {
fprintf(stderr, "[depad] failed to read the header from \"%s\".\n", argv[optind]);
ret = 1;
goto depad_end;
}
if (fai) {
h_fix = fix_header(h, fai);
} else {
fprintf(stderr, "[depad] Warning - reference lengths will not be corrected without FASTA reference\n");
h_fix = h;
}
char wmode[2];
strcat(out_mode, sam_open_mode(wmode, fn_out, NULL)==0 ? wmode : "b");
if ((out = sam_open_format(fn_out? fn_out : "-", out_mode, &ga.out)) == 0) {
fprintf(stderr, "[depad] failed to open \"%s\" for writing.\n", fn_out? fn_out : "standard output");
ret = 1;
goto depad_end;
}
// Reference-based CRAM won't work unless we also create a new reference.
// We could embed this, but for now we take the easy option.
if (ga.out.format == cram)
hts_set_opt(out, CRAM_OPT_NO_REF, 1);
if (sam_hdr_write(out, h_fix) != 0) {
fprintf(stderr, "[depad] failed to write header.\n");
ret = 1;
goto depad_end;
}
// Do the depad
ret = bam_pad2unpad(in, out, h, fai);
depad_end:
// close files, free and return
if (fai) fai_destroy(fai);
if (h) bam_hdr_destroy(h);
sam_close(in);
sam_close(out);
free(fn_list); free(fn_out);
return ret;
}
int bam_flagstat(int argc, char *argv[])
{
samFile *fp;
bam_hdr_t *header;
bam_flagstat_t *s;
char b0[16], b1[16];
int c;
enum {
INPUT_FMT_OPTION = CHAR_MAX+1,
};
sam_global_args ga = SAM_GLOBAL_ARGS_INIT;
static const struct option lopts[] = {
SAM_OPT_GLOBAL_OPTIONS('-', 0, '-', '-', '-', '@'),
{NULL, 0, NULL, 0}
};
while ((c = getopt_long(argc, argv, "@:", lopts, NULL)) >= 0) {
switch (c) {
default: if (parse_sam_global_opt(c, optarg, lopts, &ga) == 0) break;
/* else fall-through */
case '?':
usage_exit(samtools_stderr, EXIT_FAILURE);
}
}
if (argc != optind+1) {
if (argc == optind) usage_exit(samtools_stdout, EXIT_SUCCESS);
else usage_exit(samtools_stderr, EXIT_FAILURE);
}
fp = sam_open_format(argv[optind], "r", &ga.in);
if (fp == NULL) {
print_error_errno("flagstat", "Cannot open input file \"%s\"", argv[optind]);
return 1;
}
if (ga.nthreads > 0)
hts_set_threads(fp, ga.nthreads);
if (hts_set_opt(fp, CRAM_OPT_REQUIRED_FIELDS,
SAM_FLAG | SAM_MAPQ | SAM_RNEXT)) {
fprintf(samtools_stderr, "Failed to set CRAM_OPT_REQUIRED_FIELDS value\n");
return 1;
}
if (hts_set_opt(fp, CRAM_OPT_DECODE_MD, 0)) {
fprintf(samtools_stderr, "Failed to set CRAM_OPT_DECODE_MD value\n");
return 1;
}
header = sam_hdr_read(fp);
if (header == NULL) {
fprintf(samtools_stderr, "Failed to read header for \"%s\"\n", argv[optind]);
return 1;
}
s = bam_flagstat_core(fp, header);
fprintf(samtools_stdout, "%lld + %lld in total (QC-passed reads + QC-failed reads)\n", s->n_reads[0], s->n_reads[1]);
fprintf(samtools_stdout, "%lld + %lld secondary\n", s->n_secondary[0], s->n_secondary[1]);
fprintf(samtools_stdout, "%lld + %lld supplementary\n", s->n_supp[0], s->n_supp[1]);
fprintf(samtools_stdout, "%lld + %lld duplicates\n", s->n_dup[0], s->n_dup[1]);
fprintf(samtools_stdout, "%lld + %lld mapped (%s : %s)\n", s->n_mapped[0], s->n_mapped[1], percent(b0, s->n_mapped[0], s->n_reads[0]), percent(b1, s->n_mapped[1], s->n_reads[1]));
fprintf(samtools_stdout, "%lld + %lld paired in sequencing\n", s->n_pair_all[0], s->n_pair_all[1]);
fprintf(samtools_stdout, "%lld + %lld read1\n", s->n_read1[0], s->n_read1[1]);
fprintf(samtools_stdout, "%lld + %lld read2\n", s->n_read2[0], s->n_read2[1]);
fprintf(samtools_stdout, "%lld + %lld properly paired (%s : %s)\n", s->n_pair_good[0], s->n_pair_good[1], percent(b0, s->n_pair_good[0], s->n_pair_all[0]), percent(b1, s->n_pair_good[1], s->n_pair_all[1]));
fprintf(samtools_stdout, "%lld + %lld with itself and mate mapped\n", s->n_pair_map[0], s->n_pair_map[1]);
fprintf(samtools_stdout, "%lld + %lld singletons (%s : %s)\n", s->n_sgltn[0], s->n_sgltn[1], percent(b0, s->n_sgltn[0], s->n_pair_all[0]), percent(b1, s->n_sgltn[1], s->n_pair_all[1]));
fprintf(samtools_stdout, "%lld + %lld with mate mapped to a different chr\n", s->n_diffchr[0], s->n_diffchr[1]);
fprintf(samtools_stdout, "%lld + %lld with mate mapped to a different chr (mapQ>=5)\n", s->n_diffhigh[0], s->n_diffhigh[1]);
free(s);
bam_hdr_destroy(header);
sam_close(fp);
sam_global_args_free(&ga);
return 0;
}
int bam_mating(int argc, char *argv[])
{
htsThreadPool p = {NULL, 0};
samFile *in = NULL, *out = NULL;
int c, remove_reads = 0, proper_pair_check = 1, add_ct = 0, res = 1, mate_score = 0;
sam_global_args ga = SAM_GLOBAL_ARGS_INIT;
char wmode[3] = {'w', 'b', 0};
static const struct option lopts[] = {
SAM_OPT_GLOBAL_OPTIONS('-', 0, 'O', 0, 0, '@'),
{ NULL, 0, NULL, 0 }
};
// parse args
if (argc == 1) { usage(stdout); return 0; }
while ((c = getopt_long(argc, argv, "rpcmO:@:", lopts, NULL)) >= 0) {
switch (c) {
case 'r': remove_reads = 1; break;
case 'p': proper_pair_check = 0; break;
case 'c': add_ct = 1; break;
case 'm': mate_score = 1; break;
default: if (parse_sam_global_opt(c, optarg, lopts, &ga) == 0) break;
/* else fall-through */
case '?': usage(stderr); goto fail;
}
}
if (optind+1 >= argc) { usage(stderr); goto fail; }
// init
if ((in = sam_open_format(argv[optind], "rb", &ga.in)) == NULL) {
print_error_errno("fixmate", "cannot open input file");
goto fail;
}
sam_open_mode(wmode+1, argv[optind+1], NULL);
if ((out = sam_open_format(argv[optind+1], wmode, &ga.out)) == NULL) {
print_error_errno("fixmate", "cannot open output file");
goto fail;
}
if (ga.nthreads > 0) {
if (!(p.pool = hts_tpool_init(ga.nthreads))) {
fprintf(stderr, "Error creating thread pool\n");
goto fail;
}
hts_set_opt(in, HTS_OPT_THREAD_POOL, &p);
hts_set_opt(out, HTS_OPT_THREAD_POOL, &p);
}
// run
res = bam_mating_core(in, out, remove_reads, proper_pair_check, add_ct, mate_score);
// cleanup
sam_close(in);
if (sam_close(out) < 0) {
fprintf(stderr, "[bam_mating] error while closing output file\n");
res = 1;
}
if (p.pool) hts_tpool_destroy(p.pool);
sam_global_args_free(&ga);
return res;
fail:
if (in) sam_close(in);
if (out) sam_close(out);
if (p.pool) hts_tpool_destroy(p.pool);
sam_global_args_free(&ga);
return 1;
}
