samfile_t *samopen(const char *fn, const char *mode, const void *aux)
{
// hts_open() is really sam_open(), except for #define games
samFile *hts_fp = hts_open(fn, mode);
if (hts_fp == NULL) return NULL;
samfile_t *fp = malloc(sizeof (samfile_t));
if (!fp) {
sam_close(hts_fp);
return NULL;
}
fp->file = hts_fp;
fp->x.bam = hts_fp->fp.bgzf;
if (strchr(mode, 'r')) {
if (aux) {
if (hts_set_fai_filename(fp->file, aux) != 0) {
sam_close(hts_fp);
free(fp);
return NULL;
}
}
fp->header = sam_hdr_read(fp->file); // samclose() will free this
if (fp->header == NULL) {
sam_close(hts_fp);
free(fp);
return NULL;
}
fp->is_write = 0;
if (fp->header->n_targets == 0 && bam_verbose >= 1)
fprintf(samtools_stderr, "[samopen] no @SQ lines in the header.\n");
}
else {
enum htsExactFormat fmt = hts_get_format(fp->file)->format;
fp->header = (bam_hdr_t *)aux; // For writing, we won't free it
fp->is_write = 1;
if (!(fmt == text_format || fmt == sam) || strchr(mode, 'h')) {
if (sam_hdr_write(fp->file, fp->header) < 0) {
if (bam_verbose >= 1)
fprintf(samtools_stderr, "[samopen] Couldn't write header\n");
sam_close(hts_fp);
free(fp);
return NULL;
}
}
}
return fp;
}
static void naive_concat(args_t *args)
{
// only compressed BCF atm
BGZF *bgzf_out = bgzf_open(args->output_fname,"w");;
const size_t page_size = 32768;
char *buf = (char*) malloc(page_size);
kstring_t tmp = {0,0,0};
int i;
for (i=0; i<args->nfnames; i++)
{
htsFile *hts_fp = hts_open(args->fnames[i],"r");
if ( !hts_fp ) error("Failed to open: %s\n", args->fnames[i]);
htsFormat type = *hts_get_format(hts_fp);
if ( type.format==vcf ) error("The --naive option currently works only for compressed BCFs, sorry :-/\n");
if ( type.compression!=bgzf ) error("The --naive option currently works only for compressed BCFs, sorry :-/\n");
BGZF *fp = hts_get_bgzfp(hts_fp);
if ( !fp || bgzf_read_block(fp) != 0 || !fp->block_length )
error("Failed to read %s: %s\n", args->fnames[i], strerror(errno));
uint8_t magic[5];
if ( bgzf_read(fp, magic, 5) != 5 ) error("Failed to read the BCF header in %s\n", args->fnames[i]);
if (strncmp((char*)magic, "BCF\2\2", 5) != 0) error("Invalid BCF magic string in %s\n", args->fnames[i]);
if ( bgzf_read(fp, &tmp.l, 4) != 4 ) error("Failed to read the BCF header in %s\n", args->fnames[i]);
hts_expand(char,tmp.l,tmp.m,tmp.s);
if ( bgzf_read(fp, tmp.s, tmp.l) != tmp.l ) error("Failed to read the BCF header in %s\n", args->fnames[i]);
// write only the first header
if ( i==0 )
{
if ( bgzf_write(bgzf_out, "BCF\2\2", 5) !=5 ) error("Failed to write %d bytes to %s\n", 5,args->output_fname);
if ( bgzf_write(bgzf_out, &tmp.l, 4) !=4 ) error("Failed to write %d bytes to %s\n", 4,args->output_fname);
if ( bgzf_write(bgzf_out, tmp.s, tmp.l) != tmp.l) error("Failed to write %d bytes to %s\n", tmp.l,args->output_fname);
}
// Output all non-header data that were read together with the header block
int nskip = fp->block_offset;
if ( fp->block_length - nskip > 0 )
{
if ( bgzf_write(bgzf_out, fp->uncompressed_block+nskip, fp->block_length-nskip)<0 ) error("Error: %d\n",fp->errcode);
}
if ( bgzf_flush(bgzf_out)<0 ) error("Error: %d\n",bgzf_out->errcode);
// Stream the rest of the file as it is, without recompressing, but remove BGZF EOF blocks
ssize_t nread, ncached = 0, nwr;
const int neof = 28;
char cached[neof];
while (1)
{
nread = bgzf_raw_read(fp, buf, page_size);
// page_size boundary may occur in the middle of the EOF block, so we need to cache the blocks' ends
if ( nread<=0 ) break;
if ( nread<=neof ) // last block
{
if ( ncached )
{
// flush the part of the cache that won't be needed
nwr = bgzf_raw_write(bgzf_out, cached, nread);
if (nwr != nread) error("Write failed, wrote %d instead of %d bytes.\n", nwr,(int)nread);
// make space in the cache so that we can append to the end
if ( nread!=neof ) memmove(cached,cached+nread,neof-nread);
}
// fill the cache and check for eof outside this loop
memcpy(cached+neof-nread,buf,nread);
break;
}
// not the last block, flush the cache if full
if ( ncached )
{
nwr = bgzf_raw_write(bgzf_out, cached, ncached);
if (nwr != ncached) error("Write failed, wrote %d instead of %d bytes.\n", nwr,(int)ncached);
ncached = 0;
}
// fill the cache
nread -= neof;
memcpy(cached,buf+nread,neof);
ncached = neof;
nwr = bgzf_raw_write(bgzf_out, buf, nread);
if (nwr != nread) error("Write failed, wrote %d instead of %d bytes.\n", nwr,(int)nread);
}
if ( ncached && memcmp(cached,"\037\213\010\4\0\0\0\0\0\377\6\0\102\103\2\0\033\0\3\0\0\0\0\0\0\0\0\0",neof) )
{
nwr = bgzf_raw_write(bgzf_out, cached, neof);
if (nwr != neof) error("Write failed, wrote %d instead of %d bytes.\n", nwr,(int)neof);
}
if (hts_close(hts_fp)) error("Close failed: %s\n",args->fnames[i]);
}
free(buf);
free(tmp.s);
if (bgzf_close(bgzf_out) < 0) error("Error: %d\n",bgzf_out->errcode);
}
int vcf_index_stats(char *fname, int stats)
{
char *fn_out = NULL;
FILE *out;
out = fn_out ? fopen(fn_out, "w") : stdout;
const char **seq;
int i, nseq;
tbx_t *tbx = NULL;
hts_idx_t *idx = NULL;
htsFile *fp = hts_open(fname,"r");
if ( !fp ) { fprintf(stderr,"Could not read %s\n", fname); return 1; }
bcf_hdr_t *hdr = bcf_hdr_read(fp);
if ( !hdr ) { fprintf(stderr,"Could not read the header: %s\n", fname); return 1; }
if ( hts_get_format(fp)->format==vcf )
{
tbx = tbx_index_load(fname);
if ( !tbx ) { fprintf(stderr,"Could not load TBI index: %s\n", fname); return 1; }
}
else if ( hts_get_format(fp)->format==bcf )
{
idx = bcf_index_load(fname);
if ( !idx ) { fprintf(stderr,"Could not load CSI index: %s\n", fname); return 1; }
}
else
{
fprintf(stderr,"Could not detect the file type as VCF or BCF: %s\n", fname);
return 1;
}
seq = tbx ? tbx_seqnames(tbx, &nseq) : bcf_index_seqnames(idx, hdr, &nseq);
uint64_t sum = 0;
for (i=0; i<nseq; i++)
{
uint64_t records, v;
hts_idx_get_stat(tbx ? tbx->idx : idx, i, &records, &v);
sum+=records;
if (stats&2 || !records) continue;
bcf_hrec_t *hrec = bcf_hdr_get_hrec(hdr, BCF_HL_CTG, "ID", seq[i], NULL);
int hkey = hrec ? bcf_hrec_find_key(hrec, "length") : -1;
fprintf(out,"%s\t%s\t%" PRIu64 "\n", seq[i], hkey<0?".":hrec->vals[hkey], records);
}
if (!sum)
{
// No counts found.
// Is this because index version has no stored count data, or no records?
bcf1_t *rec = bcf_init1();
if (bcf_read1(fp, hdr, rec) >= 0)
{
fprintf(stderr,"%s index of %s does not contain any count metadata. Please re-index with a newer version of bcftools or tabix.\n", tbx ? "TBI" : "CSI", fname);
return 1;
}
bcf_destroy1(rec);
}
if (stats&2) fprintf(out, "%" PRIu64 "\n", sum);
free(seq);
fclose(out);
hts_close(fp);
bcf_hdr_destroy(hdr);
if (tbx)
tbx_destroy(tbx);
if (idx)
hts_idx_destroy(idx);
return 0;
}
int main_vcfindex(int argc, char *argv[])
{
int c, force = 0, tbi = 0, stats = 0;
int min_shift = BCF_LIDX_SHIFT;
static struct option loptions[] =
{
{"csi",no_argument,NULL,'c'},
{"tbi",no_argument,NULL,'t'},
{"force",no_argument,NULL,'f'},
{"min-shift",required_argument,NULL,'m'},
{"stats",no_argument,NULL,'s'},
{"nrecords",no_argument,NULL,'n'},
{NULL, 0, NULL, 0}
};
char *tmp;
while ((c = getopt_long(argc, argv, "ctfm:sn", loptions, NULL)) >= 0)
{
switch (c)
{
case 'c': tbi = 0; break;
case 't': tbi = 1; min_shift = 0; break;
case 'f': force = 1; break;
case 'm':
min_shift = strtol(optarg,&tmp,10);
if ( *tmp ) error("Could not parse argument: --min-shift %s\n", optarg);
break;
case 's': stats |= 1; break;
case 'n': stats |= 2; break;
default: usage();
}
}
if ( optind==argc ) usage();
if (stats>2)
{
fprintf(stderr, "[E::%s] expected only one of --stats or --nrecords options\n", __func__);
return 1;
}
if (tbi && min_shift>0)
{
fprintf(stderr, "[E::%s] min-shift option only expected for CSI indices \n", __func__);
return 1;
}
if (min_shift < 0 || min_shift > 30)
{
fprintf(stderr, "[E::%s] expected min_shift in range [0,30] (%d)\n", __func__, min_shift);
return 1;
}
char *fname = argv[optind];
if (stats) return vcf_index_stats(fname, stats);
htsFile *fp = hts_open(fname,"r");
if ( !fp ) error("Failed to read %s\n", fname);
htsFormat type = *hts_get_format(fp);
hts_close(fp);
if ( (type.format!=bcf && type.format!=vcf) || type.compression!=bgzf )
{
fprintf(stderr, "[E::%s] unknown filetype; expected bgzip compressed VCF or BCF\n", __func__);
if ( type.compression!=bgzf )
fprintf(stderr, "[E::%s] was the VCF/BCF compressed with bgzip?\n", __func__);
return 1;
}
if (tbi && type.format==bcf)
{
fprintf(stderr, "[Warning] TBI-index does not work for BCF files. Generating CSI instead.\n");
tbi = 0; min_shift = BCF_LIDX_SHIFT;
}
if (min_shift == 0 && type.format==bcf)
{
fprintf(stderr, "[E::%s] Require min_shift>0 for BCF files.\n", __func__);
return 1;
}
if (!tbi && type.format==vcf && min_shift == 0)
{
fprintf(stderr, "[Warning] min-shift set to 0 for VCF file. Generating TBI file.\n");
tbi = 1;
}
if (!force)
{
// Before complaining about existing index, check if the VCF file isn't newer.
char *idx_fname = (char*)alloca(strlen(fname) + 5);
strcat(strcpy(idx_fname, fname), tbi ? ".tbi" : ".csi");
struct stat stat_tbi, stat_file;
if ( stat(idx_fname, &stat_tbi)==0 )
{
stat(fname, &stat_file);
if ( stat_file.st_mtime <= stat_tbi.st_mtime )
{
fprintf(stderr,"[E::%s] the index file exists. Please use '-f' to overwrite.\n", __func__);
return 1;
}
}
}
if (type.format==bcf)
{
if ( bcf_index_build(fname, min_shift) != 0 )
{
fprintf(stderr,"[E::%s] bcf_index_build failed for %s\n", __func__, fname);
return 1;
}
}
else
{
if ( tbx_index_build(fname, min_shift, &tbx_conf_vcf) != 0 )
{
fprintf(stderr,"[E::%s] tbx_index_build failed for %s\n", __func__, fname);
return 1;
}
}
return 0;
}
static int query_regions(args_t *args, char *fname, char **regs, int nregs)
{
int i;
htsFile *fp = hts_open(fname,"r");
if ( !fp ) error("Could not read %s\n", fname);
enum htsExactFormat format = hts_get_format(fp)->format;
regidx_t *reg_idx = NULL;
if ( args->targets_fname )
{
reg_idx = regidx_init(args->targets_fname, NULL, NULL, 0, NULL);
if ( !reg_idx ) error("Could not read %s\n", args->targets_fname);
}
if ( format == bcf )
{
htsFile *out = hts_open("-","w");
if ( !out ) error("Could not open stdout\n", fname);
hts_idx_t *idx = bcf_index_load(fname);
if ( !idx ) error("Could not load .csi index of %s\n", fname);
bcf_hdr_t *hdr = bcf_hdr_read(fp);
if ( !hdr ) error("Could not read the header: %s\n", fname);
if ( args->print_header )
bcf_hdr_write(out,hdr);
if ( !args->header_only )
{
bcf1_t *rec = bcf_init();
for (i=0; i<nregs; i++)
{
hts_itr_t *itr = bcf_itr_querys(idx,hdr,regs[i]);
while ( bcf_itr_next(fp, itr, rec) >=0 )
{
if ( reg_idx && !regidx_overlap(reg_idx, bcf_seqname(hdr,rec),rec->pos,rec->pos+rec->rlen-1, NULL) ) continue;
bcf_write(out,hdr,rec);
}
tbx_itr_destroy(itr);
}
bcf_destroy(rec);
}
if ( hts_close(out) ) error("hts_close returned non-zero status for stdout\n");
bcf_hdr_destroy(hdr);
hts_idx_destroy(idx);
}
else if ( format==vcf || format==sam || format==unknown_format )
{
tbx_t *tbx = tbx_index_load(fname);
if ( !tbx ) error("Could not load .tbi/.csi index of %s\n", fname);
kstring_t str = {0,0,0};
if ( args->print_header )
{
while ( hts_getline(fp, KS_SEP_LINE, &str) >= 0 )
{
if ( !str.l || str.s[0]!=tbx->conf.meta_char ) break;
puts(str.s);
}
}
if ( !args->header_only )
{
int nseq;
const char **seq = NULL;
if ( reg_idx ) seq = tbx_seqnames(tbx, &nseq);
for (i=0; i<nregs; i++)
{
hts_itr_t *itr = tbx_itr_querys(tbx, regs[i]);
if ( !itr ) continue;
while (tbx_itr_next(fp, tbx, itr, &str) >= 0)
{
if ( reg_idx && !regidx_overlap(reg_idx,seq[itr->curr_tid],itr->curr_beg,itr->curr_end, NULL) ) continue;
puts(str.s);
}
tbx_itr_destroy(itr);
}
free(seq);
}
free(str.s);
tbx_destroy(tbx);
}
else if ( format==bam )
error("Please use \"samtools view\" for querying BAM files.\n");
if ( reg_idx ) regidx_destroy(reg_idx);
if ( hts_close(fp) ) error("hts_close returned non-zero status: %s\n", fname);
for (i=0; i<nregs; i++) free(regs[i]);
free(regs);
return 0;
}
int main(int argc, char **argv)
{
int i, n;
static struct option const long_opts[] =
{
{"out", required_argument, NULL, 1},
{"report", required_argument, NULL, 2},
{"dotasref", no_argument, NULL, 3},
{"help", no_argument, NULL, 0},
{"version", no_argument, NULL, 4},
{"export_uncov", no_argument, NULL, 5}
};
bool help = FALSE;
bool report_version = FALSE;
while ((n = getopt_long(argc, argv, "1:2:304", long_opts, NULL)) >= 0)
{
switch (n)
{
case 1 : outfile = strdup(optarg); break;
case 2 : report = strdup(optarg); break;
case 3 : dotasref = TRUE; break;
case 0 : help = TRUE; break;
case 4 : report_version = TRUE; break;
case 5 : export_uncover = TRUE; break;
default : return 1;
}
if ( help ) return usage();
if ( report_version ) return show_version();
}
n = argc - optind;
if ( n > 1 ) errabort("only accept one input vcf");
if ( export_uncover == TRUE && outfile == FALSE) {
warnings("export uncove region only used with option --out");
export_uncover = FALSE;
}
char * input;
if ( n == 0 ) input = strdup("-");
else input = strdup(argv[optind]);
htsFile * fp = read_vcf_file(input);
enum htsExactFormat fmt = hts_get_format(fp)->format;
if ( fmt != vcf && fmt != bcf ) errabort("This is not a VCF/BCF file : %s", input);
bcf_hdr_t * hdr = bcf_hdr_read(fp);
int n_samples = bcf_hdr_nsamples(hdr);
if ( n_samples != 2 ) errabort("the input VCF/BCF file must contain only two samples! %d", n_samples);
LOG("Using sample %s as ref ...", hdr->samples[0]);
LOG("Using sample %s as test ...", hdr->samples[1]);
uint32_t matrix[4][4] = { {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0} };
bcf1_t * v = bcf_init1();
kstring_t str = { 0, 0, 0 };
uint32_t line = 0;
htsFile *out = NULL;
if ( outfile && !check_filename(outfile) ) out = hts_open(outfile, mode);
if ( out != NULL ) bcf_hdr_write(out, hdr);
while ( bcf_read1(fp, hdr, v) >= 0 )
{
bcf_unpack(v, BCF_UN_STR|BCF_UN_FMT);
int k;
str.l = 0;
int tag_id = bcf_hdr_id2int(hdr, BCF_DT_ID, "GT");
if ( !bcf_hdr_idinfo_exists(hdr, BCF_HL_FMT, tag_id) ) warnings("There is no 'GT' in the header!");
for ( i = 0; i < v->n_fmt; ++i )
if ( v->d.fmt[i].id == tag_id ) break;
if ( i == v->n_fmt ) {
vcf_format1(hdr, v, &str);
LOG("There is no tag GT in this line : %s", str.s);
continue;
}
corr_t xy[2] = { {-1, -2, -2}, {-1, -2, -2} };
bcf_fmt_t * fmt = &v->d.fmt[i];
for ( i = 0; i < 2; ++i )
{
int corr = i;
if ( fmt == NULL ) {
if ( dotasref == TRUE ) xy[corr].alt = ALT_IS_REF;
else xy[corr].alt = ALT_IS_UNC;
continue;
}
int last = -2;
uint8_t *d = (uint8_t*)((char*)fmt->p + fmt->size*i);
for ( k = 0; k < fmt->n && d[k] != (uint8_t)bcf_int8_vector_end; ++k )
{
int curr = d[k]>>1;
if ( last != curr ) {
if ( curr ) {
if ( last == -2 ) xy[corr].alt = curr > 1 ? ALT_IS_HOM : ALT_IS_REF;
else xy[corr].alt = ALT_IS_HET;
} else {
xy[corr].alt = dotasref == TRUE ? ALT_IS_REF : ALT_IS_UNC;
}
} else {
if ( curr ) {
xy[corr].alt = curr > 1 ? ALT_IS_HOM : ALT_IS_REF;
} else {
xy[corr].alt = dotasref == TRUE ? ALT_IS_REF : ALT_IS_UNC;
}
}
if (last == -2 ) {
xy[corr].min = xy[corr].max = curr;
} else {
if ( curr < xy[corr].min ) xy[corr].min = curr;
else if ( curr > xy[corr].max ) xy[corr].max = curr;
}
last = curr;
}
}
matrix[xy[0].alt][xy[1].alt]++;
if ( xy[0].alt != xy[1].alt && out != NULL) {
if ( xy[0].alt == ALT_IS_UNC || xy[1].alt == ALT_IS_UNC ) {
if ( export_uncover == TRUE ) {
str.l = 0;
vcf_format1(hdr, v, &str);
vcf_write(out, hdr, v);
}
} else {
str.l = 0;
vcf_format1(hdr, v, &str);
vcf_write(out, hdr, v);
}
}
if ( xy[0].alt == ALT_IS_HET && xy[1].alt == ALT_IS_HET && (xy[0].min != xy[1].min || xy[0].max != xy[1].max ) ) {
bias++;
matrix[ALT_IS_HET][ALT_IS_HET]--;
if ( out != NULL ) {
str.l = 0;
vcf_format1(hdr, v, &str);
vcf_write(out, hdr, v);
}
}
line++;
}
if ( out ) hts_close(out);
if ( str.m ) free(str.s);
write_report(matrix, hdr);
bcf_hdr_destroy(hdr);
free(input);
bcf_destroy1(v);
if ( outfile ) free(outfile);
if ( report ) free(report);
if ( hts_close(fp) ) warnings("hts_close returned non-zero status: %s", input);
return 0;
}
/*!
@abstract Merge multiple sorted BAM.
@param is_by_qname whether to sort by query name
@param out output BAM file name
@param mode sam_open() mode to be used to create the final output file
(overrides level settings from UNCOMP and LEVEL1 flags)
@param headers name of SAM file from which to copy '@' header lines,
or NULL to copy them from the first file to be merged
@param n number of files to be merged
@param fn names of files to be merged
@param flag flags that control how the merge is undertaken
@param reg region to merge
@param n_threads number of threads to use (passed to htslib)
@discussion Padding information may NOT correctly maintained. This
function is NOT thread safe.
*/
int bam_merge_core2(int by_qname, const char *out, const char *mode, const char *headers, int n, char * const *fn, int flag, const char *reg, int n_threads)
{
samFile *fpout, **fp;
heap1_t *heap;
bam_hdr_t *hout = NULL;
int i, j, *RG_len = NULL;
uint64_t idx = 0;
char **RG = NULL;
hts_itr_t **iter = NULL;
bam_hdr_t **hdr = NULL;
trans_tbl_t *translation_tbl = NULL;
// Is there a specified pre-prepared header to use for output?
if (headers) {
samFile* fpheaders = sam_open(headers, "r");
if (fpheaders == NULL) {
const char *message = strerror(errno);
fprintf(pysamerr, "[bam_merge_core] cannot open '%s': %s\n", headers, message);
return -1;
}
hout = sam_hdr_read(fpheaders);
sam_close(fpheaders);
}
g_is_by_qname = by_qname;
fp = (samFile**)calloc(n, sizeof(samFile*));
heap = (heap1_t*)calloc(n, sizeof(heap1_t));
iter = (hts_itr_t**)calloc(n, sizeof(hts_itr_t*));
hdr = (bam_hdr_t**)calloc(n, sizeof(bam_hdr_t*));
translation_tbl = (trans_tbl_t*)calloc(n, sizeof(trans_tbl_t));
// prepare RG tag from file names
if (flag & MERGE_RG) {
RG = (char**)calloc(n, sizeof(char*));
RG_len = (int*)calloc(n, sizeof(int));
for (i = 0; i != n; ++i) {
int l = strlen(fn[i]);
const char *s = fn[i];
if (l > 4 && strcmp(s + l - 4, ".bam") == 0) l -= 4;
for (j = l - 1; j >= 0; --j) if (s[j] == '/') break;
++j; l -= j;
RG[i] = (char*)calloc(l + 1, 1);
RG_len[i] = l;
strncpy(RG[i], s + j, l);
}
}
// open and read the header from each file
for (i = 0; i < n; ++i) {
bam_hdr_t *hin;
fp[i] = sam_open(fn[i], "r");
if (fp[i] == NULL) {
int j;
fprintf(pysamerr, "[bam_merge_core] fail to open file %s\n", fn[i]);
for (j = 0; j < i; ++j) sam_close(fp[j]);
free(fp); free(heap);
// FIXME: possible memory leak
return -1;
}
hin = sam_hdr_read(fp[i]);
if (hout)
trans_tbl_init(hout, hin, translation_tbl+i, flag & MERGE_COMBINE_RG, flag & MERGE_COMBINE_PG);
else {
// As yet, no headers to merge into...
hout = bam_hdr_dup(hin);
// ...so no need to translate header into itself
trans_tbl_init(hout, hin, translation_tbl+i, true, true);
}
// TODO sam_itr_next() doesn't yet work for SAM files,
// so for those keep the headers around for use with sam_read1()
if (hts_get_format(fp[i])->format == sam) hdr[i] = hin;
else { bam_hdr_destroy(hin); hdr[i] = NULL; }
if ((translation_tbl+i)->lost_coord_sort && !by_qname) {
fprintf(pysamerr, "[bam_merge_core] Order of targets in file %s caused coordinate sort to be lost\n", fn[i]);
}
}
// Transform the header into standard form
pretty_header(&hout->text,hout->l_text);
// If we're only merging a specified region move our iters to start at that point
if (reg) {
int* rtrans = rtrans_build(n, hout->n_targets, translation_tbl);
int tid, beg, end;
const char *name_lim = hts_parse_reg(reg, &beg, &end);
char *name = malloc(name_lim - reg + 1);
memcpy(name, reg, name_lim - reg);
name[name_lim - reg] = '\0';
tid = bam_name2id(hout, name);
free(name);
if (tid < 0) {
fprintf(pysamerr, "[%s] Malformated region string or undefined reference name\n", __func__);
return -1;
}
for (i = 0; i < n; ++i) {
hts_idx_t *idx = sam_index_load(fp[i], fn[i]);
// (rtrans[i*n+tid]) Look up what hout tid translates to in input tid space
int mapped_tid = rtrans[i*hout->n_targets+tid];
if (mapped_tid != INT32_MIN) {
iter[i] = sam_itr_queryi(idx, mapped_tid, beg, end);
} else {
iter[i] = sam_itr_queryi(idx, HTS_IDX_NONE, 0, 0);
}
hts_idx_destroy(idx);
if (iter[i] == NULL) break;
}
free(rtrans);
} else {
for (i = 0; i < n; ++i) {
if (hdr[i] == NULL) {
iter[i] = sam_itr_queryi(NULL, HTS_IDX_REST, 0, 0);
if (iter[i] == NULL) break;
}
else iter[i] = NULL;
}
}
if (i < n) {
fprintf(pysamerr, "[%s] Memory allocation failed\n", __func__);
return -1;
}
// Load the first read from each file into the heap
for (i = 0; i < n; ++i) {
heap1_t *h = heap + i;
h->i = i;
h->b = bam_init1();
if ((iter[i]? sam_itr_next(fp[i], iter[i], h->b) : sam_read1(fp[i], hdr[i], h->b)) >= 0) {
bam_translate(h->b, translation_tbl + i);
h->pos = ((uint64_t)h->b->core.tid<<32) | (uint32_t)((int32_t)h->b->core.pos+1)<<1 | bam_is_rev(h->b);
h->idx = idx++;
}
else {
h->pos = HEAP_EMPTY;
bam_destroy1(h->b);
h->b = NULL;
}
}
// Open output file and write header
if ((fpout = sam_open(out, mode)) == 0) {
fprintf(pysamerr, "[%s] fail to create the output file.\n", __func__);
return -1;
}
sam_hdr_write(fpout, hout);
if (!(flag & MERGE_UNCOMP)) hts_set_threads(fpout, n_threads);
// Begin the actual merge
ks_heapmake(heap, n, heap);
while (heap->pos != HEAP_EMPTY) {
bam1_t *b = heap->b;
if (flag & MERGE_RG) {
uint8_t *rg = bam_aux_get(b, "RG");
if (rg) bam_aux_del(b, rg);
bam_aux_append(b, "RG", 'Z', RG_len[heap->i] + 1, (uint8_t*)RG[heap->i]);
}
sam_write1(fpout, hout, b);
if ((j = (iter[heap->i]? sam_itr_next(fp[heap->i], iter[heap->i], b) : sam_read1(fp[heap->i], hdr[heap->i], b))) >= 0) {
bam_translate(b, translation_tbl + heap->i);
heap->pos = ((uint64_t)b->core.tid<<32) | (uint32_t)((int)b->core.pos+1)<<1 | bam_is_rev(b);
heap->idx = idx++;
} else if (j == -1) {
heap->pos = HEAP_EMPTY;
bam_destroy1(heap->b);
heap->b = NULL;
} else fprintf(pysamerr, "[bam_merge_core] '%s' is truncated. Continue anyway.\n", fn[heap->i]);
ks_heapadjust(heap, 0, n, heap);
}
// Clean up and close
if (flag & MERGE_RG) {
for (i = 0; i != n; ++i) free(RG[i]);
free(RG); free(RG_len);
}
for (i = 0; i < n; ++i) {
trans_tbl_destroy(translation_tbl + i);
hts_itr_destroy(iter[i]);
bam_hdr_destroy(hdr[i]);
sam_close(fp[i]);
}
bam_hdr_destroy(hout);
sam_close(fpout);
free(translation_tbl); free(fp); free(heap); free(iter); free(hdr);
return 0;
}
int main_reheader(int argc, char *argv[])
{
int inplace = 0, r, add_PG = 1, c;
bam_hdr_t *h;
samFile *in;
char *arg_list = stringify_argv(argc+1, argv-1);
static const struct option lopts[] = {
{"help", no_argument, NULL, 'h'},
{"in-place", no_argument, NULL, 'i'},
{"no-PG", no_argument, NULL, 'P'},
{NULL, 0, NULL, 0}
};
while ((c = getopt_long(argc, argv, "hiP", lopts, NULL)) >= 0) {
switch (c) {
case 'P': add_PG = 0; break;
case 'i': inplace = 1; break;
case 'h': usage(stdout, 0); break;
default:
fprintf(stderr, "Invalid option '%c'\n", c);
usage(stderr, 1);
}
}
if (argc - optind != 2)
usage(stderr, 1);
{ // read the header
samFile *fph = sam_open(argv[optind], "r");
if (fph == 0) {
fprintf(stderr, "[%s] fail to read the header from %s.\n", __func__, argv[optind]);
return 1;
}
h = sam_hdr_read(fph);
sam_close(fph);
if (h == NULL) {
fprintf(stderr, "[%s] failed to read the header for '%s'.\n",
__func__, argv[1]);
return 1;
}
}
in = sam_open(argv[optind+1], inplace?"r+":"r");
if (in == 0) {
fprintf(stderr, "[%s] fail to open file %s.\n", __func__, argv[optind+1]);
return 1;
}
if (hts_get_format(in)->format == bam) {
r = bam_reheader(in->fp.bgzf, h, fileno(stdout), arg_list, add_PG);
} else {
if (inplace)
r = cram_reheader_inplace(in->fp.cram, h, arg_list, add_PG);
else
r = cram_reheader(in->fp.cram, h, arg_list, add_PG);
}
if (sam_close(in) != 0)
r = -1;
bam_hdr_destroy(h);
if (arg_list)
free(arg_list);
return -r;
}
int main_cat(int argc, char *argv[])
{
bam_hdr_t *h = 0;
char *outfn = 0;
int c, ret;
samFile *in;
while ((c = getopt(argc, argv, "h:o:")) >= 0) {
switch (c) {
case 'h': {
samFile *fph = sam_open(optarg, "r");
if (fph == 0) {
fprintf(stderr, "[%s] ERROR: fail to read the header from '%s'.\n", __func__, argv[1]);
return 1;
}
h = sam_hdr_read(fph);
if (h == NULL) {
fprintf(stderr,
"[%s] ERROR: failed to read the header for '%s'.\n",
__func__, argv[1]);
return 1;
}
sam_close(fph);
break;
}
case 'o': outfn = strdup(optarg); break;
}
}
if (argc - optind < 1) {
fprintf(stderr, "Usage: samtools cat [-h header.sam] [-o out.bam] <in1.bam> [...]\n");
return 1;
}
in = sam_open(argv[optind], "r");
if (!in) {
fprintf(stderr, "[%s] ERROR: failed to open file '%s'.\n", __func__, argv[optind]);
return 1;
}
switch (hts_get_format(in)->format) {
case bam:
sam_close(in);
ret = bam_cat(argc - optind, argv + optind, h, outfn? outfn : "-");
break;
case cram:
sam_close(in);
ret = cram_cat(argc - optind, argv + optind, h, outfn? outfn : "-");
break;
default:
sam_close(in);
fprintf(stderr, "[%s] ERROR: input is not BAM or CRAM\n", __func__);
return 1;
}
free(outfn);
if (h)
bam_hdr_destroy(h);
return ret;
}
int samthreads(samfile_t *fp, int n_threads, int n_sub_blks)
{
if (hts_get_format(fp->file)->format != bam || !fp->is_write) return -1;
bgzf_mt(fp->x.bam, n_threads, n_sub_blks);
return 0;
}
int main(int argc, char **argv)
{
if ( argc == 1 )
error("Usage : bed_annos -c config.json -O z -o output.vcf.gz input.vcf.gz");
int i;
for ( i = 1; i < argc; ) {
const char *a = argv[i++];
const char **var = 0;
if ( strcmp(a, "-c") == 0 )
var = &json_fname;
else if ( strcmp(a, "-O") == 0 )
var = &output_fname_type;
else if ( strcmp(a, "-o") == 0 )
var = &output_fname;
if ( var != 0 ) {
if ( i == argc )
error("Missing an argument after %s", a);
*var = argv[i++];
continue;
}
if ( input_fname == 0 ) {
input_fname = a;
continue;
}
error("Unknown argument : %s.", a);
}
struct vcfanno_config *con = vcfanno_config_init();
if ( vcfanno_load_config(con, json_fname) != 0 )
error("Failed to load configure file. %s : %s", json_fname, strerror(errno));
vcfanno_config_debug(con);
if ( con->beds.n_beds == 0)
error("No bed database specified.");
if ( input_fname == 0 && (!isatty(fileno(stdin))) )
input_fname = "-";
if ( input_fname == 0 )
error("No input file.");
int out_type = FT_VCF;
if ( output_fname_type != 0 ) {
switch (output_fname_type[0]) {
case 'b':
out_type = FT_BCF_GZ; break;
case 'u':
out_type = FT_BCF; break;
case 'z':
out_type = FT_VCF_GZ; break;
case 'v':
out_type = FT_VCF; break;
default :
error("The output type \"%d\" not recognised\n", out_type);
};
}
htsFile *fp = hts_open(input_fname, "r");
if ( fp == NULL )
error("Failed to open %s : %s.", input_fname, strerror(errno));
htsFormat type = *hts_get_format(fp);
if ( type.format != vcf && type.format != bcf )
error("Unsupported input format. %s", input_fname);
bcf_hdr_t *hdr = bcf_hdr_read(fp);
if ( hdr == NULL )
error("Failed to parse header.");
bcf_hdr_t *hdr_out = bcf_hdr_dup(hdr);
htsFile *fout = output_fname == 0 ? hts_open("-", hts_bcf_wmode(out_type)) : hts_open(output_fname, hts_bcf_wmode(out_type));
struct beds_options opts = { .beds_is_inited = 0,};
beds_options_init(&opts);
opts.hdr_out = hdr_out;
for ( i = 0; i < con->beds.n_beds; ++i ) {
beds_database_add(&opts, con->beds.files[i].fname, con->beds.files[i].columns);
}
bcf_hdr_write(fout, hdr_out);
bcf1_t *line = bcf_init();
while ( bcf_read(fp, hdr, line) == 0 ) {
anno_beds_core(&opts, line);
bcf_write(fout, hdr_out, line);
}
bcf_destroy(line);
bcf_hdr_destroy(hdr);
bcf_hdr_destroy(hdr_out);
beds_options_destroy(&opts);
hts_close(fp);
hts_close(fout);
return 0;
}
int main_quickcheck(int argc, char** argv)
{
int verbose = 0;
hts_verbose = 0;
const char* optstring = "v";
int opt;
while ((opt = getopt(argc, argv, optstring)) != -1) {
switch (opt) {
case 'v':
verbose++;
break;
default:
usage_quickcheck(pysamerr);
return 1;
}
}
argc -= optind;
argv += optind;
if (argc < 1) {
usage_quickcheck(stdout);
return 1;
}
if (verbose >= 2) {
fprintf(pysamerr, "verbosity set to %d\n", verbose);
}
if (verbose >= 4) {
hts_verbose = 3;
}
int ret = 0;
int i;
for (i = 0; i < argc; i++) {
char* fn = argv[i];
int file_state = 0;
if (verbose >= 3) fprintf(pysamerr, "checking %s\n", fn);
// attempt to open
htsFile *hts_fp = hts_open(fn, "r");
if (hts_fp == NULL) {
if (verbose >= 2) fprintf(pysamerr, "%s could not be opened for reading\n", fn);
file_state |= 2;
}
else {
if (verbose >= 3) fprintf(pysamerr, "opened %s\n", fn);
// make sure we have sequence data
const htsFormat *fmt = hts_get_format(hts_fp);
if (fmt->category != sequence_data ) {
if (verbose >= 2) fprintf(pysamerr, "%s was not identified as sequence data\n", fn);
file_state |= 4;
}
else {
if (verbose >= 3) fprintf(pysamerr, "%s is sequence data\n", fn);
// check header
bam_hdr_t *header = sam_hdr_read(hts_fp);
if (header->n_targets <= 0) {
if (verbose >= 2) fprintf(pysamerr, "%s had no targets in header\n", fn);
file_state |= 8;
}
else {
if (verbose >= 3) fprintf(pysamerr, "%s has %d targets in header\n", fn, header->n_targets);
}
// only check EOF on BAM for now
// TODO implement and use hts_check_EOF() to include CRAM support
if (fmt->format == bam) {
if (bgzf_check_EOF(hts_fp->fp.bgzf) <= 0) {
if (verbose >= 2) fprintf(pysamerr, "%s was missing EOF block\n", fn);
file_state |= 16;
}
else {
if (verbose >= 3) fprintf(pysamerr, "%s has good EOF block\n", fn);
}
}
}
hts_close(hts_fp);
}
if (file_state > 0 && verbose >= 1) {
fprintf(stdout, "%s\n", fn);
}
ret |= file_state;
}
return ret;
}
int convert(int argc, char **argv)
{
if (argc < 2) return convert_help();
int c;
char *in=NULL, *out=NULL, *bim=NULL, *vid=NULL, *tmp_dir=NULL, *ped=NULL;
uint32_t num_fields, num_records, col = 2;
int i_is_set = 0,
o_is_set = 0,
f_is_set = 0,
b_is_set = 0,
v_is_set = 0,
t_is_set = 0,
p_is_set = 0,
r_is_set = 0;
while((c = getopt (argc, argv, "hi:o:f:r:b:v:t:p:c:")) != -1) {
switch (c) {
case 'c':
col = atoi(optarg);
break;
case 'p':
p_is_set = 1;
ped = optarg;
break;
case 't':
t_is_set = 1;
tmp_dir = optarg;
break;
case 'v':
v_is_set = 1;
vid = optarg;
break;
case 'b':
b_is_set = 1;
bim = optarg;
break;
case 'i':
i_is_set = 1;
in = optarg;
break;
case 'o':
o_is_set = 1;
out = optarg;
break;
case 'f':
f_is_set = 1;
num_fields = atoi(optarg);
break;
case 'r':
r_is_set = 1;
num_records = atoi(optarg);
break;
case 'h':
convert_help();
return 1;
case '?':
if ( (optopt == 'i') ||
(optopt == 'f') ||
(optopt == 'r') ||
(optopt == 't') ||
(optopt == 's') ||
(optopt == 'p') ||
(optopt == 'c') ||
(optopt == 'o') )
fprintf (stderr, "Option -%c requires an argument.\n",
optopt);
else if (isprint (optopt))
fprintf (stderr, "Unknown option `-%c'.\n", optopt);
else
fprintf (stderr, "Unknown option character `\\x%x'.\n", optopt);
default:
convert_help();
return 1;
}
}
char *type = argv[0];
if (i_is_set == 0) {
printf("Input file is not set\n");
return convert_help();
}
if (strcmp(type, "bcf") == 0) {
if ( (f_is_set == 0) || (r_is_set == 0) ) {
fprintf(stderr,"Attempting to autodetect num of records "
"and fields from %s\n", in);
//Try and auto detect the sizes, need the index
tbx_t *tbx = NULL;
hts_idx_t *idx = NULL;
htsFile *fp = hts_open(in,"rb");
if ( !fp ) {
fprintf(stderr,"Could not read %s\n", in);
return 1;
}
bcf_hdr_t *hdr = bcf_hdr_read(fp);
if ( !hdr ) {
fprintf(stderr,"Could not read the header: %s\n", in);
return 1;
}
if (hts_get_format(fp)->format==vcf) {
tbx = tbx_index_load(in);
if ( !tbx ) {
fprintf(stderr,"Could not load TBI index: %s\n", in);
return 1;
}
} else if ( hts_get_format(fp)->format==bcf ) {
idx = bcf_index_load(in);
if ( !idx ) {
fprintf(stderr,"Could not load CSI index: %s\n", in);
return 1;
}
} else {
fprintf(stderr,
"Could not detect the file type as VCF or BCF: %s\n",
in);
return 1;
}
num_fields = hdr->n[BCF_DT_SAMPLE];
num_records = 0;
const char **seq;
int nseq;
seq = tbx ? tbx_seqnames(tbx, &nseq) :
bcf_index_seqnames(idx, hdr, &nseq);
int i;
uint32_t sum = 0;
for (i = 0; i < nseq; ++i) {
uint64_t records, v;
hts_idx_get_stat(tbx ? tbx->idx: idx, i, &records, &v);
num_records += records;
}
fprintf(stderr, "Number of records:%u\tNumber of fields:%u\n",
num_records, num_fields);
free(seq);
hts_close(fp);
bcf_hdr_destroy(hdr);
if (idx)
hts_idx_destroy(idx);
if (tbx)
tbx_destroy(tbx);
}
if (o_is_set == 0) {
out = (char*)malloc(strlen(in) + 5); // 5 for ext and \0
strcpy(out,in);
strcat(out, ".gqt");
}
if (b_is_set == 0) {
bim = (char*)malloc(strlen(in) + 5); // 5 for ext and \0
strcpy(bim,in);
strcat(bim, ".bim");
}
if (v_is_set == 0) {
vid = (char*)malloc(strlen(in) + 5); // 5 for ext and \0
strcpy(vid,in);
strcat(vid, ".vid");
}
if (t_is_set == 0) {
tmp_dir = (char*)malloc(3*sizeof(char)); // "./\0"
strcpy(tmp_dir,"./");
}
int r = bcf_wahbm(in, out, bim, vid, tmp_dir, num_fields, num_records);
return r;
}
if (strcmp(type, "ped") == 0) {
if (o_is_set == 0) {
if (p_is_set == 1) {
out = (char*)malloc(strlen(ped) + 4); // 4 for ext and \0
strcpy(out,ped);
strcat(out, ".db");
} else {
out = (char*)malloc(strlen(in) + 4); // 4 for ext and \0
strcpy(out,in);
strcat(out, ".db");
}
}
fprintf(stderr, "Creating sample database %s\n", out);
return ped_ped(in, ped, col, out);
}
return convert_help();
}
