int bcf_hdr_set(bcf_hdr_t *hdr, const char *fname)
{
int i, n;
char **lines = hts_readlines(fname, &n);
if ( !lines ) return 1;
for (i=0; i<n-1; i++)
{
int k;
bcf_hrec_t *hrec = bcf_hdr_parse_line(hdr,lines[i],&k);
bcf_hdr_add_hrec(hdr, hrec);
free(lines[i]);
}
bcf_hdr_parse_sample_line(hdr,lines[n-1]);
free(lines[n-1]);
free(lines);
bcf_hdr_sync(hdr);
bcf_hdr_fmt_text(hdr);
return 0;
}
static void reheader_vcf(args_t *args)
{
kstring_t hdr = {0,0,0};
htsFile *fp = hts_open(args->fname, "r"); if ( !fp ) error("Failed to open: %s\n", args->fname);
while ( hts_getline(fp, KS_SEP_LINE, &fp->line) >=0 )
{
kputc('\n',&fp->line); // hts_getline eats the newline character
if ( fp->line.s[0]!='#' ) break;
kputsn(fp->line.s,fp->line.l,&hdr);
}
int nsamples = 0;
char **samples = NULL;
if ( args->samples_fname )
samples = hts_readlines(args->samples_fname, &nsamples);
if ( args->header_fname )
{
free(hdr.s); hdr.s = NULL; hdr.l = hdr.m = 0;
read_header_file(args->header_fname, &hdr);
}
if ( samples )
{
set_samples(samples, nsamples, &hdr);
int i;
for (i=0; i<nsamples; i++) free(samples[i]);
free(samples);
}
int out = STDOUT_FILENO;
if ( write(out, hdr.s, hdr.l)!=hdr.l ) error("Failed to write %d bytes\n", hdr.l);
free(hdr.s);
if ( fp->line.l )
{
if ( write(out, fp->line.s, fp->line.l)!=fp->line.l ) error("Failed to write %d bytes\n", fp->line.l);
}
while ( hts_getline(fp, KS_SEP_LINE, &fp->line) >=0 ) // uncompressed file implies small size, we don't worry about speed
{
kputc('\n',&fp->line);
if ( write(out, fp->line.s, fp->line.l)!=fp->line.l ) error("Failed to write %d bytes\n", fp->line.l);
}
hts_close(fp);
}
int main_vcfconcat(int argc, char *argv[])
{
int c;
args_t *args = (args_t*) calloc(1,sizeof(args_t));
args->argc = argc; args->argv = argv;
args->output_fname = "-";
args->output_type = FT_VCF;
args->n_threads = 0;
args->record_cmd_line = 1;
args->min_PQ = 30;
static struct option loptions[] =
{
{"naive",no_argument,NULL,'n'},
{"compact-PS",no_argument,NULL,'c'},
{"regions",required_argument,NULL,'r'},
{"regions-file",required_argument,NULL,'R'},
{"remove-duplicates",no_argument,NULL,'D'},
{"rm-dups",required_argument,NULL,'d'},
{"allow-overlaps",no_argument,NULL,'a'},
{"ligate",no_argument,NULL,'l'},
{"output",required_argument,NULL,'o'},
{"output-type",required_argument,NULL,'O'},
{"threads",required_argument,NULL,9},
{"file-list",required_argument,NULL,'f'},
{"min-PQ",required_argument,NULL,'q'},
{"no-version",no_argument,NULL,8},
{NULL,0,NULL,0}
};
char *tmp;
while ((c = getopt_long(argc, argv, "h:?o:O:f:alq:Dd:r:R:cn",loptions,NULL)) >= 0)
{
switch (c) {
case 'c': args->compact_PS = 1; break;
case 'r': args->regions_list = optarg; break;
case 'R': args->regions_list = optarg; args->regions_is_file = 1; break;
case 'd': args->remove_dups = optarg; break;
case 'D': args->remove_dups = "none"; break;
case 'q':
args->min_PQ = strtol(optarg,&tmp,10);
if ( *tmp ) error("Could not parse argument: --min-PQ %s\n", optarg);
break;
case 'n': args->naive_concat = 1; break;
case 'a': args->allow_overlaps = 1; break;
case 'l': args->phased_concat = 1; break;
case 'f': args->file_list = optarg; break;
case 'o': args->output_fname = optarg; break;
case 'O':
switch (optarg[0]) {
case 'b': args->output_type = FT_BCF_GZ; break;
case 'u': args->output_type = FT_BCF; break;
case 'z': args->output_type = FT_VCF_GZ; break;
case 'v': args->output_type = FT_VCF; break;
default: error("The output type \"%s\" not recognised\n", optarg);
};
break;
case 9 : args->n_threads = strtol(optarg, 0, 0); break;
case 8 : args->record_cmd_line = 0; break;
case 'h':
case '?': usage(args); break;
default: error("Unknown argument: %s\n", optarg);
}
}
while ( optind<argc )
{
args->nfnames++;
args->fnames = (char **)realloc(args->fnames,sizeof(char*)*args->nfnames);
args->fnames[args->nfnames-1] = strdup(argv[optind]);
optind++;
}
if ( args->allow_overlaps && args->phased_concat ) args->allow_overlaps = 0;
if ( args->compact_PS && !args->phased_concat ) error("The -c option is intended only with -l\n");
if ( args->file_list )
{
if ( args->nfnames ) error("Cannot combine -l with file names on command line.\n");
args->fnames = hts_readlines(args->file_list, &args->nfnames);
if ( !args->fnames ) error("Could not read the file: %s\n", args->file_list);
}
if ( !args->nfnames ) usage(args);
if ( args->remove_dups && !args->allow_overlaps ) error("The -D option is supported only with -a\n");
if ( args->regions_list && !args->allow_overlaps ) error("The -r/-R option is supported only with -a\n");
if ( args->naive_concat )
{
if ( args->allow_overlaps ) error("The option --naive cannot be combined with --allow-overlaps\n");
if ( args->phased_concat ) error("The option --naive cannot be combined with --ligate\n");
naive_concat(args);
destroy_data(args);
free(args);
return 0;
}
init_data(args);
concat(args);
destroy_data(args);
free(args);
return 0;
}
int main_vcfview(int argc, char *argv[])
{
int i, c, clevel = -1, flag = 0, n_samples = -1, *imap = 0, excl_snp = 0, excl_indel = 0;
char *fn_ref = 0, *fn_out = 0, moder[8], **samples = 0;
bcf_hdr_t *h, *hsub = 0;
htsFile *in;
bcf1_t *b;
while ((c = getopt(argc, argv, "l:bSt:o:T:s:GNI")) >= 0) {
switch (c) {
case 'l': clevel = atoi(optarg); flag |= 2; break;
case 'S': flag |= 1; break;
case 'b': flag |= 2; break;
case 'G': n_samples = 0; break;
case 't': fn_ref = optarg; flag |= 1; break;
case 'o': fn_out = optarg; break;
case 's': samples = hts_readlines(optarg, &n_samples); break;
case 'N': excl_snp = 1; break;
case 'I': excl_indel = 1; break;
}
}
if (argc == optind) {
fprintf(stderr, "\nUsage: vcfview [options] <in.bcf>|<in.vcf>|<in.vcf.gz>\n\n");
fprintf(stderr, "Options: -b output in BCF\n");
fprintf(stderr, " -S input is VCF\n");
fprintf(stderr, " -o FILE output file name [stdout]\n");
fprintf(stderr, " -l INT compression level [%d]\n", clevel);
fprintf(stderr, " -t FILE list of reference names and lengths [null]\n");
fprintf(stderr, " -s FILE/STR list of samples (STR if started with ':'; FILE otherwise) [null]\n");
fprintf(stderr, " -G drop individual genotype information\n");
fprintf(stderr, " -N exclude SNPs\n");
fprintf(stderr, " -I exclude INDELs\n");
fprintf(stderr, "\n");
return 1;
}
strcpy(moder, "r");
if ((flag&1) == 0 && !(file_type(argv[optind])&(IS_VCF|IS_VCF_GZ))) strcat(moder, "b");
in = hts_open(argv[optind], moder, fn_ref);
h = vcf_hdr_read(in);
if (h == 0) {
fprintf(stderr, "[E::%s] fail to read the VCF/BCF2 header\n", __func__);
hts_close(in);
return 1;
}
if (n_samples >= 0) {
if (n_samples) imap = (int*)malloc(n_samples * sizeof(int));
hsub = bcf_hdr_subset(h, n_samples, samples, imap);
}
b = bcf_init1();
if ((flag&4) == 0) { // VCF/BCF output
htsFile *out;
char modew[8];
strcpy(modew, "w");
if (clevel >= 0 && clevel <= 9) sprintf(modew + 1, "%d", clevel);
if (flag&2) strcat(modew, "b");
out = hts_open(fn_out? fn_out : "-", modew, 0);
vcf_hdr_write(out, hsub? hsub : h);
if (optind + 1 < argc && !(flag&1)) { // BAM input and has a region
hts_idx_t *idx;
if ((idx = bcf_index_load(argv[optind])) == 0) {
fprintf(stderr, "[E::%s] fail to load the BCF index\n", __func__);
return 1;
}
for (i = optind + 1; i < argc; ++i) {
hts_itr_t *iter;
if ((iter = bcf_itr_querys(idx, h, argv[i])) == 0) {
fprintf(stderr, "[E::%s] fail to parse region '%s'\n", __func__, argv[i]);
continue;
}
while (bcf_itr_next((BGZF*)in->fp, iter, b) >= 0) {
if (excl_snp && bcf_is_snp(b)) continue;
if (excl_indel && !bcf_is_snp(b)) continue;
if (n_samples >= 0) {
bcf_subset(h, b, n_samples, imap);
vcf_write1(out, hsub, b);
} else vcf_write1(out, h, b);
}
hts_itr_destroy(iter);
}
hts_idx_destroy(idx);
} else {
while (vcf_read1(in, h, b) >= 0) {
if (excl_snp && bcf_is_snp(b)) continue;
if (excl_indel && !bcf_is_snp(b)) continue;
if (n_samples >= 0) {
bcf_subset(h, b, n_samples, imap);
vcf_write1(out, hsub, b);
} else vcf_write1(out, h, b);
}
}
hts_close(out);
}
bcf_destroy1(b);
if (n_samples > 0) {
for (i = 0; i < n_samples; ++i) free(samples[i]);
free(samples);
bcf_hdr_destroy(hsub);
free(imap);
}
bcf_hdr_destroy(h);
hts_close(in);
return 0;
}
static void reheader_bcf(args_t *args, int is_compressed)
{
htsFile *fp = hts_open(args->fname, "r"); if ( !fp ) error("Failed to open: %s\n", args->fname);
bcf_hdr_t *hdr = bcf_hdr_read(fp); if ( !hdr ) error("Failed to read the header: %s\n", args->fname);
kstring_t htxt = {0,0,0};
int hlen;
htxt.s = bcf_hdr_fmt_text(hdr, 1, &hlen);
htxt.l = hlen;
int i, nsamples = 0;
char **samples = NULL;
if ( args->samples_fname )
samples = hts_readlines(args->samples_fname, &nsamples);
if ( args->header_fname )
{
free(htxt.s); htxt.s = NULL; htxt.l = htxt.m = 0;
read_header_file(args->header_fname, &htxt);
}
if ( samples )
{
set_samples(samples, nsamples, &htxt);
for (i=0; i<nsamples; i++) free(samples[i]);
free(samples);
}
bcf_hdr_t *hdr_out = bcf_hdr_init("r");
bcf_hdr_parse(hdr_out, htxt.s);
if ( args->header_fname ) hdr_out = strip_header(hdr, hdr_out);
// write the header and the body
htsFile *fp_out = hts_open("-",is_compressed ? "wb" : "wbu");
bcf_hdr_write(fp_out, hdr_out);
bcf1_t *rec = bcf_init();
while ( bcf_read(fp, hdr, rec)==0 )
{
// sanity checking, this slows things down. Make it optional?
bcf_unpack(rec, BCF_UN_ALL);
if ( rec->rid >= hdr_out->n[BCF_DT_CTG] || strcmp(bcf_hdr_int2id(hdr,BCF_DT_CTG,rec->rid),bcf_hdr_int2id(hdr_out,BCF_DT_CTG,rec->rid)) )
error("The CHROM is not defined: \"%s\"\n", bcf_hdr_int2id(hdr,BCF_DT_CTG,rec->rid));
for (i=0; i<rec->d.n_flt; i++)
{
int id = rec->d.flt[i];
if ( id >= hdr_out->n[BCF_DT_ID] ) break;
if ( !bcf_hdr_idinfo_exists(hdr_out,BCF_HL_FLT,id) ) break;
if ( strcmp(hdr->id[BCF_DT_ID][id].key,hdr_out->id[BCF_DT_ID][id].key) )
error("FIXME: Broken FILTER ids: %s vs %s\n", hdr->id[BCF_DT_ID][id].key,hdr_out->id[BCF_DT_ID][id].key);
}
if ( i!=rec->d.n_flt )
error("The FILTER is not defined: \"%s\"\n", bcf_hdr_int2id(hdr,BCF_DT_ID,rec->d.flt[i]));
for (i=0; i<rec->n_info; i++)
{
int id = rec->d.info[i].key;
if ( id >= hdr_out->n[BCF_DT_ID] ) break;
if ( !hdr_out->id[BCF_DT_ID][id].key ) break;
if ( !bcf_hdr_idinfo_exists(hdr_out,BCF_HL_INFO,id) ) break;
if ( strcmp(hdr->id[BCF_DT_ID][id].key,hdr_out->id[BCF_DT_ID][id].key) )
error("FIXME: Broken INFO ids: %s vs %s\n", hdr->id[BCF_DT_ID][id].key,hdr_out->id[BCF_DT_ID][id].key);
}
if ( i!=rec->n_info )
error("The INFO tag is not defined: \"%s\"\n", bcf_hdr_int2id(hdr,BCF_DT_ID,rec->d.info[i].key));
for (i=0; i<rec->n_fmt; i++)
{
int id = rec->d.fmt[i].id;
if ( id >= hdr_out->n[BCF_DT_ID] ) break;
if ( !hdr_out->id[BCF_DT_ID][id].key ) break;
if ( !bcf_hdr_idinfo_exists(hdr_out,BCF_HL_FMT,id) ) break;
if ( strcmp(hdr->id[BCF_DT_ID][id].key,hdr_out->id[BCF_DT_ID][id].key) )
error("FIXME: Broken FORMAT ids: %s vs %s\n", hdr->id[BCF_DT_ID][id].key,hdr_out->id[BCF_DT_ID][id].key);
}
if ( i!=rec->n_fmt )
error("The FORMAT tag is not defined: \"%s\"\n", bcf_hdr_int2id(hdr,BCF_DT_ID,rec->d.fmt[i].id));
bcf_write(fp_out,hdr_out,rec);
}
bcf_destroy(rec);
free(htxt.s);
hts_close(fp_out);
hts_close(fp);
bcf_hdr_destroy(hdr_out);
bcf_hdr_destroy(hdr);
}
static void reheader_vcf_gz(args_t *args)
{
BGZF *fp = bgzf_open(args->fname,"r");
if ( !fp || bgzf_read_block(fp) != 0 || !fp->block_length )
error("Failed to read %s: %s\n", args->fname, strerror(errno));
kstring_t hdr = {0,0,0};
char *buffer = (char*) fp->uncompressed_block;
// Read the header and find the position of the data block
if ( buffer[0]!='#' ) error("Could not parse the header, expected '#', found '%c'\n", buffer[0]);
int skip_until = 1; // end of the header in the current uncompressed block
while (1)
{
if ( buffer[skip_until]=='\n' )
{
skip_until++;
if ( skip_until>=fp->block_length )
{
kputsn(buffer,skip_until,&hdr);
if ( bgzf_read_block(fp) != 0 || !fp->block_length ) error("FIXME: No body in the file: %s\n", args->fname);
skip_until = 0;
}
// The header has finished
if ( buffer[skip_until]!='#' )
{
kputsn(buffer,skip_until,&hdr);
break;
}
}
skip_until++;
if ( skip_until>=fp->block_length )
{
kputsn(buffer,fp->block_length,&hdr);
if (bgzf_read_block(fp) != 0 || !fp->block_length) error("FIXME: No body in the file: %s\n", args->fname);
skip_until = 0;
}
}
int nsamples = 0;
char **samples = NULL;
if ( args->samples_fname )
samples = hts_readlines(args->samples_fname, &nsamples);
if ( args->header_fname )
{
free(hdr.s); hdr.s = NULL; hdr.l = hdr.m = 0;
read_header_file(args->header_fname, &hdr);
}
if ( samples )
{
set_samples(samples, nsamples, &hdr);
int i;
for (i=0; i<nsamples; i++) free(samples[i]);
free(samples);
}
// Output the modified header
BGZF *bgzf_out = bgzf_dopen(fileno(stdout), "w");
bgzf_write(bgzf_out, hdr.s, hdr.l);
free(hdr.s);
// Output all remainig data read with the header block
if ( fp->block_length - skip_until > 0 )
{
if ( bgzf_write(bgzf_out, buffer+skip_until, fp->block_length-skip_until)<0 ) error("Error: %d\n",fp->errcode);
}
if ( bgzf_flush(bgzf_out)<0 ) error("Error: %d\n",bgzf_out->errcode);
// Stream the rest of the file without as it is, without decompressing
ssize_t nread;
int page_size = getpagesize();
char *buf = (char*) valloc(page_size);
while (1)
{
nread = bgzf_raw_read(fp, buf, page_size);
if ( nread<=0 ) break;
int count = bgzf_raw_write(bgzf_out, buf, nread);
if (count != nread) error("Write failed, wrote %d instead of %d bytes.\n", count,(int)nread);
}
if (bgzf_close(bgzf_out) < 0) error("Error: %d\n",bgzf_out->errcode);
if (bgzf_close(fp) < 0) error("Error: %d\n",fp->errcode);
free(buf);
}
int main_vcfconcat(int argc, char *argv[])
{
int c;
args_t *args = (args_t*) calloc(1,sizeof(args_t));
args->argc = argc;
args->argv = argv;
args->output_fname = "-";
args->output_type = FT_VCF;
args->min_PQ = 30;
static struct option loptions[] =
{
{"allow-overlaps",0,0,'a'},
{"ligate",0,0,'l'},
{"output",1,0,'o'},
{"output-type",1,0,'O'},
{"file-list",1,0,'f'},
{"min-PQ",1,0,'q'},
{0,0,0,0}
};
while ((c = getopt_long(argc, argv, "h:?o:O:f:alq:",loptions,NULL)) >= 0)
{
switch (c) {
case 'q':
args->min_PQ = atoi(optarg);
break;
case 'a':
args->allow_overlaps = 1;
break;
case 'l':
args->phased_concat = 1;
break;
case 'f':
args->file_list = optarg;
break;
case 'o':
args->output_fname = optarg;
break;
case 'O':
switch (optarg[0]) {
case 'b':
args->output_type = FT_BCF_GZ;
break;
case 'u':
args->output_type = FT_BCF;
break;
case 'z':
args->output_type = FT_VCF_GZ;
break;
case 'v':
args->output_type = FT_VCF;
break;
default:
error("The output type \"%s\" not recognised\n", optarg);
};
break;
case 'h':
case '?':
usage(args);
break;
default:
error("Unknown argument: %s\n", optarg);
}
}
while ( optind<argc )
{
args->nfnames++;
args->fnames = (char **)realloc(args->fnames,sizeof(char*)*args->nfnames);
args->fnames[args->nfnames-1] = strdup(argv[optind]);
optind++;
}
if ( args->allow_overlaps && args->phased_concat ) args->allow_overlaps = 0;
if ( args->file_list )
{
if ( args->nfnames ) error("Cannot combine -l with file names on command line.\n");
args->fnames = hts_readlines(args->file_list, &args->nfnames);
}
if ( !args->nfnames ) usage(args);
init_data(args);
concat(args);
destroy_data(args);
free(args);
return 0;
}
int bam_merge(int argc, char *argv[])
{
int c, is_by_qname = 0, flag = 0, ret = 0, n_threads = 0, level = -1;
char *fn_headers = NULL, *reg = NULL, mode[12];
long random_seed = (long)time(NULL);
char** fn = NULL;
int fn_size = 0;
if (argc == 1) {
merge_usage(stdout);
return 0;
}
while ((c = getopt(argc, argv, "h:nru1R:f@:l:cps:b:")) >= 0) {
switch (c) {
case 'r': flag |= MERGE_RG; break;
case 'f': flag |= MERGE_FORCE; break;
case 'h': fn_headers = strdup(optarg); break;
case 'n': is_by_qname = 1; break;
case '1': flag |= MERGE_LEVEL1; level = 1; break;
case 'u': flag |= MERGE_UNCOMP; level = 0; break;
case 'R': reg = strdup(optarg); break;
case 'l': level = atoi(optarg); break;
case '@': n_threads = atoi(optarg); break;
case 'c': flag |= MERGE_COMBINE_RG; break;
case 'p': flag |= MERGE_COMBINE_PG; break;
case 's': random_seed = atol(optarg); break;
case 'b': {
// load the list of files to read
int nfiles;
char **fn_read = hts_readlines(optarg, &nfiles);
if (fn_read) {
// Append to end of array
fn = realloc(fn, (fn_size+nfiles) * sizeof(char*));
if (fn == NULL) { ret = 1; goto end; }
memcpy(fn+fn_size, fn_read, nfiles * sizeof(char*));
fn_size += nfiles;
}
else {
fprintf(pysamerr, "[%s] Invalid file list \"%s\"\n", __func__, optarg);
ret = 1;
}
break;
}
}
}
if ( argc - optind < 1 ) {
fprintf(pysamerr, "You must at least specify the output file.\n");
merge_usage(pysamerr);
return 1;
}
srand48(random_seed);
if (!(flag & MERGE_FORCE) && strcmp(argv[optind], "-")) {
FILE *fp = fopen(argv[optind], "rb");
if (fp != NULL) {
fclose(fp);
fprintf(pysamerr, "[%s] File '%s' exists. Please apply '-f' to overwrite. Abort.\n", __func__, argv[optind]);
return 1;
}
}
int nargcfiles = argc - (optind+1);
if (nargcfiles > 0) {
// Add argc files to end of array
fn = realloc(fn, (fn_size+nargcfiles) * sizeof(char*));
if (fn == NULL) { ret = 1; goto end; }
memcpy(fn+fn_size, argv + (optind+1), nargcfiles * sizeof(char*));
}
if (fn_size+nargcfiles < 2) {
fprintf(pysamerr, "You must specify at least 2 input files.\n");
merge_usage(pysamerr);
return 1;
}
strcpy(mode, "wb");
if (level >= 0) sprintf(strchr(mode, '\0'), "%d", level < 9? level : 9);
if (bam_merge_core2(is_by_qname, argv[optind], mode, fn_headers, fn_size+nargcfiles, fn, flag, reg, n_threads) < 0) ret = 1;
end:
if (fn_size > 0) {
int i;
for (i=0; i<fn_size; i++) free(fn[i]);
free(fn);
}
free(reg);
free(fn_headers);
return ret;
}