static void write_buffer(const char *fn, const char *mode, size_t l, bam1_p *buf, const bam_header_t *h, int n_threads)
{
size_t i;
bamFile fp;
fp = strcmp(fn, "-")? bam_open(fn, mode) : bam_dopen(fileno(stdout), mode);
if (fp == 0) return;
bam_header_write(fp, h);
if (n_threads > 1) bgzf_mt(fp, n_threads, 256);
for (i = 0; i < l; ++i)
bam_write1_core(fp, &buf[i]->core, buf[i]->data_len, buf[i]->data);
bam_close(fp);
}
int main(int argc, char *argv[]) {
if (argc <= 1) {
fprintf(stderr, "Usage: thrash_threads1 input.bam\n");
exit(1);
}
int i;
for (i = 0; i < 10000; i++) {
printf("i=%d\n", i);
BGZF *fpin = bgzf_open(argv[1], "r");
bgzf_mt(fpin, 2, 256);
if (bgzf_close(fpin) < 0) abort();
}
return 0;
}
int samthreads(samfile_t *fp, int n_threads, int n_sub_blks)
{
if (!(fp->type&1) || (fp->type&2)) return -1;
bgzf_mt(fp->x.bam, n_threads, n_sub_blks);
return 0;
}
int bam_merge_core2(int by_qname, const char *out, const char *headers, int n, char * const *fn, int flag, const char *reg, int level)
#endif
{
bamFile fpout, *fp;
heap1_t *heap;
bam_header_t *hout = 0;
bam_header_t *hheaders = NULL;
int i, j, *RG_len = 0;
uint64_t idx = 0;
char **RG = 0, mode[8];
bam_iter_t *iter = 0;
if (headers) {
tamFile fpheaders = sam_open(headers);
if (fpheaders == 0) {
const char *message = strerror(errno);
fprintf(stderr, "[bam_merge_core] cannot open '%s': %s\n", headers, message);
return -1;
}
hheaders = sam_header_read(fpheaders);
sam_close(fpheaders);
}
g_is_by_qname = by_qname;
fp = (bamFile*)calloc(n, sizeof(bamFile));
heap = (heap1_t*)calloc(n, sizeof(heap1_t));
iter = (bam_iter_t*)calloc(n, sizeof(bam_iter_t));
// prepare RG tag
if (flag & MERGE_RG) {
RG = (char**)calloc(n, sizeof(void*));
RG_len = (int*)calloc(n, sizeof(int));
for (i = 0; i != n; ++i) {
int l = strlen(fn[i]);
const char *s = fn[i];
if (l > 4 && strcmp(s + l - 4, ".bam") == 0) l -= 4;
for (j = l - 1; j >= 0; --j) if (s[j] == '/') break;
++j; l -= j;
RG[i] = calloc(l + 1, 1);
RG_len[i] = l;
strncpy(RG[i], s + j, l);
}
}
// read the first
for (i = 0; i != n; ++i) {
bam_header_t *hin;
fp[i] = bam_open(fn[i], "r");
if (fp[i] == 0) {
int j;
fprintf(stderr, "[bam_merge_core] fail to open file %s\n", fn[i]);
for (j = 0; j < i; ++j) bam_close(fp[j]);
free(fp); free(heap);
// FIXME: possible memory leak
return -1;
}
hin = bam_header_read(fp[i]);
if (i == 0) { // the first BAM
hout = hin;
} else { // validate multiple baf
int min_n_targets = hout->n_targets;
if (hin->n_targets < min_n_targets) min_n_targets = hin->n_targets;
for (j = 0; j < min_n_targets; ++j)
if (strcmp(hout->target_name[j], hin->target_name[j]) != 0) {
fprintf(stderr, "[bam_merge_core] different target sequence name: '%s' != '%s' in file '%s'\n",
hout->target_name[j], hin->target_name[j], fn[i]);
return -1;
}
// If this input file has additional target reference sequences,
// add them to the headers to be output
if (hin->n_targets > hout->n_targets) {
swap_header_targets(hout, hin);
// FIXME Possibly we should also create @SQ text headers
// for the newly added reference sequences
}
bam_header_destroy(hin);
}
}
if (hheaders) {
// If the text headers to be swapped in include any @SQ headers,
// check that they are consistent with the existing binary list
// of reference information.
if (hheaders->n_targets > 0) {
if (hout->n_targets != hheaders->n_targets) {
fprintf(stderr, "[bam_merge_core] number of @SQ headers in '%s' differs from number of target sequences\n", headers);
if (!reg) return -1;
}
for (j = 0; j < hout->n_targets; ++j)
if (strcmp(hout->target_name[j], hheaders->target_name[j]) != 0) {
fprintf(stderr, "[bam_merge_core] @SQ header '%s' in '%s' differs from target sequence\n", hheaders->target_name[j], headers);
if (!reg) return -1;
}
}
swap_header_text(hout, hheaders);
bam_header_destroy(hheaders);
}
if (reg) {
int tid, beg, end;
if (bam_parse_region(hout, reg, &tid, &beg, &end) < 0) {
fprintf(stderr, "[%s] Malformated region string or undefined reference name\n", __func__);
return -1;
}
for (i = 0; i < n; ++i) {
bam_index_t *idx;
idx = bam_index_load(fn[i]);
iter[i] = bam_iter_query(idx, tid, beg, end);
bam_index_destroy(idx);
}
}
for (i = 0; i < n; ++i) {
heap1_t *h = heap + i;
h->i = i;
h->b = (bam1_t*)calloc(1, sizeof(bam1_t));
if (bam_iter_read(fp[i], iter[i], h->b) >= 0) {
h->pos = ((uint64_t)h->b->core.tid<<32) | (uint32_t)((int32_t)h->b->core.pos+1)<<1 | bam1_strand(h->b);
h->idx = idx++;
}
else h->pos = HEAP_EMPTY;
}
if (flag & MERGE_UNCOMP) level = 0;
else if (flag & MERGE_LEVEL1) level = 1;
strcpy(mode, "w");
if (level >= 0) sprintf(mode + 1, "%d", level < 9? level : 9);
if ((fpout = strcmp(out, "-")? bam_open(out, "w") : bam_dopen(fileno(stdout), "w")) == 0) {
fprintf(stderr, "[%s] fail to create the output file.\n", __func__);
return -1;
}
bam_header_write(fpout, hout);
bam_header_destroy(hout);
#ifndef _PBGZF_USE
if (!(flag & MERGE_UNCOMP)) bgzf_mt(fpout, n_threads, 256);
#endif
ks_heapmake(heap, n, heap);
while (heap->pos != HEAP_EMPTY) {
bam1_t *b = heap->b;
if (flag & MERGE_RG) {
uint8_t *rg = bam_aux_get(b, "RG");
if (rg) bam_aux_del(b, rg);
bam_aux_append(b, "RG", 'Z', RG_len[heap->i] + 1, (uint8_t*)RG[heap->i]);
}
bam_write1_core(fpout, &b->core, b->data_len, b->data);
if ((j = bam_iter_read(fp[heap->i], iter[heap->i], b)) >= 0) {
heap->pos = ((uint64_t)b->core.tid<<32) | (uint32_t)((int)b->core.pos+1)<<1 | bam1_strand(b);
heap->idx = idx++;
} else if (j == -1) {
heap->pos = HEAP_EMPTY;
free(heap->b->data); free(heap->b);
heap->b = 0;
} else fprintf(stderr, "[bam_merge_core] '%s' is truncated. Continue anyway.\n", fn[heap->i]);
ks_heapadjust(heap, 0, n, heap);
}
if (flag & MERGE_RG) {
for (i = 0; i != n; ++i) free(RG[i]);
free(RG); free(RG_len);
}
for (i = 0; i != n; ++i) {
bam_iter_destroy(iter[i]);
bam_close(fp[i]);
}
bam_close(fpout);
free(fp); free(heap); free(iter);
return 0;
}
int main(int argc, char *argv[]) {
if (argc <= 1) {
fprintf(stderr, "Usage: thrash_threads4 input.bam\n");
exit(1);
}
// Find a valid seek location ~64M into the file
int i;
ssize_t got;
BGZF *fpin = bgzf_open(argv[1], "r");
uint64_t upos = 0, uend = 0;
char buf[100000];
for (i = 0; i < 100; i++) {
if ((got = bgzf_read(fpin, buf, 65536)) < 0)
abort();
upos += got;
}
int64_t pos = bgzf_tell(fpin);
while ((got = bgzf_read(fpin, buf, 65536)) > 0) {
uend += got;
}
if (got < 0) abort();
int64_t end = bgzf_tell(fpin);
bgzf_close(fpin);
// Ensure input is big enough to avoid case 3,4 below going off the end
// of the file
if (uend < upos + 10000000) {
fprintf(stderr, "Please supply a bigger input file\n");
exit(1);
}
#define N 1000
// Spam random seeks & reads
for (i = 0; i < 1000; i++) {
printf("i=%d\t", i);
fpin = bgzf_open(argv[1], "r");
int j, eof = 0, mt = 0;
for (j = 0; j < 80; j++) {
int n = rand() % 7;
putchar('0'+n); fflush(stdout);
switch (n) {
case 0: // start
if (bgzf_seek(fpin, 0LL, SEEK_SET) < 0) puts("!");//abort();
eof = 0;
break;
case 1: // mid
if (bgzf_seek(fpin, pos, SEEK_SET) < 0) puts("!");//abort();
eof = 0;
break;
case 2: // eof
if (bgzf_seek(fpin, end, SEEK_SET) < 0) puts("!");//abort();
eof = 1;
break;
case 3: case 4: {
int l = rand()%(n==3?100000:100);
if (bgzf_read(fpin, buf, l) != l*(1-eof)) abort();
break;
}
case 5:
usleep(N);
break;
case 6:
if (!mt)
bgzf_mt(fpin, 8, 256);
mt = 1;
break;
}
}
printf("\n");
if (bgzf_close(fpin))
abort();
}
return 0;
}
int samthreads(samfile_t *fp, int n_threads, int n_sub_blks)
{
if (hts_get_format(fp->file)->format != bam || !fp->is_write) return -1;
bgzf_mt(fp->x.bam, n_threads, n_sub_blks);
return 0;
}
int main(int argc, char **argv)
{
int c, compress, pstdout, is_forced, index = 0, rebgzip = 0, reindex = 0;
BGZF *fp;
void *buffer;
long start, end, size;
char *index_fname = NULL;
int threads = 1;
static const struct option loptions[] =
{
{"help", no_argument, NULL, 'h'},
{"offset", required_argument, NULL, 'b'},
{"stdout", no_argument, NULL, 'c'},
{"decompress", no_argument, NULL, 'd'},
{"force", no_argument, NULL, 'f'},
{"index", no_argument, NULL, 'i'},
{"index-name", required_argument, NULL, 'I'},
{"reindex", no_argument, NULL, 'r'},
{"rebgzip",no_argument,NULL,'g'},
{"size", required_argument, NULL, 's'},
{"threads", required_argument, NULL, '@'},
{"version", no_argument, NULL, 1},
{NULL, 0, NULL, 0}
};
compress = 1; pstdout = 0; start = 0; size = -1; end = -1; is_forced = 0;
while((c = getopt_long(argc, argv, "cdh?fb:@:s:iI:gr",loptions,NULL)) >= 0){
switch(c){
case 'd': compress = 0; break;
case 'c': pstdout = 1; break;
case 'b': start = atol(optarg); compress = 0; pstdout = 1; break;
case 's': size = atol(optarg); pstdout = 1; break;
case 'f': is_forced = 1; break;
case 'i': index = 1; break;
case 'I': index_fname = optarg; break;
case 'g': rebgzip = 1; break;
case 'r': reindex = 1; compress = 0; break;
case '@': threads = atoi(optarg); break;
case 1:
printf(
"bgzip (htslib) %s\n"
"Copyright (C) 2017 Genome Research Ltd.\n", hts_version());
return EXIT_SUCCESS;
case 'h':
case '?': return bgzip_main_usage();
}
}
if (size >= 0) end = start + size;
if (end >= 0 && end < start) {
fprintf(stderr, "[bgzip] Illegal region: [%ld, %ld]\n", start, end);
return 1;
}
if (compress == 1) {
struct stat sbuf;
int f_src = fileno(stdin);
if ( argc>optind )
{
if ( stat(argv[optind],&sbuf)<0 )
{
fprintf(stderr, "[bgzip] %s: %s\n", strerror(errno), argv[optind]);
return 1;
}
if ((f_src = open(argv[optind], O_RDONLY)) < 0) {
fprintf(stderr, "[bgzip] %s: %s\n", strerror(errno), argv[optind]);
return 1;
}
if (pstdout)
fp = bgzf_open("-", "w");
else
{
char *name = malloc(strlen(argv[optind]) + 5);
strcpy(name, argv[optind]);
strcat(name, ".gz");
fp = bgzf_open(name, is_forced? "w" : "wx");
if (fp == NULL && errno == EEXIST && confirm_overwrite(name))
fp = bgzf_open(name, "w");
if (fp == NULL) {
fprintf(stderr, "[bgzip] can't create %s: %s\n", name, strerror(errno));
free(name);
return 1;
}
free(name);
}
}
else if (!pstdout && isatty(fileno((FILE *)stdout)) )
return bgzip_main_usage();
else if ( index && !index_fname )
{
fprintf(stderr, "[bgzip] Index file name expected when writing to stdout\n");
return 1;
}
else
fp = bgzf_open("-", "w");
if ( index && rebgzip )
{
fprintf(stderr, "[bgzip] Can't produce a index and rebgzip simultaneously\n");
return 1;
}
if ( rebgzip && !index_fname )
{
fprintf(stderr, "[bgzip] Index file name expected when writing to stdout\n");
return 1;
}
if (threads > 1)
bgzf_mt(fp, threads, 256);
if ( index ) bgzf_index_build_init(fp);
buffer = malloc(WINDOW_SIZE);
#ifdef _WIN32
_setmode(f_src, O_BINARY);
#endif
if (rebgzip){
if ( bgzf_index_load(fp, index_fname, NULL) < 0 ) error("Could not load index: %s.gzi\n", argv[optind]);
while ((c = read(f_src, buffer, WINDOW_SIZE)) > 0)
if (bgzf_block_write(fp, buffer, c) < 0) error("Could not write %d bytes: Error %d\n", c, fp->errcode);
}
else {
while ((c = read(f_src, buffer, WINDOW_SIZE)) > 0)
if (bgzf_write(fp, buffer, c) < 0) error("Could not write %d bytes: Error %d\n", c, fp->errcode);
}
if ( index )
{
if (index_fname) {
if (bgzf_index_dump(fp, index_fname, NULL) < 0)
error("Could not write index to '%s'\n", index_fname);
} else {
if (bgzf_index_dump(fp, argv[optind], ".gz.gzi") < 0)
error("Could not write index to '%s.gz.gzi'", argv[optind]);
}
}
if (bgzf_close(fp) < 0) error("Close failed: Error %d", fp->errcode);
if (argc > optind && !pstdout) unlink(argv[optind]);
free(buffer);
close(f_src);
return 0;
}
else if ( reindex )
{
if ( argc>optind )
{
fp = bgzf_open(argv[optind], "r");
if ( !fp ) error("[bgzip] Could not open file: %s\n", argv[optind]);
}
else
{
if ( !index_fname ) error("[bgzip] Index file name expected when reading from stdin\n");
fp = bgzf_open("-", "r");
if ( !fp ) error("[bgzip] Could not read from stdin: %s\n", strerror(errno));
}
buffer = malloc(BGZF_BLOCK_SIZE);
bgzf_index_build_init(fp);
int ret;
while ( (ret=bgzf_read(fp, buffer, BGZF_BLOCK_SIZE))>0 ) ;
free(buffer);
if ( ret<0 ) error("Is the file gzipped or bgzipped? The latter is required for indexing.\n");
if ( index_fname ) {
if (bgzf_index_dump(fp, index_fname, NULL) < 0)
error("Could not write index to '%s'\n", index_fname);
} else {
if (bgzf_index_dump(fp, argv[optind], ".gzi") < 0)
error("Could not write index to '%s.gzi'\n", argv[optind]);
}
if ( bgzf_close(fp)<0 ) error("Close failed: Error %d\n",fp->errcode);
return 0;
}
else
{
struct stat sbuf;
int f_dst;
if ( argc>optind )
{
if ( stat(argv[optind],&sbuf)<0 )
{
fprintf(stderr, "[bgzip] %s: %s\n", strerror(errno), argv[optind]);
return 1;
}
char *name;
int len = strlen(argv[optind]);
if ( strcmp(argv[optind]+len-3,".gz") )
{
fprintf(stderr, "[bgzip] %s: unknown suffix -- ignored\n", argv[optind]);
return 1;
}
fp = bgzf_open(argv[optind], "r");
if (fp == NULL) {
fprintf(stderr, "[bgzip] Could not open file: %s\n", argv[optind]);
return 1;
}
if (pstdout) {
f_dst = fileno(stdout);
}
else {
const int wrflags = O_WRONLY | O_CREAT | O_TRUNC;
name = strdup(argv[optind]);
name[strlen(name) - 3] = '\0';
f_dst = open(name, is_forced? wrflags : wrflags|O_EXCL, 0666);
if (f_dst < 0 && errno == EEXIST && confirm_overwrite(name))
f_dst = open(name, wrflags, 0666);
if (f_dst < 0) {
fprintf(stderr, "[bgzip] can't create %s: %s\n", name, strerror(errno));
free(name);
return 1;
}
free(name);
}
}
else if (!pstdout && isatty(fileno((FILE *)stdin)) )
return bgzip_main_usage();
else
{
f_dst = fileno(stdout);
fp = bgzf_open("-", "r");
if (fp == NULL) {
fprintf(stderr, "[bgzip] Could not read from stdin: %s\n", strerror(errno));
return 1;
}
}
if (threads > 1)
bgzf_mt(fp, threads, 256);
buffer = malloc(WINDOW_SIZE);
if ( start>0 )
{
if ( bgzf_index_load(fp, argv[optind], ".gzi") < 0 ) error("Could not load index: %s.gzi\n", argv[optind]);
if ( bgzf_useek(fp, start, SEEK_SET) < 0 ) error("Could not seek to %d-th (uncompressd) byte\n", start);
}
#ifdef _WIN32
_setmode(f_dst, O_BINARY);
#endif
while (1) {
if (end < 0) c = bgzf_read(fp, buffer, WINDOW_SIZE);
else c = bgzf_read(fp, buffer, (end - start > WINDOW_SIZE)? WINDOW_SIZE:(end - start));
if (c == 0) break;
if (c < 0) error("Could not read %d bytes: Error %d\n", (end - start > WINDOW_SIZE)? WINDOW_SIZE:(end - start), fp->errcode);
start += c;
if ( write(f_dst, buffer, c) != c ) {
#ifdef _WIN32
if (GetLastError() != ERROR_NO_DATA)
#endif
error("Could not write %d bytes\n", c);
}
if (end >= 0 && start >= end) break;
}
free(buffer);
if (bgzf_close(fp) < 0) error("Close failed: Error %d\n",fp->errcode);
if (!pstdout) unlink(argv[optind]);
return 0;
}
}
int main(int argc, char **argv)
{
if (argc < 4)
errx(1,
"usage\t:%s <bam> <split out> <discord out> (optional #threads)",
argv[0]);
char *bam_file_name = argv[1];
char *split_file_name = argv[2];
char *disc_file_name = argv[3];
int threads = 2;
if (argc == 5) {
threads = atoi(argv[4]);
}
samFile *disc = sam_open(disc_file_name, "wb");
samFile *split = sam_open(split_file_name, "wb");
samFile *in = sam_open(bam_file_name, "rb");
if(in == NULL)
errx(1, "Unable to open BAM/SAM file.");
// TODO: handle cram.
if (threads > 1) {
bgzf_mt(in->fp.bgzf, threads, 256);
}
hts_idx_t *idx = sam_index_load(in, bam_file_name);
if(idx == NULL)
errx(1,"Unable to open BAM/SAM index.");
bam_hdr_t *hdr = sam_hdr_read(in);
int r = sam_hdr_write(disc, hdr);
r = sam_hdr_write(split, hdr);
bam1_t *aln = bam_init1();
int ret;
while(ret = sam_read1(in, hdr, aln) >= 0) {
if (((aln->core.flag) & 1294) == 0)
r = sam_write1(disc, hdr, aln);
uint8_t *sa = bam_aux_get(aln, "SA");
if (sa != 0) {
char *sa_tag = strdup(bam_aux2Z(sa));
if ( count_tags(sa_tag) == 1) {
char *chrm, strand, *cigar;
uint32_t pos;
split_sa_tag(sa_tag,
&chrm,
&pos,
&strand,
&cigar);
struct line sa, al;
calcOffsets(cigar,
pos,
strand,
&sa);
sa.chrm = chrm;
sa.strand = strand;
calcAlnOffsets(bam_get_cigar(aln),
aln->core.n_cigar,
aln->core.pos,
bam_is_rev(aln) ? '-' : '+',
&al);
al.chrm = hdr->target_name[aln->core.tid];
al.strand = bam_is_rev(aln) ? '-' : '+';
struct line *left = &al, *right = &sa;
if (left->SQO > right->SQO) {
left = &sa;
right = &al;
}
int overlap = MAX(1 + MIN(left->EQO, right->EQO) -
MAX(left->SQO, right->SQO), 0);
int alen1 = 1 + left->EQO - left->SQO;
int alen2 = 1 + right->EQO - right->SQO;
int mno = MIN(alen1-overlap, alen2-overlap);
if (mno < MIN_NON_OVERLAP)
continue;
if ( (strcmp(left->chrm, right->chrm) == 0) &&
(left->strand == right->strand) ) {
int leftDiag, rightDiag, insSize;
if (left->strand == '-') {
leftDiag = left->rapos - left->sclip;
rightDiag = (right->rapos + right->raLen) -
(right->sclip + right->qaLen);
insSize = rightDiag - leftDiag;
} else {
leftDiag = (left->rapos + left->raLen) -
(left->sclip + left->qaLen);
rightDiag = right->rapos - right->sclip;
insSize = leftDiag - rightDiag;
}
int desert = right->SQO - left->EQO - 1;
if ((abs(insSize) < MIN_INDEL_SIZE) ||
((desert > 0) && (
(desert - (int)MAX(0, insSize)) >
MAX_UNMAPPED_BASES)))
continue;
}
char *qname = bam_get_qname(aln);
if ((aln->core.flag & 64) == 64)
qname[0] = 'A';
else
qname[0] = 'B';
r = sam_write1(split, hdr, aln);
}
free(sa_tag);
}
}
bam_destroy1(aln);
hts_idx_destroy(idx);
bam_hdr_destroy(hdr);
sam_close(in);
sam_close(disc);
sam_close(split);
if(ret < -1) {
errx(1, "lumpy_filter: error reading bam: %s\n", bam_file_name);
}
}
