/*
* Reads a file and outputs a new BAM file to fd with 'h' replaced as
* the header. No checks are made to the validity.
*/
int bam_reheader(BGZF *in, bam_hdr_t *h, int fd,
const char *arg_list, int add_PG)
{
BGZF *fp;
ssize_t len;
uint8_t *buf;
if (in->is_write) return -1;
buf = malloc(BUF_SIZE);
if (bam_hdr_read(in) == NULL) {
fprintf(stderr, "Couldn't read header\n");
free(buf);
return -1;
}
fp = bgzf_fdopen(fd, "w");
if (add_PG) {
// Around the houses, but it'll do until we can manipulate bam_hdr_t natively.
SAM_hdr *sh = sam_hdr_parse_(h->text, h->l_text);
if (sam_hdr_add_PG(sh, "samtools",
"VN", samtools_version(),
arg_list ? "CL": NULL,
arg_list ? arg_list : NULL,
NULL) != 0)
return -1;
free(h->text);
h->text = strdup(sam_hdr_str(sh));
h->l_text = sam_hdr_length(sh);
if (!h->text)
return -1;
sam_hdr_free(sh);
}
bam_hdr_write(fp, h);
if (in->block_offset < in->block_length) {
bgzf_write(fp, in->uncompressed_block + in->block_offset, in->block_length - in->block_offset);
bgzf_flush(fp);
}
while ((len = bgzf_raw_read(in, buf, BUF_SIZE)) > 0)
bgzf_raw_write(fp, buf, len);
free(buf);
fp->block_offset = in->block_offset = 0;
bgzf_close(fp);
return 0;
}
int bam_reheader(BGZF *in, const bam_header_t *h, int fd)
{
BGZF *fp;
bam_header_t *old;
ssize_t len;
uint8_t *buf;
if (in->is_write) return -1;
buf = malloc(BUF_SIZE);
old = bam_header_read(in);
fp = bgzf_fdopen(fd, "w");
bam_header_write(fp, h);
if (in->block_offset < in->block_length) {
bgzf_write(fp, in->uncompressed_block + in->block_offset, in->block_length - in->block_offset);
bgzf_flush(fp);
}
while ((len = bgzf_raw_read(in, buf, BUF_SIZE)) > 0)
bgzf_raw_write(fp, buf, len);
free(buf);
fp->block_offset = in->block_offset = 0;
bgzf_close(fp);
return 0;
}
static void naive_concat(args_t *args)
{
// only compressed BCF atm
BGZF *bgzf_out = bgzf_open(args->output_fname,"w");;
const size_t page_size = 32768;
char *buf = (char*) malloc(page_size);
kstring_t tmp = {0,0,0};
int i;
for (i=0; i<args->nfnames; i++)
{
htsFile *hts_fp = hts_open(args->fnames[i],"r");
if ( !hts_fp ) error("Failed to open: %s\n", args->fnames[i]);
htsFormat type = *hts_get_format(hts_fp);
if ( type.format==vcf ) error("The --naive option currently works only for compressed BCFs, sorry :-/\n");
if ( type.compression!=bgzf ) error("The --naive option currently works only for compressed BCFs, sorry :-/\n");
BGZF *fp = hts_get_bgzfp(hts_fp);
if ( !fp || bgzf_read_block(fp) != 0 || !fp->block_length )
error("Failed to read %s: %s\n", args->fnames[i], strerror(errno));
uint8_t magic[5];
if ( bgzf_read(fp, magic, 5) != 5 ) error("Failed to read the BCF header in %s\n", args->fnames[i]);
if (strncmp((char*)magic, "BCF\2\2", 5) != 0) error("Invalid BCF magic string in %s\n", args->fnames[i]);
if ( bgzf_read(fp, &tmp.l, 4) != 4 ) error("Failed to read the BCF header in %s\n", args->fnames[i]);
hts_expand(char,tmp.l,tmp.m,tmp.s);
if ( bgzf_read(fp, tmp.s, tmp.l) != tmp.l ) error("Failed to read the BCF header in %s\n", args->fnames[i]);
// write only the first header
if ( i==0 )
{
if ( bgzf_write(bgzf_out, "BCF\2\2", 5) !=5 ) error("Failed to write %d bytes to %s\n", 5,args->output_fname);
if ( bgzf_write(bgzf_out, &tmp.l, 4) !=4 ) error("Failed to write %d bytes to %s\n", 4,args->output_fname);
if ( bgzf_write(bgzf_out, tmp.s, tmp.l) != tmp.l) error("Failed to write %d bytes to %s\n", tmp.l,args->output_fname);
}
// Output all non-header data that were read together with the header block
int nskip = fp->block_offset;
if ( fp->block_length - nskip > 0 )
{
if ( bgzf_write(bgzf_out, fp->uncompressed_block+nskip, fp->block_length-nskip)<0 ) error("Error: %d\n",fp->errcode);
}
if ( bgzf_flush(bgzf_out)<0 ) error("Error: %d\n",bgzf_out->errcode);
// Stream the rest of the file as it is, without recompressing, but remove BGZF EOF blocks
ssize_t nread, ncached = 0, nwr;
const int neof = 28;
char cached[neof];
while (1)
{
nread = bgzf_raw_read(fp, buf, page_size);
// page_size boundary may occur in the middle of the EOF block, so we need to cache the blocks' ends
if ( nread<=0 ) break;
if ( nread<=neof ) // last block
{
if ( ncached )
{
// flush the part of the cache that won't be needed
nwr = bgzf_raw_write(bgzf_out, cached, nread);
if (nwr != nread) error("Write failed, wrote %d instead of %d bytes.\n", nwr,(int)nread);
// make space in the cache so that we can append to the end
if ( nread!=neof ) memmove(cached,cached+nread,neof-nread);
}
// fill the cache and check for eof outside this loop
memcpy(cached+neof-nread,buf,nread);
break;
}
// not the last block, flush the cache if full
if ( ncached )
{
nwr = bgzf_raw_write(bgzf_out, cached, ncached);
if (nwr != ncached) error("Write failed, wrote %d instead of %d bytes.\n", nwr,(int)ncached);
ncached = 0;
}
// fill the cache
nread -= neof;
memcpy(cached,buf+nread,neof);
ncached = neof;
nwr = bgzf_raw_write(bgzf_out, buf, nread);
if (nwr != nread) error("Write failed, wrote %d instead of %d bytes.\n", nwr,(int)nread);
}
if ( ncached && memcmp(cached,"\037\213\010\4\0\0\0\0\0\377\6\0\102\103\2\0\033\0\3\0\0\0\0\0\0\0\0\0",neof) )
{
nwr = bgzf_raw_write(bgzf_out, cached, neof);
if (nwr != neof) error("Write failed, wrote %d instead of %d bytes.\n", nwr,(int)neof);
}
if (hts_close(hts_fp)) error("Close failed: %s\n",args->fnames[i]);
}
free(buf);
free(tmp.s);
if (bgzf_close(bgzf_out) < 0) error("Error: %d\n",bgzf_out->errcode);
}
int reheader_file(const char *fname, const char *header, int ftype, tbx_conf_t *conf)
{
if ( ftype & IS_TXT || !ftype )
{
BGZF *fp = bgzf_open(fname,"r");
if ( !fp || bgzf_read_block(fp) != 0 || !fp->block_length ) return -1;
char *buffer = fp->uncompressed_block;
int skip_until = 0;
// Skip the header: find out the position of the data block
if ( buffer[0]==conf->meta_char )
{
skip_until = 1;
while (1)
{
if ( buffer[skip_until]=='\n' )
{
skip_until++;
if ( skip_until>=fp->block_length )
{
if ( bgzf_read_block(fp) != 0 || !fp->block_length ) error("FIXME: No body in the file: %s\n", fname);
skip_until = 0;
}
// The header has finished
if ( buffer[skip_until]!=conf->meta_char ) break;
}
skip_until++;
if ( skip_until>=fp->block_length )
{
if (bgzf_read_block(fp) != 0 || !fp->block_length) error("FIXME: No body in the file: %s\n", fname);
skip_until = 0;
}
}
}
// Output the new header
FILE *hdr = fopen(header,"r");
if ( !hdr ) error("%s: %s", header,strerror(errno));
const size_t page_size = 32768;
char *buf = malloc(page_size);
BGZF *bgzf_out = bgzf_open("-", "w");
ssize_t nread;
while ( (nread=fread(buf,1,page_size-1,hdr))>0 )
{
if ( nread<page_size-1 && buf[nread-1]!='\n' ) buf[nread++] = '\n';
if (bgzf_write(bgzf_out, buf, nread) < 0) error("Error: %d\n",bgzf_out->errcode);
}
if ( fclose(hdr) ) error("close failed: %s\n", header);
// Output all remainig data read with the header block
if ( fp->block_length - skip_until > 0 )
{
if (bgzf_write(bgzf_out, buffer+skip_until, fp->block_length-skip_until) < 0) error("Error: %d\n",fp->errcode);
}
if (bgzf_flush(bgzf_out) < 0) error("Error: %d\n",bgzf_out->errcode);
while (1)
{
nread = bgzf_raw_read(fp, buf, page_size);
if ( nread<=0 ) break;
int count = bgzf_raw_write(bgzf_out, buf, nread);
if (count != nread) error("Write failed, wrote %d instead of %d bytes.\n", count,(int)nread);
}
if (bgzf_close(bgzf_out) < 0) error("Error: %d\n",bgzf_out->errcode);
if (bgzf_close(fp) < 0) error("Error: %d\n",fp->errcode);
free(buf);
}
else
error("todo: reheader BCF, BAM\n"); // BCF is difficult, records contain pointers to the header.
return 0;
}
static void reheader_vcf_gz(args_t *args)
{
BGZF *fp = bgzf_open(args->fname,"r");
if ( !fp || bgzf_read_block(fp) != 0 || !fp->block_length )
error("Failed to read %s: %s\n", args->fname, strerror(errno));
kstring_t hdr = {0,0,0};
char *buffer = (char*) fp->uncompressed_block;
// Read the header and find the position of the data block
if ( buffer[0]!='#' ) error("Could not parse the header, expected '#', found '%c'\n", buffer[0]);
int skip_until = 1; // end of the header in the current uncompressed block
while (1)
{
if ( buffer[skip_until]=='\n' )
{
skip_until++;
if ( skip_until>=fp->block_length )
{
kputsn(buffer,skip_until,&hdr);
if ( bgzf_read_block(fp) != 0 || !fp->block_length ) error("FIXME: No body in the file: %s\n", args->fname);
skip_until = 0;
}
// The header has finished
if ( buffer[skip_until]!='#' )
{
kputsn(buffer,skip_until,&hdr);
break;
}
}
skip_until++;
if ( skip_until>=fp->block_length )
{
kputsn(buffer,fp->block_length,&hdr);
if (bgzf_read_block(fp) != 0 || !fp->block_length) error("FIXME: No body in the file: %s\n", args->fname);
skip_until = 0;
}
}
int nsamples = 0;
char **samples = NULL;
if ( args->samples_fname )
samples = hts_readlines(args->samples_fname, &nsamples);
if ( args->header_fname )
{
free(hdr.s); hdr.s = NULL; hdr.l = hdr.m = 0;
read_header_file(args->header_fname, &hdr);
}
if ( samples )
{
set_samples(samples, nsamples, &hdr);
int i;
for (i=0; i<nsamples; i++) free(samples[i]);
free(samples);
}
// Output the modified header
BGZF *bgzf_out = bgzf_dopen(fileno(stdout), "w");
bgzf_write(bgzf_out, hdr.s, hdr.l);
free(hdr.s);
// Output all remainig data read with the header block
if ( fp->block_length - skip_until > 0 )
{
if ( bgzf_write(bgzf_out, buffer+skip_until, fp->block_length-skip_until)<0 ) error("Error: %d\n",fp->errcode);
}
if ( bgzf_flush(bgzf_out)<0 ) error("Error: %d\n",bgzf_out->errcode);
// Stream the rest of the file without as it is, without decompressing
ssize_t nread;
int page_size = getpagesize();
char *buf = (char*) valloc(page_size);
while (1)
{
nread = bgzf_raw_read(fp, buf, page_size);
if ( nread<=0 ) break;
int count = bgzf_raw_write(bgzf_out, buf, nread);
if (count != nread) error("Write failed, wrote %d instead of %d bytes.\n", count,(int)nread);
}
if (bgzf_close(bgzf_out) < 0) error("Error: %d\n",bgzf_out->errcode);
if (bgzf_close(fp) < 0) error("Error: %d\n",fp->errcode);
free(buf);
}
int bam_cat(int nfn, char * const *fn, const bam_hdr_t *h, const char* outbam)
{
BGZF *fp;
uint8_t *buf;
uint8_t ebuf[BGZF_EMPTY_BLOCK_SIZE];
const int es=BGZF_EMPTY_BLOCK_SIZE;
int i;
fp = strcmp(outbam, "-")? bgzf_open(outbam, "w") : bgzf_fdopen(fileno(stdout), "w");
if (fp == 0) {
fprintf(stderr, "[%s] ERROR: fail to open output file '%s'.\n", __func__, outbam);
return 1;
}
if (h) bam_hdr_write(fp, h);
buf = (uint8_t*) malloc(BUF_SIZE);
for(i = 0; i < nfn; ++i){
BGZF *in;
bam_hdr_t *old;
int len,j;
in = strcmp(fn[i], "-")? bgzf_open(fn[i], "r") : bgzf_fdopen(fileno(stdin), "r");
if (in == 0) {
fprintf(stderr, "[%s] ERROR: fail to open file '%s'.\n", __func__, fn[i]);
return -1;
}
if (in->is_write) return -1;
old = bam_hdr_read(in);
if (old == NULL) {
fprintf(stderr, "[%s] ERROR: couldn't read header for '%s'.\n",
__func__, fn[i]);
bgzf_close(in);
return -1;
}
if (h == 0 && i == 0) bam_hdr_write(fp, old);
if (in->block_offset < in->block_length) {
bgzf_write(fp, in->uncompressed_block + in->block_offset, in->block_length - in->block_offset);
bgzf_flush(fp);
}
j=0;
while ((len = bgzf_raw_read(in, buf, BUF_SIZE)) > 0) {
if(len<es){
int diff=es-len;
if(j==0) {
fprintf(stderr, "[%s] ERROR: truncated file?: '%s'.\n", __func__, fn[i]);
return -1;
}
bgzf_raw_write(fp, ebuf, len);
memcpy(ebuf,ebuf+len,diff);
memcpy(ebuf+diff,buf,len);
} else {
if(j!=0) bgzf_raw_write(fp, ebuf, es);
len-= es;
memcpy(ebuf,buf+len,es);
bgzf_raw_write(fp, buf, len);
}
j=1;
}
/* check final gzip block */
{
const uint8_t gzip1=ebuf[0];
const uint8_t gzip2=ebuf[1];
const uint32_t isize=*((uint32_t*)(ebuf+es-4));
if(((gzip1!=GZIPID1) || (gzip2!=GZIPID2)) || (isize!=0)) {
fprintf(stderr, "[%s] WARNING: Unexpected block structure in file '%s'.", __func__, fn[i]);
fprintf(stderr, " Possible output corruption.\n");
bgzf_raw_write(fp, ebuf, es);
}
}
bam_hdr_destroy(old);
bgzf_close(in);
}
free(buf);
bgzf_close(fp);
return 0;
}