/*
* Renumbers RG numbers in a cram compression header.
*
* CRAM stores RG as the Nth number in the header, rather than a
* string holding the ID: tag. This is smaller in space, but means
* "samtools cat" to join files together that contain single but
* different RG lines needs a way of renumbering them.
*
* The file descriptor is expected to be immediately after the
* cram_container structure (ie before the cram compression header).
* Due to the nature of the CRAM format, this needs to read and write
* the blocks itself. Note that there may be multiple slices within
* the container, meaning multiple compression headers to manipulate.
* Changing RG may change the size of the compression header and
* therefore the length field in the container. Hence we rewrite all
* blocks just incase and also emit the adjusted container.
*
* The current implementation can only cope with renumbering a single
* RG (and only then if it is using HUFFMAN or BETA codecs). In
* theory it *may* be possible to renumber multiple RGs if they use
* HUFFMAN to the CORE block or use an external block unshared by any
* other data series. So we have an API that can be upgraded to
* support this, but do not implement it for now. An example
* implementation of RG as an EXTERNAL block would be to find that
* block and rewrite it, returning the number of blocks consumed.
*
* Returns 0 on success;
* -1 if unable to edit;
* -2 on other errors (eg I/O).
*/
int cram_transcode_rg(cram_fd *in, cram_fd *out,
cram_container *c,
int nrg, int *in_rg, int *out_rg) {
int new_rg = *out_rg, old_size, new_size;
cram_block *o_blk, *n_blk;
cram_block_compression_hdr *ch;
if (nrg != 1) {
hts_log_error("CRAM transcode supports only a single RG");
return -2;
}
// Produce a new block holding the updated compression header,
// with RG transcoded to a new value. (Single only supported.)
o_blk = cram_read_block(in);
old_size = cram_block_size(o_blk);
ch = cram_decode_compression_header(in, o_blk);
if (cram_block_compression_hdr_set_rg(ch, new_rg) != 0)
return -1;
cram_block_compression_hdr_decoder2encoder(in, ch);
n_blk = cram_encode_compression_header(in, c, ch);
cram_free_compression_header(ch);
/*
* Warning: this has internal knowledge of the cram compression
* header format.
*
* The decoder doesn't set c->tags_used, so the encoder puts a two
* byte blank segment. This means n_blk is too short. We skip
* through the decoded old block (o_blk) and copy from there.
*/
char *cp = cram_block_get_data(o_blk);
char *op = cp;
char *endp = cp + cram_block_get_uncomp_size(o_blk);
//fprintf(stderr, "sz = %d\n", (int)(endp-cp));
int32_t i32;
cp += safe_itf8_get(cp, endp, &i32);
cp += i32;
cp += safe_itf8_get(cp, endp, &i32);
cp += i32;
op = cp;
cp += safe_itf8_get(cp, endp, &i32);
i32 += (cp-op);
//fprintf(stderr, "remaining %d bytes\n", i32);
cram_block_set_size(n_blk, cram_block_get_size(n_blk)-2);
cram_block_append(n_blk, op, i32);
cram_block_update_size(n_blk);
new_size = cram_block_size(n_blk);
//fprintf(stderr, "size %d -> %d\n", old_size, new_size);
// Now we've constructedthe updated compression header,
// amend the container too (it may have changed size).
int32_t *landmarks, num_landmarks;
landmarks = cram_container_get_landmarks(c, &num_landmarks);
if (old_size != new_size) {
int diff = new_size - old_size, j;
for (j = 0; j < num_landmarks; j++)
landmarks[j] += diff;
//cram_container_set_landmarks(c, num_landmarks, landmarks);
cram_container_set_length(c, cram_container_get_length(c) + diff);
}
// Finally write it all out; container, compression header,
// and then all the remaining slice blocks.
if (cram_write_container(out, c) != 0)
return -2;
cram_write_block(out, n_blk);
cram_free_block(o_blk);
cram_free_block(n_blk);
// Container num_blocks can be invalid, due to a bug.
// Instead we iterate in slice context instead.
return cram_copy_slice(in, out, num_landmarks);
}
/*
* CRAM files don't store the RG:Z:ID per read in the aux field.
* Instead they have a numerical data series (RG) to point each read
* back to the Nth @RG line in the file. This means that we may need
* to edit the RG data series (if the files were produced from
* "samtools split" for example).
*
* The encoding method is stored in the compression header. Typical
* examples:
*
* RG => EXTERNAL {18} # Block content-id 18 holds RG values
* # as a series of ITF8 encoded values
*
* RG => HUFFMAN {1, 255, 255, 255, 255, 255, 1, 0}
* # One RG value #-1. (No RG)
*
* RG => HUFFMAN {1, 0, 1, 0} # One RG value #0 (always first RG)
*
* RG => HUFFMAN {2, 0, 1, 2, 1, 1}
* # Two RG values, #0 and #1, written
* # to the CORE block and possibly
* # mixed with other data series.
*
* A single value can (but may not be) implemented as a zero bit
* huffman code. In this situation we can change the meta-data in the
* compression header to renumber an RG value..
*/
int cram_cat(int nfn, char * const *fn, const bam_hdr_t *h, const char* outcram)
{
samFile *out;
cram_fd *out_c;
int i, vers_maj, vers_min;
khash_s2i *rg2id = NULL;
bam_hdr_t *new_h = NULL;
/* Check consistent versioning and compatible headers */
if (!(new_h = cram_cat_check_hdr(nfn, fn, h, &rg2id, &vers_maj, &vers_min)))
return -1;
/* Open the file with cram_vers */
char vers[100];
sprintf(vers, "%d.%d", vers_maj, vers_min);
out = sam_open(outcram, "wc");
if (out == 0) {
fprintf(stderr, "[%s] ERROR: fail to open output file '%s'.\n", __func__, outcram);
return 1;
}
out_c = out->fp.cram;
cram_set_option(out_c, CRAM_OPT_VERSION, vers);
//fprintf(stderr, "Creating cram vers %s\n", vers);
cram_fd_set_header(out_c, sam_hdr_parse_(new_h->text, new_h->l_text)); // needed?
sam_hdr_write(out, new_h);
for (i = 0; i < nfn; ++i) {
samFile *in;
cram_fd *in_c;
cram_container *c;
bam_hdr_t *old;
int new_rg = -1;
in = sam_open(fn[i], "rc");
if (in == 0) {
fprintf(stderr, "[%s] ERROR: fail to open file '%s'.\n", __func__, fn[i]);
return -1;
}
in_c = in->fp.cram;
old = sam_hdr_read(in);
khash_s2i *rg2id_in = hash_rg(old);
// Compute RG mapping if suitable for changing.
if (rg2id_in->n_id == 1) {
int _;
new_rg = hash_s2i_inc(rg2id, rg2id_in->id[0], NULL, &_);
} else {
new_rg = 0;
}
hash_s2i_free(rg2id_in);
// Copy contains and blocks within them
while ((c = cram_read_container(in_c))) {
cram_block *blk;
if (cram_container_is_empty(in_c)) {
if (cram_write_container(out_c, c) != 0)
return -1;
// Container compression header
if (!(blk = cram_read_block(in_c)))
return -1;
if (cram_write_block(out_c, blk) != 0) {
cram_free_block(blk);
return -1;
}
cram_free_block(blk);
cram_free_container(c);
continue;
}
// If we have just one RG key and new_rg != 0 then
// we need to edit the compression header. IF WE CAN.
if (new_rg) {
int zero = 0;
//fprintf(stderr, "Transcode RG %d to %d\n", 0, new_rg);
cram_transcode_rg(in_c, out_c, c, 1, &zero, &new_rg);
} else {
int32_t num_slices;
// Not switching rg so do the usual read/write loop
if (cram_write_container(out_c, c) != 0)
return -1;
// Container compression header
if (!(blk = cram_read_block(in_c)))
return -1;
if (cram_write_block(out_c, blk) != 0) {
cram_free_block(blk);
return -1;
}
cram_free_block(blk);
// Container num_blocks can be invalid, due to a bug.
// Instead we iterate in slice context instead.
(void)cram_container_get_landmarks(c, &num_slices);
cram_copy_slice(in_c, out_c, num_slices);
}
cram_free_container(c);
}
bam_hdr_destroy(old);
sam_close(in);
}
sam_close(out);
hash_s2i_free(rg2id);
bam_hdr_destroy(new_h);
return 0;
}
