/* * Reads a file and outputs a new CRAM file to stdout with 'h' * replaced as the header. No checks are made to the validity. * * FIXME: error checking */ int cram_reheader(cram_fd *in, bam_hdr_t *h, const char *arg_list, int add_PG) { htsFile *h_out = hts_open("-", "wc"); cram_fd *out = h_out->fp.cram; cram_container *c = NULL; int ret = -1; // Attempt to fill out a cram->refs[] array from @SQ headers cram_fd_set_header(out, sam_hdr_parse_(h->text, h->l_text)); if (add_PG) { if (sam_hdr_add_PG(cram_fd_get_header(out), "samtools", "VN", samtools_version(), arg_list ? "CL": NULL, arg_list ? arg_list : NULL, NULL) != 0) goto err; // Covert back to bam_hdr_t struct free(h->text); h->text = strdup(sam_hdr_str(cram_fd_get_header(out))); h->l_text = sam_hdr_length(cram_fd_get_header(out)); if (!h->text) goto err; } if (sam_hdr_write(h_out, h) != 0) goto err; cram_set_option(out, CRAM_OPT_REFERENCE, NULL); while ((c = cram_read_container(in))) { int32_t i, num_blocks = cram_container_get_num_blocks(c); if (cram_write_container(out, c) != 0) goto err; for (i = 0; i < num_blocks; i++) { cram_block *blk = cram_read_block(in); if (!blk || cram_write_block(out, blk) != 0) { if (blk) cram_free_block(blk); goto err; } cram_free_block(blk); } cram_free_container(c); } ret = 0; err: if (hts_close(h_out) != 0) ret = -1; return ret; }
/* * Reads a file and outputs a new BAM file to fd with 'h' replaced as * the header. No checks are made to the validity. */ int bam_reheader(BGZF *in, bam_hdr_t *h, int fd, const char *arg_list, int add_PG) { BGZF *fp; ssize_t len; uint8_t *buf; if (in->is_write) return -1; buf = malloc(BUF_SIZE); if (bam_hdr_read(in) == NULL) { fprintf(stderr, "Couldn't read header\n"); free(buf); return -1; } fp = bgzf_fdopen(fd, "w"); if (add_PG) { // Around the houses, but it'll do until we can manipulate bam_hdr_t natively. SAM_hdr *sh = sam_hdr_parse_(h->text, h->l_text); if (sam_hdr_add_PG(sh, "samtools", "VN", samtools_version(), arg_list ? "CL": NULL, arg_list ? arg_list : NULL, NULL) != 0) return -1; free(h->text); h->text = strdup(sam_hdr_str(sh)); h->l_text = sam_hdr_length(sh); if (!h->text) return -1; sam_hdr_free(sh); } bam_hdr_write(fp, h); if (in->block_offset < in->block_length) { bgzf_write(fp, in->uncompressed_block + in->block_offset, in->block_length - in->block_offset); bgzf_flush(fp); } while ((len = bgzf_raw_read(in, buf, BUF_SIZE)) > 0) bgzf_raw_write(fp, buf, len); free(buf); fp->block_offset = in->block_offset = 0; bgzf_close(fp); return 0; }
int main(int argc, char **argv) { cram_fd *fd; bam_file_t *bfd; bam_seq_t *bam = NULL; char mode[4] = {'w', '\0', '\0', '\0'}; char *prefix = NULL; int decode_md = 0; int C; int start, end; char ref_name[1024] = {0}, *arg_list, *ref_fn = NULL; int embed_ref = 0; while ((C = getopt(argc, argv, "bu0123456789mp:hr:R:X")) != -1) { switch (C) { case 'b': mode[1] = 'b'; break; case 'u': mode[2] = '0'; break; case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': mode[2] = C; break; case 'm': decode_md = 1; break; case 'p': prefix = optarg; break; case 'h': usage(stdout); return 0; case 'r': ref_fn = optarg; break; case 'X': embed_ref = 1; break; case 'R': { char *cp = strchr(optarg, ':'); if (cp) { *cp = 0; switch (sscanf(cp+1, "%d-%d", &start, &end)) { case 1: end = start; break; case 2: break; default: fprintf(stderr, "Malformed range format\n"); return 1; } } else { start = INT_MIN; end = INT_MAX; } strncpy(ref_name, optarg, 1023); break; } case '?': fprintf(stderr, "Unrecognised option: -%c\n", optopt); usage(stderr); return 1; } } if (argc - optind != 1 && argc - optind != 2) { usage(stderr); return 1; } if (argc - optind == 1) { if (NULL == (bfd = bam_open("-", mode))) { fprintf(stderr, "Failed to open SAM/BAM output\n."); return 1; } } else { if (NULL == (bfd = bam_open(argv[optind+1], mode))) { fprintf(stderr, "Failed to open SAM/BAM output\n."); perror(argv[optind+1]); return 1; } } if (NULL == (fd = cram_open(argv[optind], "rb"))) { fprintf(stderr, "Error opening CRAM file '%s'.\n", argv[optind]); return 1; } if (*ref_name != 0) cram_index_load(fd, argv[optind]); if (prefix) cram_set_option(fd, CRAM_OPT_PREFIX, prefix); if (decode_md) cram_set_option(fd, CRAM_OPT_DECODE_MD, decode_md); if (embed_ref) cram_set_option(fd, CRAM_OPT_EMBED_REF, embed_ref); /* Find and load reference */ cram_load_reference(fd, ref_fn); if (!fd->refs && !embed_ref) { fprintf(stderr, "Unable to find an appropriate reference.\n" "Please specify a valid reference with -r ref.fa option.\n"); return 1; } bfd->header = fd->header; if (*ref_name != 0) { cram_range r; int refid = sam_hdr_name2ref(fd->header, ref_name); if (refid == -1 && *ref_name != '*') { fprintf(stderr, "Unknown reference name '%s'\n", ref_name); return 1; } r.refid = refid; r.start = start; r.end = end; cram_set_option(fd, CRAM_OPT_RANGE, &r); } /* SAM Header */ if (!(arg_list = stringify_argv(argc, argv))) return 1; sam_hdr_add_PG(bfd->header, "cram_to_sam", "VN", PACKAGE_VERSION, "CL", arg_list, NULL); free(arg_list); bam_write_header(bfd); while (cram_get_bam_seq(fd, &bam) == 0) { bam_put_seq(bfd, bam); } if (!cram_eof(fd)) { fprintf(stderr, "Error while reading file\n"); return 1; } cram_close(fd); bfd->header = NULL; bam_close(bfd); free(bam); return 0; }
int main(int argc, char **argv) { scram_fd *in, *out; bam_seq_t *s; char imode[10], *in_f = "", omode[10], *out_f = ""; int level = '\0'; // nul terminate string => auto level int c, verbose = 0; int s_opt = 0, S_opt = 0, embed_ref = 0, ignore_md5 = 0, decode_md = 0; char *ref_fn = NULL; int start, end, multi_seq = -1, no_ref = 0; int use_bz2 = 0, use_arith = 0, use_lzma = 0; char ref_name[1024] = {0}; refs_t *refs; int nthreads = 1; t_pool *p = NULL; int max_reads = -1; enum quality_binning binning = BINNING_NONE; /* Parse command line arguments */ while ((c = getopt(argc, argv, "u0123456789hvs:S:V:r:xXeI:O:R:!MmjJZt:BN:")) != -1) { switch (c) { case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': level = c; break; case 'u': level = '0'; break; case 'h': usage(stdout); return 0; case 'v': verbose++; break; case 's': s_opt = atoi(optarg); break; case 'S': S_opt = atoi(optarg); break; case 'm': decode_md = 1; break; case 'V': if (cram_set_option(NULL, CRAM_OPT_VERSION, optarg)) return 1; break; case 'r': ref_fn = optarg; break; case 'X': fprintf(stderr, "-X is deprecated in favour of -e.\n"); case 'e': embed_ref = 1; break; case 'x': no_ref = 1; break; case 'I': in_f = parse_format(optarg); break; case 'O': out_f = parse_format(optarg); break; case 'R': { char *cp = strchr(optarg, ':'); if (cp) { *cp = 0; switch (sscanf(cp+1, "%d-%d", &start, &end)) { case 1: end = start; break; case 2: break; default: fprintf(stderr, "Malformed range format\n"); return 1; } } else { start = INT_MIN; end = INT_MAX; } strncpy(ref_name, optarg, 1023); break; } case '!': ignore_md5 = 1; break; case 'M': multi_seq = 1; break; case 'j': #ifdef HAVE_LIBBZ2 use_bz2 = 1; #else fprintf(stderr, "Warning: bzip2 support is not compiled into this" " version.\nPlease recompile.\n"); #endif break; case 'J': use_arith = 1; break; case 'Z': #ifdef HAVE_LIBLZMA use_lzma = 1; #else fprintf(stderr, "Warning: lzma support is not compiled into this" " version.\nPlease recompile.\n"); #endif break; case 't': nthreads = atoi(optarg); if (nthreads < 1) { fprintf(stderr, "Number of threads needs to be >= 1\n"); return 1; } break; case 'B': binning = BINNING_ILLUMINA; break; case 'N': // For debugging max_reads = atoi(optarg); break; case '?': fprintf(stderr, "Unrecognised option: -%c\n", optopt); usage(stderr); return 1; } } if (argc - optind > 2) { fprintf(stderr, "Usage: scramble [input_file [output_file]]\n"); return 1; } /* Open up input and output files */ sprintf(imode, "r%s%c", in_f, level); if (argc - optind > 0) { if (*in_f == 0) sprintf(imode, "r%s%c", detect_format(argv[optind]), level); if (!(in = scram_open(argv[optind], imode))) { fprintf(stderr, "Failed to open file %s\n", argv[optind]); return 1; } } else { if (!(in = scram_open("-", imode))) { fprintf(stderr, "Failed to open file %s\n", argv[optind]); return 1; } } if (!in->is_bam && ref_fn) { cram_load_reference(in->c, ref_fn); if (!in->c->refs && !embed_ref) { fprintf(stderr, "Unable to find an appropriate reference.\n" "Please specify a valid reference with " "-r ref.fa option.\n"); return 1; } } sprintf(omode, "w%s%c", out_f, level); if (argc - optind > 1) { if (*out_f == 0) sprintf(omode, "w%s%c", detect_format(argv[optind+1]), level); if (!(out = scram_open(argv[optind+1], omode))) { fprintf(stderr, "Failed to open file %s\n", argv[optind+1]); return 1; } } else { if (!(out = scram_open("-", omode))) { fprintf(stderr, "Failed to open file %s\n", argv[optind+1]); return 1; } } /* Set any format specific options */ scram_set_refs(out, refs = scram_get_refs(in)); scram_set_option(out, CRAM_OPT_VERBOSITY, verbose); if (s_opt) if (scram_set_option(out, CRAM_OPT_SEQS_PER_SLICE, s_opt)) return 1; if (S_opt) if (scram_set_option(out, CRAM_OPT_SLICES_PER_CONTAINER, S_opt)) return 1; if (embed_ref) if (scram_set_option(out, CRAM_OPT_EMBED_REF, embed_ref)) return 1; if (use_bz2) if (scram_set_option(out, CRAM_OPT_USE_BZIP2, use_bz2)) return 1; if (use_arith) if (scram_set_option(out, CRAM_OPT_USE_ARITH, use_arith)) return 1; if (use_lzma) if (scram_set_option(out, CRAM_OPT_USE_LZMA, use_lzma)) return 1; if (binning != BINNING_NONE) if (scram_set_option(out, CRAM_OPT_BINNING, binning)) return 1; if (no_ref) if (scram_set_option(out, CRAM_OPT_NO_REF, no_ref)) return 1; if (multi_seq) if (scram_set_option(out, CRAM_OPT_MULTI_SEQ_PER_SLICE, multi_seq)) return 1; if (decode_md) { if (no_ref) { fprintf(stderr, "Cannot use -m in conjunction with -x.\n"); return 1; } if (scram_set_option(in, CRAM_OPT_DECODE_MD, decode_md)) return 1; } if (nthreads > 1) { if (NULL == (p = t_pool_init(nthreads*2, nthreads))) return 1; if (scram_set_option(in, CRAM_OPT_THREAD_POOL, p)) return 1; if (scram_set_option(out, CRAM_OPT_THREAD_POOL, p)) return 1; } if (ignore_md5) if (scram_set_option(in, CRAM_OPT_IGNORE_MD5, ignore_md5)) return 1; /* Copy header and refs from in to out, for writing purposes */ scram_set_header(out, scram_get_header(in)); // Needs doing after loading the header. if (ref_fn) { if (scram_set_option(out, CRAM_OPT_REFERENCE, ref_fn)) return 1; } else { // Attempt to fill out a cram->refs[] array from @SQ headers scram_set_option(out, CRAM_OPT_REFERENCE, NULL); } if (scram_get_header(out)) { char *arg_list = stringify_argv(argc, argv); if (!arg_list) return 1; if (sam_hdr_add_PG(scram_get_header(out), "scramble", "VN", PACKAGE_VERSION, "CL", arg_list, NULL)) return 1; if (scram_write_header(out)) return 1; free(arg_list); } /* Support for sub-range queries, currently implemented for CRAM only */ if (*ref_name != 0) { cram_range r; int refid; if (in->is_bam) { fprintf(stderr, "Currently the -R option is only implemented for CRAM indices\n"); return 1; } cram_index_load(in->c, argv[optind]); refid = sam_hdr_name2ref(in->c->header, ref_name); if (refid == -1 && *ref_name != '*') { fprintf(stderr, "Unknown reference name '%s'\n", ref_name); return 1; } r.refid = refid; r.start = start; r.end = end; if (scram_set_option(in, CRAM_OPT_RANGE, &r)) return 1; } /* Do the actual file format conversion */ s = NULL; while (scram_get_seq(in, &s) >= 0) { if (-1 == scram_put_seq(out, s)) { fprintf(stderr, "Failed to encode sequence\n"); return 1; } if (max_reads >= 0) if (--max_reads == 0) break; } if (max_reads == -1) { switch(scram_eof(in)) { case 0: fprintf(stderr, "Failed to decode sequence\n"); return 1; case 2: fprintf(stderr, "Warning: no end-of-file block identified. " "File may be truncated.\n"); break; case 1: default: // expected case break; } } /* Finally tidy up and close files */ if (scram_close(in)) return 1; if (scram_close(out)) return 1; if (p) t_pool_destroy(p, 0); if (s) free(s); return 0; }
int main(int argc, char **argv) { cram_fd *out; bam_file_t *in; bam_seq_t *s = NULL; char *out_fn; int level = '\0'; // nul terminate string => auto level char out_mode[4]; int c, verbose = 0; int s_opt = 0, S_opt = 0, embed_ref = 0; char *arg_list, *ref_fn = NULL; while ((c = getopt(argc, argv, "u0123456789hvs:S:V:r:X")) != -1) { switch (c) { case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': level = c; break; case 'u': level = '0'; break; case 'h': usage(stdout); return 0; case 'v': verbose++; break; case 's': s_opt = atoi(optarg); break; case 'S': S_opt = atoi(optarg); break; case 'V': cram_set_option(NULL, CRAM_OPT_VERSION, optarg); break; case 'r': ref_fn = optarg; break; case 'X': embed_ref = 1; break; case '?': fprintf(stderr, "Unrecognised option: -%c\n", optopt); usage(stderr); return 1; } } if (argc - optind != 1 && argc - optind != 2) { usage(stderr); return 1; } /* opening */ if (NULL == (in = bam_open(argv[optind], "rb"))) { perror(argv[optind]); return 1; } out_fn = argc - optind == 2 ? argv[optind+1] : "-"; sprintf(out_mode, "wb%c", level); if (NULL == (out = cram_open(out_fn, out_mode))) { fprintf(stderr, "Error opening CRAM file '%s'.\n", out_fn); return 1; } /* SAM Header */ if (!(arg_list = stringify_argv(argc, argv))) return 1; sam_hdr_add_PG(in->header, "sam_to_cram", "VN", PACKAGE_VERSION, "CL", arg_list, NULL); free(arg_list); /* Find and load reference */ if (!ref_fn) { SAM_hdr_type *ty = sam_hdr_find(in->header, "SQ", NULL, NULL); if (ty) { SAM_hdr_tag *tag; if ((tag = sam_hdr_find_key(in->header, ty, "UR", NULL))) { ref_fn = tag->str + 3; if (strncmp(ref_fn, "file:", 5) == 0) ref_fn += 5; } } } out->header = in->header; if (ref_fn) cram_load_reference(out, ref_fn); if (!out->refs) { fprintf(stderr, "Unable to open reference.\n" "Please specify a valid reference with -r ref.fa option.\n"); return 1; } refs2id(out->refs, out->header); if (-1 == cram_write_SAM_hdr(out, in->header)) return 1; cram_set_option(out, CRAM_OPT_VERBOSITY, verbose); if (s_opt) cram_set_option(out, CRAM_OPT_SEQS_PER_SLICE, s_opt); if (S_opt) cram_set_option(out, CRAM_OPT_SLICES_PER_CONTAINER, S_opt); if (embed_ref) cram_set_option(out, CRAM_OPT_EMBED_REF, embed_ref); /* Sequence iterators */ while (bam_get_seq(in, &s) > 0) { if (-1 == cram_put_bam_seq(out, s)) { fprintf(stderr, "Failed in cram_put_bam_seq()\n"); return 1; } } bam_close(in); out->header = NULL; // freed by bam_close() if (-1 == cram_close(out)) { fprintf(stderr, "Failed in cram_close()\n"); return 1; } if (s) free(s); return 0; }
/* * Reads a version 3 CRAM file and replaces the header in-place, * provided the header is small enough to fit without growing the * entire file. * * Version 3 format has a SAM header held as an (optionally) * compressed block within the header container. Additional * uncompressed blocks or simply unallocated space (the difference * between total block sizes and the container size) are used to * provide room for growth or contraction of the compressed header. * * Returns 0 on success; * -1 on general failure; * -2 on failure due to insufficient size */ int cram_reheader_inplace3(cram_fd *fd, const bam_hdr_t *h, const char *arg_list, int add_PG) { cram_container *c = NULL; cram_block *b = NULL; SAM_hdr *hdr = NULL; off_t start, sz, end; int container_sz, max_container_sz; char *buf = NULL; int ret = -1; if (cram_major_vers(fd) < 2 || cram_major_vers(fd) > 3) { fprintf(stderr, "[%s] unsupported CRAM version %d\n", __func__, cram_major_vers(fd)); goto err; } if (!(hdr = sam_hdr_parse_(h->text, h->l_text))) goto err; if (add_PG && sam_hdr_add_PG(hdr, "samtools", "VN", samtools_version(), arg_list ? "CL": NULL, arg_list ? arg_list : NULL, NULL)) goto err; int header_len = sam_hdr_length(hdr); /* Fix M5 strings? Maybe out of scope for this tool */ // Find current size of SAM header block if ((start = hseek(cram_fd_get_fp(fd), 26, SEEK_SET)) != 26) goto err; if (!(c = cram_read_container(fd))) goto err; // +5 allows num_landmarks to increase from 0 to 1 (Cramtools) max_container_sz = cram_container_size(c)+5; sz = htell(cram_fd_get_fp(fd)) + cram_container_get_length(c) - start; end = htell(cram_fd_get_fp(fd)) + cram_container_get_length(c); // We force 1 block instead of (optionally) 2. C CRAM // implementations for v3 were writing 1 compressed block followed // by 1 uncompressed block. However this is tricky to deal with // as changing block sizes can mean the block header also changes // size due to itf8 and variable size integers. // // If we had 1 block, this doesn't change anything. // If we had 2 blocks, the new container header will be smaller by // 1+ bytes, requiring the cram_container_get_length(c) to be larger in value. // However this is an int32 instead of itf8 so the container // header structure stays the same size. This means we can always // reduce the number of blocks without running into size problems. cram_container_set_num_blocks(c, 1); int32_t *landmark; int32_t num_landmarks; landmark = cram_container_get_landmarks(c, &num_landmarks); if (num_landmarks && landmark) { num_landmarks = 1; landmark[0] = 0; } else { num_landmarks = 0; } cram_container_set_landmarks(c, num_landmarks, landmark); buf = malloc(max_container_sz); container_sz = max_container_sz; if (cram_store_container(fd, c, buf, &container_sz) != 0) goto err; if (!buf) goto err; // Proposed new length, but changing cram_container_get_length(c) may change the // container_sz and thus the remainder (cram_container_get_length(c) itself). cram_container_set_length(c, sz - container_sz); int old_container_sz = container_sz; container_sz = max_container_sz; if (cram_store_container(fd, c, buf, &container_sz) != 0) goto err; if (old_container_sz != container_sz) { fprintf(stderr, "Quirk of fate makes this troublesome! " "Please use non-inplace version.\n"); goto err; } // Version 3.0 supports compressed header b = cram_new_block(FILE_HEADER, 0); int32_put_blk(b, header_len); cram_block_append(b, sam_hdr_str(hdr), header_len); cram_block_update_size(b); cram_compress_block(fd, b, NULL, -1, -1); if (hseek(cram_fd_get_fp(fd), 26, SEEK_SET) != 26) goto err; if (cram_block_size(b) > cram_container_get_length(c)) { fprintf(stderr, "New header will not fit. Use non-inplace version" " (%d > %d)\n", (int)cram_block_size(b), cram_container_get_length(c)); ret = -2; goto err; } if (cram_write_container(fd, c) == -1) goto err; if (cram_write_block(fd, b) == -1) goto err; // Blank out the remainder int rsz = end - htell(cram_fd_get_fp(fd)); assert(rsz >= 0); if (rsz) { char *rem = calloc(1, rsz); ret = hwrite(cram_fd_get_fp(fd), rem, rsz) == rsz ? 0 : -1; free(rem); } err: if (c) cram_free_container(c); if (buf) free(buf); if (b) cram_free_block(b); if (hdr) sam_hdr_free(hdr); return ret; }
/* * Reads a version 2 CRAM file and replaces the header in-place, * provided the header is small enough to fit without growing the * entire file. * * Version 2 format has an uncompressed SAM header with multiple nul * termination bytes to permit inline header editing. * * Returns 0 on success; * -1 on general failure; * -2 on failure due to insufficient size */ int cram_reheader_inplace2(cram_fd *fd, const bam_hdr_t *h, const char *arg_list, int add_PG) { cram_container *c = NULL; cram_block *b = NULL; SAM_hdr *hdr = NULL; off_t start; int ret = -1; if (cram_major_vers(fd) < 2 || cram_major_vers(fd) > 3) { fprintf(stderr, "[%s] unsupported CRAM version %d\n", __func__, cram_major_vers(fd)); goto err; } if (!(hdr = sam_hdr_parse_(h->text, h->l_text))) goto err; if (add_PG && sam_hdr_add_PG(hdr, "samtools", "VN", samtools_version(), arg_list ? "CL": NULL, arg_list ? arg_list : NULL, NULL)) goto err; int header_len = sam_hdr_length(hdr); /* Fix M5 strings? Maybe out of scope for this tool */ // Load the existing header if ((start = hseek(cram_fd_get_fp(fd), 26, SEEK_SET)) != 26) goto err; if (!(c = cram_read_container(fd))) goto err; // Version 2.1 has a single uncompressed block which is nul // terminated with many nuls to permit growth. // // So load old block and keep all contents identical bar the // header text itself if (!(b = cram_read_block(fd))) goto err; if (cram_block_get_uncomp_size(b) < header_len+4) { fprintf(stderr, "New header will not fit. Use non-inplace version (%d > %d)\n", header_len+4, cram_block_get_uncomp_size(b)); ret = -2; goto err; } cram_block_set_offset(b, 0); // rewind block int32_put_blk(b, header_len); cram_block_append(b, sam_hdr_str(hdr), header_len); // Zero the remaining block memset(cram_block_get_data(b)+cram_block_get_offset(b), 0, cram_block_get_uncomp_size(b) - cram_block_get_offset(b)); // Make sure all sizes and byte-offsets are consistent after memset cram_block_set_offset(b, cram_block_get_uncomp_size(b)); cram_block_set_comp_size(b, cram_block_get_uncomp_size(b)); if (hseek(cram_fd_get_fp(fd), start, SEEK_SET) != start) goto err; if (cram_write_container(fd, c) == -1) goto err; if (cram_write_block(fd, b) == -1) goto err; ret = 0; err: if (c) cram_free_container(c); if (b) cram_free_block(b); if (hdr) sam_hdr_free(hdr); return ret; }