int samfetch(samfile_t *fp, const bam_index_t *idx, int tid, int beg, int end, void *data, bam_fetch_f func)
{
bam1_t *b = bam_init1();
hts_itr_t *iter = sam_itr_queryi(idx, tid, beg, end);
int ret;
while ((ret = sam_itr_next(fp->file, iter, b)) >= 0) func(b, data);
hts_itr_destroy(iter);
bam_destroy1(b);
return (ret == -1)? 0 : ret;
}
void
bam_streamer::
set_new_region(const int ref, const int beg, const int end)
{
if (nullptr != _hitr) hts_itr_destroy(_hitr);
_load_index();
if (ref < 0)
{
log_os << "ERROR: Invalid region specified for BAM/CRAM file: " << name() << "\n";
exit(EXIT_FAILURE);
}
_hitr = sam_itr_queryi(_hidx,ref,beg,end);
_is_region = true;
_region.clear();
_is_record_set = false;
_record_no = 0;
}
int main_bedcov(int argc, char *argv[])
{
gzFile fp;
kstring_t str;
kstream_t *ks;
hts_idx_t **idx;
aux_t **aux;
int *n_plp, dret, i, n, c, min_mapQ = 0;
int64_t *cnt;
const bam_pileup1_t **plp;
int usage = 0;
sam_global_args ga = SAM_GLOBAL_ARGS_INIT;
static const struct option lopts[] = {
SAM_OPT_GLOBAL_OPTIONS('-', 0, '-', '-', 0),
{ NULL, 0, NULL, 0 }
};
while ((c = getopt_long(argc, argv, "Q:", lopts, NULL)) >= 0) {
switch (c) {
case 'Q': min_mapQ = atoi(optarg); break;
default: if (parse_sam_global_opt(c, optarg, lopts, &ga) == 0) break;
/* else fall-through */
case '?': usage = 1; break;
}
if (usage) break;
}
if (usage || optind + 2 > argc) {
fprintf(pysam_stderr, "Usage: samtools bedcov [options] <in.bed> <in1.bam> [...]\n\n");
fprintf(pysam_stderr, " -Q INT Only count bases of at least INT quality [0]\n");
sam_global_opt_help(pysam_stderr, "-.--.");
return 1;
}
memset(&str, 0, sizeof(kstring_t));
n = argc - optind - 1;
aux = calloc(n, sizeof(aux_t*));
idx = calloc(n, sizeof(hts_idx_t*));
for (i = 0; i < n; ++i) {
aux[i] = calloc(1, sizeof(aux_t));
aux[i]->min_mapQ = min_mapQ;
aux[i]->fp = sam_open_format(argv[i+optind+1], "r", &ga.in);
if (aux[i]->fp)
idx[i] = sam_index_load(aux[i]->fp, argv[i+optind+1]);
if (aux[i]->fp == 0 || idx[i] == 0) {
fprintf(pysam_stderr, "ERROR: fail to open index BAM file '%s'\n", argv[i+optind+1]);
return 2;
}
// TODO bgzf_set_cache_size(aux[i]->fp, 20);
aux[i]->header = sam_hdr_read(aux[i]->fp);
if (aux[i]->header == NULL) {
fprintf(pysam_stderr, "ERROR: failed to read header for '%s'\n",
argv[i+optind+1]);
return 2;
}
}
cnt = calloc(n, 8);
fp = gzopen(argv[optind], "rb");
ks = ks_init(fp);
n_plp = calloc(n, sizeof(int));
plp = calloc(n, sizeof(bam_pileup1_t*));
while (ks_getuntil(ks, KS_SEP_LINE, &str, &dret) >= 0) {
char *p, *q;
int tid, beg, end, pos;
bam_mplp_t mplp;
for (p = q = str.s; *p && *p != '\t'; ++p);
if (*p != '\t') goto bed_error;
*p = 0; tid = bam_name2id(aux[0]->header, q); *p = '\t';
if (tid < 0) goto bed_error;
for (q = p = p + 1; isdigit(*p); ++p);
if (*p != '\t') goto bed_error;
*p = 0; beg = atoi(q); *p = '\t';
for (q = p = p + 1; isdigit(*p); ++p);
if (*p == '\t' || *p == 0) {
int c = *p;
*p = 0; end = atoi(q); *p = c;
} else goto bed_error;
for (i = 0; i < n; ++i) {
if (aux[i]->iter) hts_itr_destroy(aux[i]->iter);
aux[i]->iter = sam_itr_queryi(idx[i], tid, beg, end);
}
mplp = bam_mplp_init(n, read_bam, (void**)aux);
bam_mplp_set_maxcnt(mplp, 64000);
memset(cnt, 0, 8 * n);
while (bam_mplp_auto(mplp, &tid, &pos, n_plp, plp) > 0)
if (pos >= beg && pos < end)
for (i = 0; i < n; ++i) cnt[i] += n_plp[i];
for (i = 0; i < n; ++i) {
kputc('\t', &str);
kputl(cnt[i], &str);
}
fputs(str.s, pysam_stdout) & fputc('\n', pysam_stdout);
bam_mplp_destroy(mplp);
continue;
bed_error:
fprintf(pysam_stderr, "Errors in BED line '%s'\n", str.s);
}
free(n_plp); free(plp);
ks_destroy(ks);
gzclose(fp);
free(cnt);
for (i = 0; i < n; ++i) {
if (aux[i]->iter) hts_itr_destroy(aux[i]->iter);
hts_idx_destroy(idx[i]);
bam_hdr_destroy(aux[i]->header);
sam_close(aux[i]->fp);
free(aux[i]);
}
free(aux); free(idx);
free(str.s);
sam_global_args_free(&ga);
return 0;
}
int main_bedcov(int argc, char *argv[])
{
gzFile fp;
kstring_t str;
kstream_t *ks;
hts_idx_t **idx;
aux_t **aux;
int *n_plp, dret, i, n, c, min_mapQ = 0;
int64_t *cnt;
const bam_pileup1_t **plp;
while ((c = getopt(argc, argv, "Q:")) >= 0) {
switch (c) {
case 'Q': min_mapQ = atoi(optarg); break;
}
}
if (optind + 2 > argc) {
fprintf(stderr, "Usage: samtools bedcov <in.bed> <in1.bam> [...]\n");
return 1;
}
memset(&str, 0, sizeof(kstring_t));
n = argc - optind - 1;
aux = calloc(n, sizeof(aux_t*));
idx = calloc(n, sizeof(hts_idx_t*));
for (i = 0; i < n; ++i) {
aux[i] = calloc(1, sizeof(aux_t));
aux[i]->min_mapQ = min_mapQ;
aux[i]->fp = sam_open(argv[i+optind+1], "r");
idx[i] = sam_index_load(aux[i]->fp, argv[i+optind+1]);
if (aux[i]->fp == 0 || idx[i] == 0) {
fprintf(stderr, "ERROR: fail to open index BAM file '%s'\n", argv[i+optind+1]);
return 2;
}
// TODO bgzf_set_cache_size(aux[i]->fp, 20);
aux[i]->header = sam_hdr_read(aux[i]->fp);
}
cnt = calloc(n, 8);
fp = gzopen(argv[optind], "rb");
ks = ks_init(fp);
n_plp = calloc(n, sizeof(int));
plp = calloc(n, sizeof(bam_pileup1_t*));
while (ks_getuntil(ks, KS_SEP_LINE, &str, &dret) >= 0) {
char *p, *q;
int tid, beg, end, pos;
bam_mplp_t mplp;
for (p = q = str.s; *p && *p != '\t'; ++p);
if (*p != '\t') goto bed_error;
*p = 0; tid = bam_name2id(aux[0]->header, q); *p = '\t';
if (tid < 0) goto bed_error;
for (q = p = p + 1; isdigit(*p); ++p);
if (*p != '\t') goto bed_error;
*p = 0; beg = atoi(q); *p = '\t';
for (q = p = p + 1; isdigit(*p); ++p);
if (*p == '\t' || *p == 0) {
int c = *p;
*p = 0; end = atoi(q); *p = c;
} else goto bed_error;
for (i = 0; i < n; ++i) {
if (aux[i]->iter) hts_itr_destroy(aux[i]->iter);
aux[i]->iter = sam_itr_queryi(idx[i], tid, beg, end);
}
mplp = bam_mplp_init(n, read_bam, (void**)aux);
bam_mplp_set_maxcnt(mplp, 64000);
memset(cnt, 0, 8 * n);
while (bam_mplp_auto(mplp, &tid, &pos, n_plp, plp) > 0)
if (pos >= beg && pos < end)
for (i = 0; i < n; ++i) cnt[i] += n_plp[i];
for (i = 0; i < n; ++i) {
kputc('\t', &str);
kputl(cnt[i], &str);
}
puts(str.s);
bam_mplp_destroy(mplp);
continue;
bed_error:
fprintf(stderr, "Errors in BED line '%s'\n", str.s);
}
free(n_plp); free(plp);
ks_destroy(ks);
gzclose(fp);
free(cnt);
for (i = 0; i < n; ++i) {
if (aux[i]->iter) hts_itr_destroy(aux[i]->iter);
hts_idx_destroy(idx[i]);
bam_hdr_destroy(aux[i]->header);
sam_close(aux[i]->fp);
free(aux[i]);
}
free(aux); free(idx);
free(str.s);
return 0;
}
int bam2depth(const std::string& chromosomeName, const int startPos, const int endPos, const int minBaseQuality, const int minMappingQuality, const std::vector <std::string> & listOfFiles,
std::vector< double > & averageCoveragePerBam
)
{
int i, n, tid, beg, end, pos, *n_plp, baseQ = 0, mapQ = 0;
const bam_pileup1_t **plp;
char *reg = 0; // specified region
//void *bed = 0; // BED data structure
aux_t **data;
bam_mplp_t mplp;
// initialize the auxiliary data structures
n = listOfFiles.size(); // the number of BAMs on the command line
data = (aux_t **)calloc(n, sizeof(void*)); // data[i] for the i-th input
beg = 0; end = 1<<30; tid = -1; // set the default region
beg = startPos;
end = endPos;
for (i = 0; i < n; ++i) {
data[i] = (aux_t*)calloc(1, sizeof(aux_t));
data[i]->fp = sam_open(listOfFiles[i].c_str(), "r"); // open BAM
data[i]->min_mapQ = mapQ; // set the mapQ filter
data[i]->hdr = sam_hdr_read(data[i]->fp); // read the BAM header
tid = bam_name2id(data[i]->hdr, chromosomeName.c_str());
if (tid >= 0) { // if a region is specified and parsed successfully
hts_idx_t *idx = sam_index_load(data[i]->fp, listOfFiles[i].c_str()); // load the index
data[i]->iter = sam_itr_queryi(idx, tid, beg, end); // set the iterator
hts_idx_destroy(idx); // the index is not needed any more; phase out of the memory
}
}
// the core multi-pileup loop
mplp = bam_mplp_init(n, read_bam, (void**)data); // initialization
n_plp = (int*)calloc(n, sizeof(int)); // n_plp[i] is the number of covering reads from the i-th BAM
plp = (const bam_pileup1_t **)calloc(n, sizeof(void*)); // plp[i] points to the array of covering reads (internal in mplp)
std::vector<int> sumOfReadDepths( n, 0);
while (bam_mplp_auto(mplp, &tid, &pos, n_plp, plp) > 0) { // come to the next covered position
if (pos < beg || pos >= end) continue; // out of range; skip
for (i = 0; i < n; ++i) { // base level filters have to go here
int j, m = 0;
for (j = 0; j < n_plp[i]; ++j) {
const bam_pileup1_t *p = plp[i] + j; // DON'T modfity plp[][] unless you really know
if (p->is_del || p->is_refskip) ++m; // having dels or refskips at tid:pos
else if (bam_get_qual(p->b)[p->qpos] < baseQ) ++m; // low base quality
}
sumOfReadDepths[ i ] += n_plp[i] - m;
}
}
averageCoveragePerBam.resize( n );
for (int fileIndex=0; fileIndex< n; fileIndex++ ) {
averageCoveragePerBam[ fileIndex ] = (double)sumOfReadDepths[ fileIndex ] / (end - beg );
}
free(n_plp); free(plp);
bam_mplp_destroy(mplp);
for (i = 0; i < n; ++i) {
bam_hdr_destroy(data[i]->hdr);
sam_close(data[i]->fp);
if (data[i]->iter) hts_itr_destroy(data[i]->iter);
free(data[i]);
}
free(data); free(reg);
return 0;
}
/*!
@abstract Merge multiple sorted BAM.
@param is_by_qname whether to sort by query name
@param out output BAM file name
@param mode sam_open() mode to be used to create the final output file
(overrides level settings from UNCOMP and LEVEL1 flags)
@param headers name of SAM file from which to copy '@' header lines,
or NULL to copy them from the first file to be merged
@param n number of files to be merged
@param fn names of files to be merged
@param flag flags that control how the merge is undertaken
@param reg region to merge
@param n_threads number of threads to use (passed to htslib)
@discussion Padding information may NOT correctly maintained. This
function is NOT thread safe.
*/
int bam_merge_core2(int by_qname, const char *out, const char *mode, const char *headers, int n, char * const *fn, int flag, const char *reg, int n_threads)
{
samFile *fpout, **fp;
heap1_t *heap;
bam_hdr_t *hout = NULL;
int i, j, *RG_len = NULL;
uint64_t idx = 0;
char **RG = NULL;
hts_itr_t **iter = NULL;
bam_hdr_t **hdr = NULL;
trans_tbl_t *translation_tbl = NULL;
// Is there a specified pre-prepared header to use for output?
if (headers) {
samFile* fpheaders = sam_open(headers, "r");
if (fpheaders == NULL) {
const char *message = strerror(errno);
fprintf(pysamerr, "[bam_merge_core] cannot open '%s': %s\n", headers, message);
return -1;
}
hout = sam_hdr_read(fpheaders);
sam_close(fpheaders);
}
g_is_by_qname = by_qname;
fp = (samFile**)calloc(n, sizeof(samFile*));
heap = (heap1_t*)calloc(n, sizeof(heap1_t));
iter = (hts_itr_t**)calloc(n, sizeof(hts_itr_t*));
hdr = (bam_hdr_t**)calloc(n, sizeof(bam_hdr_t*));
translation_tbl = (trans_tbl_t*)calloc(n, sizeof(trans_tbl_t));
// prepare RG tag from file names
if (flag & MERGE_RG) {
RG = (char**)calloc(n, sizeof(char*));
RG_len = (int*)calloc(n, sizeof(int));
for (i = 0; i != n; ++i) {
int l = strlen(fn[i]);
const char *s = fn[i];
if (l > 4 && strcmp(s + l - 4, ".bam") == 0) l -= 4;
for (j = l - 1; j >= 0; --j) if (s[j] == '/') break;
++j; l -= j;
RG[i] = (char*)calloc(l + 1, 1);
RG_len[i] = l;
strncpy(RG[i], s + j, l);
}
}
// open and read the header from each file
for (i = 0; i < n; ++i) {
bam_hdr_t *hin;
fp[i] = sam_open(fn[i], "r");
if (fp[i] == NULL) {
int j;
fprintf(pysamerr, "[bam_merge_core] fail to open file %s\n", fn[i]);
for (j = 0; j < i; ++j) sam_close(fp[j]);
free(fp); free(heap);
// FIXME: possible memory leak
return -1;
}
hin = sam_hdr_read(fp[i]);
if (hout)
trans_tbl_init(hout, hin, translation_tbl+i, flag & MERGE_COMBINE_RG, flag & MERGE_COMBINE_PG);
else {
// As yet, no headers to merge into...
hout = bam_hdr_dup(hin);
// ...so no need to translate header into itself
trans_tbl_init(hout, hin, translation_tbl+i, true, true);
}
// TODO sam_itr_next() doesn't yet work for SAM files,
// so for those keep the headers around for use with sam_read1()
if (hts_get_format(fp[i])->format == sam) hdr[i] = hin;
else { bam_hdr_destroy(hin); hdr[i] = NULL; }
if ((translation_tbl+i)->lost_coord_sort && !by_qname) {
fprintf(pysamerr, "[bam_merge_core] Order of targets in file %s caused coordinate sort to be lost\n", fn[i]);
}
}
// Transform the header into standard form
pretty_header(&hout->text,hout->l_text);
// If we're only merging a specified region move our iters to start at that point
if (reg) {
int* rtrans = rtrans_build(n, hout->n_targets, translation_tbl);
int tid, beg, end;
const char *name_lim = hts_parse_reg(reg, &beg, &end);
char *name = malloc(name_lim - reg + 1);
memcpy(name, reg, name_lim - reg);
name[name_lim - reg] = '\0';
tid = bam_name2id(hout, name);
free(name);
if (tid < 0) {
fprintf(pysamerr, "[%s] Malformated region string or undefined reference name\n", __func__);
return -1;
}
for (i = 0; i < n; ++i) {
hts_idx_t *idx = sam_index_load(fp[i], fn[i]);
// (rtrans[i*n+tid]) Look up what hout tid translates to in input tid space
int mapped_tid = rtrans[i*hout->n_targets+tid];
if (mapped_tid != INT32_MIN) {
iter[i] = sam_itr_queryi(idx, mapped_tid, beg, end);
} else {
iter[i] = sam_itr_queryi(idx, HTS_IDX_NONE, 0, 0);
}
hts_idx_destroy(idx);
if (iter[i] == NULL) break;
}
free(rtrans);
} else {
for (i = 0; i < n; ++i) {
if (hdr[i] == NULL) {
iter[i] = sam_itr_queryi(NULL, HTS_IDX_REST, 0, 0);
if (iter[i] == NULL) break;
}
else iter[i] = NULL;
}
}
if (i < n) {
fprintf(pysamerr, "[%s] Memory allocation failed\n", __func__);
return -1;
}
// Load the first read from each file into the heap
for (i = 0; i < n; ++i) {
heap1_t *h = heap + i;
h->i = i;
h->b = bam_init1();
if ((iter[i]? sam_itr_next(fp[i], iter[i], h->b) : sam_read1(fp[i], hdr[i], h->b)) >= 0) {
bam_translate(h->b, translation_tbl + i);
h->pos = ((uint64_t)h->b->core.tid<<32) | (uint32_t)((int32_t)h->b->core.pos+1)<<1 | bam_is_rev(h->b);
h->idx = idx++;
}
else {
h->pos = HEAP_EMPTY;
bam_destroy1(h->b);
h->b = NULL;
}
}
// Open output file and write header
if ((fpout = sam_open(out, mode)) == 0) {
fprintf(pysamerr, "[%s] fail to create the output file.\n", __func__);
return -1;
}
sam_hdr_write(fpout, hout);
if (!(flag & MERGE_UNCOMP)) hts_set_threads(fpout, n_threads);
// Begin the actual merge
ks_heapmake(heap, n, heap);
while (heap->pos != HEAP_EMPTY) {
bam1_t *b = heap->b;
if (flag & MERGE_RG) {
uint8_t *rg = bam_aux_get(b, "RG");
if (rg) bam_aux_del(b, rg);
bam_aux_append(b, "RG", 'Z', RG_len[heap->i] + 1, (uint8_t*)RG[heap->i]);
}
sam_write1(fpout, hout, b);
if ((j = (iter[heap->i]? sam_itr_next(fp[heap->i], iter[heap->i], b) : sam_read1(fp[heap->i], hdr[heap->i], b))) >= 0) {
bam_translate(b, translation_tbl + heap->i);
heap->pos = ((uint64_t)b->core.tid<<32) | (uint32_t)((int)b->core.pos+1)<<1 | bam_is_rev(b);
heap->idx = idx++;
} else if (j == -1) {
heap->pos = HEAP_EMPTY;
bam_destroy1(heap->b);
heap->b = NULL;
} else fprintf(pysamerr, "[bam_merge_core] '%s' is truncated. Continue anyway.\n", fn[heap->i]);
ks_heapadjust(heap, 0, n, heap);
}
// Clean up and close
if (flag & MERGE_RG) {
for (i = 0; i != n; ++i) free(RG[i]);
free(RG); free(RG_len);
}
for (i = 0; i < n; ++i) {
trans_tbl_destroy(translation_tbl + i);
hts_itr_destroy(iter[i]);
bam_hdr_destroy(hdr[i]);
sam_close(fp[i]);
}
bam_hdr_destroy(hout);
sam_close(fpout);
free(translation_tbl); free(fp); free(heap); free(iter); free(hdr);
return 0;
}
int methyltest_main(int argc, char** argv)
{
parse_methyltest_options(argc, argv);
omp_set_num_threads(opt::num_threads);
Fast5Map name_map(opt::reads_file);
ModelMap models = read_models_fofn(opt::models_fofn, mtest_alphabet);
// Open the BAM and iterate over reads
// load bam file
htsFile* bam_fh = sam_open(opt::bam_file.c_str(), "r");
assert(bam_fh != NULL);
// load bam index file
std::string index_filename = opt::bam_file + ".bai";
hts_idx_t* bam_idx = bam_index_load(index_filename.c_str());
assert(bam_idx != NULL);
// read the bam header
bam_hdr_t* hdr = sam_hdr_read(bam_fh);
// load reference fai file
faidx_t *fai = fai_load(opt::genome_file.c_str());
hts_itr_t* itr;
// If processing a region of the genome, only emit events aligned to this window
int clip_start = -1;
int clip_end = -1;
if(opt::region.empty()) {
// TODO: is this valid?
itr = sam_itr_queryi(bam_idx, HTS_IDX_START, 0, 0);
} else {
fprintf(stderr, "Region: %s\n", opt::region.c_str());
itr = sam_itr_querys(bam_idx, hdr, opt::region.c_str());
hts_parse_reg(opt::region.c_str(), &clip_start, &clip_end);
}
#ifndef H5_HAVE_THREADSAFE
if(opt::num_threads > 1) {
fprintf(stderr, "You enabled multi-threading but you do not have a threadsafe HDF5\n");
fprintf(stderr, "Please recompile nanopolish's built-in libhdf5 or run with -t 1\n");
exit(1);
}
#endif
// Initialize writers
OutputHandles handles;
handles.site_writer = fopen(std::string(opt::bam_file + ".methyltest.sites.bed").c_str(), "w");
handles.read_writer = fopen(std::string(opt::bam_file + ".methyltest.reads.tsv").c_str(), "w");
handles.strand_writer = fopen(std::string(opt::bam_file + ".methyltest.strand.tsv").c_str(), "w");
// Write a header to the reads.tsv file
fprintf(handles.read_writer, "name\tsum_ll_ratio\tn_cpg\tcomplement_model\ttags\n");
// strand header
fprintf(handles.strand_writer, "name\tsum_ll_ratio\tn_cpg\tmodel\n");
// Initialize iteration
std::vector<bam1_t*> records(opt::batch_size, NULL);
for(size_t i = 0; i < records.size(); ++i) {
records[i] = bam_init1();
}
int result;
size_t num_reads_processed = 0;
size_t num_records_buffered = 0;
Progress progress("[methyltest]");
do {
assert(num_records_buffered < records.size());
// read a record into the next slot in the buffer
result = sam_itr_next(bam_fh, itr, records[num_records_buffered]);
num_records_buffered += result >= 0;
// realign if we've hit the max buffer size or reached the end of file
if(num_records_buffered == records.size() || result < 0) {
#pragma omp parallel for
for(size_t i = 0; i < num_records_buffered; ++i) {
bam1_t* record = records[i];
size_t read_idx = num_reads_processed + i;
if( (record->core.flag & BAM_FUNMAP) == 0) {
calculate_methylation_for_read(models, name_map, fai, hdr, record, read_idx, handles);
}
}
num_reads_processed += num_records_buffered;
num_records_buffered = 0;
}
} while(result >= 0);
assert(num_records_buffered == 0);
progress.end();
// cleanup records
for(size_t i = 0; i < records.size(); ++i) {
bam_destroy1(records[i]);
}
// cleanup
fclose(handles.site_writer);
fclose(handles.read_writer);
fclose(handles.strand_writer);
sam_itr_destroy(itr);
bam_hdr_destroy(hdr);
fai_destroy(fai);
sam_close(bam_fh);
hts_idx_destroy(bam_idx);
return EXIT_SUCCESS;
}
int calcCoverage(char *fName, Slice *slice, htsFile *in, hts_idx_t *idx, int flags) {
int ref;
int begRange;
int endRange;
char region[1024];
char region_name[512];
if (Slice_getChrStart(slice) != 1) {
fprintf(stderr, "Currently only allow a slice start position of 1\n");
return 1;
}
if (flags & M_UCSC_NAMING) {
sprintf(region,"chr%s", Slice_getSeqRegionName(slice));
} else {
sprintf(region,"%s", Slice_getSeqRegionName(slice));
}
bam_hdr_t *header = bam_hdr_init();
header = bam_hdr_read(in->fp.bgzf);
ref = bam_name2id(header, region);
if (ref < 0) {
fprintf(stderr, "Invalid region %s\n", region);
exit(1);
}
sprintf(region,"%s:%ld-%ld", region_name,
Slice_getSeqRegionStart(slice),
Slice_getSeqRegionEnd(slice));
if (hts_parse_reg(region, &begRange, &endRange) == NULL) {
fprintf(stderr, "Could not parse %s\n", region);
exit(2);
}
bam_hdr_destroy(header);
hts_itr_t *iter = sam_itr_queryi(idx, ref, begRange, endRange);
bam1_t *b = bam_init1();
Coverage *coverage = calloc(Slice_getLength(slice),sizeof(Coverage));
long counter = 0;
long overlapping = 0;
long bad = 0;
int startIndex = 0;
while (bam_itr_next(in, iter, b) >= 0) {
if (b->core.flag & (BAM_FUNMAP | BAM_FSECONDARY | BAM_FQCFAIL | BAM_FDUP)) {
bad++;
continue;
}
int end;
//end = bam_calend(&b->core, bam1_cigar(b));
end = bam_endpos(b);
// There is a special case for reads which have zero length and start at begRange (so end at begRange ie. before the first base we're interested in).
// That is the reason for the || end == begRange test
if (end == begRange) {
continue;
}
counter++;
if (!(counter%1000000)) {
if (verbosity > 1) { printf("."); }
fflush(stdout);
}
// Remember: b->core.pos is zero based!
int cigInd;
int refPos;
int readPos;
uint32_t *cigar = bam_get_cigar(b);
for (cigInd = readPos = 0, refPos = b->core.pos; cigInd < b->core.n_cigar; ++cigInd) {
int k;
int lenCigBlock = cigar[cigInd]>>4;
int op = cigar[cigInd]&0xf;
if (op == BAM_CMATCH || op == BAM_CEQUAL || op == BAM_CDIFF) {
for (k = 0; k < lenCigBlock; ++k) {
//if (ref[refPos+k] == 0) break; // out of boundary
coverage[refPos+k].coverage++;
}
if (k < lenCigBlock) break;
refPos += lenCigBlock; readPos += lenCigBlock;
} else if (op == BAM_CDEL) {
for (k = 0; k < lenCigBlock; ++k) {
// if (ref[refPos+k] == 0) break;
coverage[refPos+k].coverage++;
}
if (k < lenCigBlock) break;
refPos += lenCigBlock;
} else if (op == BAM_CSOFT_CLIP) {
readPos += lenCigBlock;
} else if (op == BAM_CHARD_CLIP) {
} else if (op == BAM_CINS) {
readPos += lenCigBlock;
} else if (op == BAM_CREF_SKIP) {
refPos += lenCigBlock;
}
}
#ifdef DONE
int j;
int done = 0;
int hadOverlap = 0;
for (j=startIndex; j < Vector_getNumElement(genes) && !done; j++) {
Gene *gene = Vector_getElementAt(genes,j);
if (!gene) {
continue;
}
// Remember: b->core.pos is zero based!
if (b->core.pos < Gene_getEnd(gene) && end >= Gene_getStart(gene)) {
int k;
int doneGene = 0;
for (k=0; k<Gene_getTranscriptCount(gene) && !doneGene; k++) {
Transcript *trans = Gene_getTranscriptAt(gene,k);
if (b->core.pos < Transcript_getEnd(trans) && end >= Transcript_getStart(trans)) {
int m;
for (m=0; m<Transcript_getExonCount(trans) && !doneGene; m++) {
Exon *exon = Transcript_getExonAt(trans,m);
if (b->core.pos < Exon_getEnd(exon) && end >= Exon_getStart(exon)) {
// Only count as overlapping once (could be that a read overlaps more than one gene)
if (!hadOverlap) {
overlapping++;
hadOverlap = 1;
}
gs = IDHash_getValue(geneCountsHash, Gene_getDbID(gene));
gs->score++;
doneGene = 1;
}
}
}
}
} else if (Gene_getStart(gene) > end) {
done = 1;
} else if (Gene_getEnd(gene) < b->core.pos+1) {
gs = IDHash_getValue(geneCountsHash, Gene_getDbID(gene));
printf("Gene %s (%s) score %ld\n",Gene_getStableId(gene),
Gene_getDisplayXref(gene) ? DBEntry_getDisplayId(Gene_getDisplayXref(gene)) : "",
gs->score);
if (verbosity > 1) {
printf("Removing gene %s (index %d) with extent %d to %d\n",
Gene_getStableId(gene),
gs->index,
Gene_getStart(gene),
Gene_getEnd(gene));
}
Vector_setElementAt(genes,j,NULL);
// Magic (very important for speed) - move startIndex to first non null gene
int n;
startIndex = 0;
for (n=0;n<Vector_getNumElement(genes);n++) {
void *v = Vector_getElementAt(genes,n);
if (v != NULL) {
break;
}
startIndex++;
}
if (verbosity > 1) {
printf("startIndex now %d\n",startIndex);
}
}
}
#endif
}
if (verbosity > 1) { printf("\n"); }
#ifdef DONE
// Print out read counts for what ever's left in the genes array
int n;
for (n=0;n<Vector_getNumElement(genes);n++) {
Gene *gene = Vector_getElementAt(genes,n);
if (gene != NULL) {
gs = IDHash_getValue(geneCountsHash, Gene_getDbID(gene));
printf("Gene %s (%s) score %ld\n",Gene_getStableId(gene),
Gene_getDisplayXref(gene) ? DBEntry_getDisplayId(Gene_getDisplayXref(gene)) : "",
gs->score);
}
}
#endif
printf("Read %ld reads. Number of bad reads (unmapped, qc fail, secondary, dup) %ld\n", counter, bad);
long i;
for (i=0; i< Slice_getLength(slice); i++) {
printf("%ld %ld\n", i+1, coverage[i].coverage);
}
sam_itr_destroy(iter);
bam_destroy1(b);
return 1;
}
void train_one_round(const Fast5Map& name_map, size_t round)
{
const PoreModelMap& current_models = PoreModelSet::get_models(opt::trained_model_type);
// Initialize the training summary stats for each kmer for each model
ModelTrainingMap model_training_data;
for(auto current_model_iter = current_models.begin(); current_model_iter != current_models.end(); current_model_iter++) {
// one summary entry per kmer in the model
std::vector<StateSummary> summaries(current_model_iter->second.get_num_states());
model_training_data[current_model_iter->first] = summaries;
}
// Open the BAM and iterate over reads
// load bam file
htsFile* bam_fh = sam_open(opt::bam_file.c_str(), "r");
assert(bam_fh != NULL);
// load bam index file
std::string index_filename = opt::bam_file + ".bai";
hts_idx_t* bam_idx = bam_index_load(index_filename.c_str());
assert(bam_idx != NULL);
// read the bam header
bam_hdr_t* hdr = sam_hdr_read(bam_fh);
// load reference fai file
faidx_t *fai = fai_load(opt::genome_file.c_str());
hts_itr_t* itr;
// If processing a region of the genome, only emit events aligned to this window
int clip_start = -1;
int clip_end = -1;
if(opt::region.empty()) {
// TODO: is this valid?
itr = sam_itr_queryi(bam_idx, HTS_IDX_START, 0, 0);
} else {
fprintf(stderr, "Region: %s\n", opt::region.c_str());
itr = sam_itr_querys(bam_idx, hdr, opt::region.c_str());
hts_parse_reg(opt::region.c_str(), &clip_start, &clip_end);
}
#ifndef H5_HAVE_THREADSAFE
if(opt::num_threads > 1) {
fprintf(stderr, "You enabled multi-threading but you do not have a threadsafe HDF5\n");
fprintf(stderr, "Please recompile nanopolish's built-in libhdf5 or run with -t 1\n");
exit(1);
}
#endif
// Initialize iteration
std::vector<bam1_t*> records(opt::batch_size, NULL);
for(size_t i = 0; i < records.size(); ++i) {
records[i] = bam_init1();
}
int result;
size_t num_reads_realigned = 0;
size_t num_records_buffered = 0;
Progress progress("[methyltrain]");
do {
assert(num_records_buffered < records.size());
// read a record into the next slot in the buffer
result = sam_itr_next(bam_fh, itr, records[num_records_buffered]);
num_records_buffered += result >= 0;
// realign if we've hit the max buffer size or reached the end of file
if(num_records_buffered == records.size() || result < 0) {
#pragma omp parallel for
for(size_t i = 0; i < num_records_buffered; ++i) {
bam1_t* record = records[i];
size_t read_idx = num_reads_realigned + i;
if( (record->core.flag & BAM_FUNMAP) == 0) {
add_aligned_events(name_map, fai, hdr, record, read_idx, clip_start, clip_end, round, model_training_data);
}
}
num_reads_realigned += num_records_buffered;
num_records_buffered = 0;
}
if(opt::progress) {
fprintf(stderr, "Realigned %zu reads in %.1lfs\r", num_reads_realigned, progress.get_elapsed_seconds());
}
} while(result >= 0);
assert(num_records_buffered == 0);
progress.end();
// open the summary file
std::stringstream summary_fn;
summary_fn << "methyltrain" << opt::out_suffix << ".summary";
FILE* summary_fp = fopen(summary_fn.str().c_str(), "w");
fprintf(summary_fp, "model_short_name\tkmer\tnum_matches\tnum_skips\t"
"num_stays\tnum_events_for_training\twas_trained\t"
"trained_level_mean\ttrained_level_stdv\n");
// open the tsv file with the raw training data
std::stringstream training_fn;
training_fn << "methyltrain" << opt::out_suffix << ".round" << round << ".events.tsv";
std::ofstream training_ofs(training_fn.str());
// write out a header for the training data
StateTrainingData::write_header(training_ofs);
// iterate over models: template, complement_pop1, complement_pop2
for(auto model_training_iter = model_training_data.begin();
model_training_iter != model_training_data.end(); model_training_iter++) {
// Initialize the trained model from the input model
auto current_model_iter = current_models.find(model_training_iter->first);
assert(current_model_iter != current_models.end());
std::string model_name = model_training_iter->first;
std::string model_short_name = current_model_iter->second.metadata.get_short_name();
// Initialize the new model from the current model
PoreModel updated_model = current_model_iter->second;
uint32_t k = updated_model.k;
const std::vector<StateSummary>& summaries = model_training_iter->second;
// Generate the complete set of kmers
std::string gen_kmer(k, 'A');
std::vector<std::string> all_kmers;
for(size_t ki = 0; ki < summaries.size(); ++ki) {
all_kmers.push_back(gen_kmer);
mtrain_alphabet->lexicographic_next(gen_kmer);
}
assert(gen_kmer == std::string(k, 'A'));
assert(all_kmers.front() == std::string(k, 'A'));
assert(all_kmers.back() == std::string(k, 'T'));
// Update means for each kmer
#pragma omp parallel for
for(size_t ki = 0; ki < summaries.size(); ++ki) {
assert(ki < all_kmers.size());
std::string kmer = all_kmers[ki];
// write the observed values to a tsv file
#pragma omp critical
{
for(size_t ei = 0; ei < summaries[ki].events.size(); ++ei) {
summaries[ki].events[ei].write_tsv(training_ofs, model_short_name, kmer);
}
}
bool is_m_kmer = kmer.find('M') != std::string::npos;
bool update_kmer = opt::training_target == TT_ALL_KMERS ||
(is_m_kmer && opt::training_target == TT_METHYLATED_KMERS) ||
(!is_m_kmer && opt::training_target == TT_UNMETHYLATED_KMERS);
bool trained = false;
// only train if there are a sufficient number of events for this kmer
if(update_kmer && summaries[ki].events.size() >= opt::min_number_of_events_to_train) {
// train a mixture model where a minority of k-mers aren't methylated
ParamMixture mixture;
float incomplete_methylation_rate = 0.05f;
std::string um_kmer = mtrain_alphabet->unmethylate(kmer);
size_t um_ki = mtrain_alphabet->kmer_rank(um_kmer.c_str(), k);
// Initialize the training parameters. If this is a kmer containing
// a methylation site we train a two component mixture, otherwise
// just fit a gaussian
float major_weight = is_m_kmer ? 1 - incomplete_methylation_rate : 1.0f;
mixture.log_weights.push_back(log(major_weight));
mixture.params.push_back(current_model_iter->second.get_parameters(ki));
if(is_m_kmer) {
// add second unmethylated component
mixture.log_weights.push_back(std::log(incomplete_methylation_rate));
mixture.params.push_back(current_model_iter->second.get_parameters(um_ki));
}
if(opt::verbose > 1) {
fprintf(stderr, "INIT__MIX %s\t%s\t[%.2lf %.2lf %.2lf]\t[%.2lf %.2lf %.2lf]\n", model_training_iter->first.c_str(), kmer.c_str(),
std::exp(mixture.log_weights[0]), mixture.params[0].level_mean, mixture.params[0].level_stdv,
std::exp(mixture.log_weights[1]), mixture.params[1].level_mean, mixture.params[1].level_stdv);
}
ParamMixture trained_mixture = train_gaussian_mixture(summaries[ki].events, mixture);
if(opt::verbose > 1) {
fprintf(stderr, "TRAIN_MIX %s\t%s\t[%.2lf %.2lf %.2lf]\t[%.2lf %.2lf %.2lf]\n", model_training_iter->first.c_str(), kmer.c_str(),
std::exp(trained_mixture.log_weights[0]), trained_mixture.params[0].level_mean, trained_mixture.params[0].level_stdv,
std::exp(trained_mixture.log_weights[1]), trained_mixture.params[1].level_mean, trained_mixture.params[1].level_stdv);
}
#pragma omp critical
updated_model.states[ki] = trained_mixture.params[0];
if (model_stdv()) {
ParamMixture ig_mixture;
// weights
ig_mixture.log_weights = trained_mixture.log_weights;
// states
ig_mixture.params.emplace_back(trained_mixture.params[0]);
if(is_m_kmer) {
ig_mixture.params.emplace_back(current_model_iter->second.get_parameters(um_ki));
}
// run training
auto trained_ig_mixture = train_invgaussian_mixture(summaries[ki].events, ig_mixture);
LOG("methyltrain", debug)
<< "IG_INIT__MIX " << model_training_iter->first.c_str() << " " << kmer.c_str() << " ["
<< std::fixed << std::setprecision(5) << ig_mixture.params[0].sd_mean << " "
<< ig_mixture.params[1].sd_mean << "]" << std::endl
<< "IG_TRAIN_MIX " << model_training_iter->first.c_str() << " " << kmer.c_str() << " ["
<< trained_ig_mixture.params[0].sd_mean << " "
<< trained_ig_mixture.params[1].sd_mean << "]" << std::endl;
// update state
#pragma omp critical
{
updated_model.states[ki] = trained_ig_mixture.params[0];
}
}
trained = true;
}
#pragma omp critical
{
fprintf(summary_fp, "%s\t%s\t%d\t%d\t%d\t%zu\t%d\t%.2lf\t%.2lf\n",
model_short_name.c_str(), kmer.c_str(),
summaries[ki].num_matches, summaries[ki].num_skips, summaries[ki].num_stays,
summaries[ki].events.size(), trained, updated_model.states[ki].level_mean, updated_model.states[ki].level_stdv);
}
// add the updated model into the collection (or replace what is already there)
PoreModelSet::insert_model(opt::trained_model_type, updated_model);
}
}
// cleanup records
for(size_t i = 0; i < records.size(); ++i) {
bam_destroy1(records[i]);
}
// cleanup
sam_itr_destroy(itr);
bam_hdr_destroy(hdr);
fai_destroy(fai);
sam_close(bam_fh);
hts_idx_destroy(bam_idx);
fclose(summary_fp);
}
static void iterators1(void)
{
hts_itr_destroy(sam_itr_queryi(NULL, HTS_IDX_REST, 0, 0));
hts_itr_destroy(sam_itr_queryi(NULL, HTS_IDX_NONE, 0, 0));
}
int scorereads_main(int argc, char** argv)
{
parse_scorereads_options(argc, argv);
omp_set_num_threads(opt::num_threads);
Fast5Map name_map(opt::reads_file);
ModelMap models;
if (!opt::models_fofn.empty())
models = read_models_fofn(opt::models_fofn);
// Open the BAM and iterate over reads
// load bam file
htsFile* bam_fh = sam_open(opt::bam_file.c_str(), "r");
assert(bam_fh != NULL);
// load bam index file
std::string index_filename = opt::bam_file + ".bai";
hts_idx_t* bam_idx = bam_index_load(index_filename.c_str());
assert(bam_idx != NULL);
// read the bam header
bam_hdr_t* hdr = sam_hdr_read(bam_fh);
// load reference fai file
faidx_t *fai = fai_load(opt::genome_file.c_str());
hts_itr_t* itr;
// If processing a region of the genome, only emit events aligned to this window
int clip_start = -1;
int clip_end = -1;
if(opt::region.empty()) {
// TODO: is this valid?
itr = sam_itr_queryi(bam_idx, HTS_IDX_START, 0, 0);
} else {
fprintf(stderr, "Region: %s\n", opt::region.c_str());
itr = sam_itr_querys(bam_idx, hdr, opt::region.c_str());
hts_parse_reg(opt::region.c_str(), &clip_start, &clip_end);
}
#ifndef H5_HAVE_THREADSAFE
if(opt::num_threads > 1) {
fprintf(stderr, "You enabled multi-threading but you do not have a threadsafe HDF5\n");
fprintf(stderr, "Please recompile nanopolish's built-in libhdf5 or run with -t 1\n");
exit(1);
}
#endif
// Initialize iteration
std::vector<bam1_t*> records(opt::batch_size, NULL);
for(size_t i = 0; i < records.size(); ++i) {
records[i] = bam_init1();
}
int result;
size_t num_reads_realigned = 0;
size_t num_records_buffered = 0;
do {
assert(num_records_buffered < records.size());
// read a record into the next slot in the buffer
result = sam_itr_next(bam_fh, itr, records[num_records_buffered]);
num_records_buffered += result >= 0;
// realign if we've hit the max buffer size or reached the end of file
if(num_records_buffered == records.size() || result < 0) {
#pragma omp parallel for schedule(dynamic)
for(size_t i = 0; i < num_records_buffered; ++i) {
bam1_t* record = records[i];
size_t read_idx = num_reads_realigned + i;
if( (record->core.flag & BAM_FUNMAP) == 0) {
//load read
std::string read_name = bam_get_qname(record);
std::string fast5_path = name_map.get_path(read_name);
SquiggleRead sr(read_name, fast5_path);
// TODO: early exit when have processed all of the reads in readnames
if (!opt::readnames.empty() &&
std::find(opt::readnames.begin(), opt::readnames.end(), read_name) == opt::readnames.end() )
continue;
for(size_t strand_idx = 0; strand_idx < NUM_STRANDS; ++strand_idx) {
std::vector<EventAlignment> ao = alignment_from_read(sr, strand_idx, read_idx,
models, fai, hdr,
record, clip_start, clip_end);
if (ao.size() == 0)
continue;
// Update pore model based on alignment
if ( opt::calibrate )
recalibrate_model(sr, strand_idx, ao, false);
double score = model_score(sr, strand_idx, fai, ao, 500);
if (score > 0)
continue;
#pragma omp critical(print)
std::cout << read_name << " " << ( strand_idx ? "complement" : "template" )
<< " " << sr.pore_model[strand_idx].name << " " << score << std::endl;
}
}
}
num_reads_realigned += num_records_buffered;
num_records_buffered = 0;
}
} while(result >= 0);
// cleanup records
for(size_t i = 0; i < records.size(); ++i) {
bam_destroy1(records[i]);
}
// cleanup
sam_itr_destroy(itr);
bam_hdr_destroy(hdr);
fai_destroy(fai);
sam_close(bam_fh);
hts_idx_destroy(bam_idx);
return 0;
}
