arma::rowvec asympt_calc(const arma::vec& theta,
const std::vector<std::string>& desc, const arma::field<arma::vec>& objdesc,
std::string model_type, const arma::vec& scales, const arma::mat& V, const arma::mat& omega,
const arma::vec& wv_empir, const arma::vec& theo, double obj_value){
// Take derivatives
arma::mat A = derivative_first_matrix(theta, desc, objdesc, scales);
/* A note to someone in the future...
* Yes, there is a difference in order between the diff (wv_empir-theo) for D_matrix
* and the model_score diff (theo-wv_empir).
*/
// Create the D Matrix (note this is in the analytical_matrix_derivaties.cpp file)
arma::mat D = D_matrix(theta, desc, objdesc, scales, omega*(theo - wv_empir));
arma::rowvec temp(4);
arma::vec result = model_score(A, D, omega, V, obj_value);
temp(0) = obj_value;
temp(1) = arma::as_scalar(result.row(1));
temp(2) = arma::as_scalar(result.row(0));
temp(3) = arma::as_scalar( gof_test(theta, desc, objdesc, model_type, scales, V, wv_empir).row(1) ); // GoF
return temp;
}
// Update the training data with aligned events from a read
void add_aligned_events(const Fast5Map& name_map,
const faidx_t* fai,
const bam_hdr_t* hdr,
const bam1_t* record,
size_t read_idx,
int region_start,
int region_end,
size_t round,
ModelTrainingMap& training)
{
// Load a squiggle read for the mapped read
std::string read_name = bam_get_qname(record);
std::string fast5_path = name_map.get_path(read_name);
// load read
SquiggleRead sr(read_name, fast5_path);
// replace the models that are built into the read with the current trained model
sr.replace_models(opt::trained_model_type);
for(size_t strand_idx = 0; strand_idx < NUM_STRANDS; ++strand_idx) {
// skip if 1D reads and this is the wrong strand
if(!sr.has_events_for_strand(strand_idx)) {
continue;
}
// set k
uint32_t k = sr.pore_model[strand_idx].k;
// Align to the new model
EventAlignmentParameters params;
params.sr = &sr;
params.fai = fai;
params.hdr = hdr;
params.record = record;
params.strand_idx = strand_idx;
params.alphabet = mtrain_alphabet;
params.read_idx = read_idx;
params.region_start = region_start;
params.region_end = region_end;
std::vector<EventAlignment> alignment_output = align_read_to_ref(params);
if (alignment_output.size() == 0)
return;
// Update pore model based on alignment
std::string curr_model = sr.pore_model[strand_idx].metadata.get_short_name();
double orig_score = -INFINITY;
if (opt::output_scores) {
orig_score = model_score(sr, strand_idx, fai, alignment_output, 500, NULL);
#pragma omp critical(print)
std::cout << round << " " << curr_model << " " << read_idx << " " << strand_idx << " Original " << orig_score << std::endl;
}
if ( opt::calibrate ) {
double resid = 0.;
recalibrate_model(sr, strand_idx, alignment_output, mtrain_alphabet, resid, true);
if (opt::output_scores) {
double rescaled_score = model_score(sr, strand_idx, fai, alignment_output, 500, NULL);
#pragma omp critical(print)
{
std::cout << round << " " << curr_model << " " << read_idx << " " << strand_idx << " Rescaled " << rescaled_score << std::endl;
std::cout << round << " " << curr_model << " " << read_idx << " " << strand_idx << " Delta " << rescaled_score-orig_score << std::endl;
}
}
}
// Get the training data for this model
auto& emission_map = training[curr_model];
for(size_t i = 0; i < alignment_output.size(); ++i) {
const EventAlignment& ea = alignment_output[i];
std::string model_kmer = ea.model_kmer;
// Grab the previous/next model kmer from the alignment_output table.
// If the read is from the same strand as the reference
// the next kmer comes from the next alignment_output (and vice-versa)
// other the indices are swapped
int next_stride = ea.rc ? -1 : 1;
std::string prev_kmer = "";
std::string next_kmer = "";
if(i > 0 && i < alignment_output.size() - 1) {
// check that the event indices are correct for the next expected position
assert(alignment_output[i + next_stride].event_idx - ea.event_idx == 1);
assert(alignment_output[i - next_stride].event_idx - ea.event_idx == -1);
// only set the previous/next when there was exactly one base of movement along the referenc
if( std::abs(alignment_output[i + next_stride].ref_position - ea.ref_position) == 1) {
next_kmer = alignment_output[i + next_stride].model_kmer;
}
if( std::abs(alignment_output[i - next_stride].ref_position - ea.ref_position) == 1) {
prev_kmer = alignment_output[i - next_stride].model_kmer;
}
}
// Get the rank of the kmer that we aligned to (on the sequencing strand, = model_kmer)
uint32_t rank = mtrain_alphabet->kmer_rank(model_kmer.c_str(), k);
assert(rank < emission_map.size());
auto& kmer_summary = emission_map[rank];
// We only use this event for training if its not at the end of the alignment
// (to avoid bad alignments around the read edges) and if its not too short (to
// avoid bad measurements from effecting the levels too much)
bool use_for_training = i > opt::min_distance_from_alignment_end &&
i + opt::min_distance_from_alignment_end < alignment_output.size() &&
alignment_output[i].hmm_state == 'M' &&
sr.get_duration( alignment_output[i].event_idx, strand_idx) >= opt::min_event_duration &&
sr.get_fully_scaled_level(alignment_output[i].event_idx, strand_idx) >= 1.0;
if(use_for_training) {
StateTrainingData std(sr, ea, rank, prev_kmer, next_kmer);
#pragma omp critical(kmer)
kmer_summary.events.push_back(std);
}
if(ea.hmm_state == 'M') {
#pragma omp atomic
kmer_summary.num_matches += 1;
} else if(ea.hmm_state == 'E') {
#pragma omp atomic
kmer_summary.num_stays += 1;
}
}
} // for strands
}
int scorereads_main(int argc, char** argv)
{
parse_scorereads_options(argc, argv);
omp_set_num_threads(opt::num_threads);
Fast5Map name_map(opt::reads_file);
ModelMap models;
if (!opt::models_fofn.empty())
models = read_models_fofn(opt::models_fofn);
// Open the BAM and iterate over reads
// load bam file
htsFile* bam_fh = sam_open(opt::bam_file.c_str(), "r");
assert(bam_fh != NULL);
// load bam index file
std::string index_filename = opt::bam_file + ".bai";
hts_idx_t* bam_idx = bam_index_load(index_filename.c_str());
assert(bam_idx != NULL);
// read the bam header
bam_hdr_t* hdr = sam_hdr_read(bam_fh);
// load reference fai file
faidx_t *fai = fai_load(opt::genome_file.c_str());
hts_itr_t* itr;
// If processing a region of the genome, only emit events aligned to this window
int clip_start = -1;
int clip_end = -1;
if(opt::region.empty()) {
// TODO: is this valid?
itr = sam_itr_queryi(bam_idx, HTS_IDX_START, 0, 0);
} else {
fprintf(stderr, "Region: %s\n", opt::region.c_str());
itr = sam_itr_querys(bam_idx, hdr, opt::region.c_str());
hts_parse_reg(opt::region.c_str(), &clip_start, &clip_end);
}
#ifndef H5_HAVE_THREADSAFE
if(opt::num_threads > 1) {
fprintf(stderr, "You enabled multi-threading but you do not have a threadsafe HDF5\n");
fprintf(stderr, "Please recompile nanopolish's built-in libhdf5 or run with -t 1\n");
exit(1);
}
#endif
// Initialize iteration
std::vector<bam1_t*> records(opt::batch_size, NULL);
for(size_t i = 0; i < records.size(); ++i) {
records[i] = bam_init1();
}
int result;
size_t num_reads_realigned = 0;
size_t num_records_buffered = 0;
do {
assert(num_records_buffered < records.size());
// read a record into the next slot in the buffer
result = sam_itr_next(bam_fh, itr, records[num_records_buffered]);
num_records_buffered += result >= 0;
// realign if we've hit the max buffer size or reached the end of file
if(num_records_buffered == records.size() || result < 0) {
#pragma omp parallel for schedule(dynamic)
for(size_t i = 0; i < num_records_buffered; ++i) {
bam1_t* record = records[i];
size_t read_idx = num_reads_realigned + i;
if( (record->core.flag & BAM_FUNMAP) == 0) {
//load read
std::string read_name = bam_get_qname(record);
std::string fast5_path = name_map.get_path(read_name);
SquiggleRead sr(read_name, fast5_path);
// TODO: early exit when have processed all of the reads in readnames
if (!opt::readnames.empty() &&
std::find(opt::readnames.begin(), opt::readnames.end(), read_name) == opt::readnames.end() )
continue;
for(size_t strand_idx = 0; strand_idx < NUM_STRANDS; ++strand_idx) {
std::vector<EventAlignment> ao = alignment_from_read(sr, strand_idx, read_idx,
models, fai, hdr,
record, clip_start, clip_end);
if (ao.size() == 0)
continue;
// Update pore model based on alignment
if ( opt::calibrate )
recalibrate_model(sr, strand_idx, ao, false);
double score = model_score(sr, strand_idx, fai, ao, 500);
if (score > 0)
continue;
#pragma omp critical(print)
std::cout << read_name << " " << ( strand_idx ? "complement" : "template" )
<< " " << sr.pore_model[strand_idx].name << " " << score << std::endl;
}
}
}
num_reads_realigned += num_records_buffered;
num_records_buffered = 0;
}
} while(result >= 0);
// cleanup records
for(size_t i = 0; i < records.size(); ++i) {
bam_destroy1(records[i]);
}
// cleanup
sam_itr_destroy(itr);
bam_hdr_destroy(hdr);
fai_destroy(fai);
sam_close(bam_fh);
hts_idx_destroy(bam_idx);
return 0;
}
