int IonstatsTestFragments(int argc, const char *argv[])
{
OptArgs opts;
opts.ParseCmdLine(argc, argv);
string input_bam_filename = opts.GetFirstString('i', "input", "");
string fasta_filename = opts.GetFirstString('r', "ref", "");
string output_json_filename = opts.GetFirstString('o', "output", "ionstats_tf.json");
int histogram_length = opts.GetFirstInt ('h', "histogram-length", 400);
if(argc < 2 or input_bam_filename.empty() or fasta_filename.empty()) {
IonstatsTestFragmentsHelp();
return 1;
}
//
// Prepare for metric calculation
//
map<string,string> tf_sequences;
PopulateReferenceSequences(tf_sequences, fasta_filename);
BamReader input_bam;
if (!input_bam.Open(input_bam_filename)) {
fprintf(stderr, "[ionstats] ERROR: cannot open %s\n", input_bam_filename.c_str());
return 1;
}
int num_tfs = input_bam.GetReferenceCount();
SamHeader sam_header = input_bam.GetHeader();
if(!sam_header.HasReadGroups()) {
fprintf(stderr, "[ionstats] ERROR: no read groups in %s\n", input_bam_filename.c_str());
return 1;
}
string flow_order;
string key;
for (SamReadGroupIterator rg = sam_header.ReadGroups.Begin(); rg != sam_header.ReadGroups.End(); ++rg) {
if(rg->HasFlowOrder())
flow_order = rg->FlowOrder;
if(rg->HasKeySequence())
key = rg->KeySequence;
}
// Need these metrics stratified by TF.
vector<ReadLengthHistogram> called_histogram(num_tfs);
vector<ReadLengthHistogram> aligned_histogram(num_tfs);
vector<ReadLengthHistogram> AQ10_histogram(num_tfs);
vector<ReadLengthHistogram> AQ17_histogram(num_tfs);
vector<SimpleHistogram> error_by_position(num_tfs);
vector<MetricGeneratorSNR> system_snr(num_tfs);
vector<MetricGeneratorHPAccuracy> hp_accuracy(num_tfs);
for (int tf = 0; tf < num_tfs; ++tf) {
called_histogram[tf].Initialize(histogram_length);
aligned_histogram[tf].Initialize(histogram_length);
AQ10_histogram[tf].Initialize(histogram_length);
AQ17_histogram[tf].Initialize(histogram_length);
error_by_position[tf].Initialize(histogram_length);
}
vector<uint16_t> flow_signal_fz(flow_order.length());
vector<int16_t> flow_signal_zm(flow_order.length());
const RefVector& refs = input_bam.GetReferenceData();
// Missing:
// - hp accuracy - tough, copy verbatim from TFMapper?
BamAlignment alignment;
vector<char> MD_op;
vector<int> MD_len;
MD_op.reserve(1024);
MD_len.reserve(1024);
string MD_tag;
//
// Main loop over mapped reads in the input BAM
//
while(input_bam.GetNextAlignment(alignment)) {
if (!alignment.IsMapped() or !alignment.GetTag("MD",MD_tag))
continue;
// The check below eliminates unexpected alignments
if (alignment.IsReverseStrand() or alignment.Position > 5)
continue;
int current_tf = alignment.RefID;
//
// Step 1. Parse MD tag
//
MD_op.clear();
MD_len.clear();
for (const char *MD_ptr = MD_tag.c_str(); *MD_ptr;) {
int item_length = 0;
if (*MD_ptr >= '0' and *MD_ptr <= '9') { // Its a match
MD_op.push_back('M');
for (; *MD_ptr and *MD_ptr >= '0' and *MD_ptr <= '9'; ++MD_ptr)
item_length = 10*item_length + *MD_ptr - '0';
} else {
if (*MD_ptr == '^') { // Its a deletion
MD_ptr++;
MD_op.push_back('D');
} else // Its a substitution
MD_op.push_back('X');
for (; *MD_ptr and *MD_ptr >= 'A' and *MD_ptr <= 'Z'; ++MD_ptr)
item_length++;
}
MD_len.push_back(item_length);
}
//
// Step 2. Synchronously scan through Cigar and MD, doing error accounting
//
int MD_idx = alignment.IsReverseStrand() ? MD_op.size()-1 : 0;
int cigar_idx = alignment.IsReverseStrand() ? alignment.CigarData.size()-1 : 0;
int increment = alignment.IsReverseStrand() ? -1 : 1;
int AQ10_bases = 0;
int AQ17_bases = 0;
int num_bases = 0;
int num_errors = 0;
while (cigar_idx < (int)alignment.CigarData.size() and MD_idx < (int) MD_op.size() and cigar_idx >= 0 and MD_idx >= 0) {
if (alignment.CigarData[cigar_idx].Length == 0) { // Try advancing cigar
cigar_idx += increment;
continue;
}
if (MD_len[MD_idx] == 0) { // Try advancing MD
MD_idx += increment;
continue;
}
// Match
if (alignment.CigarData[cigar_idx].Type == 'M' and MD_op[MD_idx] == 'M') {
int advance = min((int)alignment.CigarData[cigar_idx].Length, MD_len[MD_idx]);
num_bases += advance;
alignment.CigarData[cigar_idx].Length -= advance;
MD_len[MD_idx] -= advance;
// Insertion (read has a base, reference doesn't)
} else if (alignment.CigarData[cigar_idx].Type == 'I') {
int advance = alignment.CigarData[cigar_idx].Length;
for (int cnt = 0; cnt < advance; ++cnt) {
error_by_position[current_tf].Add(num_bases);
num_bases++;
num_errors++;
}
alignment.CigarData[cigar_idx].Length -= advance;
// Deletion (reference has a base, read doesn't)
} else if (alignment.CigarData[cigar_idx].Type == 'D' and MD_op[MD_idx] == 'D') {
int advance = min((int)alignment.CigarData[cigar_idx].Length, MD_len[MD_idx]);
for (int cnt = 0; cnt < advance; ++cnt) {
error_by_position[current_tf].Add(num_bases);
num_errors++;
}
alignment.CigarData[cigar_idx].Length -= advance;
MD_len[MD_idx] -= advance;
// Substitution
} else if (MD_op[MD_idx] == 'X') {
int advance = min((int)alignment.CigarData[cigar_idx].Length, MD_len[MD_idx]);
for (int cnt = 0; cnt < advance; ++cnt) {
error_by_position[current_tf].Add(num_bases);
num_bases++;
num_errors++;
}
alignment.CigarData[cigar_idx].Length -= advance;
MD_len[MD_idx] -= advance;
} else {
printf("ionstats tf: Unexpected OP combination: %s Cigar=%c, MD=%c !\n",
alignment.Name.c_str(), alignment.CigarData[cigar_idx].Type, MD_op[MD_idx]);
break;
}
if (num_errors*10 <= num_bases) AQ10_bases = num_bases;
if (num_errors*50 <= num_bases) AQ17_bases = num_bases;
}
//
// Step 3. Profit
//
called_histogram[current_tf].Add(alignment.Length);
aligned_histogram[current_tf].Add(num_bases);
AQ10_histogram[current_tf].Add(AQ10_bases);
AQ17_histogram[current_tf].Add(AQ17_bases);
if(alignment.GetTag("ZM", flow_signal_zm))
system_snr[current_tf].Add(flow_signal_zm, key.c_str(), flow_order);
else if(alignment.GetTag("FZ", flow_signal_fz))
system_snr[current_tf].Add(flow_signal_fz, key.c_str(), flow_order);
// HP accuracy - keeping it simple
if (!alignment.IsReverseStrand()) {
string genome = key + tf_sequences[refs[current_tf].RefName];
string calls = key + alignment.QueryBases;
const char *genome_ptr = genome.c_str();
const char *calls_ptr = calls.c_str();
for (int flow = 0; flow < (int)flow_order.length() and *genome_ptr and *calls_ptr; ++flow) {
int genome_hp = 0;
int calls_hp = 0;
while (*genome_ptr == flow_order[flow]) {
genome_hp++;
genome_ptr++;
}
while (*calls_ptr == flow_order[flow]) {
calls_hp++;
calls_ptr++;
}
hp_accuracy[current_tf].Add(genome_hp, calls_hp);
}
}
}
//
// Processing complete, generate ionstats_tf.json
//
Json::Value output_json(Json::objectValue);
output_json["meta"]["creation_date"] = get_time_iso_string(time(NULL));
output_json["meta"]["format_name"] = "ionstats_tf";
output_json["meta"]["format_version"] = "1.0";
output_json["results_by_tf"] = Json::objectValue;
for (int tf = 0; tf < num_tfs; ++tf) {
if (aligned_histogram[tf].num_reads() < 1000)
continue;
called_histogram[tf].SaveToJson(output_json["results_by_tf"][refs[tf].RefName]["full"]);
aligned_histogram[tf].SaveToJson(output_json["results_by_tf"][refs[tf].RefName]["aligned"]);
AQ10_histogram[tf].SaveToJson(output_json["results_by_tf"][refs[tf].RefName]["AQ10"]);
AQ17_histogram[tf].SaveToJson(output_json["results_by_tf"][refs[tf].RefName]["AQ17"]);
error_by_position[tf].SaveToJson(output_json["results_by_tf"][refs[tf].RefName]["error_by_position"]);
system_snr[tf].SaveToJson(output_json["results_by_tf"][refs[tf].RefName]);
hp_accuracy[tf].SaveToJson(output_json["results_by_tf"][refs[tf].RefName]);
output_json["results_by_tf"][refs[tf].RefName]["sequence"] = tf_sequences[refs[tf].RefName];
}
input_bam.Close();
ofstream out(output_json_filename.c_str(), ios::out);
if (out.good()) {
out << output_json.toStyledString();
return 0;
} else {
fprintf(stderr, "ERROR: unable to write to '%s'\n", output_json_filename.c_str());
return 1;
}
}
int IonstatsBasecaller(int argc, const char *argv[])
{
OptArgs opts;
opts.ParseCmdLine(argc, argv);
string input_bam_filename = opts.GetFirstString('i', "input", "");
string output_json_filename = opts.GetFirstString('o', "output", "ionstats_basecaller.json");
int histogram_length = opts.GetFirstInt ('h', "histogram-length", 400);
if(argc < 2 or input_bam_filename.empty()) {
IonstatsBasecallerHelp();
return 1;
}
BamReader input_bam;
if (!input_bam.Open(input_bam_filename)) {
fprintf(stderr, "[ionstats] ERROR: cannot open %s\n", input_bam_filename.c_str());
return 1;
}
SamHeader sam_header = input_bam.GetHeader();
if(!sam_header.HasReadGroups()) {
fprintf(stderr, "[ionstats] ERROR: no read groups in %s\n", input_bam_filename.c_str());
return 1;
}
ReadLengthHistogram total_full_histo;
ReadLengthHistogram total_insert_histo;
ReadLengthHistogram total_Q17_histo;
ReadLengthHistogram total_Q20_histo;
total_full_histo.Initialize(histogram_length);
total_insert_histo.Initialize(histogram_length);
total_Q17_histo.Initialize(histogram_length);
total_Q20_histo.Initialize(histogram_length);
MetricGeneratorSNR system_snr;
BaseQVHistogram qv_histogram;
string flow_order;
string key;
for (SamReadGroupIterator rg = sam_header.ReadGroups.Begin(); rg != sam_header.ReadGroups.End(); ++rg) {
if(rg->HasFlowOrder())
flow_order = rg->FlowOrder;
if(rg->HasKeySequence())
key = rg->KeySequence;
}
double qv_to_error_rate[256];
for (int qv = 0; qv < 256; qv++)
qv_to_error_rate[qv] = pow(10.0,-0.1*(double)qv);
BamAlignment alignment;
string read_group;
vector<uint16_t> flow_signal_fz(flow_order.length());
vector<int16_t> flow_signal_zm(flow_order.length());
while(input_bam.GetNextAlignment(alignment)) {
// Record read length
unsigned int full_length = alignment.Length;
total_full_histo.Add(full_length);
// Record insert length
int insert_length = 0;
if (alignment.GetTag("ZA",insert_length))
total_insert_histo.Add(insert_length);
// Compute and record Q17 and Q20
int Q17_length = 0;
int Q20_length = 0;
double num_accumulated_errors = 0.0;
for(int pos = 0; pos < alignment.Length; ++pos) {
num_accumulated_errors += qv_to_error_rate[(int)alignment.Qualities[pos] - 33];
if (num_accumulated_errors / (pos + 1) <= 0.02)
Q17_length = pos + 1;
if (num_accumulated_errors / (pos + 1) <= 0.01)
Q20_length = pos + 1;
}
total_Q17_histo.Add(Q17_length);
total_Q20_histo.Add(Q20_length);
// Record data for system snr
if(alignment.GetTag("ZM", flow_signal_zm))
system_snr.Add(flow_signal_zm, key.c_str(), flow_order);
else if(alignment.GetTag("FZ", flow_signal_fz))
system_snr.Add(flow_signal_fz, key.c_str(), flow_order);
// Record qv histogram
qv_histogram.Add(alignment.Qualities);
}
input_bam.Close();
Json::Value output_json(Json::objectValue);
output_json["meta"]["creation_date"] = get_time_iso_string(time(NULL));
output_json["meta"]["format_name"] = "ionstats_basecaller";
output_json["meta"]["format_version"] = "1.0";
system_snr.SaveToJson(output_json);
qv_histogram.SaveToJson(output_json);
total_full_histo.SaveToJson(output_json["full"]);
total_insert_histo.SaveToJson(output_json["insert"]);
total_Q17_histo.SaveToJson(output_json["Q17"]);
total_Q20_histo.SaveToJson(output_json["Q20"]);
ofstream out(output_json_filename.c_str(), ios::out);
if (out.good()) {
out << output_json.toStyledString();
return 0;
} else {
fprintf(stderr, "ERROR: unable to write to '%s'\n", output_json_filename.c_str());
return 1;
}
}
