int main(int argc, char* argv[]) {
string program = "samtom4";
string versionString = VERSION;
AppendPerforceChangelist(PERFORCE_VERSION_STRING, versionString);
string samFileName, refFileName, outFileName;
bool printHeader = false;
bool parseSmrtTitle = false;
bool useShortRefName = false;
CommandLineParser clp;
clp.SetProgramName(program);
clp.SetVersion(versionString);
clp.SetProgramSummary("Converts a SAM file generated by blasr to M4 format.");
clp.RegisterStringOption("in.sam", &samFileName,
"Input SAM file, which is produced by blasr.");
clp.RegisterStringOption("reference.fasta", &refFileName,
"Reference used to generate file.sam.");
clp.RegisterStringOption("out.m4", &outFileName,
"Output in blasr M4 format.");
clp.RegisterPreviousFlagsAsHidden();
clp.RegisterFlagOption("header", &printHeader,
"Print M4 header.");
clp.RegisterFlagOption("useShortRefName", &useShortRefName,
"Use abbreviated reference names obtained "
"from file.sam instead of using full names "
"from reference.fasta.");
//clp.SetExamples(program + " file.sam reference.fasta out.m4");
clp.ParseCommandLine(argc, argv);
ostream * outFilePtr = &cout;
ofstream outFileStrm;
if (outFileName != "") {
CrucialOpen(outFileName, outFileStrm, std::ios::out);
outFilePtr = &outFileStrm;
}
SAMReader<SAMFullReferenceSequence, SAMReadGroup, SAMAlignment> samReader;
FASTAReader fastaReader;
//
// Initialize samReader and fastaReader.
//
samReader.Initialize(samFileName);
fastaReader.Initialize(refFileName);
//
// Configure the file log.
//
string command;
CommandLineParser::CommandLineToString(argc, argv, command);
//
// Read necessary input.
//
vector<FASTASequence> references;
fastaReader.ReadAllSequences(references);
AlignmentSet<SAMFullReferenceSequence, SAMReadGroup, SAMAlignment> alignmentSet;
samReader.ReadHeader(alignmentSet);
//
// The order of references in vector<FASTASequence> references and
// AlignmentSet<, , >alignmentSet.references can be different.
// Rearrange alignmentSet.references such that it is ordered in
// exactly the same way as vector<FASTASequence> references.
//
alignmentSet.RearrangeReferences(references);
//
// Map short names for references obtained from file.sam to
// full names obtained from reference.fasta
//
map<string, string> shortRefNameToFull;
map<string, string>::iterator it;
assert(references.size() == alignmentSet.references.size());
if (!useShortRefName) {
for (size_t i = 0; i < references.size(); i++) {
string shortRefName = alignmentSet.references[i].GetSequenceName();
string fullRefName(references[i].title);
if (shortRefNameToFull.find(shortRefName) != shortRefNameToFull.end()) {
cout << "ERROR, Found more than one reference " << shortRefName << "in sam header" << endl;
exit(1);
}
shortRefNameToFull[shortRefName] = fullRefName;
alignmentSet.references[i].sequenceName = fullRefName;
}
}
// Map reference name obtained from SAM file to indices
map<string, int> refNameToIndex;
for (size_t i = 0; i < references.size(); i++) {
string refName = alignmentSet.references[i].GetSequenceName();
refNameToIndex[refName] = i;
}
//
// Store the alignments.
//
SAMAlignment samAlignment;
size_t alignIndex = 0;
//
// For 150K, each chip produces about 300M sequences
// (not including quality values and etc.).
// Let's assume that the sam file and reference data can
// fit in the memory.
// Need to scale for larger sequal data in the future.
//
if (printHeader)
IntervalOutput::PrintHeader(*outFilePtr);
// The socre matrix does not matter because we will use the
// aligner's score from SAM file anyway.
DistanceMatrixScoreFunction<DNASequence, DNASequence> distScoreFn;
while (samReader.GetNextAlignment(samAlignment)) {
if (samAlignment.rName == "*") {
continue;
}
if (!useShortRefName) {
//convert shortRefName to fullRefName
it = shortRefNameToFull.find(samAlignment.rName);
if (it == shortRefNameToFull.end()) {
cout << "ERROR, Could not find " << samAlignment.rName << " in the reference repository." << endl;
exit(1);
}
samAlignment.rName = (*it).second;
}
// The padding character 'P' is not supported
if (samAlignment.cigar.find('P') != string::npos) {
cout << "WARNING. Could not process sam record with 'P' in its cigar string."
<< endl;
continue;
}
vector<AlignmentCandidate<> > convertedAlignments;
//
// Keep reference as forward.
// So if IsReverseComplement(sam.flag)==true, then qStrand is reverse
// and tStrand is forward.
//
bool keepRefAsForward = false;
SAMAlignmentsToCandidates(samAlignment, references, refNameToIndex,
convertedAlignments, parseSmrtTitle,
keepRefAsForward);
if (convertedAlignments.size() > 1) {
cout << "WARNING. Ignore an alignment which has multiple segments." << endl;
continue;
}
//all alignments are unique single-ended alignments.
for (int i = 0; i < 1; i++) {
AlignmentCandidate<> & alignment = convertedAlignments[i];
ComputeAlignmentStats(alignment, alignment.qAlignedSeq.seq,
alignment.tAlignedSeq.seq, distScoreFn);
// Use aligner's score from SAM file anyway.
alignment.score = samAlignment.as;
alignment.mapQV = samAlignment.mapQV;
// Since SAM only has the aligned sequence, many info of the
// original query (e.g. the full length) is missing.
// Overwrite alignment.qLength (which is length of the query
// in the SAM alignment) with xq (which is the length of the
// original query sequence saved by blasr) right before printing
// the output so that one can reconstruct a blasr m4 record from
// a blasr sam alignment.
if (samAlignment.xq!=0)
alignment.qLength = samAlignment.xq;
IntervalOutput::PrintFromSAM(alignment, *outFilePtr);
alignment.FreeSubsequences();
}
++alignIndex;
}
if (outFileName != "") {
outFileStrm.close();
}
return 0;
}
int main(int argc, char* argv[]) {
string program = "samtoh5";
string versionString = VERSION;
AppendPerforceChangelist(PERFORCE_VERSION_STRING, versionString);
string samFileName, cmpFileName, refFileName;
bool parseSmrtTitle = false;
bool useShortRefName = false;
CommandLineParser clp;
string readType = "standard";
int verbosity = 0;
clp.SetProgramName(program);
clp.SetProgramSummary("Converts in.sam file to out.cmp.h5 file.");
clp.SetVersion(versionString);
clp.RegisterStringOption("in.sam", &samFileName,
"Input SAM file.", true);
clp.RegisterStringOption("reference.fasta", &refFileName,
"Reference used to generate reads.", true);
clp.RegisterStringOption("out.cmp.h5", &cmpFileName,
"Output cmp.h5 file.", true);
clp.RegisterPreviousFlagsAsHidden();
clp.RegisterFlagOption("smrtTitle", &parseSmrtTitle,
"Use this option when converting alignments "
"generated from reads produced by the "
"pls2fasta from bas.h5 files by parsing read "
"coordinates from the SMRT read title. The title "
"is in the format /name/hole/coordinates, where "
"coordinates are in the format \\d+_\\d+, and "
"represent the interval of the read that was "
"aligned.");
clp.RegisterStringOption("readType", &readType,
"Set the read type: 'standard', 'strobe', 'CCS', "
"or 'cDNA'");
clp.RegisterIntOption("verbosity", &verbosity,
"Set desired verbosity.",
CommandLineParser::PositiveInteger);
clp.RegisterFlagOption("useShortRefName", &useShortRefName,
"Use abbreviated reference names obtained "
"from file.sam instead of using full names "
"from reference.fasta.");
string description = ("Because SAM has optional tags that have different "
"meanings in different programs, careful usage is required in order to "
"have proper output. The \"xs\" tag in bwa-sw is used to show the "
"suboptimal score, but in PacBio SAM (blasr) it is defined as the start "
"in the query sequence of the alignment.\nWhen \"-smrtTitle\" is "
"specified, the xs tag is ignored, but when it is not specified, the "
"coordinates given by the xs and xe tags are used to define the interval "
"of a read that is aligned. The CIGAR string is relative to this interval.");
clp.SetExamples(description);
clp.ParseCommandLine(argc, argv);
if (readType != "standard" and readType != "strobe" and
readType != "cDNA" and readType != "CCS") {
cout << "ERROR. Read type '" << readType
<< "' must be one of either 'standard', 'strobe', 'cDNA' or 'CCS'."
<< endl;
exit(1);
}
cerr << "[INFO] " << GetTimestamp() << " [" << program << "] started." << endl;
SAMReader<SAMFullReferenceSequence, SAMReadGroup, SAMPosAlignment> samReader;
FASTAReader fastaReader;
HDFCmpFile<AlignmentCandidate<FASTASequence, FASTASequence> > cmpFile;
//
// Initialize input/output files.
//
samReader.Initialize(samFileName);
fastaReader.Initialize(refFileName);
cmpFile.Create(cmpFileName);
//
// Configure the file log.
//
string command;
CommandLineParser::CommandLineToString(argc, argv, command);
string log = "Convert sam to cmp.h5";
cmpFile.fileLogGroup.AddEntry(command, log, program, GetTimestamp(), versionString);
//
// Set the readType
//
cmpFile.SetReadType(readType);
//
// Read necessary input.
//
vector<FASTASequence> references;
fastaReader.ReadAllSequences(references);
//
// This should probably be handled by the alignmentSetAdapter, but
// time constraints...
//
AlignmentSet<SAMFullReferenceSequence, SAMReadGroup, SAMPosAlignment> alignmentSet;
samReader.ReadHeader(alignmentSet);
//
// The order of references in vector<FASTASequence> references and
// AlignmentSet<, , >alignmentSet.references can be different.
// Rearrange alignmentSet.references such that it is ordered in
// exactly the same way as vector<FASTASequence> references.
//
alignmentSet.RearrangeReferences(references);
//
// Always recompute the MD5 values even if they exist in the input
// sam file. Because MD5 is defined differently in sam and cmp.h5 files.
// The SAM convention uppercases and normalizes before computing the MD5.
// For cmp.h5, we compute the MD5 on the sequence 'as is'.
//
for(int i = 0; i < alignmentSet.references.size(); i++) {
MakeMD5((const char*)&references[i].seq[0],
(unsigned int)references[i].length, alignmentSet.references[i].md5);
}
//
// Map short names for references obtained from file.sam to full names obtained from reference.fasta
//
map<string, string> shortRefNameToFull;
map<string, string>::iterator it;
assert(references.size() == alignmentSet.references.size());
if (!useShortRefName) {
for (int i = 0; i < references.size(); i++) {
string shortRefName = alignmentSet.references[i].GetSequenceName();
string fullRefName(references[i].title);
if (shortRefNameToFull.find(shortRefName) != shortRefNameToFull.end()) {
cout << "ERROR, Found more than one reference " << shortRefName << "in sam header" << endl;
exit(1);
}
shortRefNameToFull[shortRefName] = fullRefName;
alignmentSet.references[i].sequenceName = fullRefName;
}
}
//
// Start setting up the cmp.h5 file.
//
AlignmentSetToCmpH5Adapter<HDFCmpFile<AlignmentCandidate<FASTASequence, FASTASequence> > > alignmentSetAdapter;
alignmentSetAdapter.Initialize();
alignmentSetAdapter.StoreReferenceInfo(alignmentSet.references, cmpFile);
//
// Store the alignments.
//
SAMAlignment samAlignment;
int alignIndex = 0;
while (samReader.GetNextAlignment(samAlignment)) {
if (samAlignment.rName == "*") {
continue;
}
if (!useShortRefName) {
//convert shortRefName to fullRefName
it = shortRefNameToFull.find(samAlignment.rName);
if (it == shortRefNameToFull.end()) {
cout << "ERROR, Could not find " << samAlignment.rName << " in the reference repository." << endl;
exit(1);
}
samAlignment.rName = (*it).second;
}
vector<AlignmentCandidate<> > convertedAlignments;
if (verbosity > 0) {
cout << "Storing alignment for " << samAlignment.qName << endl;
}
SAMAlignmentsToCandidates(samAlignment,
references, alignmentSetAdapter.refNameToIndex,
convertedAlignments, parseSmrtTitle, false);
alignmentSetAdapter.StoreAlignmentCandidateList(convertedAlignments, cmpFile, alignIndex);
int a;
for (a = 0; a < convertedAlignments.size(); a++) {
convertedAlignments[a].FreeSubsequences();
}
++alignIndex;
/* if (alignIndex == 100) {
return 0;
}*/
}
cerr << "[INFO] " << GetTimestamp() << " [" << program << "] ended." << endl;
return 0;
}
int main(int argc, char* argv[]) {
#ifdef USE_GOOGLE_PROFILER
char *profileFileName = getenv("CPUPROFILE");
if (profileFileName != NULL) {
ProfilerStart(profileFileName);
}
else {
ProfilerStart("google_profile.txt");
}
#endif
// Register inputs and outputs.
string samFileName, refFileName, outFileName;
CommandLineParser clp;
clp.RegisterStringOption("file.sam", &samFileName,
"Input SAM file.");
clp.RegisterStringOption("reference.fasta", &refFileName,
"Reference used to generate reads.");
clp.RegisterStringOption("out.sam", &outFileName,
"Output SAM file.");
clp.RegisterPreviousFlagsAsHidden();
// Register filter criteria options.
int minAlnLength = 50;
float minPctSimilarity = 70, minPctAccuracy = 70;
string hitPolicyStr = "randombest";
bool useScoreCutoff = false;
int scoreCutoff = INF_INT;
int scoreSignInt = -1;
RegisterFilterOptions(clp, minAlnLength, minPctSimilarity,
minPctAccuracy, hitPolicyStr, useScoreCutoff,
scoreSignInt, scoreCutoff);
int seed = 1;
clp.RegisterIntOption("seed", &seed,
"(1) Seed for random number generator.\n"
"If seed is 0, then use current time as seed.",
CommandLineParser::Integer);
string holeNumberStr;
Ranges holeNumberRanges;
clp.RegisterStringOption("holeNumbers", &holeNumberStr,
"A string of comma-delimited hole number ranges to output hits, "
"such as '1,2,10-12'. "
"This requires hit titles to be in SMRT read title format.");
bool parseSmrtTitle = false;
clp.RegisterFlagOption("smrtTitle", &parseSmrtTitle,
"Use this option when filtering alignments generated by "
"programs other than blasr, e.g. bwa-sw or gmap. "
" Parse read coordinates from the SMRT read title. "
"The title is in the format /name/hole/coordinates, where"
" coordinates are in the format \\d+_\\d+, and represent "
"the interval of the read that was aligned.");
/* This experimental option can be useful for metagenomics, in which case
* there are hundreds of sequences in the target, of which many titles are
* long and may contain white spaces (e.g., ' ', '\t').
* In order to save disc space and avoid the (possibly) none unique mapping
* between full and short reference names, one may call blasr with
* -titleTable option to represent all target sequences in the output
* by their indices in the title table.*/
string titleTableName = "";
clp.RegisterStringOption("titleTable", &titleTableName,
"Use this experimental option when filtering alignments generated by "
"blasr with -titleTable titleTableName, in which case "
"reference titles in SAM are represented by their "
"indices (e.g., 0, 1, 2, ...) in the title table.");
string adapterGffFileName = "";
clp.RegisterStringOption("filterAdapterOnly", &adapterGffFileName,
"Use this option to remove reads which can only map to adapters "
"specified in the GFF file.");
bool verbose = false;
clp.RegisterFlagOption("v", &verbose, "Be verbose.");
clp.SetExamples(
"Because SAM has optional tags that have different meanings"
" in different programs, careful usage is required in order "
"to have proper output. The \"xs\" tag in bwa-sw is used to "
"show the suboptimal score, but in PacBio SAM (blasr) it is "
"defined as the start in the query sequence of the alignment.\n"
"When \"-smrtTitle\" is specified, the xs tag is ignored, but "
"when it is not specified, the coordinates given by the xs and "
"xe tags are used to define the interval of a read that is "
"aligned. The CIGAR string is relative to this interval.");
clp.ParseCommandLine(argc, argv);
// Set random number seed.
if (seed == 0) {
srand(time(NULL));
} else {
srand(seed);
}
scoreSign = (scoreSignInt == -1)?ScoreSign::NEGATIVE:ScoreSign::POSITIVE;
Score s(static_cast<float>(scoreCutoff), scoreSign);
FilterCriteria filterCriteria(minAlnLength, minPctSimilarity,
minPctAccuracy, true, s);
filterCriteria.Verbose(verbose);
HitPolicy hitPolicy(hitPolicyStr, scoreSign);
string errMsg;
if (not filterCriteria.MakeSane(errMsg)) {
cout << errMsg << endl;
exit(1);
}
// Parse hole number ranges.
if (holeNumberStr.size() != 0) {
if (not holeNumberRanges.setRanges(holeNumberStr)) {
cout << "Could not parse hole number ranges: "
<< holeNumberStr << "." << endl;
exit(1);
}
}
// Open output file.
ostream * outFilePtr = &cout;
ofstream outFileStrm;
if (outFileName != "") {
CrucialOpen(outFileName, outFileStrm, std::ios::out);
outFilePtr = &outFileStrm;
}
GFFFile adapterGffFile;
if (adapterGffFileName != "")
adapterGffFile.ReadAll(adapterGffFileName);
SAMReader<SAMFullReferenceSequence, SAMReadGroup, SAMAlignment> samReader;
FASTAReader fastaReader;
//
// Initialize samReader and fastaReader.
//
samReader.Initialize(samFileName);
fastaReader.Initialize(refFileName);
//
// Configure the file log.
//
string command;
CommandLineParser::CommandLineToString(argc, argv, command);
string log = "Filter sam hits.";
string program = "samFilter";
string versionString = VERSION;
AppendPerforceChangelist(PERFORCE_VERSION_STRING, versionString);
//
// Read necessary input.
//
vector<FASTASequence> references;
fastaReader.ReadAllSequences(references);
// If the SAM file is generated by blasr with -titleTable,
// then references in the SAM are represented by
// their corresponding indices in the title table.
// In that case, we need to convert reference titles in fasta file
// to their corresponding indices in the title table, such that
// references in both SAM and fasta files are represented
// by title table indices and therefore can match.
if (titleTableName != "") {
ConvertTitlesToTitleTableIndices(references, titleTableName);
}
AlignmentSet<SAMFullReferenceSequence, SAMReadGroup, SAMAlignment> alignmentSet;
vector<string> allHeaders = samReader.ReadHeader(alignmentSet);
// Process SAM Header.
string commandLineString;
clp.CommandLineToString(argc, argv, commandLineString);
allHeaders.push_back("@PG\tID:SAMFILTER\tVN:" + versionString + \
"\tCL:" + program + " " + commandLineString);
for (int i = 0; i < allHeaders.size(); i++) {
outFileStrm << allHeaders[i] << endl;
}
//
// The order of references in vector<FASTASequence> references and
// AlignmentSet<, , >alignmentSet.references can be different.
// Rearrange alignmentSet.references such that they are ordered in
// exactly the same way as vector<FASTASequence> references.
//
alignmentSet.RearrangeReferences(references);
// Map reference name obtained from SAM file to indices
map<string, int> refNameToIndex;
for (int i = 0; i < references.size(); i++) {
string refName = alignmentSet.references[i].GetSequenceName();
refNameToIndex[refName] = i;
}
//
// Store the alignments.
//
SAMAlignment samAlignment;
int alignIndex = 0;
//
// For 150K, each chip produces about 300M sequences
// (not including quality values and etc.).
// Let's assume that the sam file and reference data can
// fit in the memory.
// Need to scale for larger sequal data in the future.
//
vector<SAMAlignment> allSAMAlignments;
while (samReader.GetNextAlignment(samAlignment)) {
if (samAlignment.rName == "*") {
continue;
}
if (parseSmrtTitle and holeNumberStr.size() != 0) {
string movieName;
int thisHoleNumber;
if (not ParsePBIReadName(samAlignment.qName,
movieName,
thisHoleNumber)) {
cout << "ERROR, could not parse SMRT title: "
<< samAlignment.qName << "." << endl;
exit(1);
}
if (not holeNumberRanges.contains(UInt(thisHoleNumber))) {
if (verbose)
cout << thisHoleNumber << " is not in range." << endl;
continue;
}
}
if (samAlignment.cigar.find('P') != string::npos) {
cout << "WARNING. Could not process SAM record with 'P' in "
<< "its cigar string." << endl;
continue;
}
vector<AlignmentCandidate<> > convertedAlignments;
SAMAlignmentsToCandidates(samAlignment,
references, refNameToIndex,
convertedAlignments, parseSmrtTitle, false);
if (convertedAlignments.size() > 1) {
cout << "WARNING. Ignore multiple segments." << endl;
continue;
}
for (int i = 0; i < 1; i++) {
AlignmentCandidate<> & alignment = convertedAlignments[i];
//score func does not matter
DistanceMatrixScoreFunction<DNASequence, DNASequence> distFunc;
ComputeAlignmentStats(alignment, alignment.qAlignedSeq.seq,
alignment.tAlignedSeq.seq, distFunc);
// Check whether this alignment can only map to adapters in
// the adapter GFF file.
if (adapterGffFileName != "" and
CheckAdapterOnly(adapterGffFile, alignment, refNameToIndex)) {
if (verbose)
cout << alignment.qName << " filter adapter only."
<< endl;
continue;
}
// Assign score to samAlignment.
samAlignment.score = samAlignment.as;
if (not filterCriteria.Satisfy(static_cast<AlignmentCandidate<> *>(&alignment))) {
continue;
}
allSAMAlignments.push_back( samAlignment );
alignment.FreeSubsequences();
}
++alignIndex;
}
// Sort all SAM alignments by qName, score and target position.
sort(allSAMAlignments.begin(), allSAMAlignments.end(),
byQNameScoreTStart);
unsigned int groupBegin = 0;
unsigned int groupEnd = -1;
vector<SAMAlignment> filteredSAMAlignments;
while(groupBegin < allSAMAlignments.size()) {
// Get the next group of SAM alignments which have the same qName
// from allSAMAlignments[groupBegin ... groupEnd)
GetNextSAMAlignmentGroup(allSAMAlignments, groupBegin, groupEnd);
vector<unsigned int> hitIndices = ApplyHitPolicy(
hitPolicy, allSAMAlignments, groupBegin, groupEnd);
for(unsigned int i = 0; i < hitIndices.size(); i++) {
filteredSAMAlignments.push_back(allSAMAlignments[hitIndices[i]]);
}
groupBegin = groupEnd;
}
// Sort all SAM alignments by reference name and query name
sort(filteredSAMAlignments.begin(), filteredSAMAlignments.end(),
byRNameQName);
for(unsigned int i = 0; i < filteredSAMAlignments.size(); i++) {
filteredSAMAlignments[i].PrintSAMAlignment(outFileStrm);
}
if (outFileName != "") {
outFileStrm.close();
}
#ifdef USE_GOOGLE_PROFILER
ProfilerStop();
#endif
return 0;
}