int Stats::execute(int argc, char **argv)
{
// Extract command line arguments.
String inFile = "";
String indexFile = "";
bool basic = false;
bool noeof = false;
bool params = false;
bool qual = false;
bool phred = false;
int maxNumReads = -1;
bool unmapped = false;
String pBaseQC = "";
String cBaseQC = "";
String regionList = "";
int excludeFlags = 0;
int requiredFlags = 0;
bool withinRegion = false;
int minMapQual = 0;
String dbsnp = "";
PosList *dbsnpListPtr = NULL;
bool baseSum = false;
int bufferSize = PileupHelper::DEFAULT_WINDOW_SIZE;
ParameterList inputParameters;
BEGIN_LONG_PARAMETERS(longParameterList)
LONG_PARAMETER_GROUP("Required Parameters")
LONG_STRINGPARAMETER("in", &inFile)
LONG_PARAMETER_GROUP("Types of Statistics")
LONG_PARAMETER("basic", &basic)
LONG_PARAMETER("qual", &qual)
LONG_PARAMETER("phred", &phred)
LONG_STRINGPARAMETER("pBaseQC", &pBaseQC)
LONG_STRINGPARAMETER("cBaseQC", &cBaseQC)
LONG_PARAMETER_GROUP("Optional Parameters")
LONG_INTPARAMETER("maxNumReads", &maxNumReads)
LONG_PARAMETER("unmapped", &unmapped)
LONG_STRINGPARAMETER("bamIndex", &indexFile)
LONG_STRINGPARAMETER("regionList", ®ionList)
LONG_INTPARAMETER("excludeFlags", &excludeFlags)
LONG_INTPARAMETER("requiredFlags", &requiredFlags)
LONG_PARAMETER("noeof", &noeof)
LONG_PARAMETER("params", ¶ms)
LONG_PARAMETER_GROUP("Optional phred/qual Only Parameters")
LONG_PARAMETER("withinRegion", &withinRegion)
LONG_PARAMETER_GROUP("Optional BaseQC Only Parameters")
LONG_PARAMETER("baseSum", &baseSum)
LONG_INTPARAMETER("bufferSize", &bufferSize)
LONG_INTPARAMETER("minMapQual", &minMapQual)
LONG_STRINGPARAMETER("dbsnp", &dbsnp)
END_LONG_PARAMETERS();
inputParameters.Add(new LongParameters ("Input Parameters",
longParameterList));
inputParameters.Read(argc-1, &(argv[1]));
// If no eof block is required for a bgzf file, set the bgzf file type to
// not look for it.
if(noeof)
{
// Set that the eof block is not required.
BgzfFileType::setRequireEofBlock(false);
}
// Check to see if the in file was specified, if not, report an error.
if(inFile == "")
{
usage();
inputParameters.Status();
// In file was not specified but it is mandatory.
std::cerr << "--in is a mandatory argument for stats, "
<< "but was not specified" << std::endl;
return(-1);
}
// Use the index file if unmapped or regionList is not empty.
bool useIndex = (unmapped|| (!regionList.IsEmpty()));
// IndexFile is required, so check to see if it has been set.
if(useIndex && (indexFile == ""))
{
// In file was not specified, so set it to the in file
// + ".bai"
indexFile = inFile + ".bai";
}
////////////////////////////////////////
// Setup in case pileup is used.
Pileup<PileupElementBaseQCStats> pileup(bufferSize);
// Initialize start/end positions.
myStartPos = 0;
myEndPos = -1;
// Open the output qc file if applicable.
IFILE baseQCPtr = NULL;
if(!pBaseQC.IsEmpty() && !cBaseQC.IsEmpty())
{
usage();
inputParameters.Status();
// Cannot specify both types of baseQC.
std::cerr << "Cannot specify both --pBaseQC & --cBaseQC." << std::endl;
return(-1);
}
else if(!pBaseQC.IsEmpty())
{
baseQCPtr = ifopen(pBaseQC, "w");
PileupElementBaseQCStats::setPercentStats(true);
}
else if(!cBaseQC.IsEmpty())
{
baseQCPtr = ifopen(cBaseQC, "w");
PileupElementBaseQCStats::setPercentStats(false);
}
if(baseQCPtr != NULL)
{
PileupElementBaseQCStats::setOutputFile(baseQCPtr);
PileupElementBaseQCStats::printHeader();
}
if((baseQCPtr != NULL) || baseSum)
{
PileupElementBaseQCStats::setMapQualFilter(minMapQual);
PileupElementBaseQCStats::setBaseSum(baseSum);
}
if(params)
{
inputParameters.Status();
}
// Open the file for reading.
SamFile samIn;
if(!samIn.OpenForRead(inFile))
{
fprintf(stderr, "%s\n", samIn.GetStatusMessage());
return(samIn.GetStatus());
}
samIn.SetReadFlags(requiredFlags, excludeFlags);
// Set whether or not basic statistics should be generated.
samIn.GenerateStatistics(basic);
// Read the sam header.
SamFileHeader samHeader;
if(!samIn.ReadHeader(samHeader))
{
fprintf(stderr, "%s\n", samIn.GetStatusMessage());
return(samIn.GetStatus());
}
// Open the bam index file for reading if we are
// doing unmapped reads (also set the read section).
if(useIndex)
{
samIn.ReadBamIndex(indexFile);
if(unmapped)
{
samIn.SetReadSection(-1);
}
if(!regionList.IsEmpty())
{
myRegionList = ifopen(regionList, "r");
}
}
//////////////////////////
// Read dbsnp if specified and doing baseQC
if(((baseQCPtr != NULL) || baseSum) && (!dbsnp.IsEmpty()))
{
// Read the dbsnp file.
IFILE fdbSnp;
fdbSnp = ifopen(dbsnp,"r");
// Determine how many entries.
const SamReferenceInfo& refInfo = samHeader.getReferenceInfo();
int maxRefLen = 0;
for(int i = 0; i < refInfo.getNumEntries(); i++)
{
int refLen = refInfo.getReferenceLength(i);
if(refLen >= maxRefLen)
{
maxRefLen = refLen + 1;
}
}
dbsnpListPtr = new PosList(refInfo.getNumEntries(),maxRefLen);
if(fdbSnp==NULL)
{
std::cerr << "Open dbSNP file " << dbsnp.c_str() << " failed!\n";
}
else if(dbsnpListPtr == NULL)
{
std::cerr << "Failed to init the memory allocation for the dbsnpList.\n";
}
else
{
// Read the dbsnp file.
StringArray tokens;
String buffer;
int position = 0;
int refID = 0;
// Loop til the end of the file.
while (!ifeof(fdbSnp))
{
// Read the next line.
buffer.ReadLine(fdbSnp);
// If it does not have at least 2 columns,
// continue to the next line.
if (buffer.IsEmpty() || buffer[0] == '#') continue;
tokens.AddTokens(buffer);
if(tokens.Length() < 2) continue;
if(!tokens[1].AsInteger(position))
{
std::cerr << "Improperly formatted region line, start position "
<< "(2nd column) is not an integer: "
<< tokens[1]
<< "; Skipping to the next line.\n";
continue;
}
// Look up the reference name.
refID = samHeader.getReferenceID(tokens[0]);
if(refID != SamReferenceInfo::NO_REF_ID)
{
// Reference id was found, so add it to the dbsnp
dbsnpListPtr->addPosition(refID, position);
}
tokens.Clear();
buffer.Clear();
}
}
ifclose(fdbSnp);
}
// Read the sam records.
SamRecord samRecord;
int numReads = 0;
//////////////////////
// Setup in case doing a quality count.
// Quality histogram.
const int MAX_QUAL = 126;
const int START_QUAL = 33;
uint64_t qualCount[MAX_QUAL+1];
for(int i = 0; i <= MAX_QUAL; i++)
{
qualCount[i] = 0;
}
const int START_PHRED = 0;
const int PHRED_DIFF = START_QUAL - START_PHRED;
const int MAX_PHRED = MAX_QUAL - PHRED_DIFF;
uint64_t phredCount[MAX_PHRED+1];
for(int i = 0; i <= MAX_PHRED; i++)
{
phredCount[i] = 0;
}
int refPos = 0;
Cigar* cigarPtr = NULL;
char cigarChar = '?';
// Exclude clips from the qual/phred counts if unmapped reads are excluded.
bool qualExcludeClips = excludeFlags & SamFlag::UNMAPPED;
//////////////////////////////////
// When not reading by sections, getNextSection returns true
// the first time, then false the next time.
while(getNextSection(samIn))
{
// Keep reading records from the file until SamFile::ReadRecord
// indicates to stop (returns false).
while(((maxNumReads < 0) || (numReads < maxNumReads)) && samIn.ReadRecord(samHeader, samRecord))
{
// Another record was read, so increment the number of reads.
++numReads;
// See if the quality histogram should be genereated.
if(qual || phred)
{
// Get the quality.
const char* qual = samRecord.getQuality();
// Check for no quality ('*').
if((qual[0] == '*') && (qual[1] == 0))
{
// This record does not have a quality string, so no
// quality processing is necessary.
}
else
{
int index = 0;
cigarPtr = samRecord.getCigarInfo();
cigarChar = '?';
refPos = samRecord.get0BasedPosition();
if(!qualExcludeClips && (cigarPtr != NULL))
{
// Offset the reference position by any soft clips
// by subtracting the queryIndex of this start position.
// refPos is now the start position of the clips.
refPos -= cigarPtr->getQueryIndex(0);
}
while(qual[index] != 0)
{
// Skip this quality if it is clipped and we are skipping clips.
if(cigarPtr != NULL)
{
cigarChar = cigarPtr->getCigarCharOpFromQueryIndex(index);
}
if(qualExcludeClips && Cigar::isClip(cigarChar))
{
// Skip a clipped quality.
++index;
// Increment the position.
continue;
}
if(withinRegion && (myEndPos != -1) && (refPos >= myEndPos))
{
// We have hit the end of the region, stop processing this
// quality string.
break;
}
if(withinRegion && (refPos < myStartPos))
{
// This position is not in the target.
++index;
// Update the position if this is found in the reference or a clip.
if(Cigar::foundInReference(cigarChar) || Cigar::isClip(cigarChar))
{
++refPos;
}
continue;
}
// Check for valid quality.
if((qual[index] < START_QUAL) || (qual[index] > MAX_QUAL))
{
if(qual)
{
std::cerr << "Invalid Quality found: " << qual[index]
<< ". Must be between "
<< START_QUAL << " and " << MAX_QUAL << ".\n";
}
if(phred)
{
std::cerr << "Invalid Phred Quality found: " << qual[index] - PHRED_DIFF
<< ". Must be between "
<< START_QUAL << " and " << MAX_QUAL << ".\n";
}
// Skip an invalid quality.
++index;
// Update the position if this is found in the reference or a clip.
if(Cigar::foundInReference(cigarChar) || Cigar::isClip(cigarChar))
{
++refPos;
}
continue;
}
// Increment the count for this quality.
++(qualCount[(int)(qual[index])]);
++(phredCount[(int)(qual[index]) - PHRED_DIFF]);
// Update the position if this is found in the reference or a clip.
if(Cigar::foundInReference(cigarChar) || Cigar::isClip(cigarChar))
{
++refPos;
}
++index;
}
}
}
// Check the next thing to do for the read.
if((baseQCPtr != NULL) || baseSum)
{
// Pileup the bases for this read.
pileup.processAlignmentRegion(samRecord, myStartPos, myEndPos, dbsnpListPtr);
}
}
// Done with a section, move on to the next one.
// New section, so flush the pileup.
pileup.flushPileup();
}
// Flush the rest of the pileup.
if((baseQCPtr != NULL) || baseSum)
{
// Pileup the bases.
pileup.processAlignmentRegion(samRecord, myStartPos, myEndPos, dbsnpListPtr);
PileupElementBaseQCStats::printSummary();
ifclose(baseQCPtr);
}
std::cerr << "Number of records read = " <<
samIn.GetCurrentRecordCount() << std::endl;
if(basic)
{
std::cerr << std::endl;
samIn.PrintStatistics();
}
// Print the quality stats.
if(qual)
{
std::cerr << std::endl;
std::cerr << "Quality\tCount\n";
for(int i = START_QUAL; i <= MAX_QUAL; i++)
{
std::cerr << i << "\t" << qualCount[i] << std::endl;
}
}
// Print the phred quality stats.
if(phred)
{
std::cerr << std::endl;
std::cerr << "Phred\tCount\n";
for(int i = START_PHRED; i <= MAX_PHRED; i++)
{
std::cerr << i << "\t" << phredCount[i] << std::endl;
}
}
SamStatus::Status status = samIn.GetStatus();
if(status == SamStatus::NO_MORE_RECS)
{
// A status of NO_MORE_RECS means that all reads were successful.
status = SamStatus::SUCCESS;
}
return(status);
}
int Bam2FastQ::execute(int argc, char **argv)
{
// Extract command line arguments.
String inFile = "";
bool readName = false;
String refFile = "";
String firstOut = "";
String secondOut = "";
String unpairedOut = "";
bool interleave = false;
bool noeof = false;
bool gzip = false;
bool params = false;
myOutBase = "";
myNumMateFailures = 0;
myNumPairs = 0;
myNumUnpaired = 0;
mySplitRG = false;
myQField = "";
myNumQualTagErrors = 0;
myReverseComp = true;
myRNPlus = false;
myFirstRNExt = DEFAULT_FIRST_EXT;
mySecondRNExt = DEFAULT_SECOND_EXT;
myCompression = InputFile::DEFAULT;
ParameterList inputParameters;
BEGIN_LONG_PARAMETERS(longParameterList)
LONG_PARAMETER_GROUP("Required Parameters")
LONG_STRINGPARAMETER("in", &inFile)
LONG_PARAMETER_GROUP("Optional Parameters")
LONG_PARAMETER("readName", &readName)
LONG_PARAMETER("splitRG", &mySplitRG)
LONG_STRINGPARAMETER("qualField", &myQField)
LONG_PARAMETER("merge", &interleave)
LONG_STRINGPARAMETER("refFile", &refFile)
LONG_STRINGPARAMETER("firstRNExt", &myFirstRNExt)
LONG_STRINGPARAMETER("secondRNExt", &mySecondRNExt)
LONG_PARAMETER("rnPlus", &myRNPlus)
LONG_PARAMETER("noReverseComp", &myReverseComp)
LONG_PARAMETER("gzip", &gzip)
LONG_PARAMETER("noeof", &noeof)
LONG_PARAMETER("params", ¶ms)
LONG_PARAMETER_GROUP("Optional OutputFile Names")
LONG_STRINGPARAMETER("outBase", &myOutBase)
LONG_STRINGPARAMETER("firstOut", &firstOut)
LONG_STRINGPARAMETER("secondOut", &secondOut)
LONG_STRINGPARAMETER("unpairedOut", &unpairedOut)
LONG_PHONEHOME(VERSION)
END_LONG_PARAMETERS();
inputParameters.Add(new LongParameters ("Input Parameters",
longParameterList));
// parameters start at index 2 rather than 1.
inputParameters.Read(argc, argv, 2);
// If no eof block is required for a bgzf file, set the bgzf file type to
// not look for it.
if(noeof)
{
// Set that the eof block is not required.
BgzfFileType::setRequireEofBlock(false);
}
if(gzip)
{
myCompression = InputFile::GZIP;
}
// Check to see if the in file was specified, if not, report an error.
if(inFile == "")
{
usage();
inputParameters.Status();
// In file was not specified but it is mandatory.
std::cerr << "--in is a mandatory argument, "
<< "but was not specified" << std::endl;
return(-1);
}
// Cannot specify both interleaved & secondOut since secondOut would be N/A.
if(interleave && !secondOut.IsEmpty())
{
usage();
inputParameters.Status();
std::cerr << "ERROR: Cannot specify --merge & --secondOut.\n";
return(-1);
}
// Cannot specify both interleaved & secondOut since secondOut would be N/A.
if(interleave && !secondOut.IsEmpty())
{
usage();
inputParameters.Status();
std::cerr << "ERROR: Cannot specify --merge & --secondOut.\n";
return(-1);
}
// Cannot specify both splitRG & firstOut/secondOut/unpairedOut
// since it needs a different file for each RG.
if(mySplitRG && (!firstOut.IsEmpty() ||
!secondOut.IsEmpty() || !unpairedOut.IsEmpty()))
{
usage();
inputParameters.Status();
std::cerr << "ERROR: Cannot specify --splitRG & --firstOut/--secondOut/--unpairedOut.\n";
std::cerr << "Use --outBase instead.\n";
return(-1);
}
// Cannot specify splitRG & output to stdout.
if(mySplitRG && (myOutBase[0] == '-'))
{
usage();
inputParameters.Status();
std::cerr << "ERROR: Cannot specify --splitRG & write to stdout.\n";
return(-1);
}
// Check to see if the out file was specified, if not, generate it from
// the input filename.
if(myOutBase == "")
{
// Just remove the extension from the input filename.
int extStart = inFile.FastFindLastChar('.');
if(extStart <= 0)
{
myOutBase = inFile;
}
else
{
myOutBase = inFile.Left(extStart);
}
}
if(mySplitRG)
{
std::string fqList = myOutBase.c_str();
fqList += ".list";
myFqList = ifopen(fqList.c_str(), "w");
ifprintf(myFqList, "MERGE_NAME\tFASTQ1\tFASTQ2\tRG\n");
}
// Check to see if the first/second/single-ended were specified and
// if not, set them.
myFirstFileNameExt = "_1.fastq";
mySecondFileNameExt = "_2.fastq";
myUnpairedFileNameExt = ".fastq";
if(interleave)
{
myFirstFileNameExt = "_interleaved.fastq";
myFirstFileNameExt = "_interleaved.fastq";
}
getFileName(firstOut, myFirstFileNameExt);
getFileName(secondOut, mySecondFileNameExt);
getFileName(unpairedOut, myUnpairedFileNameExt);
if(params)
{
inputParameters.Status();
}
// Open the files for reading/writing.
// Open prior to opening the output files,
// so if there is an error, the outputs don't get created.
SamFile samIn;
samIn.OpenForRead(inFile, &mySamHeader);
// Skip non-primary reads.
samIn.SetReadFlags(0, 0x0100);
// Open the output files if not splitting RG
if(!mySplitRG)
{
myUnpairedFile = ifopen(unpairedOut, "w", myCompression);
// Only open the first file if it is different than an already opened file.
if(firstOut != unpairedOut)
{
myFirstFile = ifopen(firstOut, "w", myCompression);
}
else
{
myFirstFile = myUnpairedFile;
}
// If it is interleaved or the 2nd file is not a new name, set it appropriately.
if(interleave || secondOut == firstOut)
{
mySecondFile = myFirstFile;
}
else if(secondOut == unpairedOut)
{
mySecondFile = myUnpairedFile;
}
else
{
mySecondFile = ifopen(secondOut, "w", myCompression);
}
if(myUnpairedFile == NULL)
{
std::cerr << "Failed to open " << unpairedOut
<< " so can't convert bam2FastQ.\n";
return(-1);
}
if(myFirstFile == NULL)
{
std::cerr << "Failed to open " << firstOut
<< " so can't convert bam2FastQ.\n";
return(-1);
}
if(mySecondFile == NULL)
{
std::cerr << "Failed to open " << secondOut
<< " so can't convert bam2FastQ.\n";
return(-1);
}
}
if((readName) || (strcmp(mySamHeader.getSortOrder(), "queryname") == 0))
{
readName = true;
}
else
{
// defaulting to coordinate sorted.
samIn.setSortedValidation(SamFile::COORDINATE);
}
// Setup the '=' translation if the reference was specified.
if(!refFile.IsEmpty())
{
GenomeSequence* refPtr = new GenomeSequence(refFile);
samIn.SetReadSequenceTranslation(SamRecord::BASES);
samIn.SetReference(refPtr);
}
SamRecord* recordPtr;
int16_t samFlag;
SamStatus::Status returnStatus = SamStatus::SUCCESS;
while(returnStatus == SamStatus::SUCCESS)
{
recordPtr = myPool.getRecord();
if(recordPtr == NULL)
{
// Failed to allocate a new record.
throw(std::runtime_error("Failed to allocate a new SAM/BAM record"));
}
if(!samIn.ReadRecord(mySamHeader, *recordPtr))
{
// Failed to read a record.
returnStatus = samIn.GetStatus();
continue;
}
// Have a record. Check to see if it is a pair or unpaired read.
samFlag = recordPtr->getFlag();
if(SamFlag::isPaired(samFlag))
{
if(readName)
{
handlePairedRN(*recordPtr);
}
else
{
handlePairedCoord(*recordPtr);
}
}
else
{
++myNumUnpaired;
writeFastQ(*recordPtr, myUnpairedFile,
myUnpairedFileNameExt);
}
}
// Flush All
cleanUpMateMap(0, true);
if(returnStatus == SamStatus::NO_MORE_RECS)
{
returnStatus = SamStatus::SUCCESS;
}
samIn.Close();
closeFiles();
// Output the results
std::cerr << "\nFound " << myNumPairs << " read pairs.\n";
std::cerr << "Found " << myNumUnpaired << " unpaired reads.\n";
if(myNumMateFailures != 0)
{
std::cerr << "Failed to find mates for " << myNumMateFailures
<< " reads, so they were written as unpaired\n"
<< " (not included in either of the above counts).\n";
}
if(myNumQualTagErrors != 0)
{
std::cerr << myNumQualTagErrors << " records did not have tag "
<< myQField.c_str() << " or it was invalid, so the quality field was used for those records.\n";
}
return(returnStatus);
}
void testFlagRead(const char* fileName)
{
SamFile inSam;
SamFileHeader samHeader;
SamRecord samRecord;
////////////////////////////////////////////////////////////
// Required flag 0x48 (only flag 73 matches)
// Exclude nothing
assert(inSam.OpenForRead(fileName));
assert(inSam.ReadHeader(samHeader));
validateHeader(samHeader);
inSam.SetReadFlags(0x48, 0x0);
assert(inSam.ReadRecord(samHeader, samRecord) == true);
validateRead1(samRecord);
assert(inSam.ReadRecord(samHeader, samRecord) == false);
inSam.Close();
////////////////////////////////////////////////////////////
// No required flags.
// Exclude 0x48. This leaves just the one read with flag 133.
assert(inSam.OpenForRead(fileName));
assert(inSam.ReadHeader(samHeader));
validateHeader(samHeader);
inSam.SetReadFlags(0x0, 0x48);
assert(inSam.ReadRecord(samHeader, samRecord) == true);
validateRead2(samRecord);
assert(inSam.ReadRecord(samHeader, samRecord) == false);
inSam.Close();
////////////////////////////////////////////////////////////
// Required flag 0x40
// Exclude 0x48.
// This will not find any records since the exclude and required conflict.
assert(inSam.OpenForRead(fileName));
assert(inSam.ReadHeader(samHeader));
validateHeader(samHeader);
inSam.SetReadFlags(0x40, 0x48);
assert(inSam.ReadRecord(samHeader, samRecord) == false);
inSam.Close();
////////////////////////////////////////////////////////////
// Required flag 0x4
// Exclude 0x8.
// Only finds flag 133.
assert(inSam.OpenForRead(fileName));
assert(inSam.ReadHeader(samHeader));
validateHeader(samHeader);
inSam.SetReadFlags(0x4, 0x8);
assert(inSam.ReadRecord(samHeader, samRecord) == true);
validateRead2(samRecord);
assert(inSam.ReadRecord(samHeader, samRecord) == false);
inSam.Close();
////////////////////////////////////////////////////////////
// Required flag 0x4
// Exclude nothing
// Finds flags 133 & 141.
assert(inSam.OpenForRead(fileName));
assert(inSam.ReadHeader(samHeader));
validateHeader(samHeader);
inSam.SetReadFlags(0x4, 0x0);
assert(inSam.ReadRecord(samHeader, samRecord) == true);
validateRead2(samRecord);
assert(inSam.ReadRecord(samHeader, samRecord) == true);
validateRead8(samRecord);
assert(inSam.ReadRecord(samHeader, samRecord) == true);
validateRead10(samRecord);
assert(inSam.ReadRecord(samHeader, samRecord) == false);
inSam.Close();
}