void Recab::addRecabSpecificParameters(LongParamContainer& params)
{
params.addGroup("Required Recab Parameters");
params.addString("refFile", &myRefFile);
params.addGroup("Optional Recab Parameters");
params.addString("dbsnp", &myDbsnpFile);
params.addInt("minBaseQual", &myMinBaseQual);
params.addInt("maxBaseQual", &myMaxBaseQual);
params.addInt("blended", &myBlendedWeight);
params.addBool("fitModel", &myFitModel);
params.addBool("fast", &myFast);
params.addBool("keepPrevDbsnp", &myKeepPrevDbsnp);
params.addBool("keepPrevNonAdjacent", &myKeepPrevNonAdjacent);
params.addBool("useLogReg", &myLogReg);
params.addString("qualField", &myQField);
params.addString("storeQualTag", &myStoreQualTag);
params.addString("buildExcludeFlags", &myBuildExcludeFlags);
params.addString("applyExcludeFlags", &myApplyExcludeFlags);
myParamsSetup = false;
mySqueeze.addBinningParameters(params);
}
int Dedup_LowMem::execute(int argc, char** argv)
{
/* --------------------------------
* process the arguments
* -------------------------------*/
String inFile, outFile, logFile;
myDoRecab = false;
bool removeFlag = false;
bool verboseFlag = false;
myForceFlag = false;
myNumMissingMate = 0;
myMinQual = DEFAULT_MIN_QUAL;
String excludeFlags = "0xB04";
uint16_t intExcludeFlags = 0;
bool noeof = false;
bool params = false;
LongParamContainer parameters;
parameters.addGroup("Required Parameters");
parameters.addString("in", &inFile);
parameters.addString("out", &outFile);
parameters.addGroup("Optional Parameters");
parameters.addInt("minQual", & myMinQual);
parameters.addString("log", &logFile);
parameters.addBool("oneChrom", &myOneChrom);
parameters.addBool("recab", &myDoRecab);
parameters.addBool("rmDups", &removeFlag);
parameters.addBool("force", &myForceFlag);
parameters.addString("excludeFlags", &excludeFlags);
parameters.addBool("verbose", &verboseFlag);
parameters.addBool("noeof", &noeof);
parameters.addBool("params", ¶ms);
parameters.addPhoneHome(VERSION);
myRecab.addRecabSpecificParameters(parameters);
ParameterList inputParameters;
inputParameters.Add(new LongParameters ("Input Parameters",
parameters.getLongParameterList()));
// parameters start at index 2 rather than 1.
inputParameters.Read(argc, argv, 2);
// If no eof block is required for a bgzf file, set the bgzf file type to
// not look for it.
if(noeof)
{
// Set that the eof block is not required.
BgzfFileType::setRequireEofBlock(false);
}
if(inFile.IsEmpty())
{
printUsage(std::cerr);
inputParameters.Status();
std::cerr << "Specify an input file" << std::endl;
return EXIT_FAILURE;
}
if(outFile.IsEmpty())
{
printUsage(std::cerr);
inputParameters.Status();
std::cerr << "Specify an output file" << std::endl;
return EXIT_FAILURE;
}
intExcludeFlags = excludeFlags.AsInteger();
if(myForceFlag && SamFlag::isDuplicate(intExcludeFlags))
{
printUsage(std::cerr);
inputParameters.Status();
std::cerr << "Cannot specify --force and Duplicate in the excludeFlags. Since --force indicates to override"
<< " previous duplicate setting and the excludeFlags says to skip those, you can't do both.\n";
return EXIT_FAILURE;
}
if(!SamFlag::isSecondary(intExcludeFlags))
{
printUsage(std::cerr);
inputParameters.Status();
std::cerr << "ERROR: Secondary reads must be excluded, edit --excludeFlags to include 0x0100\n";
return EXIT_FAILURE;
}
if(!(intExcludeFlags & SamFlag::SUPPLEMENTARY_ALIGNMENT))
{
printUsage(std::cerr);
inputParameters.Status();
std::cerr << "ERROR: Supplementary reads must be excluded, edit --excludeFlags to include 0x0800\n";
return EXIT_FAILURE;
}
if(logFile.IsEmpty())
{
logFile = outFile + ".log";
}
if(myDoRecab)
{
int status = myRecab.processRecabParam();
if(status != 0)
{
inputParameters.Status();
return(status);
}
}
if(params)
{
inputParameters.Status();
}
Logger::gLogger = new Logger(logFile.c_str(), verboseFlag);
/* -------------------------------------------------------------------
* The arguments are processed. Prepare the input BAM file,
* instantiate dedup_LowMem, and construct the read group library map
* ------------------------------------------------------------------*/
SamFile samIn;
samIn.OpenForRead(inFile.c_str());
// If the file isn't sorted it will throw an exception.
samIn.setSortedValidation(SamFile::COORDINATE);
SamFileHeader header;
samIn.ReadHeader(header);
buildReadGroupLibraryMap(header);
lastReference = -1;
lastCoordinate = -1;
// for keeping some basic statistics
uint32_t recordCount = 0;
uint32_t pairedCount = 0;
uint32_t properPairCount = 0;
uint32_t unmappedCount = 0;
uint32_t reverseCount = 0;
uint32_t qualCheckFailCount = 0;
uint32_t secondaryCount = 0;
uint32_t supplementaryCount = 0;
uint32_t excludedCount = 0;
// Now we start reading records
SamRecord* recordPtr;
SamStatus::Status returnStatus = SamStatus::SUCCESS;
while(returnStatus == SamStatus::SUCCESS)
{
recordPtr = mySamPool.getRecord();
if(recordPtr == NULL)
{
std::cerr << "Failed to allocate enough records\n";
return(-1);
}
if(!samIn.ReadRecord(header, *recordPtr))
{
returnStatus = samIn.GetStatus();
continue;
}
// Take note of properties of this record
int flag = recordPtr->getFlag();
if(SamFlag::isPaired(flag)) ++pairedCount;
if(SamFlag::isProperPair(flag)) ++properPairCount;
if(SamFlag::isReverse(flag)) ++reverseCount;
if(SamFlag::isQCFailure(flag)) ++qualCheckFailCount;
if(SamFlag::isSecondary(flag)) ++secondaryCount;
if(flag & SamFlag::SUPPLEMENTARY_ALIGNMENT) ++supplementaryCount;
if(!SamFlag::isMapped(flag)) ++unmappedCount;
// put the record in the appropriate maps:
// single reads go in myFragmentMap
// paired reads go in myPairedMap
recordCount = samIn.GetCurrentRecordCount();
// if we have moved to a new position, look back at previous reads for duplicates
if (hasPositionChanged(*recordPtr))
{
cleanupPriorReads(recordPtr);
}
// Determine if this read should be checked for duplicates.
if((!SamFlag::isMapped(flag)) || ((flag & intExcludeFlags) != 0))
{
++excludedCount;
// No deduping done on this record, but still build the recab table.
if(myDoRecab)
{
myRecab.processReadBuildTable(*recordPtr);
}
// Nothing more to do with this record, so
// release the pointer.
mySamPool.releaseRecord(recordPtr);
}
else
{
if(SamFlag::isDuplicate(flag) && !myForceFlag)
{
// Error: Marked duplicates, and duplicates aren't excluded.
Logger::gLogger->error("There are records already duplicate marked.");
Logger::gLogger->error("Use -f to clear the duplicate flag and start the dedup_LowMem procedure over");
}
checkDups(*recordPtr, recordCount);
mySamPool.releaseRecord(recordPtr);
}
// let the user know we're not napping
if (verboseFlag && (recordCount % 100000 == 0))
{
Logger::gLogger->writeLog("recordCount=%u singleKeyMap=%u pairedKeyMap=%u, dictSize=%u",
recordCount, myFragmentMap.size(),
myPairedMap.size(),
myMateMap.size());
}
}
// we're finished reading record so clean up the duplicate search and
// close the input file
cleanupPriorReads(NULL);
samIn.Close();
// print some statistics
Logger::gLogger->writeLog("--------------------------------------------------------------------------");
Logger::gLogger->writeLog("SUMMARY STATISTICS OF THE READS");
Logger::gLogger->writeLog("Total number of reads: %u",recordCount);
Logger::gLogger->writeLog("Total number of paired-end reads: %u",
pairedCount);
Logger::gLogger->writeLog("Total number of properly paired reads: %u",
properPairCount);
Logger::gLogger->writeLog("Total number of unmapped reads: %u",
unmappedCount);
Logger::gLogger->writeLog("Total number of reverse strand mapped reads: %u",
reverseCount);
Logger::gLogger->writeLog("Total number of QC-failed reads: %u",
qualCheckFailCount);
Logger::gLogger->writeLog("Total number of secondary reads: %u",
secondaryCount);
Logger::gLogger->writeLog("Total number of supplementary reads: %u",
supplementaryCount);
Logger::gLogger->writeLog("Size of singleKeyMap (must be zero): %u",
myFragmentMap.size());
Logger::gLogger->writeLog("Size of pairedKeyMap (must be zero): %u",
myPairedMap.size());
Logger::gLogger->writeLog("Total number of missing mates: %u",
myNumMissingMate);
Logger::gLogger->writeLog("Total number of reads excluded from duplicate checking: %u",
excludedCount);
Logger::gLogger->writeLog("--------------------------------------------------------------------------");
Logger::gLogger->writeLog("Sorting the indices of %d duplicated records",
myDupList.size());
// sort the indices of duplicate records
std::sort(myDupList.begin(), myDupList.end(),
std::less<uint32_t> ());
// get ready to write the output file by making a second pass
// through the input file
samIn.OpenForRead(inFile.c_str());
samIn.ReadHeader(header);
SamFile samOut;
samOut.OpenForWrite(outFile.c_str());
samOut.WriteHeader(header);
// If we are recalibrating, output the model information.
if(myDoRecab)
{
myRecab.modelFitPrediction(outFile);
}
// an iterator to run through the duplicate indices
int currentDupIndex = 0;
bool moreDups = !myDupList.empty();
// let the user know what we're doing
Logger::gLogger->writeLog("\nWriting %s", outFile.c_str());
// count the duplicate records as a check
uint32_t singleDuplicates(0), pairedDuplicates(0);
// start reading records and writing them out
SamRecord record;
while(samIn.ReadRecord(header, record))
{
uint32_t currentIndex = samIn.GetCurrentRecordCount();
bool foundDup = moreDups &&
(currentIndex == myDupList[currentDupIndex]);
// modify the duplicate flag and write out the record,
// if it's appropriate
int flag = record.getFlag();
if (foundDup)
{
// this record is a duplicate, so mark it.
record.setFlag( flag | 0x400 );
currentDupIndex++;
// increment duplicate counters to verify we found them all
if ( ( ( flag & 0x0001 ) == 0 ) || ( flag & 0x0008 ) )
{ // unpaired or mate unmapped
singleDuplicates++;
}
else
{
pairedDuplicates++;
}
// recalibrate if necessary.
if(myDoRecab)
{
myRecab.processReadApplyTable(record);
}
// write the record if we are not removing duplicates
if (!removeFlag ) samOut.WriteRecord(header, record);
}
else
{
if(myForceFlag)
{
// this is not a duplicate we've identified but we want to
// remove any duplicate marking
record.setFlag( flag & 0xfffffbff ); // unmark duplicate
}
// Not a duplicate, so recalibrate if necessary.
if(myDoRecab)
{
myRecab.processReadApplyTable(record);
}
samOut.WriteRecord(header, record);
}
// Let the user know we're still here
if (verboseFlag && (currentIndex % 100000 == 0)) {
Logger::gLogger->writeLog("recordCount=%u", currentIndex);
}
}
// We're done. Close the files and print triumphant messages.
samIn.Close();
samOut.Close();
Logger::gLogger->writeLog("Successfully %s %u unpaired and %u paired duplicate reads",
removeFlag ? "removed" : "marked" ,
singleDuplicates,
pairedDuplicates/2);
Logger::gLogger->writeLog("\nDedup_LowMem complete!");
return 0;
}
int Recab::execute(int argc, char *argv[])
{
bool verboseFlag = false;
String inFile,outFile,logFile;
bool noeof = false;
bool params = false;
SamFile samIn,samOut;
ParameterList inputParameters;
LongParamContainer parameters;
parameters.addGroup("Required Generic Parameters");
parameters.addString("in", &inFile);
parameters.addString("out", &outFile);
parameters.addGroup("Optional Generic Parameters");
parameters.addString("log", &logFile);
parameters.addBool("verbose", &verboseFlag);
parameters.addBool("noeof", &noeof);
parameters.addBool("params", ¶ms);
parameters.addPhoneHome(VERSION);
addRecabSpecificParameters(parameters);
inputParameters.Add(new LongParameters ("Input Parameters",
parameters.getLongParameterList()));
// parameters start at index 2 rather than 1.
inputParameters.Read(argc, argv, 2);
// If no eof block is required for a bgzf file, set the bgzf file type to
// not look for it.
if(noeof)
{
// Set that the eof block is not required.
BgzfFileType::setRequireEofBlock(false);
}
if(inFile.IsEmpty())
{
usage();
inputParameters.Status();
std::cerr << "Missing required --in parameter" << std::endl;
return EXIT_FAILURE;
}
if(outFile.IsEmpty())
{
usage();
inputParameters.Status();
std::cerr << "Missing required --out parameter" << std::endl;
return EXIT_FAILURE;
}
int status = processRecabParam();
if(status != 0)
{
inputParameters.Status();
return(status);
}
if ( logFile.IsEmpty() )
{
logFile = outFile + ".log";
}
if(params)
{
inputParameters.Status();
}
Logger::gLogger = new Logger(logFile.c_str(), verboseFlag);
////////////////
////// Errormodel
Logger::gLogger->writeLog("Initialize errormodel structure...");
////////////////////////////////////////
// SAM/BAM file open
////////////////////////////////////////
////////////////////////////////////////
// Iterate SAM records
if(!samIn.OpenForRead(inFile.c_str()))
{
Logger::gLogger->error("Failed to open SAM/BAM file %s",inFile.c_str() );
return EXIT_FAILURE;
}
Logger::gLogger->writeLog("Start iterating SAM/BAM file %s",inFile.c_str());
time_t now = time(0);
tm* localtm = localtime(&now);
Logger::gLogger->writeLog("Start: %s", asctime(localtm));
SamRecord samRecord;
SamFileHeader samHeader;
samIn.ReadHeader(samHeader);
srand (time(NULL));
int numRecs = 0;
while(samIn.ReadRecord(samHeader, samRecord) == true)
{
processReadBuildTable(samRecord);
//Status info
numRecs++;
if(verboseFlag)
{
if(numRecs%10000000==0)
Logger::gLogger->writeLog("%ld records processed", numRecs);
}
}
now = time(0);
localtm = localtime(&now);
Logger::gLogger->writeLog("End: %s", asctime(localtm));
if((outFile[0] == '-') && (logFile[0] != '-'))
{
// Since outFile is to stdout, and logfile isn't, pass logfile name
modelFitPrediction(logFile);
}
else
{
modelFitPrediction(outFile);
}
Logger::gLogger->writeLog("Writing recalibrated file %s",outFile.c_str());
////////////////////////
////////////////////////
//// Write file
samIn.OpenForRead(inFile.c_str());
samOut.OpenForWrite(outFile.c_str());
samIn.ReadHeader(samHeader);
samOut.WriteHeader(samHeader);
while(samIn.ReadRecord(samHeader, samRecord) == true)
{
// Recalibrate.
processReadApplyTable(samRecord);
samOut.WriteRecord(samHeader, samRecord);
}
Logger::gLogger->writeLog("Total # Reads recab table not applied to: %ld", myNumApplySkipped);
Logger::gLogger->writeLog("Total # Reads recab table applied to: %ld", myNumApplyReads);
Logger::gLogger->writeLog("Recalibration successfully finished");
return EXIT_SUCCESS;
}