inline
int
ScanInterpretTest::scanReadVerify(Ndb* pNdb,
int records,
int parallelism,
ScanFilter& filter){
int retryAttempt = 0;
const int retryMax = 100;
int check;
NdbConnection *pTrans;
NdbScanOperation *pOp;
while (true){
if (retryAttempt >= retryMax){
ndbout << "ERROR: has retried this operation " << retryAttempt
<< " times, failing!" << endl;
return NDBT_FAILED;
}
pTrans = pNdb->startTransaction();
if (pTrans == NULL) {
const NdbError err = pNdb->getNdbError();
if (err.status == NdbError::TemporaryError){
ERR(err);
NdbSleep_MilliSleep(50);
retryAttempt++;
continue;
}
ERR(err);
return NDBT_FAILED;
}
pOp = pTrans->getNdbScanOperation(tab.getName());
if (pOp == NULL) { if (pOp->getValue("KOL2") == 0){
ERR(pNdb->getNdbError());
return NDBT_FAILED;
}
ERR(pTrans->getNdbError());
pNdb->closeTransaction(pTrans);
return NDBT_FAILED;
}
if( pOp->readTuples(NdbScanOperation::LM_Read, 0, parallelism) ) {
ERR(pTrans->getNdbError());
pNdb->closeTransaction(pTrans);
return NDBT_FAILED;
}
check = pOp->interpret_exit_ok();
if (check == -1) {
ERR(pTrans->getNdbError());
pNdb->closeTransaction(pTrans);
return NDBT_FAILED;
}
// Read all attributes
for(int a = 0; a<tab.getNoOfColumns(); a++){
if((row.attributeStore(a) =
pOp->getValue(tab.getColumn(a)->getName())) == 0) {
ERR(pTrans->getNdbError());
pNdb->closeTransaction(pTrans);
return NDBT_FAILED;
}
}
check = pTrans->execute(NoCommit);
if( check == -1 ) {
ERR(pTrans->getNdbError());
pNdb->closeTransaction(pTrans);
return NDBT_FAILED;
}
int eof;
int rows = 0;
int rowsNoExist = 0;
int rowsExist = 0;
int existingRecordsNotFound = 0;
int nonExistingRecordsFound = 0;
NdbConnection* pExistTrans;
NdbConnection* pNoExistTrans;
while((eof = pOp->nextResult(true)) == 0){
pExistTrans = pNdb->startTransaction();
if (pExistTrans == NULL) {
const NdbError err = pNdb->getNdbError();
ERR(err);
return NDBT_FAILED;
}
pNoExistTrans = pNdb->startTransaction();
if (pNoExistTrans == NULL) {
const NdbError err = pNdb->getNdbError();
ERR(err);
return NDBT_FAILED;
}
do {
rows++;
if (filter.verifyRecord(row) == NDBT_OK){
rowsExist++;
if (addRowToCheckTrans(pNdb, pExistTrans) != 0){
pNdb->closeTransaction(pTrans);
pNdb->closeTransaction(pExistTrans);
pNdb->closeTransaction(pNoExistTrans);
return NDBT_FAILED;
}
}else{
rowsNoExist++;
if (addRowToCheckTrans(pNdb, pNoExistTrans) != 0){
pNdb->closeTransaction(pTrans);
pNdb->closeTransaction(pExistTrans);
pNdb->closeTransaction(pNoExistTrans);
return NDBT_FAILED;
}
}
} while((eof = pOp->nextResult(false)) == 0);
// Execute the transaction containing reads of
// all the records that should be in the result table
check = pExistTrans->execute(Commit);
if( check == -1 ) {
const NdbError err = pExistTrans->getNdbError();
ERR(err);
if (err.code != 626){
pNdb->closeTransaction(pExistTrans);
pNdb->closeTransaction(pNoExistTrans);
pNdb->closeTransaction(pTrans);
return NDBT_FAILED;
}else{
// Some of the records expected to be found wasn't
// there
existingRecordsNotFound = 1;
}
}
pNdb->closeTransaction(pExistTrans);
// Execute the transaction containing reads of
// all the records that should NOT be in the result table
check = pNoExistTrans->execute(Commit, CommitAsMuchAsPossible);
if( check == -1 ) {
const NdbError err = pNoExistTrans->getNdbError();
// The transactions error code should be zero
if (err.code != 626){
ERR(err);
pNdb->closeTransaction(pNoExistTrans);
pNdb->closeTransaction(pTrans);
return NDBT_FAILED;
}
// Loop through the no existing transaction and check that no
// operations where successful
const NdbOperation* pOp2 = NULL;
while ((pOp2 = pNoExistTrans->getNextCompletedOperation(pOp2)) != NULL){
const NdbError err = pOp2->getNdbError();
if (err.code != 626){
ndbout << "err.code = " << err.code<< endl;
nonExistingRecordsFound = 1;
}
}
}
pNdb->closeTransaction(pNoExistTrans);
}
if (eof == -1) {
const NdbError err = pTrans->getNdbError();
if (err.status == NdbError::TemporaryError){
ERR(err);
pNdb->closeTransaction(pTrans);
NdbSleep_MilliSleep(50);
retryAttempt++;
continue;
}
ERR(err);
pNdb->closeTransaction(pTrans);
return NDBT_FAILED;
}
int testResult = NDBT_OK;
int rowsResult = 0;
UtilTransactions utilTrans(restab);
if (utilTrans.selectCount(pNdb,
240,
&rowsResult) != 0){
return NDBT_FAILED;
}
if (existingRecordsNotFound == 1){
ndbout << "!!! Expected records not found" << endl;
testResult = NDBT_FAILED;
}
if (nonExistingRecordsFound == 1){
ndbout << "!!! Unxpected records found" << endl;
testResult = NDBT_FAILED;
}
ndbout << rows << " rows scanned("
<< rowsExist << " found, " << rowsResult<<" expected)" << endl;
if (rowsResult != rowsExist){
ndbout << "!!! Number of rows in result table different from expected" << endl;
testResult = NDBT_FAILED;
}
return testResult;
}
return NDBT_FAILED;
}
inline
int
ScanInterpretTest::scanRead(Ndb* pNdb,
int records,
int parallelism,
ScanFilter& filter){
int retryAttempt = 0;
int retryMax = 100;
int check;
NdbConnection *pTrans;
NdbScanOperation *pOp;
while (true){
if (retryAttempt >= retryMax){
ndbout << "ERROR: has retried this operation " << retryAttempt
<< " times, failing!" << endl;
return NDBT_FAILED;
}
pTrans = pNdb->startTransaction();
if (pTrans == NULL) {
const NdbError err = pNdb->getNdbError();
if (err.status == NdbError::TemporaryError){
ERR(err);
NdbSleep_MilliSleep(50);
retryAttempt++;
continue;
}
ERR(err);
return NDBT_FAILED;
}
pOp = pTrans->getNdbScanOperation(tab.getName());
if (pOp == NULL) {
ERR(pTrans->getNdbError());
pNdb->closeTransaction(pTrans);
return NDBT_FAILED;
}
if( pOp->readTuples(NdbScanOperation::LM_Read, 0, parallelism) ) {
ERR(pTrans->getNdbError());
pNdb->closeTransaction(pTrans);
return NDBT_FAILED;
}
if (filter.filterOp(pOp) != 0){
ERR(pTrans->getNdbError());
pNdb->closeTransaction(pTrans);
return NDBT_FAILED;
}
// Read all attributes
for(int a = 0; a<tab.getNoOfColumns(); a++){
if((row.attributeStore(a) =
pOp->getValue(tab.getColumn(a)->getName())) == 0) {
ERR(pTrans->getNdbError());
pNdb->closeTransaction(pTrans);
return NDBT_FAILED;
}
}
check = pTrans->execute(NoCommit);
if( check == -1 ) {
ERR(pTrans->getNdbError());
pNdb->closeTransaction(pTrans);
return NDBT_FAILED;
}
int eof;
int rows = 0;
NdbConnection* pInsTrans;
while((eof = pOp->nextResult(true)) == 0){
do {
rows++;
if (addRowToInsert(pNdb, pTrans) != 0){
pNdb->closeTransaction(pTrans);
return NDBT_FAILED;
}
} while((eof = pOp->nextResult(false)) == 0);
check = pTrans->execute(Commit);
if( check == -1 ) {
const NdbError err = pTrans->getNdbError();
ERR(err);
pNdb->closeTransaction(pTrans);
return NDBT_FAILED;
}
}
if (eof == -1) {
const NdbError err = pTrans->getNdbError();
if (err.status == NdbError::TemporaryError){
ERR(err);
pNdb->closeTransaction(pTrans);
NdbSleep_MilliSleep(50);
retryAttempt++;
continue;
}
ERR(err);
pNdb->closeTransaction(pTrans);
return NDBT_FAILED;
}
pNdb->closeTransaction(pTrans);
g_info << rows << " rows have been scanned" << endl;
return NDBT_OK;
}
return NDBT_FAILED;
}
PWIZ_API_DECL void ChromatogramList_Thermo::createIndex() const
{
for (int controllerType = Controller_MS;
controllerType <= Controller_UV;
++controllerType)
{
long numControllers = rawfile_->getNumberOfControllersOfType((ControllerType) controllerType);
for (long n=1; n <= numControllers; ++n)
{
rawfile_->setCurrentController((ControllerType) controllerType, n);
// skip this controller if it has no spectra
if (rawfile_->value(NumSpectra) == 0)
continue;
switch ((ControllerType) controllerType)
{
case Controller_MS:
{
// support file-level TIC for all file types
index_.push_back(IndexEntry());
IndexEntry& ci = index_.back();
ci.controllerType = (ControllerType) controllerType;
ci.controllerNumber = n;
ci.filter = "";
ci.index = index_.size()-1;
ci.id = "TIC";
ci.chromatogramType = MS_TIC_chromatogram;
idMap_[ci.id] = ci.index;
// for certain filter types, support additional chromatograms
auto_ptr<StringArray> filterArray = rawfile_->getFilters();
ScanFilter filterParser;
for (size_t i=0, ic=filterArray->size(); i < ic; ++i)
{
string filterString = filterArray->item(i);
filterParser.initialize();
filterParser.parse(filterString);
switch (filterParser.scanType_)
{
case ScanType_SRM:
{
// produce spectra rather than a chromatogram
if (config_.srmAsSpectra)
break;
string precursorMZ = (format("%.10g") % filterParser.precursorMZs_[0]).str();
/*index_.push_back(IndexEntry());
IndexEntry& ci = index_.back();
ci.controllerType = (ControllerType) controllerType;
ci.controllerNumber = n;
ci.filter = filterString;
ci.index = index_.size()-1;
ci.id = "SRM TIC " + precursorMZ;
ci.q1 = filterParser.precursorMZs_[0];
idMap_[ci.id] = ci.index;*/
for (size_t j=0, jc=filterParser.scanRangeMin_.size(); j < jc; ++j)
{
index_.push_back(IndexEntry());
IndexEntry& ci = index_.back();
ci.chromatogramType = MS_SRM_chromatogram;
ci.controllerType = (ControllerType) controllerType;
ci.controllerNumber = n;
ci.filter = filterString;
ci.index = index_.size()-1;
ci.q1 = filterParser.precursorMZs_[0];
ci.q3 = (filterParser.scanRangeMin_[j] + filterParser.scanRangeMax_[j]) / 2.0;
ci.id = (format("SRM SIC %s,%.10g")
% precursorMZ
% ci.q3
).str();
ci.q3Offset = (filterParser.scanRangeMax_[j] - filterParser.scanRangeMin_[j]) / 2.0;
idMap_[ci.id] = ci.index;
}
}
break; // case ScanType_SRM
case ScanType_SIM:
{
// produce spectra rather than a chromatogram
if (config_.simAsSpectra)
break;
for (size_t j=0, jc=filterParser.scanRangeMin_.size(); j < jc; ++j)
{
index_.push_back(IndexEntry());
IndexEntry& ci = index_.back();
ci.chromatogramType = MS_SIM_chromatogram;
ci.controllerType = (ControllerType) controllerType;
ci.controllerNumber = n;
ci.filter = filterString;
ci.index = index_.size()-1;
ci.q1 = (filterParser.scanRangeMin_[j] + filterParser.scanRangeMax_[j]) / 2.0;
ci.id = (format("SIM SIC %.10g")
% ci.q1
).str();
// this should be q1Offset
ci.q3Offset = (filterParser.scanRangeMax_[j] - filterParser.scanRangeMin_[j]) / 2.0;
idMap_[ci.id] = ci.index;
}
}
break; // case ScanType_SIM
default:
case ScanType_Full:
/*{
string precursorMZ = lexical_cast<string>(filterParser.precursorMZs_[0]);
index_.push_back(make_pair(ChromatogramIdentity(), filterString));
ChromatogramIdentity& ci = index_.back().first;
ci.index = index_.size()-1;
ci.id = "SIC " + precursorMZ;
idMap_[ci.id] = ci.index;
}*/
break;
}
}
}
break; // case Controller_MS
case Controller_PDA:
{
// "Total Scan" appears to be the equivalent of the TIC
index_.push_back(IndexEntry());
IndexEntry& ci = index_.back();
ci.controllerType = (ControllerType) controllerType;
ci.controllerNumber = n;
ci.index = index_.size()-1;
ci.id = "Total Scan";
ci.chromatogramType = MS_absorption_chromatogram;
idMap_[ci.id] = ci.index;
}
break; // case Controller_PDA
case Controller_Analog:
{
// "ECD" appears to be the equivalent of the TIC
index_.push_back(IndexEntry());
IndexEntry& ci = index_.back();
ci.controllerType = (ControllerType) controllerType;
ci.controllerNumber = n;
ci.index = index_.size()-1;
ci.id = "ECD";
ci.chromatogramType = MS_emission_chromatogram;
idMap_[ci.id] = ci.index;
}
default:
// TODO: are there sensible default chromatograms for other controller types?
break;
}
}
}
/*ostringstream imStream;
std::auto_ptr<LabelValueArray> imArray = rawfile_->getInstrumentMethods();
for(size_t i=0, end=imArray->size(); i < end; ++i)
imStream << imArray->label(i) << imArray->value(i) << endl;
string im = imStream.str();
// Parent Center Width Time CE Q1PW Q3PW TubeLens
boost::regex scanEventRegex("^\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+$");*/
}
PWIZ_API_DECL ChromatogramPtr ChromatogramList_Thermo::chromatogram(size_t index, bool getBinaryData) const
{
boost::call_once(indexInitialized_.flag, boost::bind(&ChromatogramList_Thermo::createIndex, this));
if (index>size())
throw runtime_error(("[ChromatogramList_Thermo::chromatogram()] Bad index: "
+ lexical_cast<string>(index)).c_str());
const IndexEntry& ci = index_[index];
ChromatogramPtr result(new Chromatogram);
result->index = ci.index;
result->id = ci.id;
result->set(ci.chromatogramType);
rawfile_->setCurrentController(ci.controllerType, ci.controllerNumber);
switch (ci.chromatogramType)
{
default:
break;
case MS_TIC_chromatogram:
{
ChromatogramDataPtr cd = rawfile_->getChromatogramData(
Type_TIC, Operator_None, Type_MassRange,
"", "", "", 0,
0, rawfile_->rt(rawfile_->value(NumSpectra)),
Smoothing_None, 0);
pwiz::msdata::TimeIntensityPair* data = reinterpret_cast<pwiz::msdata::TimeIntensityPair*>(cd->data());
if (getBinaryData) result->setTimeIntensityPairs(data, cd->size(), UO_minute, MS_number_of_counts);
else result->defaultArrayLength = cd->size();
}
break;
case MS_SIC_chromatogram: // generate SIC for <precursor>
{
ChromatogramDataPtr cd = rawfile_->getChromatogramData(
Type_BasePeak, Operator_None, Type_MassRange,
index_[index].filter, "", "", 0,
0, rawfile_->rt(rawfile_->value(NumSpectra)),
Smoothing_None, 0);
pwiz::msdata::TimeIntensityPair* data = reinterpret_cast<pwiz::msdata::TimeIntensityPair*>(cd->data());
if (getBinaryData) result->setTimeIntensityPairs(data, cd->size(), UO_minute, MS_number_of_counts);
else result->defaultArrayLength = cd->size();
}
break;
/*case 3: // generate SRM TIC for <precursor>
{
vector<string> tokens;
bal::split(tokens, ci.id, bal::is_any_of(" "));
ChromatogramDataPtr cd = rawfile_->getChromatogramData(
Type_TIC, Operator_None, Type_MassRange,
"ms2 " + tokens[2], "", "", 0,
0, rawfile_->rt(rawfile_->value(NumSpectra)),
Smoothing_None, 0);
pwiz::msdata::TimeIntensityPair* data = reinterpret_cast<pwiz::msdata::TimeIntensityPair*>(cd->data());
if (getBinaryData) result->setTimeIntensityPairs(data, cd->size(), UO_minute, MS_number_of_counts);
else result->defaultArrayLength = cd->size();
}
break;*/
case MS_SRM_chromatogram:
{
result->precursor.isolationWindow.set(MS_isolation_window_target_m_z, ci.q1, MS_m_z);
ScanFilter filterParser;
filterParser.parse(ci.filter);
ActivationType activationType = filterParser.activationType_;
if (activationType == ActivationType_Unknown)
activationType = ActivationType_CID; // assume CID
SetActivationType(activationType, result->precursor.activation);
if (filterParser.activationType_ == ActivationType_CID)
result->precursor.activation.set(MS_collision_energy, filterParser.precursorEnergies_[0]);
result->product.isolationWindow.set(MS_isolation_window_target_m_z, ci.q3, MS_m_z);
result->product.isolationWindow.set(MS_isolation_window_lower_offset, ci.q3Offset, MS_m_z);
result->product.isolationWindow.set(MS_isolation_window_upper_offset, ci.q3Offset, MS_m_z);
string q1 = (format("%.10g") % ci.q1).str();
string q3Range = (format("%.10g-%.10g")
% (ci.q3 - ci.q3Offset)
% (ci.q3 + ci.q3Offset)
).str();
ChromatogramDataPtr cd = rawfile_->getChromatogramData(
Type_MassRange, Operator_None, Type_MassRange,
"SRM ms2 " + q1, q3Range, "", 0,
0, rawfile_->rt(rawfile_->value(NumSpectra)),
Smoothing_None, 0);
pwiz::msdata::TimeIntensityPair* data = reinterpret_cast<pwiz::msdata::TimeIntensityPair*>(cd->data());
if (getBinaryData) result->setTimeIntensityPairs(data, cd->size(), UO_minute, MS_number_of_counts);
else result->defaultArrayLength = cd->size();
}
break;
case MS_SIM_chromatogram:
{
result->precursor.isolationWindow.set(MS_isolation_window_target_m_z, ci.q1, MS_m_z);
result->precursor.isolationWindow.set(MS_isolation_window_lower_offset, ci.q3Offset, MS_m_z);
result->precursor.isolationWindow.set(MS_isolation_window_upper_offset, ci.q3Offset, MS_m_z);
string q1Range = (format("%.10g-%.10g")
% (ci.q1 - ci.q3Offset)
% (ci.q1 + ci.q3Offset)
).str();
ChromatogramDataPtr cd = rawfile_->getChromatogramData(
Type_MassRange, Operator_None, Type_MassRange,
"SIM ms [" + q1Range + "]", q1Range, "", 0,
0, rawfile_->rt(rawfile_->value(NumSpectra)),
Smoothing_None, 0);
pwiz::msdata::TimeIntensityPair* data = reinterpret_cast<pwiz::msdata::TimeIntensityPair*>(cd->data());
if (getBinaryData) result->setTimeIntensityPairs(data, cd->size(), UO_minute, MS_number_of_counts);
else result->defaultArrayLength = cd->size();
}
break;
case MS_absorption_chromatogram: // generate "Total Scan" chromatogram for entire run
{
ChromatogramDataPtr cd = rawfile_->getChromatogramData(
Type_TotalScan, Operator_None, Type_MassRange,
"", "", "", 0,
0, rawfile_->rt(rawfile_->value(NumSpectra)),
Smoothing_None, 0);
pwiz::msdata::TimeIntensityPair* data = reinterpret_cast<pwiz::msdata::TimeIntensityPair*>(cd->data());
if (getBinaryData) result->setTimeIntensityPairs(data, cd->size(), UO_minute, MS_number_of_counts);
else result->defaultArrayLength = cd->size();
}
break;
case MS_mass_chromatogram: // generate "ECD" chromatogram for entire run
{
ChromatogramDataPtr cd = rawfile_->getChromatogramData(
Type_ECD, Operator_None, Type_MassRange,
"", "", "", 0,
0, std::numeric_limits<double>::max(),
Smoothing_None, 0);
pwiz::msdata::TimeIntensityPair* data = reinterpret_cast<pwiz::msdata::TimeIntensityPair*>(cd->data());
if (getBinaryData) result->setTimeIntensityPairs(data, cd->size(), UO_minute, MS_number_of_counts);
else result->defaultArrayLength = cd->size();
}
break;
}
return result;
}
int main(int argc, char* argv[])
{
try
{
if (argc > 2)
throw runtime_error("Usage: ScanFilterTest [Thermo RAW filename]");
else if (argc == 1)
{
// unit test static strings
for (size_t i=0; i < testScanFiltersSize; ++i)
{
const TestScanFilter& f = testScanFilters[i];
try
{
vector<double> cidParentMass = parseDoubleArray(f.cidParentMassArray);
vector<double> cidParentEnergy = parseDoubleArray(f.cidEnergyArray);
vector<double> scanRangeMin = parseDoubleArray(f.scanRangeMinArray);
vector<double> scanRangeMax = parseDoubleArray(f.scanRangeMaxArray);
double compensationVoltage = 0.0;
if (string(f.compensationVoltage) != "")
compensationVoltage = lexical_cast<double> (f.compensationVoltage);
ScanFilter scanFilter;
scanFilter.parse(f.filter);
testFilter(scanFilter,
f.scanSegment,
f.scanEvent,
f.accurateMassType,
f.coronaOn,
f.detectorSet,
f.photoIonizationOn,
f.sourceCIDOn,
f.turboScanOn,
f.supplementalCIDOn,
f.widebandOn,
f.enhancedOn,
f.dependentActive,
f.lockMassOn,
f.faimsOn,
compensationVoltage,
f.massAnalyzerType,
f.polarityType,
f.dataPointType,
f.ionizationType,
f.scanType,
f.msLevel,
cidParentMass,
cidParentEnergy,
scanRangeMin,
scanRangeMax);
}
catch (exception& e)
{
cout << "Unit test on filter \"" << f.filter << "\" failed:\n" << e.what() << endl;
}
}
}
return 0;
}
catch (exception& e)
{
cout << "Caught exception: " << e.what() << endl;
}
catch (...)
{
cout << "Caught unknown exception.\n";
}
return 1;
}