void KKStrTest::AssertAreEqual (const char* expected, const KKStr& found, const KKStr& testName)
{
bool equal = strcmp(expected, found.Str ()) == 0;
KKStr msg;
msg << "Expected: " << expected << "\t" << "Found: " << found.Str ();
Assert(equal, testName, msg);
}
void KKJob::ProcessStatusStr (const KKStr& statusStr)
{
log.Level (30) << "KKJob::ProcessStatusStr[" << statusStr << "]" << endl;
KKStr fieldName;
KKStr fieldValue;
VectorKKStr fields = statusStr.Split ('\t');
kkuint32 fieldNum = 0;
while (fieldNum < fields.size ())
{
fieldName = fields[fieldNum];
fieldNum++;
if (fieldNum < fields.size ())
{
fieldValue = fields[fieldNum];
fieldNum++;
}
else
{
fieldValue = "";
}
fieldName.Upper ();
fieldValue.TrimLeft ("\n\r\t ");
fieldValue.TrimRight ("\n\r\t ");
if (fieldName.CompareIgnoreCase ("JOBID") == 0)
jobId = atoi (fieldValue.Str ());
else if (fieldName.CompareIgnoreCase ("PARENTID") == 0)
parentId = atoi (fieldValue.Str ());
else if (fieldName.CompareIgnoreCase ("STATUS") == 0)
status = JobStatusFromStr (fieldValue);
else if (fieldName.CompareIgnoreCase ("NumProcessors") == 0)
numProcessors = fieldValue.ToInt ();
else if (fieldName.CompareIgnoreCase ("NumPorcessesAllowed") == 0)
numPorcessesAllowed = fieldValue.ToInt ();
else if (fieldName.CompareIgnoreCase ("Prerequisites") == 0)
PrerequisitesFromStr (fieldValue);
else
{
ProcessStatusField (fieldName, fieldValue);
}
}
} /* ProcessStatusStr */
void FeatureFileIO::SaveFeatureFile (const KKStr& _fileName,
FeatureNumListConst& _selFeatures,
FeatureVectorList& _examples,
kkuint32& _numExamplesWritten,
VolConstBool& _cancelFlag,
bool& _successful,
RunLog& _log
)
{
_log.Level (10) << "FeatureFileIO::SaveFeatureFile - File[" << _fileName << "]." << endl;
ofstream out (_fileName.Str ());
if (!out.is_open())
{
_log.Level (-1) << "***ERROR***, SaveFeatureFile, Opening File[" << _fileName << "]" << endl;
_successful = false;
}
out.precision (9);
KKStr errorMessage;
SaveFile (_examples, _fileName, _selFeatures, out, _numExamplesWritten, _cancelFlag, _successful, errorMessage, _log);
out.close ();
} /* SaveFeatureFile */
void CmdLineExpander::ExtractParametersFromFile (const KKStr& cmdFileName,
VectorKKStr& cmdFileParameters,
bool& validFile
)
{
FILE* in = osFOPEN (cmdFileName.Str (), "r");
if (!in)
{
log.Level (-1) << endl << endl << endl
<< "ExtractParametersFromFile *** EROR ***" << endl
<< endl
<< " Invalid CmdFile[" << cmdFileName << "]" << endl
<< endl;
validFile = false;
return;
}
KKStr token;
bool eof = false;
token = osReadNextQuotedStr (in, " \n\r", eof);
while (!eof)
{
cmdFileParameters.push_back (token);
token = osReadNextQuotedStr (in, " \n\r", eof);
}
std::fclose (in);
} /* ExtractParametersFromFile */
void DataBaseServerList::WriteConfigFile () const
{
osCreateDirectoryPath (PicesVariables::ConfigurationDirectory ());
KKStr configFileName = osAddSlash (PicesVariables::ConfigurationDirectory ()) + "MySql.cfg";
log.Level (10) << "DataBaseServerList::WriteConfigFile Writing ConfigFileName[" << configFileName << "]" << endl;
ofstream o (configFileName.Str ());
if (!o.is_open ())
{
log.Level (-1) << endl << endl
<< "DataBaseServerList::WriteConfigFile ***ERROR*** Opening ConfigFile[" << configFileName << "]" << endl
<< endl;
return;
}
o << "// PICES MySQL configuration file. There will be one entry for each known MySQL database server." << std::endl
<< "//" << std::endl
<< "// Date Written: " << osGetLocalDateTime () << std::endl
<< "//" << std::endl
<< "// Number Entries: " << QueueSize () << std::endl
<< "//" << std::endl;
DataBaseServerList::const_iterator idx;
for (idx = begin (); idx != end (); idx++)
o << "Server" << "\t" << (*idx)->ToParameterStr () << endl;
o << "//" << std::endl
<< "DefaultServer" << "\t" << defaultServerDescription << std::endl;
o.close ();
} /* WriteConfigFile */
HTMLReport& KKB::operator<< (HTMLReport& htmlReport,
const DateTime& right
)
{
KKStr s = right.Date ().YYYY_MM_DD () + "-" + right.Time ().HH_MM_SS ();
htmlReport.Append (s.Str ());
return htmlReport;
}
KKStrParser::KKStrParser (const KKStr& _str):
len (_str.Len ()),
nextPos (0),
str (_str.Str ()),
trimWhiteSpace (false),
weOwnStr (false),
whiteSpace (NULL)
{
}
/**
*@brief Adds Instrument data to the underlying Scanner files as text.
*/
void ScannerFile::WriteInstrumentDataWord (uchar idNum,
kkuint32 scanLineNum,
WordFormat32Bits dataWord
)
{
KKStr s (100);
s << "InstrumentDataWord" << "\t" << (int)idNum << "\t" << scanLineNum << "\t" << dataWord.unsigned32BitInt;
WriteTextBlock ((const uchar*)s.Str (), s.Len ());
} /* WriteInstrumentDataWord */
FeatureVectorListPtr FeatureFileIO::LoadFeatureFile
(const KKStr& _fileName,
MLClassList& _mlClasses,
OptionUInt32 _maxCount,
VolConstBool& _cancelFlag, // will be monitored, if set to True Load will terminate.
bool& _successful,
bool& _changesMade,
RunLog& _log
)
{
_log.Level (10) << "LoadFeatureFile File[" << _fileName << "] FileFormat[" << driverName << "]" << endl;
_changesMade = false;
kkint32 estimatedNumOfDataItems = -1;
_successful = true;
ifstream in (_fileName.Str (), ios_base::in);
if (!in.is_open ())
{
_log.Level (-1) << "LoadFeatureFile ***ERROR*** Error Opening File[" << _fileName << "]." << endl;
_successful = false;
return NULL;
}
KKStr errorMessage;
FileDescConstPtr fileDescFromFile = GetFileDesc (_fileName, in, &_mlClasses, estimatedNumOfDataItems, errorMessage, _log);
if (fileDescFromFile == NULL)
{
_log.Level (-1) << endl << endl
<< "FeatureFileIO::LoadFeatureFile ***ERROR*** Loading Feature File[" << _fileName << "]" << endl
<< endl;
_successful = false;
return NULL;
}
FileDescConstPtr fileDesc = FileDesc::GetExistingFileDesc (fileDescFromFile);
in.clear ();
in.seekg (0, ios::beg);
FeatureVectorListPtr examples = LoadFile (_fileName, fileDesc, _mlClasses, in, _maxCount, _cancelFlag, _changesMade, errorMessage, _log);
if (examples == NULL)
{
_successful = false;
}
else
{
_successful = true;
}
in.close ();
return examples;
} /* LoadFeatureFile */
HTMLReport& KKB::operator<< (HTMLReport& htmlReport,
kkuint64 right
)
{
KKStr s (30);
s = StrFormatInt64 (right, "0");
htmlReport.Append (s.Str ());
return htmlReport;
}
void InstrumentDataGPS::ParseDegreeMinutesStr (const KKStr& str,
double& degrees,
double& minutes
) const
{
degrees = 0.0;
minutes = 0.0;
auto x = str.LocateCharacter ('.');
if (!x)
return;
KKStr degStr = str.SubStrSeg (0, x - 2);
KKStr minStr = str.SubStrPart (x - 2);
degrees = atof (degStr.Str ());
minutes = atof (minStr.Str ());
} /* ParseDegreeMinutesStr */
void ScannerFile::WriteHeader ()
{
KKStr ln (100);
ln << "ScannerFile" << "\t" << FileFormatStr () << "\n";
fwrite (ln.Str (), 1, ln.Len (), file);
ScannerHeaderFields::const_iterator idx;
for (idx = headerFields->begin (); idx != headerFields->end (); ++idx)
{
ln = "";
ln << idx->first << "\t" << idx->second << "\n";
fwrite (ln.Str (), 1, ln.Len (), file);
}
// Write End of text Marker
char ch = 0;
fwrite (&ch, 1, 1, file);
headerDataWritten = true;
} /* WriteHeader */
void Histogram::Save (KKStr fileName) const
{
ofstream o (fileName.Str ());
kkint32 bucketIDX;
float avgVal = totalVal / totalCount;
o << "MinValue" << "\t" << "\t" << "BucketSize" << "\t" << "NumOfBuckets" << "\t" << "TotalCount" << "\t" << "Average" << std::endl;
o << minValue << "\t" << "\t" << bucketSize << "\t" << numOfBuckets << "\t" << totalCount << "\t" << avgVal << std::endl;
o << std::endl;
o << std::endl;
float avg;
float percentage;
float bucketVal = minValue + bucketSize;
o << "BucketValue" << "\t"
<< "Count" << "\t"
<< "AvgValue" << "\t"
<< "%OfTotalCount"
<< std::endl
<< std::endl;
for (bucketIDX = 0; bucketIDX < numOfBuckets; bucketIDX++)
{
o << bucketVal;
o << "\t" << buckets[bucketIDX];
if (buckets[bucketIDX] > (float)0.0)
avg = bucketTotals[bucketIDX] / buckets[bucketIDX];
else
avg = 0.0;
o << "\t" << avg;
percentage = (float)100.0 * buckets[bucketIDX] / totalCount;
o << "\t" << percentage << "%";
bucketVal += bucketSize;
o << std::endl;
}
o << std::endl;
o << std::endl;
o.close ();
} /* Save */
ActiveLearningReport::ActiveLearningReport (RunLog& _log,
MLClassList& _mlClasses,
KKStr _subDirName
):
mlClasses (_mlClasses),
log (_log),
subDirName (_subDirName),
results (true, 10),
baseResultsFileName ("ActiveLearningResults")
{
if (!subDirName.Empty ())
{
KKStr w (subDirName);
osAddLastSlash (w);
w << baseResultsFileName;
baseResultsFileName = w;
}
// C:\users\kkramer\GradSchool\Plankton\ActiveLearning\Results\010-IPC\2003-12-03_AllOrders_010-IPC_50-IPR
int x = subDirName.LocateLastOccurrence ('_');
if (x > 5)
{
KKStr leftSide = subDirName.SubStrPart (0, x - 1);
KKStr rightSide = subDirName.SubStrPart (x + 1);
x = rightSide.LocateCharacter ('-');
KKStr IPR = rightSide.SubStrPart (0, x - 1);
imagesPerRetraining = atoi (IPR.Str ());
x = leftSide.LocateLastOccurrence ('_');
rightSide = subDirName.SubStrPart (x + 1);
x = rightSide.LocateCharacter ('-');
KKStr IPC = rightSide.SubStrPart (0, x - 1);
initialImagesPerClass = atoi (IPC.Str ());
}
}
void GoalKeeper::ReportBlockedStats ()
{
HANDLE mutexCreateHandle = CreateMutex (NULL, /**< default security attributes. */
false, /**< initially not owned. */
"GoalKeeper_ReportBlockedStats"
);
WaitForSingleObject (mutexCreateHandle, INFINITE);
KKStr path = "C:\\Temp\\BlockedStats";
KKB::osCreateDirectoryPath (path);
KKStr fileName = osAddSlash (path) + "BlockedStats.txt";
ofstream o (fileName.Str (), ios_base::app);
o << "Name"
<< "\t" << "Count"
<< "\t" << "BlockedThreadId"
<< "\t" << "BlockerThreadId"
<< "\t" << "DateTimeBlocked"
<< "\t" << "MilliSecsBlocked"
<< "\t" << "NumBlockedThreads"
<< endl;
int c = 0;
BlockedStatList::iterator idx;
for (idx = blockedStats->begin (), c = 0; idx != blockedStats->end (); ++idx, ++c)
{
BlockedStat* bs = *idx;
o << name
<< "\t" << c
<< "\t" << bs->blockedThreadId
<< "\t" << bs->blockerThreadId
<< "\t" << bs->dateTimeBlocked.Date () << " " << bs->dateTimeBlocked.Time ()
<< "\t" << bs->milliSecsBlocked
<< "\t" << bs->numBlockedThreads
<< endl;
}
o.close ();
ReleaseMutex (mutexCreateHandle);
CloseHandle(mutexCreateHandle);
} /* ReportBlockedStats */
void ActiveLearningReport::Save (KKStr fileName)
{
ofstream f (fileName.Str ());
if (!f.is_open ())
{
log.Level (-1) << endl
<< "ActiveLearningReport::Save *** ERROR ***" << endl
<< endl
<< " Could noty open File[" << fileName << "]" << endl
<< endl;
osWaitForEnter ();
exit (-1);
}
f << "\\InitialTrainingImagesPerClass" << "\t" << initialImagesPerClass << endl;
f << "\\ImagesPerRetraining" << "\t" << imagesPerRetraining << endl;
SortOrderResultsListIterator soIDX (results);
SortOrderResultsPtr sortOrderResults = NULL;
for (soIDX.Reset (); sortOrderResults = soIDX.CurPtr (); ++soIDX)
{
f << "\\SortOrder" << "\t" << SortOrderDescr (sortOrderResults->SortOrder ()) << endl;
f << "\\NumOfRetrainings" << "\t" << sortOrderResults->QueueSize () << endl;
f << "\\Fields" << "\t" << "NumOfTrainingImages"
<< "\t" << "Accuracy"
<< "\t" << "NumSupportVectors"
<< endl;
RetrainingResultsListIterator rIDX (*sortOrderResults);
RetrainingResultsPtr retraining = NULL;
for (rIDX.Reset (); retraining = rIDX.CurPtr (); ++rIDX)
{
f << retraining->NumOfTrainingImages () << "\t"
<< retraining->Accuracy () << "\t"
<< retraining->NumSupportVectors ()
<< endl;
}
}
f.close ();
} /* Save */
ScannerFile::Format ScannerFile::GuessFormatOfFile (const KKStr& _fileName,
RunLog& _log
)
{
// Will guess what file format by trying to open each one until one is considered valid.
FILE* f = osFOPEN (_fileName.Str (), "rb");
if (!f)
return Format::sfUnKnown;
bool endOfText = false;
KKStr ln (100);
ReadHeaderOneLine (f, endOfText, ln);
fclose (f);
KKStr fieldName = ln.ExtractToken2 ("\t");
if (!fieldName.EqualIgnoreCase ("ScannerFile"))
return Format::sfUnKnown;
KKStr scannerFileFormatStr = ln.ExtractToken2 ("\t");
return ScannerFileFormatFromStr (scannerFileFormatStr);
} /* GuessFormatOfFile */
void FeatureFileIO::AppendToFile (const KKStr& _fileName,
FeatureNumListConst& _selFeatures,
FeatureVectorList& _examples,
kkuint32& _numExamplesWritten,
VolConstBool& _cancelFlag,
bool& _successful,
RunLog& _log
)
{
_log.Level (10) << "FeatureFileIO::AppendToFile - File[" << _fileName << "]." << endl;
_successful = true;
ofstream out (_fileName.Str (), ios::app);
if (!out.is_open())
{
KKStr err;
err << "AppendToFile Error Opening File[" << _fileName << "]";
_log.Level (-1) << endl
<< "FeatureFileIO::AppendToFile ***ERROR***" << endl
<< endl
<< " " << err << endl
<< endl;
osDisplayWarning (err);
_successful = false;
return;
}
KKStr errorMessage;
SaveFile (_examples, _fileName, _selFeatures, out, _numExamplesWritten, _cancelFlag, _successful, errorMessage, _log);
out.close ();
return;
} /* AppendToFile */
void OurNeighbors::RandomReport (ImageFeaturesList& images)
{
double allClassesMeanNNDAnyClass = 0.0f;
double allClassesMeanStdDevAnyClass = 0.0f;
ClassSummaryList classSummaries (log);
MLClassList::iterator classIdx;
VectorKKStr zScoreSummaryLines;
for (classIdx = mlClasses->begin (); classIdx != mlClasses->end (); classIdx++)
{
MLClassPtr mlClass = *classIdx;
if (fromPlankton && (fromPlankton != mlClass))
continue;
double randomMeanNND = 0.0f;
double randomStdDevNND = 0.0f;
double realDataU2Stat = 0.0f;
double sampleMeanNND = 0.0f;
double sampleStdDevNND = 0.0f;
double sampleMaxDist = 0.0f;
double sampleMinDist = 0.0f;
ImageFeaturesListPtr imagesInClass = images.ExtractExamplesForAGivenClass (mlClass);
if (imagesInClass->QueueSize () > 0)
{
// We are going to make a list of images that has duplicate instances of 'ImageFeatures' objects
// This way when we Randomize the locations of each 'sfCentroidRow' and 'sfCentroidCol' we do not
// imapct on the original data.
ImageFeaturesListPtr imagesToRandomize = new ImageFeaturesList (*imagesInClass,
true // 'true means we want to own the data so new instances will be created.
);
LLoydsEntryListPtr lloydsEntries = DeriveAllLLoydsBins (*imagesToRandomize);
{
// We will now get mean and stdDev of nnd for this class
NeighborList neighbors (*imagesToRandomize, log);
neighbors.FindNearestNeighbors (neighborType, mlClass);
neighbors.CalcStatistics (sampleMeanNND, sampleStdDevNND, sampleMaxDist, sampleMinDist);
}
KKStr randomReportFileName;
if (reportFileName.Empty ())
randomReportFileName = "RandomDistanceReport";
else
randomReportFileName = osRemoveExtension (reportFileName) + "_Random";
randomReportFileName << "_" << mlClass->Name () << ".txt";
ofstream randomReport (randomReportFileName.Str ());
randomReport << "Random Distribution Report" << endl
<< endl;
randomReport << "Source Directory [" << sourceRootDirPath << "]" << endl;
randomReport << "Class [" << mlClass->Name () << "]" << endl;
RandomNND randomizeLocations (lastScanLine,
*imagesToRandomize,
numOfIterations,
numOfBuckets,
bucketSize,
randomReport,
log
);
randomizeLocations.GenerateReport ();
randomMeanNND = randomizeLocations.NND_Mean ();
randomStdDevNND = randomizeLocations.NND_StdDev ();
realDataU2Stat = randomizeLocations.RealDataU2Stat ();
//double sampleMeanNND = 0.0f;
//double sampleStdDevNND = 0.0f;
double z_Score = Z_Score (sampleMeanNND, randomMeanNND, randomStdDevNND, imagesToRandomize->QueueSize ());
randomReport << endl << endl << endl
<< "Z-Score" << endl
<< "=======" << endl
<< endl
<< "SampleMeanNND " << "\t" << sampleMeanNND << endl
<< "SampleStdDevNND " << "\t" << sampleStdDevNND << endl
<< "RandomMeanNND " << "\t" << randomMeanNND << endl
<< "RandomStdDevNND " << "\t" << randomStdDevNND << endl
<< "------- Z-Score " << "\t" << z_Score << endl
<< endl;
KKStr zScoreSummaryLine = mlClass->Name () + "\t" +
StrFormatDouble (sampleMeanNND, "###,##0.00") + "\t" +
StrFormatDouble (sampleStdDevNND, "###,##0.00") + "\t" +
StrFormatDouble (randomMeanNND, "###,##0.00") + "\t" +
StrFormatDouble (randomStdDevNND, "###,##0.00") + "\t" +
StrFormatDouble (z_Score, "###,##0.000");
zScoreSummaryLines.push_back (zScoreSummaryLine);
// The new instance on 'ClassSummary' that is aboiut to be created will take ownership
// of lloydsBins.
classSummaries.PushOnBack (new ClassSummary (mlClass, lloydsEntries, (float)realDataU2Stat, (float)z_Score));
delete imagesToRandomize; imagesToRandomize = NULL;
}
delete imagesInClass; imagesInClass = NULL;
}
if (!fromPlankton)
{
LLoydsEntryListPtr allClassesLLoydsEntries = DeriveAllLLoydsBins (images);
// Create a report for all classes
KKStr randomReportFileName;
if (reportFileName.Empty ())
randomReportFileName = "RandomDistanceReport_All.txt";
else
randomReportFileName = osRemoveExtension (reportFileName) + "_Random_All.txt";
ofstream randomReport (randomReportFileName.Str ());
randomReport << "Source Directory [" << sourceRootDirPath << "]" << endl;
randomReport << "Class [" << "All" << "]" << endl;
{
// Find the mean and stddev of Nearest Neighbor regardless of class.
NeighborList allClassesNeighbors (images, log);
allClassesNeighbors.FindNearestNeighbors (NeighborType::AnyPlankton, fromPlankton);
double allClassesMinDistAnyClass = 0.0f;
double allClassesMaxDistAnyClass = 0.0f;
allClassesNeighbors.CalcStatistics (allClassesMeanNNDAnyClass,
allClassesMeanStdDevAnyClass,
allClassesMinDistAnyClass,
allClassesMaxDistAnyClass
);
}
RandomNND randomizeLocations (lastScanLine,
images,
numOfIterations,
numOfBuckets,
bucketSize,
randomReport,
log
);
randomizeLocations.GenerateReport ();
// All classes Z-Score
double allClassesRandomMeanNND = randomizeLocations.NND_Mean ();
double allClassesRandomStdDevNND = randomizeLocations.NND_StdDev ();
double allClassesRealDataU2Stat = randomizeLocations.RealDataU2Stat ();
double z_Score = Z_Score (allClassesMeanNNDAnyClass, allClassesRandomMeanNND, allClassesRandomStdDevNND, images.QueueSize ());
KKStr zScoreSummaryLine = KKStr ("All-Classes") + "\t" +
StrFormatDouble (allClassesMeanNNDAnyClass, "###,##0.00") + "\t" +
StrFormatDouble (allClassesMeanStdDevAnyClass, "###,##0.00") + "\t" +
StrFormatDouble (allClassesRandomMeanNND, "###,##0.00") + "\t" +
StrFormatDouble (allClassesRandomStdDevNND, "###,##0.00") + "\t" +
StrFormatDouble (z_Score, "###,##0.00");
zScoreSummaryLines.push_back (zScoreSummaryLine);
randomReport << endl << endl << endl
<< "Z-Score" << endl
<< "=======" << endl
<< endl
<< "SampleMeanNND " << "\t" << allClassesMeanNNDAnyClass << endl
<< "SampleStdDevNND " << "\t" << allClassesMeanStdDevAnyClass << endl
<< "RandomMeanNND " << "\t" << allClassesRandomMeanNND << endl
<< "RandomStdDevNND " << "\t" << allClassesRandomStdDevNND << endl
<< "------- Z-Score " << "\t" << z_Score << endl
<< endl;
classSummaries.PushOnBack (new ClassSummary (MLClass::CreateNewMLClass (KKStr ("AllClasses")),
allClassesLLoydsEntries,
(float)allClassesRealDataU2Stat,
(float)z_Score
)
);
}
{
// Z-Score Summary Report
KKB::kkuint32 x;
*report << std::endl << std::endl
<< "Z-Score Summary By Class" << std::endl
<< std::endl
<< "ClassName" << "\t" << "SampleMean" << "\t" << "SampleStdDev" << "\t" << "RandomMean" << "\t" << "RandomStdDev" << "\t" << "Z-Score" << std::endl
<< "=========" << "\t" << "==========" << "\t" << "============" << "\t" << "==========" << "\t" << "============" << "\t" << "=======" << std::endl;
for (x = 0; x < zScoreSummaryLines.size (); x++)
*report << zScoreSummaryLines[x] << std::endl;
}
*report << endl << endl << endl;
classSummaries.SummaryReport (*report);
*report << endl << endl << endl;
classSummaries.SpatialOverlapReport (*report);
classSummaries.SaveLLoydsBinsData (lloydsBinsFileName, sourceRootDirPath, lastScanLine, baseLLoydsBinSize);
} /* RandomReport */
void ActiveLearningReport::Load (KKStr fileName,
bool& validFormat
)
{
validFormat = false;
FILE* f = osFOPEN (fileName.Str (), "r");
if (!f)
{
log.Level (-1) << endl
<< "ActiveLearningReport::Load *** ERROR ***" << endl
<< endl
<< " Could noty open File[" << fileName << "]" << endl
<< endl;
return;
}
char buff[10240];
bool fileParmsLoaded = false;
{
// Get Frst Row wicj should be InitialTrainingImagesPerClass
if (!fgets (buff, sizeof (buff), f))
{
log.Level (-1) << endl
<< "ActiveLearningReport::Load *** ERROR ***" << endl
<< endl
<< " *** No Data ***" << endl
<< endl;
fclose (f);
return;
}
KKStr l (buff);
l.Upper ();
KKStr field = l.ExtractToken ("\n\t\r ");
if (field != "\\INITIALTRAININGIMAGESPERCLASS")
{
log.Level (-1) << endl
<< "ActiveLearningReport::Load *** ERROR ***" << endl
<< endl
<< " First Liine Not Valid[" << l << "]" << endl
<< endl;
fclose (f);
return;
}
initialImagesPerClass = l.ExtractTokenInt ("\n\t\r ");
}
{
// Get second Row wicj should be ImagesPerRetraining
if (!fgets (buff, sizeof (buff), f))
{
log.Level (-1) << endl
<< "ActiveLearningReport::Load *** ERROR ***" << endl
<< endl
<< " *** Missing Second Line ***" << endl
<< endl;
fclose (f);
return;
}
KKStr l (buff);
l.Upper ();
KKStr field = l.ExtractToken ("\n\t\r ");
if (field != "\\IMAGESPERRETRAINING")
{
log.Level (-1) << endl
<< "ActiveLearningReport::Load *** ERROR ***" << endl
<< endl
<< " Second Liine Not Valid[" << l << "]" << endl
<< endl;
fclose (f);
return;
}
imagesPerRetraining = l.ExtractTokenInt ("\n\t\r ");
}
SortOrderResultsPtr curSortOrderResults = NULL;
while (fgets (buff, sizeof (buff), f))
{
KKStr l (buff);
if (l.SubStrPart (0, 0) == "\\")
{
if (curSortOrderResults)
{
results.PushOnBack (curSortOrderResults);
curSortOrderResults = NULL;
}
SortOrderType nextSortOrder = NULL_SortOrder;
int numOfRetrainings = 0;
while (l.SubStrPart (0, 0) == "\\")
{
l = l.SubStrPart (1);
l.Upper ();
KKStr fieldName = l.ExtractToken ("\n\t\r ");
if (fieldName == "SORTORDER")
{
KKStr sortOrderDesc = l.ExtractToken ("\n\t\r ");
nextSortOrder = SortOrderTypeFromStr (sortOrderDesc);
}
else if (fieldName == "NUMOFRETRAININGS")
{
numOfRetrainings = l.ExtractTokenInt ("\n\t\r ");
}
if (fgets (buff, sizeof (buff), f))
l = buff;
else
l = "";
}
if (nextSortOrder == NULL_SortOrder)
{
log.Level (-1) << "ActiveLearningReport::Load *** ERROR ***" << endl
<< endl
<< " Invalid Sort Order Defined." << endl
<< endl;
fclose (f);
return;
}
curSortOrderResults = new SortOrderResults (nextSortOrder);
}
if (!curSortOrderResults)
{
log.Level (-1) << "ActiveLearningReport::Load *** ERROR ***" << endl
<< endl
<< " Invalid Sort Order Defined." << endl
<< endl;
fclose (f);
return;
}
float numOfTrainingImages = (float)l.ExtractTokenDouble ("\n\t\r ");
float accuracy = (float)l.ExtractTokenDouble ("\n\t\r ");
float numOfSupportVectors = (float)l.ExtractTokenDouble ("\n\t\r ");
RetrainingResultsPtr retrainingResults = new RetrainingResults (numOfTrainingImages,
accuracy,
numOfSupportVectors
);
curSortOrderResults->PushOnBack (retrainingResults);
}
if (curSortOrderResults)
{
results.PushOnBack (curSortOrderResults);
curSortOrderResults = NULL;
}
fclose (f);
validFormat = true;
return;
} /* Load */
void ForestCover_AddMnemonicClassName ()
{
const char* classNames[] = {"Spruce_Fir",
"Lodgepole_Pine",
"Ponderosa_Pine",
"Cottonwood_Willow",
"Aspen",
"Douglas_fir",
"Krummholz",
NULL
};
FILE* in = osFOPEN ("K:\\Plankton\\Papers\\BitReduction\\DataSets\\ForestCover\\covtype.data", "r");
ofstream o ("K:\\Plankton\\Papers\\BitReduction\\DataSets\\ForestCover\\covtype_alpha.data");
char buff[20480];
int lc = 0;
while (fgets (buff, sizeof (buff), in))
{
KKStr ln (buff);
vector<KKStr> tokens;
while (!ln.Empty ())
{
KKStr token = ln.ExtractToken (",\n\t\r");
tokens.push_back (token);
}
int lastFieldNum = (int)tokens.size () - 1;
KKStr lastToken = tokens[lastFieldNum];
int classNum = atoi (lastToken.Str ()) - 1;
if ((classNum < 0) || (classNum > 6))
{
// Something has gone very wrong.
cerr << endl
<< " *** ERROR ***" << endl
<< endl
<< buff << endl
<< endl
<< "ClassNum[" << classNum << "] Out of range." << endl
<< endl;
osWaitForEnter ();
exit (-1);
}
KKStr className = classNames[classNum];
tokens[lastFieldNum] = className;
int x = 0;
vector<KKStr>::const_iterator idx;
for (idx = tokens.begin (); idx != tokens.end (); idx++)
{
if (x > 0)
o << ",";
o << *idx;
x++;
}
o << endl;
lc++;
if ((lc % 100) == 0)
cout << lc << endl;
}
fclose (in);
o.close ();
} /* ForestCover_AddMnemonicClassName */
void RandomSampleJob::ProcessStatusStr (KKStr statusStr)
{
log.Level (30) << "RandomSampleJob::ProcessStatusStr[" << statusStr << "]" << endl;
KKStr fieldName;
KKStr fieldValue;
compMethod = BRNull;
fieldName = statusStr.ExtractToken ("\t\n\r");
while (!fieldName.Empty ())
{
fieldName.Upper ();
fieldValue = statusStr.ExtractToken ("\t\n\r");
fieldValue.TrimLeft ("\n\r\t ");
fieldValue.TrimRight ("\n\r\t ");
if (fieldName == "JOBID")
jobId = atoi (fieldValue.Str ());
else if (fieldName == "STATUS")
status = JobStatusFromStr (fieldValue);
else if (fieldName == "KERNEL")
kernelType = KernalTypeFromStr (fieldValue);
else if (fieldName == "ENCODING")
encodingMethod = EncodingMethodFromStr (fieldValue);
else if (fieldName == "COMPMETHOD")
compMethod = CompressionMethodFromStr (fieldValue);
else if (fieldName == "C_PARAM")
c = atoi (fieldValue.Str ());
else if (fieldName == "GAMMA")
gamma = atof (fieldValue.Str ());
else if (fieldName == "ORDERINGNUM")
orderingNum = atoi (fieldValue.Str ());
else if (fieldName == "EXAMPLESTOKEEP")
numExamplesToKeep = atoi (fieldValue.Str ());
else if (fieldName == "ACCURACY")
accuracy = (float)atof (fieldValue.Str ());
else if (fieldName == "TRAINTIME")
{
trainTime = atof (fieldValue.Str ());
}
else if (fieldName == "PREDTIME")
testTime = (float)atof (fieldValue.Str ());
else if (fieldName == "SUPPORTVECTORS")
supportVectors = (float)atof (fieldValue.Str ());
else
{
log.Level (-1) << "RandomSampleJob::ProcessStatusStr Invalid Field Name[" << fieldName << "]." << endl;
}
fieldName = statusStr.ExtractToken ("\t\n\r");
}
} /* ProcessStatusStr */
void ClassSummaryList::SaveLLoydsBinsData (const KKStr& fileName,
const KKStr& srcDirName,
kkint32 lastScanLine,
kkint32 baseLLoydsBinSize
)
{
ofstream o (fileName.Str (), ios::out);
if (!o.is_open ())
{
log.Level (-1) << endl << endl << endl
<< "ClassSummaryList::SaveLLoydsBinsData *** Invalid File[" << fileName << "]." << endl
<< endl;
return;
}
ClassSummaryList::const_iterator idx;
o << "// LLoyds Bins Data" << endl
<< "//" << endl
<< "// Run Date [" << osGetLocalDateTime () << "]" << endl
<< "// SrcDir [" << srcDirName << "]" << endl
<< "// LastScanLin [" << lastScanLine << "]" << endl
<< "// Base Bin Size [" << baseLLoydsBinSize << "]" << endl
<< "//" << endl;
o << "BinSize" << "\t" << "BinNum" << "\t" << "ScanLines";
kkuint32 numOfBins = NumOfLLoydsBins ();
for (idx = begin (); idx != end (); idx++)
{
ClassSummaryPtr cs = *idx;
o << "\t" << cs->ClassName ();
}
o << endl;
VectorInt binSizes = LLoydsBinSizes ();
size_t binSizeIndex = 0;
for (binSizeIndex = 0; binSizeIndex < binSizes.size (); binSizeIndex++)
{
kkint32 binSize = binSizes[binSizeIndex];
kkint32 startScanLinNum = 0;
kkint32 endScanLineNum = startScanLinNum + binSize - 1;
kkuint32 binNum = 0;
for (binNum = 0; binNum < numOfBins; binNum++)
{
o << binSize << "\t" << binNum <<"\t" << startScanLinNum << "-" << endScanLineNum;
for (idx = begin (); idx != end (); idx++)
{
kkint32 lloydsBin = 0;
ClassSummaryPtr cs = *idx;
LLoydsEntryPtr lloydsEntry = cs->LLoydsEntryByBinSize (binSize);
if (lloydsEntry != NULL)
lloydsBin = lloydsEntry->LLoydsBin (binNum);
o << "\t" << lloydsBin;
}
startScanLinNum += binSize;
endScanLineNum += binSize;
o << endl;
}
}
o.close ();
} /* SaveLLoydsBinsData */
void DataBaseServerList::ReadConfigFile ()
{
KKStr configFileName = osAddSlash (PicesVariables::ConfigurationDirectory ()) + "MySql.cfg";
log.Level (10) << "DataBaseServerList::ReadConfigFile ConfigFileName[" << configFileName << "]" << endl;
FILE* in = osFOPEN (configFileName.Str (), "r");
if (!in)
{
log.Level (-1) << "DataBaseServerList::ReadConfigFile Error loading ConfigFileName[" << configFileName << "]" << endl;
return;
}
defaultServerDescription = "";
bool defaultServerFound = false;
char buff[1024];
while (fgets (buff, sizeof (buff), in))
{
buff[sizeof(buff) - 1] = 0; // Just to make sure that buffer not completely full
KKStr ln (buff);
if (ln.Len () < 3)
continue;
if ((ln[0] == '/') && (ln[1] == '/'))
continue;
KKStr lineName = ln.ExtractToken2 ("\n\r\t");
if (lineName.EqualIgnoreCase ("Server"))
{
DataBaseServerPtr server = new DataBaseServer (ln);
PushOnBack (server);
}
else if (lineName.EqualIgnoreCase ("DefaultServer"))
{
defaultServerDescription = ln.ExtractToken2 ("\n\r\t");
defaultServerFound = true;
}
}
if (!defaultServerFound)
{
DataBaseServerPtr defaultServer = LookUpByDescription ("Local");
if (!defaultServer)
defaultServer = LookUpByDescription ("localhost");
if (!defaultServer)
{
defaultServer = new DataBaseServer ();
defaultServer->UserName ("Root");
defaultServer->Description ("localhost");
defaultServer->Embedded (false);
defaultServer->HostName ("localhost");
defaultServer->PortNum (3306),
defaultServer->MySqlDataDir ("");
defaultServer->UserName ("root");
defaultServer->PassWord ("dasani30");
defaultServer->DataBaseName ("pices_new");
defaultServerDescription = defaultServer->Description ();
PushOnBack (defaultServer);
}
defaultServerDescription = defaultServer->Description ();
}
fclose (in);
} /* ReadConfigFile */
/******************************************************************************
* ProcessCmdLineParamters
* DESC: Extracts parameters from the command line
******************************************************************************/
bool OurNeighbors::ProcessCmdLineParameter (const KKStr& parmSwitch,
const KKStr& parmValue
)
{
bool validParm = true;
if (parmSwitch.EqualIgnoreCase ("-BUCKETCOUNT") || parmSwitch.EqualIgnoreCase ("-BC"))
{
numOfBuckets = atoi (parmValue.Str ());
if (numOfBuckets < 2)
{
log.Level (-1) << "ProcessCmdLineParameter *** Invalid Buckets[" << numOfBuckets << "] parameter" << endl;
validParm = false;
}
}
else if (parmSwitch.EqualIgnoreCase ("-BUCKETSIZE") || parmSwitch.EqualIgnoreCase ("-BS"))
{
bucketSize = atoi (parmValue.Str ());
if (bucketSize < 1)
{
log.Level (-1) << "ProcessCmdLineParameter *** Invalid BucketSize[" << bucketSize << "] parameter" << endl;
Abort (true);
}
}
else if (parmSwitch.EqualIgnoreCase ("-EXCLUDECLASS") || parmSwitch.EqualIgnoreCase ("-EC"))
{
if (parmValue.Empty ())
{
log.Level (-1) << endl << endl
<< "ProcessCmdLineParameter '-ExcludeClass' missing class name to exclude." << endl
<< endl;
Abort (true);
}
else
{
if (!excludedClasses)
excludedClasses = new MLClassList ();
if (excludedClasses->LookUpByName (parmValue) != NULL)
{
log.Level (-1) << "ProcessCmdLineParameter '-ExcludeClass[" << parmValue << "]' Specified more than once." << endl;
}
else
{
excludedClasses->PushOnBack (MLClass::CreateNewMLClass (parmValue));
}
}
}
else if (parmSwitch.EqualIgnoreCase ("-FROMPLANKTON") || parmSwitch.EqualIgnoreCase ("-FP"))
{
fromPlanktonName = parmValue;
}
else if (parmSwitch.EqualIgnoreCase ("-H") || parmSwitch.EqualIgnoreCase ("-HTML"))
htmlFileName = parmValue;
else if (parmSwitch.EqualIgnoreCase ("-ITERATIONS") || parmSwitch.EqualIgnoreCase ("-I"))
{
numOfIterations = atoi (parmValue.Str ());
if (numOfIterations < 1)
{
log.Level (-1) << "ProcessCmdLineParameter *** Invalid Iterations[" << numOfIterations << "] parameter" << endl;
Abort (true);
}
}
else if (parmSwitch.EqualIgnoreCase ("-LLOYDSBINSIZE") ||
parmSwitch.EqualIgnoreCase ("-LBS") ||
parmSwitch.EqualIgnoreCase ("-BASELLOYDSBINSIZE") ||
parmSwitch.EqualIgnoreCase ("-BLBS")
)
{
baseLLoydsBinSize = atoi (parmValue.Str ());
if (baseLLoydsBinSize < 10)
{
log.Level (-1) << endl << endl
<< "ProcessCmdLineParameter *** Invalid BaseLLoydsBinSize[" << baseLLoydsBinSize << "] parameter ***" << endl
<< endl;
Abort (true);
}
}
else if (parmSwitch.EqualIgnoreCase ("-NEARESTNEIGHBORTYPE") || parmSwitch.EqualIgnoreCase ("-NNT"))
{
if (parmValue.EqualIgnoreCase ("0") || parmValue.EqualIgnoreCase ("ANY"))
neighborType = NeighborType::AnyPlankton;
else if (parmValue.EqualIgnoreCase ("1") || parmValue.EqualIgnoreCase ("SAME"))
neighborType = NeighborType::SamePlankton;
}
else if (parmSwitch.EqualIgnoreCase ("-R") || parmSwitch.EqualIgnoreCase ("-REPORT"))
reportFileName = parmValue;
else if (parmSwitch.EqualIgnoreCase ("-RAND") || parmSwitch.EqualIgnoreCase ("-RANDOM"))
randomizeLocations = true;
else if (parmSwitch.EqualIgnoreCase ("-S") || parmSwitch.EqualIgnoreCase ("-SRC"))
sourceRootDirPath = parmValue;
else
validParm = PicesApplication::ProcessCmdLineParameter(parmSwitch, parmValue);
return !Abort ();
} /* ProcessCmdLineParameter */
void ImportValidatedClassInfo ()
{
RunLog log;
KKStr srcFileName = "D:\\Users\\kurt\\OneDrive\\Sipper\\FromAndrewToKurt\\Validation\\2014-09-16\\ValidatedImagesList.txt";
KKStr classNameFileName = "D:\\Users\\kurt\\OneDrive\\Sipper\\FromAndrewToKurt\\Validation\\2014-09-16\\ClassList.txt";
map<int, KKStr> classNameLookup;
map<int, KKStr>::iterator idx;
FILE* cnl = KKB::osFOPEN(classNameFileName.Str (), "r");
KKStrPtr line = KKB::osReadNextLine (cnl);
while (line)
{
if (!line->Empty())
{
auto fields = line->Parse("\t");
if (fields.size() > 2)
{
int classId = fields[0].ToInt32();
KKStr className = fields[1];
classNameLookup.insert(pair<int, KKStr>(classId, className));
}
}
delete line;
line = KKB::osReadNextLine (cnl);
}
ifstream i (srcFileName.Str ());
if (!i.is_open ())
return;
DataBasePtr dbConn = new DataBase (log);
if (!dbConn)
return;
char buff[20480];
int count = 0;
KKStr s (1024);
KKStr imageFileName;
int validatedClassId;
KKStr validatedClassName;
KKStr errMsg = "";
while (i.getline (buff, sizeof (buff)))
{
count++;
s = buff;
imageFileName = s.ExtractQuotedStr (",\n\t\r", false);
if (imageFileName.StartsWith("ETP"))
{
validatedClassId = s.ExtractTokenInt (",\n\t\r");
//validatedClassName = s.ExtractQuotedStr (",\n\t\r", false);
idx = classNameLookup.find(validatedClassId);
if (idx != classNameLookup.end())
{
validatedClassName = idx->second;
MLClassPtr c = MLClass::CreateNewMLClass (validatedClassName);
dbConn->ImagesUpdateValidatedClass (imageFileName, c);
if (!dbConn->Valid ())
errMsg = dbConn->LastErrorDesc ();
cout << count << "\t" << imageFileName << "\t" << validatedClassName << "\t" << errMsg << endl;
}
errMsg = "";
}
}
} /* ImportValidatedClassInfo */
void ImportGPSDataGPGGA (const KKStr& fileName)
{
RunLog log;
DataBasePtr dbConn = new DataBase (log);
ifstream i (fileName.Str ());
if (!i.is_open ())
{
log.Level (-1) << endl << endl
<< "ImpotrtGPSData Could not open file[" << fileName << "]" << endl
<< endl;
return;
}
log.Level (10) << endl << endl << endl << endl << endl
<< "ImpotrtGPSData FileName[" << fileName << "]" << endl << endl
<< endl;
char buff[20480];
bool firstPass = true;
int lastMinute = 0;
int linesRead = 0;
KKStr ln (256);
DateTime lastDateTime;
while (i.getline (buff, sizeof (buff)))
{
linesRead++;
ln = buff;
ln.TrimLeft ();
if (!ln.LocateStr ("GPGGA"))
continue;
VectorKKStr fields = ln.Parse (",");
if (fields.size () < 8)
continue;
if (!fields[2].EqualIgnoreCase ("$GPGGA"))
continue;
/*
0 1 2 3 4 5 6 7 8
06/01/2010, 23:59:59.818, $GPGGA, 235958, 2840.927, N, 08828.458, W, 2, 09,22.10,0,M,,,14,0000*12
06/02/2010, 00:00:10.818, $GPGGA, 000009, 2840.931, N, 08828.482, W, 1, 09,0.89,0,M,,,,*2D
06/02/2010, 00:00:21.802, $GPGGA, 000020, 2840.929, N, 08828.505, W, 1, 09,0.89,0,M,,,,*21
06/02/2010, 00:00:31.818, $GPGGA, 000030, 2840.924, N, 08828.526, W, 1, 09,0.89,0,M,,,,*2C
06/02/2010, 00:00:42.818, $GPGGA, 000041, 2840.917, N, 08828.547, W, 1, 09,0.89,0,M,,,,*2D
06/02/2010, 00:00:53.802, $GPGGA, 000052, 2840.906, N, 08828.568, W, 1, 09,1.00,0,M,,,,*22
06/02/2010, 00:01:03.802, $GPGGA, 000102, 2840.895, N, 08828.585, W, 1, 09,0.89,0,M,,,,*2E
06/02/2010, 00:01:13.818, $GPGGA, 000112, 2840.883, N, 08828.600, W, 1, 09,0.89,0,M,,,,*26
*/
KKStr dateStr = fields[0];
KKStr timeStr = fields[1];
KKStr latStr = fields[4];
KKStr logStr = fields[6];
auto x = latStr.LocateCharacter ('.');
if (!x)
continue;
KKStr latMinStr = latStr.SubStrPart (x - 2);
KKStr latDegStr = latStr.SubStrSeg (0, x - 2);
double latitude = latDegStr.ToDouble () + latMinStr.ToDouble () / 60.0;
if (fields[5].EqualIgnoreCase ("S"))
latitude = 0.0 - latitude;
x = logStr.LocateCharacter ('.');
if (!x)
continue;
KKStr logMinStr = logStr.SubStrPart (x - 2);
KKStr logDegStr = logStr.SubStrSeg (0, x - 2);
double longitude = logDegStr.ToDouble () + logMinStr.ToDouble () / 60.0;
if (fields[7].EqualIgnoreCase ("W"))
longitude = 0.0 - longitude;
DateType gmtDate (dateStr);
TimeType gmtTime (timeStr);
DateTime gmtDateTime (gmtDate, gmtTime);
DateTime localTime = gmtDateTime;
localTime.HoursAdd (-4);
DateTime startDT = localTime;
DateTime endDT = localTime;
if (firstPass)
{
firstPass = false;
startDT.SecondsAdd (-180);
}
else
{
DateTime deltaDT = localTime - lastDateTime;
long deltaSecs = (long)deltaDT.Seconds ();
startDT.SecondsAdd (-(deltaSecs / 2));
}
endDT.SecondsAdd (30);
if (gmtTime.Minute () != lastMinute)
{
lastMinute = gmtTime.Minute ();
log.Level (10) << "LinesRead[" << linesRead << "] File[" << osGetRootName (fileName) << "] GMT Time[" << gmtDate.MMM_DD_YYYY () << " - " << gmtTime.HH_MM_SS () << "]" << endl;
}
if ((endDT.Month () < 6) && (endDT.Day () < 28))
{
}
else
{
dbConn->InstrumentDataUpdateLatitudeAndLongitude (startDT, endDT, latitude, longitude);
}
lastDateTime = localTime;
}
i.close ();
delete dbConn;
dbConn = NULL;
} /* ImportGPSDataGPGGA */
void RandomSplitJobManager::GenerateFinalResultsReport ()
{
KKStr reportFileName = osGetRootName (ManagerName ()) + "_Results.html;";
ofstream f (reportFileName.Str ());
f << "Run Time Parameters" << endl
<< "Run Time" << "\t" << osGetLocalDateTime () << endl
<< "configFileName" << "\t" << configFileName << endl
<< "DataFileName" << "\t" << dataFileName << endl
<< "Format" << "\t" << format->DriverName () << endl
<< "DataIndexFileName" << "\t" << dataIndexFileName << endl
<< "NumFolds" << "\t" << numFolds << endl
<< "NumSplits" << "\t" << numSplits << endl
<< "splitFraction" << "\t" << splitFraction << endl
<< endl;
KKJobList::const_iterator idx;
ConfusionMatrix2 avgResults (*(this->MLClasses ()));
KKB::uint x = 0;
for (idx = Jobs ()->begin (); idx != Jobs ()->end (); idx++)
{
RandomSplitJobPtr j = dynamic_cast<RandomSplitJobPtr> (*idx);
if (j->RandomSplitsResults () != NULL)
{
f << endl
<< "Random Split[" << j->SplitNum () << "]" << endl;
j->RandomSplitsResults ()->PrintConfusionMatrixTabDelimited (f);
f << endl << endl;
j->PrintClassCounts (f);
f << endl << endl;
avgResults.AddIn (*(j->RandomSplitsResults ()), log);
x++;
}
}
f << endl << "Mean Average of all random Splits." << endl;
avgResults.FactorCounts (1.0 / (double)x);
avgResults.PrintConfusionMatrixTabDelimited (f);
f << endl
<< endl
<< endl
<< endl
<< "Class Counts" << endl
<< endl;
kkuint32 numClasses = (kkuint32)mlClasses->size ();
VectorFloat classAccs;
VectorDouble knownCounts;
VectorDouble predCounts;
VectorDouble adjCounts;
VectorDouble adjCountsStdError;
VectorDouble predDelta;
VectorDouble adjDelta;
KKStr l1, l2, l3;
mlClasses->ExtractThreeTitleLines (l1, l2, l3);
VectorKKStr knownCountLines;
VectorKKStr predCountLines;
VectorKKStr adjCountLines;
VectorKKStr deltaPredCountLines;
VectorKKStr deltaAdjCountLines;
VectorKKStr accLines;
ConfusionMatrix2 totalCM (*MLClasses ());
int totalCmCount = 0;
// Known Counts
for (idx = Jobs ()->begin (); idx != Jobs ()->end (); idx++)
{
RandomSplitJobPtr j = dynamic_cast<RandomSplitJobPtr> (*idx);
if (j->RandomSplitsResults () != NULL)
{
KKStr splitNumStr = StrFormatInt (j->SplitNum (), "ZZZ0");
j->GetClassCounts (classAccs, knownCounts, predCounts, adjCounts, adjCountsStdError, predDelta, adjDelta);
totalCM.AddIn (*(j->RandomSplitsResults ()), log);
totalCmCount++;
KKStr accLine = "Acc By Class\t" + splitNumStr;
KKStr knownLine = "Known\t" + splitNumStr;
KKStr predLine = "Predicted\t" + splitNumStr;
KKStr adjLine = "Adjusted\t" + splitNumStr;
KKStr deltaPredLine = "Delta Pred\t" + splitNumStr;
KKStr deltaAdjLine = "Delta Adj\t" + splitNumStr;
double totalAcc = 0.0;
double totalDeltaPred = 0.0;
double totalDeltaAdj = 0.0;
for (x = 0; x < numClasses; x++)
{
accLine << "\t" << StrFormatDouble (classAccs [x], "zz0.00") << "%";
knownLine << "\t" << StrFormatDouble (knownCounts [x], "-Z,ZZZ,ZZ0.0");
predLine << "\t" << StrFormatDouble (predCounts [x], "-Z,ZZZ,ZZ0.0");
adjLine << "\t" << StrFormatDouble (adjCounts [x], "-Z,ZZZ,ZZ0.0");
deltaPredLine << "\t" << StrFormatDouble (predDelta [x], "-Z,ZZZ,ZZ0.0");
deltaAdjLine << "\t" << StrFormatDouble (adjDelta [x], "-Z,ZZZ,ZZ0.0");
totalAcc += classAccs [x];
totalDeltaPred += fabs (predDelta[x]);
totalDeltaAdj += fabs (adjDelta[x]);
}
accLine << "\t" << StrFormatDouble ((totalAcc / (double)classAccs.size ()), "ZZ0.00") << "%";
deltaPredLine << "\t" << StrFormatDouble ((totalDeltaPred / (double)predDelta.size ()), "ZZ0.00");
deltaAdjLine << "\t" << StrFormatDouble ((totalDeltaAdj / (double)adjDelta.size ()), "ZZ0.00");
accLines.push_back (accLine);
knownCountLines.push_back (knownLine);
predCountLines.push_back (predLine);
adjCountLines.push_back (adjLine);
deltaPredCountLines.push_back (deltaPredLine);
deltaAdjCountLines.push_back (deltaAdjLine);
}
}
double factor = 0.0;
if (totalCmCount > 0)
factor = 1.0 / (double)totalCmCount;
totalCM.FactorCounts (factor);
f << endl << endl
<< "Average Confusion Matrix" << endl
<< endl;
totalCM.PrintConfusionMatrixTabDelimited (f);
f << "" << "\t" << "" << "\t" << l1 << endl
<< "" << "\t" << "Split" << "\t" << l2 << endl
<< "Description" << "\t" << "Num" << "\t" << l3 << endl;
f << endl << endl;
for (x = 0; x < knownCountLines.size (); x++)
f << knownCountLines[x] << endl;
f << endl << endl;
for (x = 0; x < predCountLines.size (); x++)
f << predCountLines[x] << endl;
f << endl << endl;
for (x = 0; x < adjCountLines.size (); x++)
f << adjCountLines[x] << endl;
f << endl << endl;
for (x = 0; x < deltaPredCountLines.size (); x++)
f << deltaPredCountLines[x] << endl;
f << endl << endl;
for (x = 0; x < deltaAdjCountLines.size (); x++)
f << deltaAdjCountLines[x] << endl;
f << endl << endl;
for (x = 0; x < knownCountLines.size (); x++)
f << accLines[x] << endl;
VectorFloat avgAccuracies = totalCM.AccuracyByClass ();
f << "Avg-Accuracies";
for (x = 0; x < avgAccuracies.size (); x++)
f << "\t" << StrFormatDouble (avgAccuracies[x], "zz0.00") << "%";
f << "\t" << StrFormatDouble (totalCM.Accuracy (), "zz0.00") << "%";
f << endl;
f << endl << endl;
f.close ();
} /* GenerateFinalResultsReport */
void ImpotrtGPSData (const KKStr& fileName)
{
RunLog log;
DataBasePtr dbConn = new DataBase (log);
ifstream i (fileName.Str ());
if (!i.is_open ())
{
log.Level (-1) << endl << endl
<< "ImpotrtGPSData Could not open file[" << fileName << "]" << endl
<< endl;
return;
}
log.Level (10) << endl << endl << endl << endl << endl
<< "ImpotrtGPSData FileName[" << fileName << "]" << endl << endl
<< endl;
char buff[20480];
bool firstPass = true;
int lastHour = 0;
DateTime lastDateTime;
while (i.getline (buff, sizeof (buff)))
{
KKStr ln = buff;
ln.TrimLeft ();
char ch = ln[0];
if ((ch < '0') || (ch > '9'))
continue;
KKStr dateStr = ln.ExtractToken2 (",");
KKStr timeStr = ln.ExtractToken2 (",");
KKStr WXTP_TaStr = ln.ExtractToken2 (",");
KKStr WXTS_TaStr = ln.ExtractToken2 (",");
KKStr WXTS_PaStr = ln.ExtractToken2 (",");
KKStr WXTP_PaStr = ln.ExtractToken2 (",");
KKStr dec_latStr = ln.ExtractToken2 (",");
KKStr dec_lonStr = ln.ExtractToken2 (",");
KKStr cogStr = ln.ExtractToken2 (",");
KKStr sogStr = ln.ExtractToken2 (",");
DateType gmtDate (dateStr);
TimeType gmtTime (timeStr);
DateTime gmtDateTime (gmtDate, gmtTime);
DateTime localTime = gmtDateTime;
localTime.HoursAdd (-4);
DateTime startDT = localTime;
DateTime endDT = localTime;
if (firstPass)
{
firstPass = false;
startDT.SecondsAdd (-180);
}
else
{
DateTime deltaDT = localTime - lastDateTime;
long deltaSecs = (long)deltaDT.Seconds ();
startDT.SecondsAdd (-(deltaSecs / 2));
}
endDT.SecondsAdd (120);
dbConn->InstrumentDataUpdateLatitudeAndLongitude (startDT, endDT, dec_latStr.ToDouble (), dec_lonStr.ToDouble ());
lastDateTime = localTime;
if (gmtTime.Minute () != lastHour)
{
lastHour = gmtTime.Hour ();
log.Level (10) << "File[" << osGetRootName (fileName) << "] GMT Time[" << gmtDate.MMM_DD_YYYY () << " - " << gmtTime.HH_MM_SS () << "]" << endl;
}
}
i.close ();
delete dbConn;
dbConn = NULL;
} /* ImpotrtGPSData */
/******************************************************************************
* ProcessCmdLineParamters
* DESC: Extracts parameters from the command line
******************************************************************************/
bool RandomNND::ProcessCmdLineParameter (
char parmSwitchCode,
KKStr parmSwitch,
KKStr parmValue
)
{
KKStr parmValueUpper (parmValue);
parmValueUpper.Upper ();
parmSwitch.Upper ();
if ((parmSwitch == "-BUCKETS") || (parmSwitch == "-B"))
{
numOfBuckets = atoi (parmValue.Str ());
if (numOfBuckets < 3)
{
log.Level (-1) << endl
<< "ProcessCmdLineParameter *** ERROR ***" << endl
<< " NumOfBuckets[" << numOfBuckets << "] Must be breater than 2" << endl
<< endl;
Abort (true);
}
}
else if ((parmSwitch == "-BUCKETSIZE") || (parmSwitch == "-BS"))
{
bucketSize = atoi (parmValue.Str ());
if (bucketSize < 1)
{
log.Level (-1) << endl
<< "ProcessCmdLineParameter *** ERROR ***" << endl
<< " BucketSize[" << bucketSize << "] Must be breater than 0" << endl
<< endl;
Abort (true);
}
}
else if ((parmSwitch == "-MINSIZE") || (parmSwitch == "-MS"))
{
minSize = (double)atof (parmValue.Str ());
if (minSize < 1.0)
{
log.Level (-1) << endl
<< "ProcessCmdLineParameter *** ERROR ***" << endl
<< " MinSize[" << minSize << "] Must be breater than 1.0" << endl
<< endl;
Abort (true);
}
}
else if ((parmSwitch == "-PARTICLES") || (parmSwitch == "-P"))
{
numOfParticles = atoi (parmValue.Str ());
if (numOfParticles < 2)
{
log.Level (-1) << endl
<< "ProcessCmdLineParameter *** ERROR ***" << endl
<< " numOfParticles[" << numOfParticles << "] Must be breater than 1." << endl
<< endl;
Abort (true);
}
}
else if ((parmSwitch == "-ITERATIONS") || (parmSwitch == "-I"))
{
numIterations = atoi (parmValue.Str ());
if (numIterations < 1)
{
log.Level (-1) << endl
<< "ProcessCmdLineParameter *** ERROR ***" << endl
<< " NumIterations[" << numIterations << "] Must be breater than 0." << endl
<< endl;
Abort (true);
}
}
else if ((parmSwitch == "-R") || (parmSwitch == "-REPORT"))
reportFileName = parmValue;
else if ((parmSwitch == "-SCANLINES") || (parmSwitch == "-SL"))
{
scanLines = atoi (parmValue.Str ());
if (scanLines < 100)
{
log.Level (-1) << endl
<< "ProcessCmdLineParameter *** ERROR ***" << endl
<< " scanLines[" << scanLines << "] Must be breater than 100." << endl
<< endl;
Abort (true);
}
}
else
{
log.Level (-1) << endl
<< "ProcessCmdLineParameter Invalid Parameter[" << parmSwitch << "]" << endl
<< endl;
Abort (true);
}
return !Abort ();
} /* ProcessCmdLineParameter */
