void KKStrTest::AssertAreEqual (const char* expected, const KKStr& found, const KKStr& testName)
{
bool equal = strcmp(expected, found.Str ()) == 0;
KKStr msg;
msg << "Expected: " << expected << "\t" << "Found: " << found.Str ();
Assert(equal, testName, msg);
}
void KKJobManager::StatusFileProcessLineJobStatusChange (KKStr& statusLineStr)
{
kkint32 expandedJobId = statusLineStr.ExtractTokenInt ("\t");
KKJobPtr j = jobs->LookUpByJobId (expandedJobId);
if (!j)
{
log.Level (-1) << endl << endl << endl
<< "ProcessStatusLineJobStatusChange ***Error*** Could not locate Expanded" << endl
<< endl
<< " JobId[" << expandedJobId << "]" << endl
<< endl;
EndBlock ();
osWaitForEnter ();
exit (-1);
}
KKStr statusStr = statusLineStr.ExtractToken2 ("\t");
statusStr.TrimLeft ();
statusStr.TrimRight ();
KKJob::JobStatus status = KKJob::JobStatusFromStr (statusStr);
if (status == jsNULL)
{
log.Level (-1) << endl << endl << endl
<< "ProcessStatusLineJobStatusChange ***Error*** Invalid Status Specified" << endl
<< endl
<< " JobId[" << expandedJobId << "]" << endl
<< " Status[" << statusStr << "]" << endl
<< endl;
EndBlock ();
osWaitForEnter ();
exit (-1);
}
j->Status (status);
} /* ProcessStatusLineJobStatusChange */
void StripOutAnyComments (KKStr& line)
{
bool found = false;
kkint32 len = line.Len ();
kkint32 x = 0;
while ((x < (len - 1)) && (!found))
{
if ((line[x] == '/') &&
(line[x + 1] == '/'))
found = true;
else
x++;
}
if (found)
{
if (x == 0)
line = "";
else
line = line.SubStrPart (0, x - 1);
}
} /* StripOutAnyComments */
void DataBaseServerList::WriteConfigFile () const
{
osCreateDirectoryPath (PicesVariables::ConfigurationDirectory ());
KKStr configFileName = osAddSlash (PicesVariables::ConfigurationDirectory ()) + "MySql.cfg";
log.Level (10) << "DataBaseServerList::WriteConfigFile Writing ConfigFileName[" << configFileName << "]" << endl;
ofstream o (configFileName.Str ());
if (!o.is_open ())
{
log.Level (-1) << endl << endl
<< "DataBaseServerList::WriteConfigFile ***ERROR*** Opening ConfigFile[" << configFileName << "]" << endl
<< endl;
return;
}
o << "// PICES MySQL configuration file. There will be one entry for each known MySQL database server." << std::endl
<< "//" << std::endl
<< "// Date Written: " << osGetLocalDateTime () << std::endl
<< "//" << std::endl
<< "// Number Entries: " << QueueSize () << std::endl
<< "//" << std::endl;
DataBaseServerList::const_iterator idx;
for (idx = begin (); idx != end (); idx++)
o << "Server" << "\t" << (*idx)->ToParameterStr () << endl;
o << "//" << std::endl
<< "DefaultServer" << "\t" << defaultServerDescription << std::endl;
o.close ();
} /* WriteConfigFile */
bool ImportFullSizeFromSqlBackUp::ProcessCmdLineParameter (
char parmSwitchCode,
KKStr parmSwitch,
KKStr parmValue
)
{
KKStr parmValueUpper (parmValue);
parmValueUpper.Upper ();
parmSwitch.Upper ();
if ((parmSwitch == "-S") ||
(parmSwitch == "-SRC") ||
(parmSwitch == "-SOURCE") ||
(parmSwitch.EqualIgnoreCase ("-SrcFile"))
)
{
srcFileName = parmValue;
if (!osFileExists (srcFileName))
{
log.Level (-1) << "ProcessCmdLineParameter ***ERROR*** Invalid '-SrcFile' [" << srcFileName << "] file." << endl;
Abort (true);
}
}
return !Abort ();
} /* ProcessCmdLineParameter */
HTMLReport& KKB::operator<< (HTMLReport& htmlReport,
const DateTime& right
)
{
KKStr s = right.Date ().YYYY_MM_DD () + "-" + right.Time ().HH_MM_SS ();
htmlReport.Append (s.Str ());
return htmlReport;
}
KKStrParser::KKStrParser (const KKStr& _str):
len (_str.Len ()),
nextPos (0),
str (_str.Str ()),
trimWhiteSpace (false),
weOwnStr (false),
whiteSpace (NULL)
{
}
/**
*@brief Adds Instrument data to the underlying Scanner files as text.
*/
void ScannerFile::WriteInstrumentDataWord (uchar idNum,
kkuint32 scanLineNum,
WordFormat32Bits dataWord
)
{
KKStr s (100);
s << "InstrumentDataWord" << "\t" << (int)idNum << "\t" << scanLineNum << "\t" << dataWord.unsigned32BitInt;
WriteTextBlock ((const uchar*)s.Str (), s.Len ());
} /* WriteInstrumentDataWord */
HTMLReport& KKB::operator<< (HTMLReport& htmlReport,
kkuint64 right
)
{
KKStr s (30);
s = StrFormatInt64 (right, "0");
htmlReport.Append (s.Str ());
return htmlReport;
}
bool AbundanceCorrectionStatsBuilder::ProcessCmdLineParameter (char parmSwitchCode,
KKStr parmSwitch,
KKStr parmValue
)
{
KKStr parmValueUpper (parmValue);
parmValueUpper.Upper ();
if (parmSwitch.EqualIgnoreCase ("-c") || parmSwitch.EqualIgnoreCase ("-config"))
{
configFileName = parmValue;
configFileFullPath = TrainingConfiguration2::GetEffectiveConfigFileName (configFileName);
if (!osFileExists (configFileFullPath))
{
log.Level (-1) << "ProcessCmdLineParameter ***ERROR*** Invalid '-config' [" << configFileName << "] file." << endl;
Abort (true);
}
}
else if (parmSwitch.EqualIgnoreCase ("-folds") || parmSwitch.EqualIgnoreCase ("-numFolds") || parmSwitch.EqualIgnoreCase ("-numOfFolds"))
{
numOfFolds = parmValue.ToInt ();
if ((numOfFolds < 2) || (numOfFolds > 1000))
{
log.Level (-1) << "ProcessCmdLineParameter ***ERROR*** Invalid '-NumFolds' [" << numOfFolds << "] Invlaid; valid values (2 - 1000)." << endl;
Abort (true);
}
}
else if (parmSwitch.EqualIgnoreCase ("-r") || parmSwitch.EqualIgnoreCase ("-report") || parmSwitch.EqualIgnoreCase ("-ReportFileName"))
reportFileName = parmValue;
return !Abort ();
} /* ProcessCmdLineParameter */
void KKJobManager::StatusFileRefresh ()
{
// we only want to read in any new changes to the status file.
log.Level (10) << "KKJobManager::StatusFileRefresh statusFileName[" << statusFileName << "]" << endl;
ifstream* statusFile = new ifstream (statusFileName.Str ());
if (!statusFile->is_open ())
{
log.Level (-1) << endl
<< "KKJobManager::LoadCurrentStatusFile ***ERROR*** Can not open Status File["
<< statusFileName << "]."
<< endl;
EndBlock ();
delete statusFile; statusFile = NULL;
osWaitForEnter ();
exit (-1);
}
char buff[20480];
KKStr statusStr (512);
if (statusFileNextByte >= 0)
statusFile->seekg (statusFileNextByte);
//fseek (statusFile, statusFileNextByte, SEEK_SET);
statusFile->getline (buff, sizeof (buff));
while (!statusFile->eof ())
{
statusStr = buff;
if (statusStr.SubStrPart (0, 4) == "<KKJob ")
{
ProcessJobXmlBlockOfText (statusStr, *statusFile);
}
else
{
StatusFileProcessLine (statusStr, *statusFile);
}
long zed = statusFile->tellg ();
if (zed >= 0)
statusFileNextByte = zed;
if (!statusFile->eof ())
statusFile->getline (buff, sizeof (buff));
}
statusFile->close ();
delete statusFile;
statusFile = NULL;
log.Level (20) << "KKJobManager::StatusFileRefresh Exiting." << endl;
} /* StatusFileRefresh */
void Strip ()
{
bool cancelFlag = false;
bool successful = false;
bool changesMade = false;
RunLog log;
FeatureFileIOPtr driver = FeatureFileIO::FileFormatFromStr ("C45");
MLClassList mlClasses;
FeatureVectorListPtr data =
driver->LoadFeatureFile ("D:\\Pices\\Reports\\FeatureDataFiles\\AllValidatedImages_ForJonathon\\AllValidatedDataNorm.data",
mlClasses,
-1,
cancelFlag,
successful,
changesMade,
log
);
FeatureVectorListPtr stripped = new FeatureVectorList (data->FileDesc (), false);
FeatureVectorList::const_iterator idx;
for (idx = data->begin (); idx != data->end (); ++idx)
{
FeatureVectorPtr fv = *idx;
KKStr fn = fv->ExampleFileName ();
if (fn.StartsWith ("SML") || (fn.StartsWith ("SMP")))
{
}
else
{
stripped->PushOnBack (fv);
}
}
kkuint32 numExamplesWritten = 90;
driver->SaveFeatureFile ("D:\\Pices\\Reports\\FeatureDataFiles\\AllValidatedImages_ForJonathon\\AllValidatedData1209.data",
data->AllFeatures (),
*stripped,
numExamplesWritten,
cancelFlag,
successful,
log
);
}
void KKJob::PrerequisitesFromStr (const KKStr& s)
{
prerequisites.clear ();
if (s.CompareIgnoreCase ("None") != 0)
{
VectorKKStr fields = s.Split (',');
for (kkuint32 x = 0; x < fields.size (); ++x)
{
kkint32 p = fields[x].ToInt ();
prerequisites.push_back (p);
}
}
} /* PrerequisitesFromStr */
CmdLineExpander::CmdLineExpander (const KKStr& _applicationName,
RunLog& _log,
const KKStr& _cmdLine
):
applicationName (_applicationName),
log (_log)
{
VectorKKStr initialParameters;
KKStr cmdLine (_cmdLine);
cmdLine.TrimLeft ();
while (!cmdLine.Empty ())
{
KKStr nextField = cmdLine.ExtractQuotedStr ("\n\r\t ", false); // false = Do not decode escape characters
nextField.TrimRight ();
if (!nextField.Empty ())
initialParameters.push_back (nextField);
cmdLine.TrimLeft ();
}
BuildCmdLineParameters (initialParameters);
BuildExpandedParameterPairs ();
}
uchar DateType::MonthFromStr (const KKStr& monthStr)
{
uchar month = 0;
for (month = 1; month < 13; month++)
{
if (monthStr.CompareIgnoreCase (monthlyShortNames[month]) == 0)
break;
if (monthStr.CompareIgnoreCase (monthlyNames[month]) == 0)
break;
}
return month;
} /* MonthFromStr */
/****************************************************************************************
* DetermineClassFromFileName
*
* Given an Image File Name(fileName) will return and mlClass. Will use the
* path name in fileName to determine class. If the class does not exist in
* mlClasses, will create a new instance and add it to mlClasses.
*
* After talking to the user it has come to light that the first sub-directory in
* a path will dictate what class he/she feels a image belongs to.
* Ex:
* .../Protista/
* .../Protista/SubDir1
* .../Trichodesmium
* .../Trichodesmium/SubDir1
* .../Trichodesmium/SubDir2
*
* The idea is that when trying to determine what class a image really is we look
* at the first sub-dir name in the path. We may also have to deal with seq-num's
* as part of the name, in that case we strip the _ and following numbers from the
* name to get the correct class name.
****************************************************************************************/
MLClassPtr OurNeighbors::DetermineClassFromFileName (const KKStr& fileName)
{
KKStr filename_copy = fileName;
// If there are no path separator characters('\' or '/') characters in name
// then we will not be able to determine the class.
auto x = osLocateFirstSlashChar (filename_copy);
if (!x || (x.value () < 1))
return mlClasses->GetUnKnownClass ();
KKStr className = filename_copy.SubStrSeg (0, x);
// now lets get rid of any possible trailing seq number.
// We are assuming that a underscore{"_") character separates the calcs name from the seq number.
// So if there is an underscore character, and all the characters to the right of it are
// underscore characters, then we will remove the underscore and the following numbers.
x = className.LocateLastOccurrence ('_');
if (x)
{
// Now lets eliminate any sequence number in name
// We are assuming that a underscore{"_") character separates the class name from the seq number.
// So if there is an underscore character, and all the characters to the right of it are
// numeric characters, then we will remove the underscore and the following numbers.
kkuint32 y = x.value () + 1;
bool allFollowingCharsAreNumeric = true;
while ((y < className.Len ()) && (allFollowingCharsAreNumeric))
{
char ch = className[y];
allFollowingCharsAreNumeric = ((ch >= '0') && (ch <= '9'));
y++;
}
if (allFollowingCharsAreNumeric)
{
className = className.SubStrSeg (0, x);
}
}
// Now that we have a string with the class name, lets get a pointer
// to a mlClass object from mlClasses , if none there then we get
// to create a new class.
MLClassPtr mlClass = mlClasses->GetMLClassPtr (className);
return mlClass;
} /* DetermineClassFromFileName */
KKJob::JobStatus KKJob::JobStatusFromStr (const KKStr& statusStr)
{
if (statusStr.CompareIgnoreCase ("OPEN") == 0)
return jsOpen;
if (statusStr.CompareIgnoreCase ("STARTED") == 0)
return jsStarted;
if (statusStr.CompareIgnoreCase ("DONE") == 0)
return jsDone;
if (statusStr.CompareIgnoreCase ("Expanded") == 0)
return jsExpanded;
return jsNULL;
} /* JobStatusToStr */
StartStopPoint::StartStopType StartStopPoint::StartStopTypeFromStr (const KKStr& s)
{
for (int x = (int)StartStopType::Null; x < (int)StartStopType::Invalid; ++x)
if (s.EqualIgnoreCase (startStopPointStrs[x]))
return (StartStopType)x;
return StartStopType::Null;
}
void FeatureFileIO::SaveFeatureFile (const KKStr& _fileName,
FeatureNumListConst& _selFeatures,
FeatureVectorList& _examples,
kkuint32& _numExamplesWritten,
VolConstBool& _cancelFlag,
bool& _successful,
RunLog& _log
)
{
_log.Level (10) << "FeatureFileIO::SaveFeatureFile - File[" << _fileName << "]." << endl;
ofstream out (_fileName.Str ());
if (!out.is_open())
{
_log.Level (-1) << "***ERROR***, SaveFeatureFile, Opening File[" << _fileName << "]" << endl;
_successful = false;
}
out.precision (9);
KKStr errorMessage;
SaveFile (_examples, _fileName, _selFeatures, out, _numExamplesWritten, _cancelFlag, _successful, errorMessage, _log);
out.close ();
} /* SaveFeatureFile */
void DataBaseServer::ParseParameterStr (const KKStr& parameterStr)
{
VectorKKStr parameterPairs = parameterStr.Split ("\t");
VectorKKStr::iterator idx;
for (idx = parameterPairs.begin (); idx != parameterPairs.end (); idx++)
{
VectorKKStr fields = (*idx).Split (":="); // Split by either ':' or '='
if (fields.size () < 2)
{
// Should be two fields; line must be malformed.
continue;
}
KKStr parameterName = fields[0].ToUpper ();
if ((parameterName == "EMBEDDED") || (parameterName == "EMB") || (parameterName == "E")) embedded = fields[1].ToBool ();
else if ((parameterName == "MYSQLDATADIR") || (parameterName == "MYSQL") || (parameterName == "MDD")) mySqlDataDir = fields[1];
else if ((parameterName == "DESCRIPTION") || (parameterName == "DESC") || (parameterName == "D")) description = fields[1];
else if ((parameterName == "HOSTNAME") || (parameterName == "HOST") || (parameterName == "H")) hostName = fields[1];
else if ((parameterName == "USERNAME") || (parameterName == "USER") || (parameterName == "U")) userName = fields[1];
else if ((parameterName == "PASSWORD") || (parameterName == "PW") || (parameterName == "P")) passWord = fields[1];
else if ((parameterName == "PORTNUM") || (parameterName == "PN")) portNum = fields[1].ToUint32 ();
else if ((parameterName == "DATABASENAME") || (parameterName == "DATABASE") || (parameterName == "DB")) dataBaseName = fields[1];
}
if (description.EqualIgnoreCase ("Embedded"))
embedded = true;
} /* ParseParameterStr */
void CmdLineExpander::ExtractParametersFromFile (const KKStr& cmdFileName,
VectorKKStr& cmdFileParameters,
bool& validFile
)
{
FILE* in = osFOPEN (cmdFileName.Str (), "r");
if (!in)
{
log.Level (-1) << endl << endl << endl
<< "ExtractParametersFromFile *** EROR ***" << endl
<< endl
<< " Invalid CmdFile[" << cmdFileName << "]" << endl
<< endl;
validFile = false;
return;
}
KKStr token;
bool eof = false;
token = osReadNextQuotedStr (in, " \n\r", eof);
while (!eof)
{
cmdFileParameters.push_back (token);
token = osReadNextQuotedStr (in, " \n\r", eof);
}
std::fclose (in);
} /* ExtractParametersFromFile */
void FeatureNumList::ParseToString (const KKStr& _str,
bool& _valid
)
{
_valid = true;
delete featureNums;
featureNumsAllocatedSize = 0;
maxFeatureNum = 0;
numOfFeatures = 0;
featureNums = NULL;
if (_str.EqualIgnoreCase ("NONE"))
{
maxFeatureNum = 1;
AllocateArraySize ((IntType)(maxFeatureNum + 1));
return;
}
VectorIntType* list = StrToUInt16Vetor (_str);
if (list)
{
sort(list->begin (), list->end ());
maxFeatureNum = list->back ();
AllocateArraySize ((IntType)list->size ());
for (auto idx: *list)
AddFeature (idx);
}
else
{
_valid = false;
}
delete list;
list = NULL;
} /* ParseToString */
bool DumpSipper3InstrumentData::ProcessCmdLineParameter (char parmSwitchCode,
KKStr parmSwitch,
KKStr parmValue
)
{
parmSwitch.Upper ();
if ((parmSwitch == "-REPORT") ||
(parmSwitch == "-REPORTFILE") ||
(parmSwitch == "-RF") ||
(parmSwitch == "-R")
)
{
reportFileName = parmValue;
}
else
if ((parmSwitch == "-S") ||
(parmSwitch == "-SIPPERFILE") ||
(parmSwitch == "-SF")
)
{
sipperFileName = parmValue;
}
return !Abort ();
} /* ProcessCmdLineParameter */
bool CmdLineExpander::ParameterIsASwitch (const KKStr& parm)
{
if (parm.Len () < 1)
return false;
if (parm[(kkint16)0] != '-')
return false;
if (parm.Len () == 1)
return true;
double parmValue = 0.0;
if (parm.ValidNum (parmValue))
return false;
return true;
} /* ParameterIsASwitch */
bool ImageDimensionComputations::ProcessCmdLineParameter (const KKStr& parmSwitch,
const KKStr& parmValue
)
{
bool validParm = true;
if (parmSwitch.EqualIgnoreCase ("-R") || parmSwitch.EqualIgnoreCase ("-Report"))
{
reportFileName = parmValue;
}
else
{
validParm = PicesApplication::ProcessCmdLineParameter (parmSwitch, parmValue);
}
return validParm;
} /* ProcessCmdLineParameter */
void GoalKeeper::ReportBlockedStats ()
{
HANDLE mutexCreateHandle = CreateMutex (NULL, /**< default security attributes. */
false, /**< initially not owned. */
"GoalKeeper_ReportBlockedStats"
);
WaitForSingleObject (mutexCreateHandle, INFINITE);
KKStr path = "C:\\Temp\\BlockedStats";
KKB::osCreateDirectoryPath (path);
KKStr fileName = osAddSlash (path) + "BlockedStats.txt";
ofstream o (fileName.Str (), ios_base::app);
o << "Name"
<< "\t" << "Count"
<< "\t" << "BlockedThreadId"
<< "\t" << "BlockerThreadId"
<< "\t" << "DateTimeBlocked"
<< "\t" << "MilliSecsBlocked"
<< "\t" << "NumBlockedThreads"
<< endl;
int c = 0;
BlockedStatList::iterator idx;
for (idx = blockedStats->begin (), c = 0; idx != blockedStats->end (); ++idx, ++c)
{
BlockedStat* bs = *idx;
o << name
<< "\t" << c
<< "\t" << bs->blockedThreadId
<< "\t" << bs->blockerThreadId
<< "\t" << bs->dateTimeBlocked.Date () << " " << bs->dateTimeBlocked.Time ()
<< "\t" << bs->milliSecsBlocked
<< "\t" << bs->numBlockedThreads
<< endl;
}
o.close ();
ReleaseMutex (mutexCreateHandle);
CloseHandle(mutexCreateHandle);
} /* ReportBlockedStats */
void ScannerFile::ReadHeader ()
{
delete headerFields;
headerFields = new ScannerHeaderFields ();
bool endOfText = false;
KKStr ln (100);
ReadHeaderOneLine (file, endOfText, ln);
while (!endOfText)
{
KKStr fieldName = ln.ExtractToken2 ("\t");
KKStr fieldValue = ln.ExtractToken2 ("\t");
headerFields->Add (fieldName, fieldValue);
ExtractHeaderField (fieldName, fieldValue);
ReadHeaderOneLine (file, endOfText, ln);
}
} /* ReadHeader */
void JobValidation::ProcessStatusField (const KKStr& fieldName,
const KKStr& fieldValue
)
{
// 'JobValidation' does not have any fields.
if (fieldName.EqualIgnoreCase ("configFileName"))
configFileName = fieldValue;
} /* ProcessStatusField */
KKStr KKStrParser::GetRestOfLine ()
{
if (trimWhiteSpace)
{
while ((nextPos < len) && (strchr (whiteSpace, str[nextPos]) != NULL))
nextPos++;
}
if (nextPos >= len)
return KKStr::EmptyStr ();
KKStr result (1 + len - nextPos);
char lastChar = 0;
while (nextPos < len)
{
lastChar = str[nextPos];
if (lastChar == '\n')
break;
else if (lastChar == '\r')
break;
result.Append (lastChar);
nextPos++;
}
if (nextPos < len)
{
if (lastChar == '\n')
{
if (str[nextPos] == '\r')
nextPos++;
}
else if (lastChar == '\r')
{
if (str[nextPos] == '\n')
nextPos ++;
}
}
if (trimWhiteSpace)
result.TrimRight (whiteSpace);
return result;
} /* GetRestOfLine */
/**
* @brief Processes a GPGLL string that contains Latitude and Longitudinal info.
* @param[in] _str '$GPGLL' string with '$GPGLL' stripped off front.
*/
void InstrumentDataGPS::ProcessGPGLL (const KKStr& _str)
{
KKStr str (_str);
KKStr field = str.ExtractToken2 (",");
if (field == "$GPGLL")
field = str.ExtractToken2 (",");
double latitudeDegrees;
double latitudeMinutes;
ParseDegreeMinutesStr (field, latitudeDegrees, latitudeMinutes);
double latitude = latitudeDegrees + (latitudeMinutes / 60.0);
field = str.ExtractToken2 (",");
if (field == "S")
latitude = -latitude;
field = str.ExtractToken2 (",");
double longitudeDegrees;
double longitudeMinutes;
ParseDegreeMinutesStr (field, longitudeDegrees, longitudeMinutes);
double longitude = longitudeDegrees + (longitudeMinutes / 60.0);
field = str.ExtractToken2 (",");
if (field == "E")
longitude = -longitude;
field = str.ExtractToken2 (",");
TimeType gpsTime (field);
manager->Location (curTextLineStartScanLine, latitude, longitude, gpsTime);
} /* ProcessGPGLL */
