CmdLineExpander::CmdLineExpander (const KKStr& _applicationName, RunLog& _log, const KKStr& _cmdLine ): applicationName (_applicationName), log (_log) { VectorKKStr initialParameters; KKStr cmdLine (_cmdLine); cmdLine.TrimLeft (); while (!cmdLine.Empty ()) { KKStr nextField = cmdLine.ExtractQuotedStr ("\n\r\t ", false); // false = Do not decode escape characters nextField.TrimRight (); if (!nextField.Empty ()) initialParameters.push_back (nextField); cmdLine.TrimLeft (); } BuildCmdLineParameters (initialParameters); BuildExpandedParameterPairs (); }
void KKJobManager::StatusFileProcessLineJobStatusChange (KKStr& statusLineStr) { kkint32 expandedJobId = statusLineStr.ExtractTokenInt ("\t"); KKJobPtr j = jobs->LookUpByJobId (expandedJobId); if (!j) { log.Level (-1) << endl << endl << endl << "ProcessStatusLineJobStatusChange ***Error*** Could not locate Expanded" << endl << endl << " JobId[" << expandedJobId << "]" << endl << endl; EndBlock (); osWaitForEnter (); exit (-1); } KKStr statusStr = statusLineStr.ExtractToken2 ("\t"); statusStr.TrimLeft (); statusStr.TrimRight (); KKJob::JobStatus status = KKJob::JobStatusFromStr (statusStr); if (status == jsNULL) { log.Level (-1) << endl << endl << endl << "ProcessStatusLineJobStatusChange ***Error*** Invalid Status Specified" << endl << endl << " JobId[" << expandedJobId << "]" << endl << " Status[" << statusStr << "]" << endl << endl; EndBlock (); osWaitForEnter (); exit (-1); } j->Status (status); } /* ProcessStatusLineJobStatusChange */
void KKJob::ProcessStatusStr (const KKStr& statusStr) { log.Level (30) << "KKJob::ProcessStatusStr[" << statusStr << "]" << endl; KKStr fieldName; KKStr fieldValue; VectorKKStr fields = statusStr.Split ('\t'); kkuint32 fieldNum = 0; while (fieldNum < fields.size ()) { fieldName = fields[fieldNum]; fieldNum++; if (fieldNum < fields.size ()) { fieldValue = fields[fieldNum]; fieldNum++; } else { fieldValue = ""; } fieldName.Upper (); fieldValue.TrimLeft ("\n\r\t "); fieldValue.TrimRight ("\n\r\t "); if (fieldName.CompareIgnoreCase ("JOBID") == 0) jobId = atoi (fieldValue.Str ()); else if (fieldName.CompareIgnoreCase ("PARENTID") == 0) parentId = atoi (fieldValue.Str ()); else if (fieldName.CompareIgnoreCase ("STATUS") == 0) status = JobStatusFromStr (fieldValue); else if (fieldName.CompareIgnoreCase ("NumProcessors") == 0) numProcessors = fieldValue.ToInt (); else if (fieldName.CompareIgnoreCase ("NumPorcessesAllowed") == 0) numPorcessesAllowed = fieldValue.ToInt (); else if (fieldName.CompareIgnoreCase ("Prerequisites") == 0) PrerequisitesFromStr (fieldValue); else { ProcessStatusField (fieldName, fieldValue); } } } /* ProcessStatusStr */
KKStr KKStrParser::GetRestOfLine () { if (trimWhiteSpace) { while ((nextPos < len) && (strchr (whiteSpace, str[nextPos]) != NULL)) nextPos++; } if (nextPos >= len) return KKStr::EmptyStr (); KKStr result (1 + len - nextPos); char lastChar = 0; while (nextPos < len) { lastChar = str[nextPos]; if (lastChar == '\n') break; else if (lastChar == '\r') break; result.Append (lastChar); nextPos++; } if (nextPos < len) { if (lastChar == '\n') { if (str[nextPos] == '\r') nextPos++; } else if (lastChar == '\r') { if (str[nextPos] == '\n') nextPos ++; } } if (trimWhiteSpace) result.TrimRight (whiteSpace); return result; } /* GetRestOfLine */
// Will scan for fields that are specific to 'JobValidation' void JobValidation::ProcessStatusStr (KKStrParser& statusStr) { log.Level (30) << "JobValidation::ProcessStatusStr[" << statusStr.Str () << "]" << endl; KKStr fieldName; KKStr fieldValue; while (statusStr.MoreTokens ()) { fieldName = statusStr.GetNextToken ("\t\n\r"); fieldName.Upper (); fieldValue = statusStr.GetNextToken ("\t"); fieldValue.TrimLeft ("\n\r\t "); fieldValue.TrimRight ("\n\r\t "); ProcessStatusField (fieldName, fieldValue); } } /* ProcessStatusStr */
KKStr KKStrParser::GetRestOfStr () { if (trimWhiteSpace) { while ((nextPos < len) && (strchr (whiteSpace, str[nextPos]) != NULL)) nextPos++; } if (nextPos >= len) return KKStr::EmptyStr (); KKStr result (1 + len - nextPos); while (nextPos < len) { result.Append (str[nextPos]); nextPos++; } if (trimWhiteSpace) result.TrimRight (whiteSpace); return result; } /* GetRestOfStr */
/****************************************************************************** * ProcessCmdLineParamters * DESC: Extracts parameters from the command line ******************************************************************************/ bool FeatureFileConverter::ProcessCmdLineParameter (char parmSwitchCode, KKStr parmSwitch, KKStr parmValue ) { KKStr parmValueUpper (parmValue); parmValueUpper.Upper (); parmSwitch.Upper (); if ((parmSwitch == "-DEST") || (parmSwitch == "-D") || (parmSwitch == "-DESTFILE") || (parmSwitch == "-DF") || (parmSwitch == "-DESTFILENAME") || (parmSwitch == "-DFN") ) destFileName = parmValue; else if ((parmSwitch == "-OUTFORMAT") || (parmSwitch == "-OF") || (parmSwitch == "-OUTFILEFORMAT") || (parmSwitch == "-OFF") || (parmSwitch == "-DESTFILEFORMAT") || (parmSwitch == "-DESTFORMAT") || (parmSwitch == "-DFF") ) { destFileFormat = FeatureFileIO::FileFormatFromStr (parmValue, false, // Don't care if valid read format true // must be valid write format ); if (destFileFormat == NULL) { log.Level (-1) << endl << endl << "Invalid Write File Format Specified[" << parmValue << endl << endl << "Valid Formats are <" << FeatureFileIO::FileFormatsWrittenOptionsStr () << ">" << endl << endl; Abort (true); } } else if ((parmSwitch == "-ENCODEFEATUREDATA") || (parmSwitch == "-ENCODEFEATURES") || (parmSwitch == "-EF") ) { encodingMethod = ModelParam::EncodingMethodFromStr (parmValue); if (encodingMethod != Encoding_NULL) { encodeFeatureData = true; } else { if ((parmValue == "NO") || (parmValue == "OFF") || (parmValue == "N") ) encodeFeatureData = false; else { encodingMethod = ModelParam::BinaryEncoding; encodeFeatureData = true; } } } else if ((parmSwitch == "-ENUMERATECLASSES") || (parmSwitch == "-ENUMERATECLASS") || (parmSwitch == "-EC") ) { if ((parmValue == "NO") || (parmValue == "OFF") || (parmValue == "N") ) enumerateClasses = false; else enumerateClasses = true; } else if (parmSwitch == "-FEATURES") { featureStr = parmValue; } else if ((parmSwitch == "-IF") || (parmSwitch == "-INPUTFORMAT") || (parmSwitch == "-IFF") || (parmSwitch == "-INFILEFORMAT") || (parmSwitch == "-SFF") || (parmSwitch == "-SrcFileFormat") ) { srcFileFormat = FeatureFileIO::FileFormatFromStr (parmValue, true, // Valid Read Format false // Don't care if valid write format ); if (srcFileFormat == NULL) { log.Level (-1) << endl << endl << "Invalid Source File Format Specified[" << parmValue << endl << endl << "Valid Formats are <" << FeatureFileIO::FileFormatsReadOptionsStr () << ">" << endl << endl; Abort (true); } } else if ((parmSwitch == "-NORM") || (parmSwitch == "-NORMALIZE")) { normalizeData = true; parmValue.TrimLeft (); parmValue.TrimRight (); if (!parmValue.Empty ()) nornParmsFileName = parmValue; } else if ((parmSwitch == "-R") || (parmSwitch == "-REPORT")) reportFileName = parmValue; else if ((parmSwitch == "-S") || (parmSwitch == "-SRC") || (parmSwitch == "-SF") || (parmSwitch == "-SOURCFILE") || (parmSwitch == "-SFN") || (parmSwitch == "-SRCFILENAME") || (parmSwitch == "-SOURCFILENAME") ) srcFileName = parmValue; else if ((parmSwitch == "-STAT") || (parmSwitch == "-STATS") || (parmSwitch == "-STATISTICS") ) statistics = true; else { log.Level (-1) << endl << "ProcessCmdLineParameter Invalid Parameter[" << parmSwitch << "]" << endl << endl; Abort (true); } return !Abort (); } /* ProcessCmdLineParameter */
void BinaryClass::ProcessStatusStr (KKStr statusStr, MLClassListPtr mlClasses ) { KKStr fieldName; class1 = NULL; class2 = NULL; fieldName = statusStr.ExtractToken2 ("\t"); while (!fieldName.Empty ()) { fieldName.TrimLeft ("\n\r\t "); fieldName.TrimRight ("\n\r\t "); fieldName.Upper (); if (fieldName == "ALLCLASSES") { class1 = NULL; class2 = NULL; } else if (fieldName == "BINARYCLASSES") { class1 = mlClasses->GetMLClassPtr (statusStr.ExtractToken2 ("\t")); class2 = mlClasses->GetMLClassPtr (statusStr.ExtractToken2 ("\t")); if ((class1 == NULL) || (class2== NULL)) { cout << "Class1 or Class2 == NULL" << endl; } else { if ((class1->Name ().Empty ()) || (class1->Name ().Empty ())) { cout << "Class1Name or Class2Name are empty" << endl; } } } else if (fieldName == "STATUS") status = BinaryClassStatusFromStr (statusStr.ExtractToken2 ("\t")); else if (fieldName == "NUMPROCESSORS") numProcessors = statusStr.ExtractTokenInt ("\t"); else if (fieldName.CompareIgnoreCase ("FinalResultType") == 0) resultType = FinalResultTypeFromStr (statusStr.ExtractToken2 ("\t")); else if (fieldName.CompareIgnoreCase ("ResultsFileName") == 0) finalResultsFileName = statusStr.ExtractToken2 ("\t"); else if (fieldName.CompareIgnoreCase ("SelectionMethod") == 0) selectionMethod = SelectionMethodFromStr (statusStr.ExtractToken2 ("\t")); else if (fieldName.CompareIgnoreCase ("configFileName") == 0) configFileName = statusStr.ExtractToken2 ("\t"); fieldName = statusStr.ExtractToken2 ("\t"); } } /* ProcessStatusStr */
void Configuration::LoadFile (RunLog& log) { log.Level (10) << "Configuration::LoadFile: " << fileName << endl; kkint32 lastLineNum = 0; if (fileName == "") { log.Level (-1) << endl << "Configuration::LoadFile ***ERROR*** File-Name is blank" << endl << endl; FormatGood (false); return; } FILE* inFile = osFOPEN (fileName.Str (), "r"); if (!inFile) { log.Level (-1) << endl << "Configuration::LoadFile ***ERROR*** Opening File: " << fileName << endl << endl; FormatGood (false); return; } char buff[10240]; kkint32 lineCount = 0; curSectionName = ""; ConfSectionPtr curSection = NULL; while (fgets (buff, sizeof (buff), inFile)) { lastLineNum++; KKStr line (buff); line.TrimRight (); line.TrimLeft (); StripOutAnyComments (line); log.Level (70) << line << endl; StripOutAnyComments (line); if (line.Empty ()) { // If we have a blank line, we do nothing. } else if (line.FirstChar () == '[') { // Looks like definition of new section. if (line.LastChar () == ']') { curSectionName = line.SubStrPart (1, line.Len () - 2); curSectionName.TrimLeft (); curSectionName.TrimRight (); curSectionName.Upper (); curSection = new ConfSection (curSectionName, lastLineNum); sections->AddConfSection (curSection); log.Level (30) << "LoadFile SectionName[" << curSectionName << "]." << endl; } else { log.Level (-1) << endl << "Configuration::LoadFile ***ERROR*** LineNumber[" << lastLineNum << "] Improper Section Name[" << line << "]." << endl << endl; formatGood = false; } } else { if (!curSection) { log.Level (-1) << endl << "Configuration::LoadFile ***ERROR*** Format Error LineNumber[" << lastLineNum << "]" << endl << " No Section Defined." << endl << endl; formatGood = false; curSectionName = "GLOBAL"; curSection = new ConfSection (curSectionName, lastLineNum); sections->AddConfSection (curSection); } kkint32 equalIdx = line.LocateCharacter ('='); if (equalIdx < 0) { // We have a improperly formated line. log.Level (-1) << endl << "Configuration::LoadFile ***ERROR*** LineNumber[" << lastLineNum << "] Improperly Formated Line[" << line << "]." << endl; formatGood = false; } else { KKStr settingName (line.SubStrPart (0, equalIdx - 1)); settingName.TrimLeft (); settingName.TrimRight (); settingName.Upper (); KKStr settingValue (line.SubStrPart (equalIdx + 1)); settingValue.TrimLeft (); settingValue.TrimRight (); log.Level (30) << "LoadFile SectionName[" << curSectionName << "], " << "Setting[" << settingName << "], Value[" << settingValue << "]." << endl; curSection->AddSetting (settingName, settingValue, lastLineNum); } lineCount++; } } fclose (inFile); } /* LoadFile */
void KKJobManager::ProcessJobXmlBlockOfText (const KKStr& startStr, istream& i ) { if ((startStr.SubStrPart (0, 4) != "<KKJob ") || (startStr.LastChar () != '>')) { log.Level (-1) << endl << "KKJobManager::ProcessJobXmlBlockOfText ***ERROR*** StartStr[" << startStr << "] is not a KKJob String." << endl << endl; return; } KKStr s = startStr.SubStrPart (5); s.TrimLeft (); s.ChopLastChar (); KKStr jobTypeStr = ""; kkint32 jobId = -1; VectorKKStr parameters = s.Split (','); for (kkuint32 x = 0; x < parameters.size (); ++x) { KKStr parameterStr = parameters[x]; parameterStr.TrimLeft (); parameterStr.TrimRight (); KKStr fieldName = parameterStr.ExtractToken2 ("="); fieldName.TrimLeft (); fieldName.TrimRight (); KKStr fieldValue = parameterStr.ExtractToken2 ("="); fieldValue.TrimLeft (); fieldValue.TrimRight (); if (fieldName.CompareIgnoreCase ("JobType") == 0) jobTypeStr = fieldValue; else if (fieldName.CompareIgnoreCase ("JobId") == 0) jobId = fieldValue.ToInt (); } if (jobTypeStr.Empty () || (jobId < 0)) { log.Level (-1) << endl << "KKJobManager::ProcessJobXmlBlockOfText ***ERROR*** StartStr[" << startStr << "]." << endl << " JobType and/or JobId were not provided." << endl << endl; return; } KKJobPtr j = jobs->LookUpByJobId (jobId); if (j == NULL) { // We do not have this job in memory yet. We will have to create it now. KKStr emptyStatusStr = "JobId\t" + StrFormatInt (jobId, "ZZZZ0"); j = KKJob::CallAppropriateConstructor (this, jobTypeStr, emptyStatusStr); } j->CompletedJobDataRead (i); } /* ProcessJobXmlBlockOfText */
void KKJobManager::StatusFileProcessLine (const KKStr& ln, istream& statusFile ) { if (ln.SubStrPart (0, 1) == "//") { // A coment line; we can ignore it. return; } KKStr statusStr (ln); KKStr fieldName = statusStr.ExtractToken2 ("\t"); if (fieldName.Empty ()) { // A empty line we will ignore it. return; } statusStr.TrimLeft ("\n\r\t "); statusStr.TrimRight ("\n\r\t "); if (fieldName.CompareIgnoreCase ("JOB") == 0) { // We have a KKJob entr line; the next field determines JobType fllowed by parameters for that JobType constructor. KKStr jobTypeName = fieldName = statusStr.ExtractToken2 ("\t"); KKJobPtr j = KKJob::CallAppropriateConstructor (this, jobTypeName, statusStr); KKJobPtr existingJob = jobs->LookUpByJobId (j->JobId ()); if (existingJob) { existingJob->ReFresh (*j); delete j; j = NULL; } else { jobs->PushOnBack (j); } } else if (fieldName.EqualIgnoreCase ("CPUTIMEUSED")) { double cpuTimeUsed = statusStr.ExtractTokenDouble ("\t"); cpuTimeTotalUsed += cpuTimeUsed; } else if (fieldName.EqualIgnoreCase ("CURRENTDATETIME")) { KKB::DateTime dateTime = KKB::DateTime (statusStr); if (!dateTimeFirstOneFound) { dateTimeFirstOneFound = true; dateTimeStarted = dateTime; } dateTimeEnded = dateTime; } else if (fieldName.EqualIgnoreCase ("ExpansionCount")) expansionCount = statusStr.ToInt (); else if (fieldName.EqualIgnoreCase ("ExpansionFirstJobId")) expansionFirstJobId = statusStr.ToInt (); else if (fieldName.EqualIgnoreCase ("JobStatusChange")) StatusFileProcessLineJobStatusChange (statusStr); else if (fieldName.EqualIgnoreCase ("NextJobId")) nextJobId = statusStr.ExtractTokenInt ("\t"); else if (fieldName.EqualIgnoreCase ("QuitRunning")) quitRunning = true; else if (fieldName.EqualIgnoreCase ("Restart")) restart = false; else if (fieldName.EqualIgnoreCase ("Status")) status = KKJob::JobStatusFromStr (statusStr); else { log.Level (-1) << "KKJobManager::StatusFileProcessLine Invalid Field Name[" << fieldName << "]." << endl; } } /* StatusFileProcessLine */
void RandomSampleJob::ProcessStatusStr (KKStr statusStr) { log.Level (30) << "RandomSampleJob::ProcessStatusStr[" << statusStr << "]" << endl; KKStr fieldName; KKStr fieldValue; compMethod = BRNull; fieldName = statusStr.ExtractToken ("\t\n\r"); while (!fieldName.Empty ()) { fieldName.Upper (); fieldValue = statusStr.ExtractToken ("\t\n\r"); fieldValue.TrimLeft ("\n\r\t "); fieldValue.TrimRight ("\n\r\t "); if (fieldName == "JOBID") jobId = atoi (fieldValue.Str ()); else if (fieldName == "STATUS") status = JobStatusFromStr (fieldValue); else if (fieldName == "KERNEL") kernelType = KernalTypeFromStr (fieldValue); else if (fieldName == "ENCODING") encodingMethod = EncodingMethodFromStr (fieldValue); else if (fieldName == "COMPMETHOD") compMethod = CompressionMethodFromStr (fieldValue); else if (fieldName == "C_PARAM") c = atoi (fieldValue.Str ()); else if (fieldName == "GAMMA") gamma = atof (fieldValue.Str ()); else if (fieldName == "ORDERINGNUM") orderingNum = atoi (fieldValue.Str ()); else if (fieldName == "EXAMPLESTOKEEP") numExamplesToKeep = atoi (fieldValue.Str ()); else if (fieldName == "ACCURACY") accuracy = (float)atof (fieldValue.Str ()); else if (fieldName == "TRAINTIME") { trainTime = atof (fieldValue.Str ()); } else if (fieldName == "PREDTIME") testTime = (float)atof (fieldValue.Str ()); else if (fieldName == "SUPPORTVECTORS") supportVectors = (float)atof (fieldValue.Str ()); else { log.Level (-1) << "RandomSampleJob::ProcessStatusStr Invalid Field Name[" << fieldName << "]." << endl; } fieldName = statusStr.ExtractToken ("\t\n\r"); } } /* ProcessStatusStr */
void ClassificationBiasMatrix::ReadSimpleConfusionMatrix (istream& sr, MLClassListPtr fileClasses ) { // 'classes' - The class order that the owner of this object is expecting. // 'fileClasses' - The order that the classes are stored in the text file. if ((classes == NULL) || (fileClasses == NULL)) { KKStr errMsg = "ReadSimpleConfusionMatrix ***ERROR*** The 'Classes' line was never provided."; runLog.Level (-1) << errMsg << endl; valid = false; throw KKException (errMsg); } kkint32 classesColIdx = 0; char buff[10240]; KKStr l; while (!sr.eof ()) { sr.getline (buff, sizeof (buff)); l = buff; l.TrimLeft (); l.TrimRight (); if (l.CompareIgnoreCase ("</SimpleConfusionMatrix>") == 0) break; KKStr lineName = l.ExtractToken2 ("\t"); if (lineName.CompareIgnoreCase ("DataRow") == 0) { if (fileClasses == NULL) { KKStr errMsg = "ReadSimpleConfusionMatrix ***ERROR*** 'Classes' was not provided before 'DataRow'."; runLog.Level (-1) << errMsg << endl; valid = false; throw KKException (errMsg); } KKStr className = l.ExtractToken2 ("\t"); KKStr data = l.ExtractToken2 ("\t"); MLClassPtr pc = MLClass::CreateNewMLClass (className); kkint32 classesIdx = classes->PtrToIdx (pc); kkint32 fileClassesIdx = fileClasses->PtrToIdx (pc); if (classesIdx < 0) { KKStr errMsg = "ReadSimpleConfusionMatrix ***ERROR*** DataRow specifies class[" + className + "] which is not defined by caller"; runLog.Level (-1) << errMsg << endl; valid = false; throw KKException (errMsg); } if (fileClassesIdx < 0) { KKStr errMsg = "ReadSimpleConfusionMatrix ***ERROR*** DataRow specifies class[" + className + "] was not defined in 'Classes' line."; runLog.Level (-1) << errMsg << endl; valid = false; throw KKException (errMsg); } kkint32 classesRowIdx = classesIdx; VectorKKStr dataFields = data.Split (','); if (dataFields.size () != (kkuint32)numClasses) { KKStr errMsg = "ReadSimpleConfusionMatrix ***ERROR*** DataRow Class[" + className + "] number[" + StrFormatInt ((kkint32)dataFields.size (), "ZZZ0") + "] of values provided does not match number of Classes."; runLog.Level (-1) << errMsg << endl; valid = false; throw KKException (errMsg); } for (kkint32 c = 0; c < numClasses; c++) { pc = fileClasses->IdxToPtr (c); classesColIdx = classes->PtrToIdx (pc); VectorKKStr parts = dataFields[c].Split (':'); if (parts.size () > 1) { (*counts) [classesRowIdx][classesColIdx] = parts[0].ToDouble (); (*probabilities)[classesRowIdx][classesColIdx] = parts[1].ToDouble (); } } } } } /* ReadSimpleConfusionMatrix */
void ClassificationBiasMatrix::ReadXML (istream& sr) { char buff[10240]; KKStr l (512); if (sr.eof ()) return; MLClassListPtr fileClasses = NULL; sr.getline (buff, sizeof (buff)); while (!sr.eof ()) { l = buff; l.TrimRight (); if (l.CompareIgnoreCase ("</ClassificationBiasMatrix>") == 0) break; KKStr lineName = l.ExtractToken2 ("\t"); if (!lineName.Empty ()) { KKStr fieldValue = l.ExtractToken2 ("\t"); if (lineName.CompareIgnoreCase ("Classes") == 0) { delete fileClasses; fileClasses = NULL; fileClasses = MLClassList::BuildListFromDelimtedStr (fieldValue, ','); if (classes == NULL) classes = new MLClassList (*fileClasses); } else if (lineName.CompareIgnoreCase ("ConfigDateTime") == 0) { configDateTime = fieldValue; } else if (lineName.CompareIgnoreCase ("ConfigFileName") == 0) { configFileNameFromMatrixBiasFile = fieldValue; } else if (lineName.CompareIgnoreCase ("ConfigFileDateTime") == 0) { configDateTime = fieldValue; } else if (lineName.CompareIgnoreCase ("DateTime") == 0) { dateTimeFileWritten = fieldValue; } else if (lineName.CompareIgnoreCase ("DateTimeFileWritten") == 0) { dateTimeFileWritten = fieldValue; } else if (lineName.CompareIgnoreCase ("FileName") == 0) { } else if (lineName.CompareIgnoreCase ("NumClasses") == 0) { numClasses = fieldValue.ToInt (); } else if (lineName.CompareIgnoreCase ("<SimpleConfusionMatrix>") == 0) { ReadSimpleConfusionMatrix (sr, fileClasses); } } if (!sr.eof ()) sr.getline (buff, sizeof (buff)); } } /* ReadXML */