CmdLineExpander::CmdLineExpander (const KKStr& _applicationName,
RunLog& _log,
const KKStr& _cmdLine
):
applicationName (_applicationName),
log (_log)
{
VectorKKStr initialParameters;
KKStr cmdLine (_cmdLine);
cmdLine.TrimLeft ();
while (!cmdLine.Empty ())
{
KKStr nextField = cmdLine.ExtractQuotedStr ("\n\r\t ", false); // false = Do not decode escape characters
nextField.TrimRight ();
if (!nextField.Empty ())
initialParameters.push_back (nextField);
cmdLine.TrimLeft ();
}
BuildCmdLineParameters (initialParameters);
BuildExpandedParameterPairs ();
}
XmlTag::XmlTag (istream& i)
{
tagType = tagNULL;
if (i.peek () == '<')
i.get ();
KKStr tagStr (100);
ReadWholeTag (i, tagStr);
if (tagStr.FirstChar () == '/')
{
tagStr.ChopFirstChar ();
tagType = tagEnd;
}
if (tagStr.EndsWith ("/>"))
{
tagType = tagEmpty;
tagStr.ChopLastChar ();
tagStr.ChopLastChar ();
}
else if (tagStr.LastChar () != '>')
{
tagType = tagStart;
}
else
{
if (tagType == tagNULL)
tagType = tagStart;
tagStr.ChopLastChar ();
}
name.TrimLeft ();
name.TrimRight ();
name = tagStr.ExtractToken2 (" \n\r\t");
KKStr attributeName (20);
KKStr attributeValue (20);
while (!tagStr.Empty ())
{
ExtractAttribute (tagStr, attributeName, attributeValue);
if (!attributeName.Empty ())
attributes.push_back (XmlAttribute (attributeName, attributeValue));
}
}
KKStr FeatureNumList::ToString () const
{
KKStr featureNumStr (numOfFeatures * 6);
if (numOfFeatures <= 0)
return featureNumStr;
kkint32 nextIdx = 0;
while (nextIdx < numOfFeatures)
{
kkint32 startOfGroup = nextIdx;
kkint32 endOfGroup = nextIdx;
while ((endOfGroup < (numOfFeatures - 1)) &&
(featureNums[endOfGroup] == (featureNums[endOfGroup + 1] - 1))
)
{
endOfGroup++;
}
if ((endOfGroup - startOfGroup) < 3)
{
kkint32 x;
for (x = startOfGroup; x <= endOfGroup; x++)
{
if (!featureNumStr.Empty ())
featureNumStr << ",";
featureNumStr << featureNums[x];
}
}
else
{
if (!featureNumStr.Empty ())
featureNumStr << ",";
featureNumStr << featureNums[startOfGroup] << "-" << featureNums[endOfGroup];
}
nextIdx = endOfGroup + 1;
}
return featureNumStr;
} /* ToString */
void ModelParamUsfCasCor::ParseCmdLineParameter (const KKStr& parameter,
const KKStr& value,
bool& parameterUsed,
RunLog& log
)
{
parameterUsed = true;
if (parameter.EqualIgnoreCase ("-InLimit") ||
parameter.EqualIgnoreCase ("-IL") ||
parameter.EqualIgnoreCase ("-I")
)
in_limit = value.ToInt ();
else if (parameter.EqualIgnoreCase ("-OutLimit") ||
parameter.EqualIgnoreCase ("-OL") ||
parameter.EqualIgnoreCase ("-O")
)
out_limit = value.ToInt ();
else if (parameter.EqualIgnoreCase ("-NumberOfRounds") ||
parameter.EqualIgnoreCase ("-NOR") ||
parameter.EqualIgnoreCase ("-Rounds") ||
parameter.EqualIgnoreCase ("-R")
)
number_of_rounds = value.ToInt ();
else if (parameter.EqualIgnoreCase ("-NumberOfTrials") ||
parameter.EqualIgnoreCase ("-NOT") ||
parameter.EqualIgnoreCase ("-T")
)
number_of_trials = value.ToInt ();
else if (parameter.EqualIgnoreCase ("-RandomSeed") ||
parameter.EqualIgnoreCase ("-RS") ||
parameter.EqualIgnoreCase ("-S")
)
random_seed = value.ToInt ();
else if (parameter.EqualIgnoreCase ("-UseCache") ||
parameter.EqualIgnoreCase ("-UC") ||
parameter.EqualIgnoreCase ("-Cache")
)
{
if (value.Empty ())
useCache = true;
else
useCache = value.ToBool ();
}
else
parameterUsed = false;
} /* ParseCmdLineParameter */
FeatureNumList::VectorIntType* FeatureNumList::StrToUInt16Vetor (const KKStr& s)
{
bool valid = true;
VectorIntType* results = new VectorUint16 ();
KKStrParser parser (s);
parser.TrimWhiteSpace (" ");
while (parser.MoreTokens ())
{
KKStr field = parser.GetNextToken (",\t");
if (field.Empty ())
continue;
auto dashPos = field.LocateCharacter ('-');
if (!dashPos)
{
kkint32 n = field.ToInt32 ();
if ((n < 0) || ((kkuint32)n > maxIntType))
{
valid = false;
break;
}
results->push_back ((IntType)n);
}
else
{
// We are looking at a range
kkint32 startNum = field.SubStrSeg (0, dashPos).ToInt32 ();
kkint32 endNum = field.SubStrPart (dashPos + 1).ToInt32 ();
if ((startNum > endNum) || (startNum < 0) || ((kkuint32)endNum > maxIntType))
{
valid = false;
break;
}
for (kkint32 z = startNum; z <= endNum; ++z)
results->push_back ((IntType)z);
}
}
if (!valid)
{
delete results;
results = NULL;
}
else
{
sort (results->begin (), results->end ());
}
return results;
} /* StrToUInt16Vetor */
DateTime::DateTime (const KKStr& s):
date(),
time ()
{
if (s.Empty ())
return;
KKStrParser parser (s);
KKStr field1 = parser.GetNextToken ("- ");
KKStr field2 = parser.GetNextToken ("- ");
date = DateType (field1);
time = TimeType (field2);
}
void FeatureEncoder::WriteXML (const KKStr& varName,
ostream& o
) const
{
XmlTag tagStart ("TrainingClassList", XmlTag::TagTypes::tagStart);
if (!varName.Empty ())
tagStart.AddAtribute ("VarName", varName);
tagStart.WriteXML (o);
o << endl;
XmlElementInt32::WriteXML (codedNumOfFeatures, "CodedNumOfFeatures", o);
XmlElementDouble::WriteXML (c_Param, "c_Param", o);
XmlElementInt32::WriteXML (numEncodedFeatures, "NumEncodedFeatures", o);
XmlElementInt32::WriteXML (numOfFeatures, "NumOfFeatures", o);
XmlElementInt32::WriteXML (xSpaceNeededPerExample, "xSpaceNeededPerExample", o);
if (cardinalityDest)
XmlElementArrayInt32::WriteXML (numOfFeatures, cardinalityDest, "CardinalityDest", o);
if (class1) class1->Name ().WriteXML ("Class1", o);
if (class2) class2->Name ().WriteXML ("Class2", o);
if (destFeatureNums)
XmlElementArrayInt32::WriteXML (numOfFeatures, destFeatureNums, "DestFeatureNums", o);
if (fileDesc) fileDesc->WriteXML ("FileDesc", o);
if (destFileDesc) destFileDesc->WriteXML ("DestFileDesc", o);
if (destWhatToDo)
{
VectorInt32 v;
for (kkint32 x = 0; x < numOfFeatures; ++x)
v.push_back ((kkint32)(destWhatToDo[x]));
XmlElementVectorInt32::WriteXML (v, "DestWhatToDo", o);
}
EncodingMethodToStr (encodingMethod).WriteXML ("EncodingMethod", o);
selectedFeatures.WriteXML ("selectedFeatures", o);
if (srcFeatureNums)
XmlElementArrayInt32::WriteXML (numOfFeatures, srcFeatureNums, "SrcFeatureNums", o);
XmlTag tagEnd ("TrainingClassList", XmlTag::TagTypes::tagEnd);
tagEnd.WriteXML (o);
o << endl;
}
void FeatureNumList::WriteXML (const KKStr& varName,
ostream& o
) const
{
XmlTag startTag ("FeatureNumList", XmlTag::TagTypes::tagStart);
if (!varName.Empty ())
startTag.AddAtribute ("VarName", varName);
startTag.AddAtribute ("MaxFeatureNum", maxFeatureNum);
startTag.AddAtribute ("NumOfFeatures", numOfFeatures);
startTag.WriteXML (o);
o << ToString ();
XmlTag endTag ("FeatureNumList", XmlTag::TagTypes::tagEnd);
endTag.WriteXML (o);
o << endl;
}
void ModelUsfCasCor::WriteXML (const KKStr& varName,
ostream& o
) const
{
XmlTag startTag ("ModelUsfCasCor", XmlTag::TagTypes::tagStart);
if (!varName.Empty ())
startTag.AddAtribute ("VarName", varName);
startTag.WriteXML (o);
o << endl;
WriteModelXMLFields (o); // Write the base class data fields 1st.
usfCasCorClassifier->WriteXML ("UsfCasCorClassifier", o);
XmlTag endTag ("ModelUsfCasCor", XmlTag::TagTypes::tagEnd);
endTag.WriteXML (o);
o << endl;
} /* WriteXML */
KKStr FeatureFileIO::FileFormatsReadAndWriteOptionsStr ()
{
KKStr driversThatCanReadAndWrite (128);
vector<FeatureFileIOPtr>* drivers = RegisteredDrivers ();
vector<FeatureFileIOPtr>::const_iterator idx;
for (idx = drivers->begin (); idx != drivers->end (); idx++)
{
FeatureFileIOPtr driver = *idx;
if (driver->CanWrite () && driver->CanWrite ())
{
if (!driversThatCanReadAndWrite.Empty ())
driversThatCanReadAndWrite << ", ";
driversThatCanReadAndWrite << driver->DriverName ();
}
}
return driversThatCanReadAndWrite;
} /* FileFormatsReadAndWriteOptionsStr */
void ModelParamUsfCasCor::WriteXML (const KKStr& varName,
ostream& o
) const
{
XmlTag startTag ("ModelParamUsfCasCor", XmlTag::TagTypes::tagStart);
if (!varName.Empty ())
startTag.AddAtribute ("VarName", varName);
startTag.WriteXML (o);
o << endl;
WriteXMLFields (o);
XmlElementInt32::WriteXML (in_limit, "in_limit", o);
XmlElementInt32::WriteXML (number_of_rounds, "number_of_rounds", o);
XmlElementInt32::WriteXML (number_of_trials, "number_of_trials", o);
XmlElementInt64::WriteXML (random_seed, "random_seed", o);
XmlElementBool::WriteXML (useCache, "useCache", o);
XmlTag endTag ("ModelParamUsfCasCor", XmlTag::TagTypes::tagEnd);
endTag.WriteXML (o);
o << endl;
} /* WriteXML */
void LogicalFrame::PerformMorphOperations ()
{
KKStr ops = parms.MorphOperations ();
memcpy (origFrameArea, frameArea, frameTotalPixels);
while (!ops.Empty ())
{
KKStr nextOp = ops.ExtractToken (",\n\r\t");
nextOp.Upper ();
if (nextOp == "O")
PerformOpening ();
else if (nextOp == "C")
PerformClosing ();
else if (nextOp == "D")
PerformDialation ();
else if (nextOp == "E")
PerformErosion ();
}
} /* PerformMorphOperations */
FeatureVectorListPtr FeatureFileIO::LoadInSubDirectoryTree
(FactoryFVProducerPtr _fvProducerFactory,
KKStr _rootDir,
MLClassList& _mlClasses,
bool _useDirectoryNameForClassName,
VolConstBool& _cancelFlag,
bool _rewiteRootFeatureFile,
RunLog& _log
)
{
_log.Level (10) << "FeatureFileIO::LoadInSubDirectoryTree rootDir[" << _rootDir << "]." << endl;
osAddLastSlash (_rootDir);
KKStr featureFileName ("");
KKStr fullFeatureFileName ("");
if (!_rootDir.Empty ())
{
featureFileName = osGetRootNameOfDirectory (_rootDir) + ".data";
fullFeatureFileName = _rootDir + featureFileName;
}
else
{
featureFileName = "Root.data";
fullFeatureFileName = "Root.data";
}
MLClassPtr unKnownClass = _mlClasses.GetUnKnownClass ();
if (_useDirectoryNameForClassName)
{
KKStr className = MLClass::GetClassNameFromDirName (_rootDir);
unKnownClass = _mlClasses.GetMLClassPtr (className);
}
bool changesMade = false;
FeatureVectorListPtr dirImages = NULL;
if (_rewiteRootFeatureFile)
{
DateTime timeStamp;
dirImages = FeatureDataReSink (_fvProducerFactory,
_rootDir,
featureFileName,
unKnownClass,
_useDirectoryNameForClassName,
_mlClasses,
_cancelFlag,
changesMade,
timeStamp,
_log
);
if (_useDirectoryNameForClassName)
{
FeatureVectorList::iterator idx;
for (idx = dirImages->begin (); idx != dirImages->end (); idx++)
{
if ((*idx)->MLClass () != unKnownClass)
{
(*idx)->MLClass (unKnownClass);
changesMade = true;
}
}
if (changesMade)
{
KKStr fullFileName = osAddSlash (_rootDir) + featureFileName;
kkuint32 numExamplesWritten = 0;
bool cancel = false;
bool successful = false;
SaveFeatureFile (fullFileName,
dirImages->AllFeatures (),
*dirImages,
numExamplesWritten,
cancel,
successful,
_log
);
}
}
}
else
{
dirImages = _fvProducerFactory->ManufacturFeatureVectorList (true);
}
// Now that we have processed all image files in "rootDir",
// lets process any sub-directories.
KKStr dirSearchPath = osAddSlash (_rootDir) + "*.*";
KKStrListPtr subDirectories = osGetListOfDirectories (dirSearchPath);
if (subDirectories)
{
KKStrList::iterator idx;
for (idx = subDirectories->begin (); (idx != subDirectories->end () && (!_cancelFlag)); idx++)
{
KKStr subDirName (**idx);
if (subDirName == "BorderImages")
{
// We ignore this director
continue;
}
KKStr newDirPath = osAddSlash (_rootDir) + subDirName;
FeatureVectorListPtr subDirExamples = LoadInSubDirectoryTree (_fvProducerFactory,
newDirPath,
_mlClasses,
_useDirectoryNameForClassName,
_cancelFlag,
true, // true = ReWriteRootFeatureFile
_log
);
osAddLastSlash (subDirName);
// We want to add the directory path to the ExampleFileName so that we can later locate the source image.
for (auto fv: *subDirExamples)
{
KKStr newImageFileName = subDirName + fv->ExampleFileName ();
fv->ExampleFileName (newImageFileName);
}
dirImages->AddQueue (*subDirExamples);
subDirExamples->Owner (false);
delete subDirExamples;
subDirExamples = NULL;
}
delete subDirectories; subDirectories = NULL;
}
_log.Level (10) << "LoadInSubDirectoryTree - Done" << endl;
return dirImages;
} /* LoadInSubDirectoryTree */
/******************************************************************************
* ProcessCmdLineParamters
* DESC: Extracts parameters from the command line
******************************************************************************/
bool OurNeighbors::ProcessCmdLineParameter (const KKStr& parmSwitch,
const KKStr& parmValue
)
{
bool validParm = true;
if (parmSwitch.EqualIgnoreCase ("-BUCKETCOUNT") || parmSwitch.EqualIgnoreCase ("-BC"))
{
numOfBuckets = atoi (parmValue.Str ());
if (numOfBuckets < 2)
{
log.Level (-1) << "ProcessCmdLineParameter *** Invalid Buckets[" << numOfBuckets << "] parameter" << endl;
validParm = false;
}
}
else if (parmSwitch.EqualIgnoreCase ("-BUCKETSIZE") || parmSwitch.EqualIgnoreCase ("-BS"))
{
bucketSize = atoi (parmValue.Str ());
if (bucketSize < 1)
{
log.Level (-1) << "ProcessCmdLineParameter *** Invalid BucketSize[" << bucketSize << "] parameter" << endl;
Abort (true);
}
}
else if (parmSwitch.EqualIgnoreCase ("-EXCLUDECLASS") || parmSwitch.EqualIgnoreCase ("-EC"))
{
if (parmValue.Empty ())
{
log.Level (-1) << endl << endl
<< "ProcessCmdLineParameter '-ExcludeClass' missing class name to exclude." << endl
<< endl;
Abort (true);
}
else
{
if (!excludedClasses)
excludedClasses = new MLClassList ();
if (excludedClasses->LookUpByName (parmValue) != NULL)
{
log.Level (-1) << "ProcessCmdLineParameter '-ExcludeClass[" << parmValue << "]' Specified more than once." << endl;
}
else
{
excludedClasses->PushOnBack (MLClass::CreateNewMLClass (parmValue));
}
}
}
else if (parmSwitch.EqualIgnoreCase ("-FROMPLANKTON") || parmSwitch.EqualIgnoreCase ("-FP"))
{
fromPlanktonName = parmValue;
}
else if (parmSwitch.EqualIgnoreCase ("-H") || parmSwitch.EqualIgnoreCase ("-HTML"))
htmlFileName = parmValue;
else if (parmSwitch.EqualIgnoreCase ("-ITERATIONS") || parmSwitch.EqualIgnoreCase ("-I"))
{
numOfIterations = atoi (parmValue.Str ());
if (numOfIterations < 1)
{
log.Level (-1) << "ProcessCmdLineParameter *** Invalid Iterations[" << numOfIterations << "] parameter" << endl;
Abort (true);
}
}
else if (parmSwitch.EqualIgnoreCase ("-LLOYDSBINSIZE") ||
parmSwitch.EqualIgnoreCase ("-LBS") ||
parmSwitch.EqualIgnoreCase ("-BASELLOYDSBINSIZE") ||
parmSwitch.EqualIgnoreCase ("-BLBS")
)
{
baseLLoydsBinSize = atoi (parmValue.Str ());
if (baseLLoydsBinSize < 10)
{
log.Level (-1) << endl << endl
<< "ProcessCmdLineParameter *** Invalid BaseLLoydsBinSize[" << baseLLoydsBinSize << "] parameter ***" << endl
<< endl;
Abort (true);
}
}
else if (parmSwitch.EqualIgnoreCase ("-NEARESTNEIGHBORTYPE") || parmSwitch.EqualIgnoreCase ("-NNT"))
{
if (parmValue.EqualIgnoreCase ("0") || parmValue.EqualIgnoreCase ("ANY"))
neighborType = NeighborType::AnyPlankton;
else if (parmValue.EqualIgnoreCase ("1") || parmValue.EqualIgnoreCase ("SAME"))
neighborType = NeighborType::SamePlankton;
}
else if (parmSwitch.EqualIgnoreCase ("-R") || parmSwitch.EqualIgnoreCase ("-REPORT"))
reportFileName = parmValue;
else if (parmSwitch.EqualIgnoreCase ("-RAND") || parmSwitch.EqualIgnoreCase ("-RANDOM"))
randomizeLocations = true;
else if (parmSwitch.EqualIgnoreCase ("-S") || parmSwitch.EqualIgnoreCase ("-SRC"))
sourceRootDirPath = parmValue;
else
validParm = PicesApplication::ProcessCmdLineParameter(parmSwitch, parmValue);
return !Abort ();
} /* ProcessCmdLineParameter */
void CmdLineExpander::BuildCmdLineParameters (const VectorKKStr& argv)
{
kkuint32 x = 0;
while (x < argv.size ())
{
KKStr s = argv[x];
x++;
KKStr sUpper = s.ToUpper();
if ((sUpper == "-L") || (sUpper == "-LOGFILE"))
{
if (x < argv.size ())
{
if (argv[x][(kkint16)0] != '-')
{
logFileName = argv[x];
if (!logFileName.Empty ())
log.AttachFile (logFileName);
x++;
}
}
if (logFileName.Empty ())
{
log.Level (-1) << std::endl << std::endl;
log.Level (-1) << applicationName << " - Invalid Log File Parameter (-L)." << endl;
log.Level (-1) << " Name of log file required." << endl;
log.Level (-1) << endl;
parmsGood = false;
}
}
else if (sUpper == "-CMDFILE")
{
KKStr cmdFileName = "";
if (x < argv.size ())
{
if (argv[x][(kkint16)0] != '-')
{
cmdFileName = argv[x];
x++;
}
}
if (cmdFileName.Empty ())
{
log.Level (-1) << endl << endl << endl
<< applicationName << " " << "BuildCmdLineParameters *** ERROR ***" << endl << endl
<< "-CMDFILE option did not define a file name." << endl
<< endl;
parmsGood = false;
}
else
{
if (FileInStack (cmdFileName, cmdFileStack))
{
log.Level (-1) << endl << endl << endl
<< applicationName << " BuildCmdLineParameters *** ERROR ***" << endl
<< endl
<< "-CMDFILE [" << cmdFileName << "] is being called recursively." << endl
<< endl;
parmsGood = false;
}
else
{
bool validFile = true;
cmdFileStack.push_back (cmdFileName);
VectorKKStr cmdFileParameters;
ExtractParametersFromFile (cmdFileName, cmdFileParameters, validFile);
BuildCmdLineParameters (cmdFileParameters);
cmdFileStack.pop_back ();
if (!validFile)
parmsGood = false;
}
}
}
else
{
expandedParameters.push_back (s);
}
}
} /* BuildCmdLineParameters */
bool MergeFeatureFiles::ProcessCmdLineParameter (const KKStr& parmSwitch,
const KKStr& parmValue
)
{
bool validParm = true;
if (parmSwitch.EqualIgnoreCase ("-S") ||
parmSwitch.EqualIgnoreCase ("-SRC") ||
parmSwitch.EqualIgnoreCase ("-SOURCE")
)
{
KKStr srcFileName = parmValue;
if (!osFileExists (srcFileName))
{
log.Level (-1) << "ProcessCmdLineParameter ***ERROR*** Invalid '-src' [" << srcFileName << "] file." << endl;
Abort (true);
validParm = false;
}
srcFileNames.push_back (srcFileName);
srcFormats.push_back (currentDefaultFormat);
if (!currentDefaultFormat->CanRead ())
{
log.Level (-1) << "ProcessCmdLineParameter ***ERROR*** Format[" << currentDefaultFormat->DriverName () << "] does not support read." << endl;
Abort (true);
validParm = false;
}
}
else if
(parmSwitch.EqualIgnoreCase ("-DEST") ||
parmSwitch.EqualIgnoreCase ("-D") ||
parmSwitch.EqualIgnoreCase ("-DESTINATION")
)
{
if (!destFileName.Empty ())
{
log.Level (-1) << "ProcessCmdLineParameter ***ERROR*** More than one destination file was specified." << endl;
Abort (true);
validParm= false;
}
destFileName = parmValue;
if (osFileExists (destFileName))
{
log.Level (-1) << "ProcessCmdLineParameter ***ERROR*** Destination File[" << destFileName << "] already exists." << endl;
Abort (true);
validParm = false;
}
if (!currentDefaultFormat->CanWrite ())
{
log.Level (-1) << "ProcessCmdLineParameter ***ERROR*** Format[" << currentDefaultFormat->DriverName () << "] does not support write." << endl;
Abort (true);
validParm = false;
}
destFormat = currentDefaultFormat;
}
else if
(parmSwitch.EqualIgnoreCase ("-F") ||
parmSwitch.EqualIgnoreCase ("-FORMAT")
)
{
FeatureFileIOPtr newFormat = FeatureFileIO::FileFormatFromStr (parmValue);
if (!newFormat)
{
log.Level (-1) << "ProcessCmdLineParameter ***ERROR*** No such format as[" << parmValue<< "]." << endl;
Abort (true);
validParm = false;
}
else
{
currentDefaultFormat = newFormat;
}
}
else if
(parmSwitch.EqualIgnoreCase ("-STRAT") ||
parmSwitch.EqualIgnoreCase ("-STRATIFY")
)
{
stratify = true;
if (!parmValue.Empty ())
{
numOfFolds = parmValue.ToInt ();
if ((numOfFolds < 1) || (numOfFolds > 1000))
{
log.Level (-1) << "ProcessCmdLineParameter ***ERROR*** -Folds[" << numOfFolds << "] must be 1 or greater and less than 10000." << endl;
Abort (true);
validParm = false;
}
}
}
else if
(parmSwitch.EqualIgnoreCase ("-RAND") ||
parmSwitch.EqualIgnoreCase ("-RANDOM") ||
parmSwitch.EqualIgnoreCase ("-RANDOMIZE")
)
{
randomize = true;
}
else
{
validParm = PicesApplication::ProcessCmdLineParameter (parmSwitch, parmValue);
}
return validParm;
} /* ProcessCmdLineParameter */
void UpdateFullSizeImages ()
{
RunLog log;
DataBasePtr dbConn = new DataBase (log);
SipperFileListPtr sipperFiles = dbConn->SipperFileLoad ("ETP2008", "", "");
SipperFileList::iterator idx;
bool cancelFlag = false;
KKB::uint numImages = 0;
KKB::uint numImagesSaved = 0;
for (idx = sipperFiles->begin (); idx != sipperFiles->end (); idx++)
{
SipperFilePtr sf = *idx;
KKStr fullSipperFileName = InstrumentDataFileManager::GetFullSipperFileName (sf->SipperFileName ());
if (fullSipperFileName.Empty ())
{
// we do not have access to the sipper file; no point proessing this file.
continue;
}
int imagesThisSipperFile = 0;
cout << "Sipper File [" << sf->SipperFileName () << "]" << std::endl;
bool cancelFlag = false;
//DataBaseImageListPtr images = dbConn->ImagesQuery (NULL, sf->SipperFileName (), NULL, 0.0f, 1.0f, 0, 0, 0.0f, 0.0f, "ETP2008_1AA_01_04560000_0000", 1000, false, cancelFlag);
DataBaseImageListPtr images = dbConn->ImagesQuery (NULL, sf->SipperFileName (), NULL, 'P', 0.0f, 1.0f, 0, 0, 0.0f, 0.0f, 0, -1, false, cancelFlag);
map<KKStr, KKStr>* imagesAlreadtInFullSize = GetImagesAlreadyInImagesFullSize (sf->SipperFileName (), dbConn);
map<KKStr, KKStr>::iterator fullSizeIdx;
DataBaseImageList::iterator idx;
for (idx = images->begin (); idx != images->end (); idx++)
{
imagesThisSipperFile++;
numImages++;
DataBaseImagePtr i = *idx;
KKB::uint maxDim = Max (i->Height (), i->Width ());
if (maxDim > 100)
{
fullSizeIdx = imagesAlreadtInFullSize->find (i->ImageFileName ());
if (fullSizeIdx == imagesAlreadtInFullSize->end ())
{
// We need to get the full size version of this image.
RasterPtr r = i->GetOrigImageFromSipperFile (log);
if (r)
{
dbConn->ImageFullSizeSave (i->ImageFileName (), *r);
delete r;
r = NULL;
}
numImagesSaved++;
}
}
if ((numImages % 100) == 0)
{
cout << sf->SipperFileName () << "\t"
<< StrFormatInt (images->QueueSize (), "ZZ,ZZZ,ZZ0") << "\t"
<< StrFormatInt (imagesThisSipperFile, "ZZ,ZZZ,ZZ0") << "\t"
<< StrFormatInt (numImages, "ZZ,ZZZ,ZZ0") << "\t"
<< StrFormatInt (numImagesSaved, "ZZ,ZZZ,ZZ0")
<< endl;
}
}
delete imagesAlreadtInFullSize;
imagesAlreadtInFullSize = NULL;
delete images;
images = NULL;
}
} /* UpdateFullSizeImages */
void Configuration::LoadFile (RunLog& log)
{
log.Level (10) << "Configuration::LoadFile: " << fileName << endl;
kkint32 lastLineNum = 0;
if (fileName == "")
{
log.Level (-1) << endl
<< "Configuration::LoadFile ***ERROR*** File-Name is blank" << endl
<< endl;
FormatGood (false);
return;
}
FILE* inFile = osFOPEN (fileName.Str (), "r");
if (!inFile)
{
log.Level (-1) << endl
<< "Configuration::LoadFile ***ERROR*** Opening File: " << fileName << endl
<< endl;
FormatGood (false);
return;
}
char buff[10240];
kkint32 lineCount = 0;
curSectionName = "";
ConfSectionPtr curSection = NULL;
while (fgets (buff, sizeof (buff), inFile))
{
lastLineNum++;
KKStr line (buff);
line.TrimRight ();
line.TrimLeft ();
StripOutAnyComments (line);
log.Level (70) << line << endl;
StripOutAnyComments (line);
if (line.Empty ())
{
// If we have a blank line, we do nothing.
}
else if (line.FirstChar () == '[')
{
// Looks like definition of new section.
if (line.LastChar () == ']')
{
curSectionName = line.SubStrPart (1, line.Len () - 2);
curSectionName.TrimLeft ();
curSectionName.TrimRight ();
curSectionName.Upper ();
curSection = new ConfSection (curSectionName, lastLineNum);
sections->AddConfSection (curSection);
log.Level (30) << "LoadFile SectionName[" << curSectionName << "]." << endl;
}
else
{
log.Level (-1) << endl
<< "Configuration::LoadFile ***ERROR*** LineNumber[" << lastLineNum << "] Improper Section Name[" << line << "]." << endl
<< endl;
formatGood = false;
}
}
else
{
if (!curSection)
{
log.Level (-1) << endl
<< "Configuration::LoadFile ***ERROR*** Format Error LineNumber[" << lastLineNum << "]" << endl
<< " No Section Defined." << endl
<< endl;
formatGood = false;
curSectionName = "GLOBAL";
curSection = new ConfSection (curSectionName, lastLineNum);
sections->AddConfSection (curSection);
}
kkint32 equalIdx = line.LocateCharacter ('=');
if (equalIdx < 0)
{
// We have a improperly formated line.
log.Level (-1) << endl
<< "Configuration::LoadFile ***ERROR*** LineNumber[" << lastLineNum << "] Improperly Formated Line[" << line << "]."
<< endl;
formatGood = false;
}
else
{
KKStr settingName (line.SubStrPart (0, equalIdx - 1));
settingName.TrimLeft ();
settingName.TrimRight ();
settingName.Upper ();
KKStr settingValue (line.SubStrPart (equalIdx + 1));
settingValue.TrimLeft ();
settingValue.TrimRight ();
log.Level (30) << "LoadFile SectionName[" << curSectionName << "], "
<< "Setting[" << settingName << "], Value[" << settingValue << "]."
<< endl;
curSection->AddSetting (settingName, settingValue, lastLineNum);
}
lineCount++;
}
}
fclose (inFile);
} /* LoadFile */
void RandomSampleJob::ProcessStatusStr (KKStr statusStr)
{
log.Level (30) << "RandomSampleJob::ProcessStatusStr[" << statusStr << "]" << endl;
KKStr fieldName;
KKStr fieldValue;
compMethod = BRNull;
fieldName = statusStr.ExtractToken ("\t\n\r");
while (!fieldName.Empty ())
{
fieldName.Upper ();
fieldValue = statusStr.ExtractToken ("\t\n\r");
fieldValue.TrimLeft ("\n\r\t ");
fieldValue.TrimRight ("\n\r\t ");
if (fieldName == "JOBID")
jobId = atoi (fieldValue.Str ());
else if (fieldName == "STATUS")
status = JobStatusFromStr (fieldValue);
else if (fieldName == "KERNEL")
kernelType = KernalTypeFromStr (fieldValue);
else if (fieldName == "ENCODING")
encodingMethod = EncodingMethodFromStr (fieldValue);
else if (fieldName == "COMPMETHOD")
compMethod = CompressionMethodFromStr (fieldValue);
else if (fieldName == "C_PARAM")
c = atoi (fieldValue.Str ());
else if (fieldName == "GAMMA")
gamma = atof (fieldValue.Str ());
else if (fieldName == "ORDERINGNUM")
orderingNum = atoi (fieldValue.Str ());
else if (fieldName == "EXAMPLESTOKEEP")
numExamplesToKeep = atoi (fieldValue.Str ());
else if (fieldName == "ACCURACY")
accuracy = (float)atof (fieldValue.Str ());
else if (fieldName == "TRAINTIME")
{
trainTime = atof (fieldValue.Str ());
}
else if (fieldName == "PREDTIME")
testTime = (float)atof (fieldValue.Str ());
else if (fieldName == "SUPPORTVECTORS")
supportVectors = (float)atof (fieldValue.Str ());
else
{
log.Level (-1) << "RandomSampleJob::ProcessStatusStr Invalid Field Name[" << fieldName << "]." << endl;
}
fieldName = statusStr.ExtractToken ("\t\n\r");
}
} /* ProcessStatusStr */
/******************************************************************************
* ProcessCmdLineParamters
* DESC: Extracts parameters from the command line
******************************************************************************/
bool FeatureFileConverter::ProcessCmdLineParameter (char parmSwitchCode,
KKStr parmSwitch,
KKStr parmValue
)
{
KKStr parmValueUpper (parmValue);
parmValueUpper.Upper ();
parmSwitch.Upper ();
if ((parmSwitch == "-DEST") ||
(parmSwitch == "-D") ||
(parmSwitch == "-DESTFILE") ||
(parmSwitch == "-DF") ||
(parmSwitch == "-DESTFILENAME") ||
(parmSwitch == "-DFN")
)
destFileName = parmValue;
else if ((parmSwitch == "-OUTFORMAT") ||
(parmSwitch == "-OF") ||
(parmSwitch == "-OUTFILEFORMAT") ||
(parmSwitch == "-OFF") ||
(parmSwitch == "-DESTFILEFORMAT") ||
(parmSwitch == "-DESTFORMAT") ||
(parmSwitch == "-DFF")
)
{
destFileFormat = FeatureFileIO::FileFormatFromStr (parmValue,
false, // Don't care if valid read format
true // must be valid write format
);
if (destFileFormat == NULL)
{
log.Level (-1) << endl
<< endl
<< "Invalid Write File Format Specified[" << parmValue << endl
<< endl
<< "Valid Formats are <" << FeatureFileIO::FileFormatsWrittenOptionsStr () << ">" << endl
<< endl;
Abort (true);
}
}
else if ((parmSwitch == "-ENCODEFEATUREDATA") ||
(parmSwitch == "-ENCODEFEATURES") ||
(parmSwitch == "-EF")
)
{
encodingMethod = ModelParam::EncodingMethodFromStr (parmValue);
if (encodingMethod != Encoding_NULL)
{
encodeFeatureData = true;
}
else
{
if ((parmValue == "NO") ||
(parmValue == "OFF") ||
(parmValue == "N")
)
encodeFeatureData = false;
else
{
encodingMethod = ModelParam::BinaryEncoding;
encodeFeatureData = true;
}
}
}
else if ((parmSwitch == "-ENUMERATECLASSES") ||
(parmSwitch == "-ENUMERATECLASS") ||
(parmSwitch == "-EC")
)
{
if ((parmValue == "NO") ||
(parmValue == "OFF") ||
(parmValue == "N")
)
enumerateClasses = false;
else
enumerateClasses = true;
}
else if (parmSwitch == "-FEATURES")
{
featureStr = parmValue;
}
else if ((parmSwitch == "-IF") ||
(parmSwitch == "-INPUTFORMAT") ||
(parmSwitch == "-IFF") ||
(parmSwitch == "-INFILEFORMAT") ||
(parmSwitch == "-SFF") ||
(parmSwitch == "-SrcFileFormat")
)
{
srcFileFormat = FeatureFileIO::FileFormatFromStr (parmValue,
true, // Valid Read Format
false // Don't care if valid write format
);
if (srcFileFormat == NULL)
{
log.Level (-1) << endl
<< endl
<< "Invalid Source File Format Specified[" << parmValue << endl
<< endl
<< "Valid Formats are <" << FeatureFileIO::FileFormatsReadOptionsStr () << ">" << endl
<< endl;
Abort (true);
}
}
else if ((parmSwitch == "-NORM") || (parmSwitch == "-NORMALIZE"))
{
normalizeData = true;
parmValue.TrimLeft ();
parmValue.TrimRight ();
if (!parmValue.Empty ())
nornParmsFileName = parmValue;
}
else if ((parmSwitch == "-R") || (parmSwitch == "-REPORT"))
reportFileName = parmValue;
else if ((parmSwitch == "-S") ||
(parmSwitch == "-SRC") ||
(parmSwitch == "-SF") ||
(parmSwitch == "-SOURCFILE") ||
(parmSwitch == "-SFN") ||
(parmSwitch == "-SRCFILENAME") ||
(parmSwitch == "-SOURCFILENAME")
)
srcFileName = parmValue;
else if ((parmSwitch == "-STAT") ||
(parmSwitch == "-STATS") ||
(parmSwitch == "-STATISTICS")
)
statistics = true;
else
{
log.Level (-1) << endl
<< "ProcessCmdLineParameter Invalid Parameter[" << parmSwitch << "]" << endl
<< endl;
Abort (true);
}
return !Abort ();
} /* ProcessCmdLineParameter */
void KKJobManager::StatusFileProcessLine (const KKStr& ln,
istream& statusFile
)
{
if (ln.SubStrPart (0, 1) == "//")
{
// A coment line; we can ignore it.
return;
}
KKStr statusStr (ln);
KKStr fieldName = statusStr.ExtractToken2 ("\t");
if (fieldName.Empty ())
{
// A empty line we will ignore it.
return;
}
statusStr.TrimLeft ("\n\r\t ");
statusStr.TrimRight ("\n\r\t ");
if (fieldName.CompareIgnoreCase ("JOB") == 0)
{
// We have a KKJob entr line; the next field determines JobType fllowed by parameters for that JobType constructor.
KKStr jobTypeName = fieldName = statusStr.ExtractToken2 ("\t");
KKJobPtr j = KKJob::CallAppropriateConstructor (this, jobTypeName, statusStr);
KKJobPtr existingJob = jobs->LookUpByJobId (j->JobId ());
if (existingJob)
{
existingJob->ReFresh (*j);
delete j; j = NULL;
}
else
{
jobs->PushOnBack (j);
}
}
else if (fieldName.EqualIgnoreCase ("CPUTIMEUSED"))
{
double cpuTimeUsed = statusStr.ExtractTokenDouble ("\t");
cpuTimeTotalUsed += cpuTimeUsed;
}
else if (fieldName.EqualIgnoreCase ("CURRENTDATETIME"))
{
KKB::DateTime dateTime = KKB::DateTime (statusStr);
if (!dateTimeFirstOneFound)
{
dateTimeFirstOneFound = true;
dateTimeStarted = dateTime;
}
dateTimeEnded = dateTime;
}
else if (fieldName.EqualIgnoreCase ("ExpansionCount"))
expansionCount = statusStr.ToInt ();
else if (fieldName.EqualIgnoreCase ("ExpansionFirstJobId"))
expansionFirstJobId = statusStr.ToInt ();
else if (fieldName.EqualIgnoreCase ("JobStatusChange"))
StatusFileProcessLineJobStatusChange (statusStr);
else if (fieldName.EqualIgnoreCase ("NextJobId"))
nextJobId = statusStr.ExtractTokenInt ("\t");
else if (fieldName.EqualIgnoreCase ("QuitRunning"))
quitRunning = true;
else if (fieldName.EqualIgnoreCase ("Restart"))
restart = false;
else if (fieldName.EqualIgnoreCase ("Status"))
status = KKJob::JobStatusFromStr (statusStr);
else
{
log.Level (-1) << "KKJobManager::StatusFileProcessLine Invalid Field Name[" << fieldName << "]." << endl;
}
} /* StatusFileProcessLine */
void KKJobManager::ProcessJobXmlBlockOfText (const KKStr& startStr,
istream& i
)
{
if ((startStr.SubStrPart (0, 4) != "<KKJob ") || (startStr.LastChar () != '>'))
{
log.Level (-1) << endl
<< "KKJobManager::ProcessJobXmlBlockOfText ***ERROR*** StartStr[" << startStr << "] is not a KKJob String." << endl
<< endl;
return;
}
KKStr s = startStr.SubStrPart (5);
s.TrimLeft ();
s.ChopLastChar ();
KKStr jobTypeStr = "";
kkint32 jobId = -1;
VectorKKStr parameters = s.Split (',');
for (kkuint32 x = 0; x < parameters.size (); ++x)
{
KKStr parameterStr = parameters[x];
parameterStr.TrimLeft ();
parameterStr.TrimRight ();
KKStr fieldName = parameterStr.ExtractToken2 ("=");
fieldName.TrimLeft (); fieldName.TrimRight ();
KKStr fieldValue = parameterStr.ExtractToken2 ("=");
fieldValue.TrimLeft (); fieldValue.TrimRight ();
if (fieldName.CompareIgnoreCase ("JobType") == 0)
jobTypeStr = fieldValue;
else if (fieldName.CompareIgnoreCase ("JobId") == 0)
jobId = fieldValue.ToInt ();
}
if (jobTypeStr.Empty () || (jobId < 0))
{
log.Level (-1) << endl
<< "KKJobManager::ProcessJobXmlBlockOfText ***ERROR*** StartStr[" << startStr << "]." << endl
<< " JobType and/or JobId were not provided." << endl
<< endl;
return;
}
KKJobPtr j = jobs->LookUpByJobId (jobId);
if (j == NULL)
{
// We do not have this job in memory yet. We will have to create it now.
KKStr emptyStatusStr = "JobId\t" + StrFormatInt (jobId, "ZZZZ0");
j = KKJob::CallAppropriateConstructor (this, jobTypeStr, emptyStatusStr);
}
j->CompletedJobDataRead (i);
} /* ProcessJobXmlBlockOfText */
void BinaryClass::ProcessStatusStr (KKStr statusStr,
MLClassListPtr mlClasses
)
{
KKStr fieldName;
class1 = NULL;
class2 = NULL;
fieldName = statusStr.ExtractToken2 ("\t");
while (!fieldName.Empty ())
{
fieldName.TrimLeft ("\n\r\t ");
fieldName.TrimRight ("\n\r\t ");
fieldName.Upper ();
if (fieldName == "ALLCLASSES")
{
class1 = NULL;
class2 = NULL;
}
else if (fieldName == "BINARYCLASSES")
{
class1 = mlClasses->GetMLClassPtr (statusStr.ExtractToken2 ("\t"));
class2 = mlClasses->GetMLClassPtr (statusStr.ExtractToken2 ("\t"));
if ((class1 == NULL) || (class2== NULL))
{
cout << "Class1 or Class2 == NULL" << endl;
}
else
{
if ((class1->Name ().Empty ()) || (class1->Name ().Empty ()))
{
cout << "Class1Name or Class2Name are empty" << endl;
}
}
}
else if (fieldName == "STATUS")
status = BinaryClassStatusFromStr (statusStr.ExtractToken2 ("\t"));
else if (fieldName == "NUMPROCESSORS")
numProcessors = statusStr.ExtractTokenInt ("\t");
else if (fieldName.CompareIgnoreCase ("FinalResultType") == 0)
resultType = FinalResultTypeFromStr (statusStr.ExtractToken2 ("\t"));
else if (fieldName.CompareIgnoreCase ("ResultsFileName") == 0)
finalResultsFileName = statusStr.ExtractToken2 ("\t");
else if (fieldName.CompareIgnoreCase ("SelectionMethod") == 0)
selectionMethod = SelectionMethodFromStr (statusStr.ExtractToken2 ("\t"));
else if (fieldName.CompareIgnoreCase ("configFileName") == 0)
configFileName = statusStr.ExtractToken2 ("\t");
fieldName = statusStr.ExtractToken2 ("\t");
}
} /* ProcessStatusStr */
FeatureVectorListPtr FeatureFileIO::FeatureDataReSink (FactoryFVProducerPtr _fvProducerFactory,
const KKStr& _dirName,
const KKStr& _fileName,
MLClassPtr _unknownClass,
bool _useDirectoryNameForClassName,
MLClassList& _mlClasses,
VolConstBool& _cancelFlag,
bool& _changesMade,
KKB::DateTime& _timeStamp,
RunLog& _log
)
{
_changesMade = false;
_timeStamp = DateTime ();
if (_unknownClass == NULL)
_unknownClass = MLClass::GetUnKnownClassStatic ();
KKStr className = _unknownClass->Name ();
_log.Level (10) << "FeatureFileIO::FeatureDataReSink dirName: " << _dirName << endl
<< " fileName: " << _fileName << " UnKnownClass: " << className << endl;
KKStr fullFeatureFileName = osAddSlash (_dirName) + _fileName;
bool successful = true;
KKStr fileNameToOpen;
if (_dirName.Empty ())
fileNameToOpen = _fileName;
else
fileNameToOpen = osAddSlash (_dirName) + _fileName;
bool versionsAreSame = false;
FeatureVectorListPtr origFeatureVectorData
= LoadFeatureFile (fileNameToOpen, _mlClasses, -1, _cancelFlag, successful, _changesMade, _log);
if (origFeatureVectorData == NULL)
{
successful = false;
origFeatureVectorData = _fvProducerFactory->ManufacturFeatureVectorList (true);
}
if (_cancelFlag)
{
delete origFeatureVectorData; origFeatureVectorData = NULL;
return _fvProducerFactory->ManufacturFeatureVectorList (true);
}
FeatureVectorListPtr origFeatureData = NULL;
if (successful &&
(&typeid (*origFeatureVectorData) == _fvProducerFactory->FeatureVectorListTypeId ()) &&
((*(origFeatureVectorData->FileDesc ())) == (*(_fvProducerFactory->FileDesc ())))
)
{
origFeatureData = origFeatureVectorData;
}
else
{
origFeatureData = _fvProducerFactory->ManufacturFeatureVectorList (true);
delete origFeatureVectorData;
origFeatureVectorData = NULL;
}
KKStr fileSpec = osAddSlash (_dirName) + "*.*";
KKStrListPtr fileNameList = osGetListOfFiles (fileSpec);
if (!fileNameList)
{
// There are no Image Files, so we need to return a Empty List of Image Features.
if (origFeatureData->QueueSize () > 0)
_changesMade = true;
delete origFeatureData; origFeatureData = NULL;
return _fvProducerFactory->ManufacturFeatureVectorList (true);
}
FeatureVectorProducerPtr fvProducer = _fvProducerFactory->ManufactureInstance (_log);
if (successful)
{
if (origFeatureData->Version () == fvProducer->Version ())
{
versionsAreSame = true;
_timeStamp = osGetFileDateTime (fileNameToOpen);
}
else
{
_changesMade = true;
}
}
else
{
delete origFeatureData;
origFeatureData = _fvProducerFactory->ManufacturFeatureVectorList (true);
}
origFeatureData->SortByRootName (false);
FeatureVectorListPtr extractedFeatures = _fvProducerFactory->ManufacturFeatureVectorList (true);
extractedFeatures->Version (fvProducer->Version ());
fileNameList->Sort (false);
KKStrList::iterator fnIDX;
fnIDX = fileNameList->begin (); // fileNameList
KKStrPtr imageFileName;
kkuint32 numImagesFoundInOrigFeatureData = 0;
kkuint32 numOfNewFeatureExtractions = 0;
for (fnIDX = fileNameList->begin (); (fnIDX != fileNameList->end ()) && (!_cancelFlag); ++fnIDX)
{
imageFileName = *fnIDX;
// pv414-_002_20140414-162243_02068814-1261.bmp
KKStr rootName = osGetRootName (*imageFileName);
if (rootName == "pv414-_002_20140414-162243_02068814-1261")
cout << "Stop Here." << endl;
bool validImageFileFormat = SupportedImageFileFormat (*imageFileName);
if (!validImageFileFormat)
continue;
bool featureVectorCoputaionSuccessful = false;
FeatureVectorPtr origFV = origFeatureData->BinarySearchByName (*imageFileName);
if (origFV)
numImagesFoundInOrigFeatureData++;
if (origFV && versionsAreSame)
{
featureVectorCoputaionSuccessful = true;
if (_useDirectoryNameForClassName)
{
if (origFV->MLClass () != _unknownClass)
{
_changesMade = true;
origFV->MLClass (_unknownClass);
}
}
else if ((origFV->MLClass ()->UnDefined ()) && (origFV->MLClass () != _unknownClass))
{
_changesMade = true;
origFV->MLClass (_unknownClass);
}
extractedFeatures->PushOnBack (origFV);
origFeatureData->DeleteEntry (origFV);
}
else
{
// We either DON'T have an original image or versions are not the same.
KKStr fullFileName = osAddSlash (_dirName) + (*imageFileName);
FeatureVectorPtr fv = NULL;
try
{
RasterPtr image = ReadImage (fullFileName);
if (image)
fv = fvProducer->ComputeFeatureVector (*image, _unknownClass, NULL, 1.0f, _log);
delete image;
image = NULL;
if (fv)
featureVectorCoputaionSuccessful = true;
else
featureVectorCoputaionSuccessful = false;
}
catch (...)
{
_log.Level (-1) << endl << endl
<< "FeatureDataReSink ***ERROR***" << endl
<< " Exception occurred calling constructor 'ComputeFeatureVector'." << endl
<< endl;
featureVectorCoputaionSuccessful = false;
fv = NULL;
}
if (!featureVectorCoputaionSuccessful)
{
_log.Level (-1) << " FeatureFileIOKK::FeatureDataReSink *** ERROR ***, Processing Image File["
<< imageFileName << "]."
<< endl;
delete fv;
fv = NULL;
}
else
{
_changesMade = true;
fv->ExampleFileName (*imageFileName);
_log.Level (30) << fv->ExampleFileName () << " " << fv->OrigSize () << endl;
extractedFeatures->PushOnBack (fv);
numOfNewFeatureExtractions++;
if ((numOfNewFeatureExtractions % 100) == 0)
cout << numOfNewFeatureExtractions << " Images Extracted." << endl;
}
}
}
if (numImagesFoundInOrigFeatureData != extractedFeatures->QueueSize ())
_changesMade = true;
extractedFeatures->Version (fvProducer->Version ());
if ((_changesMade) && (!_cancelFlag))
{
//extractedFeatures->WriteImageFeaturesToFile (fullFeatureFileName, RawFormat, FeatureNumList::AllFeatures (extractedFeatures->FileDesc ()));
kkuint32 numExamplesWritten = 0;
SaveFeatureFile (fullFeatureFileName,
FeatureNumList::AllFeatures (extractedFeatures->FileDesc ()),
*extractedFeatures,
numExamplesWritten,
_cancelFlag,
successful,
_log
);
_timeStamp = osGetLocalDateTime ();
}
delete fvProducer; fvProducer = NULL;
delete fileNameList; fileNameList = NULL;
delete origFeatureData; origFeatureData = NULL;
_log.Level (10) << "FeatureDataReSink Exiting Dir: " << _dirName << endl;
return extractedFeatures;
} /* FeatureDataReSink */
void ForestCover_AddMnemonicClassName ()
{
const char* classNames[] = {"Spruce_Fir",
"Lodgepole_Pine",
"Ponderosa_Pine",
"Cottonwood_Willow",
"Aspen",
"Douglas_fir",
"Krummholz",
NULL
};
FILE* in = osFOPEN ("K:\\Plankton\\Papers\\BitReduction\\DataSets\\ForestCover\\covtype.data", "r");
ofstream o ("K:\\Plankton\\Papers\\BitReduction\\DataSets\\ForestCover\\covtype_alpha.data");
char buff[20480];
int lc = 0;
while (fgets (buff, sizeof (buff), in))
{
KKStr ln (buff);
vector<KKStr> tokens;
while (!ln.Empty ())
{
KKStr token = ln.ExtractToken (",\n\t\r");
tokens.push_back (token);
}
int lastFieldNum = (int)tokens.size () - 1;
KKStr lastToken = tokens[lastFieldNum];
int classNum = atoi (lastToken.Str ()) - 1;
if ((classNum < 0) || (classNum > 6))
{
// Something has gone very wrong.
cerr << endl
<< " *** ERROR ***" << endl
<< endl
<< buff << endl
<< endl
<< "ClassNum[" << classNum << "] Out of range." << endl
<< endl;
osWaitForEnter ();
exit (-1);
}
KKStr className = classNames[classNum];
tokens[lastFieldNum] = className;
int x = 0;
vector<KKStr>::const_iterator idx;
for (idx = tokens.begin (); idx != tokens.end (); idx++)
{
if (x > 0)
o << ",";
o << *idx;
x++;
}
o << endl;
lc++;
if ((lc % 100) == 0)
cout << lc << endl;
}
fclose (in);
o.close ();
} /* ForestCover_AddMnemonicClassName */
void ClassificationBiasMatrix::ReadXML (istream& sr)
{
char buff[10240];
KKStr l (512);
if (sr.eof ())
return;
MLClassListPtr fileClasses = NULL;
sr.getline (buff, sizeof (buff));
while (!sr.eof ())
{
l = buff;
l.TrimRight ();
if (l.CompareIgnoreCase ("</ClassificationBiasMatrix>") == 0)
break;
KKStr lineName = l.ExtractToken2 ("\t");
if (!lineName.Empty ())
{
KKStr fieldValue = l.ExtractToken2 ("\t");
if (lineName.CompareIgnoreCase ("Classes") == 0)
{
delete fileClasses; fileClasses = NULL;
fileClasses = MLClassList::BuildListFromDelimtedStr (fieldValue, ',');
if (classes == NULL)
classes = new MLClassList (*fileClasses);
}
else if (lineName.CompareIgnoreCase ("ConfigDateTime") == 0)
{
configDateTime = fieldValue;
}
else if (lineName.CompareIgnoreCase ("ConfigFileName") == 0)
{
configFileNameFromMatrixBiasFile = fieldValue;
}
else if (lineName.CompareIgnoreCase ("ConfigFileDateTime") == 0)
{
configDateTime = fieldValue;
}
else if (lineName.CompareIgnoreCase ("DateTime") == 0)
{
dateTimeFileWritten = fieldValue;
}
else if (lineName.CompareIgnoreCase ("DateTimeFileWritten") == 0)
{
dateTimeFileWritten = fieldValue;
}
else if (lineName.CompareIgnoreCase ("FileName") == 0)
{
}
else if (lineName.CompareIgnoreCase ("NumClasses") == 0)
{
numClasses = fieldValue.ToInt ();
}
else if (lineName.CompareIgnoreCase ("<SimpleConfusionMatrix>") == 0)
{
ReadSimpleConfusionMatrix (sr, fileClasses);
}
}
if (!sr.eof ())
sr.getline (buff, sizeof (buff));
}
} /* ReadXML */
