void AbundanceCorrectionStatsBuilder::GenerateReportAndStats ()
{
AbundanceCorrectionMatrix abundanceCorrMatrix (*trainLibDataClasses, otherClass, log);
ConfusionMatrix2 confusionMatrix (*allClasses);
TrainTestThreadList::iterator idx;
for (idx = queueDone->begin (); idx != queueDone->end (); ++idx)
{
TrainTestThreadPtr thread = *idx;
const AbundanceCorrectionMatrixPtr acm = thread->AbundanceCorMatrix ();
const ConfusionMatrix2Ptr cm = thread->ConfusionMatrix ();
if (acm) abundanceCorrMatrix.AddIn (*acm, log);
if (cm) confusionMatrix.AddIn (*cm, log);
}
*report << endl << endl << endl;
abundanceCorrMatrix.PrintConfusionMatrixTabDelimited (*report);
*report << endl << endl << endl << endl;
confusionMatrix.PrintConfusionMatrixTabDelimited (*report);
*report << endl << endl << endl;
abundanceCorrMatrix.SaveForConfigFileName (configFileName);
} /* GenerateReportAndStats */
void ClassificationBiasMatrix::BuildFromConfusionMatrix (const ConfusionMatrix2& cm)
{
kkint32 classesRowIdx = 0;
kkint32 classesColIdx = 0;
for (classesRowIdx = 0; classesRowIdx < numClasses; classesRowIdx++)
{
double knownCount = cm.CountsByKnownClass (classesRowIdx);
for (classesColIdx = 0; classesColIdx < numClasses; classesColIdx++)
{
double predCount = cm.PredictedCountsCM (classesRowIdx, classesColIdx);
double prob = 0.0;
if (knownCount != 0.0)
prob = predCount / knownCount;
(*counts) [classesRowIdx][classesColIdx] = predCount;
(*probabilities)[classesRowIdx][classesColIdx] = prob;
}
}
} /* BuildFromConfusionMatrix*/
ClassificationBiasMatrix::ClassificationBiasMatrix (const ConfusionMatrix2& cm,
RunLog& _runLog
):
biasFileName (),
classes (NULL),
configFileName (),
configFileNameFromMatrixBiasFile (),
configDateTime (),
counts (NULL),
dateTimeFileWritten (),
errMsgs (),
numClasses (0),
probabilities (NULL),
runLog (_runLog),
valid (true)
{
classes = new MLClassList (cm.MLClasses ());
numClasses = classes->QueueSize ();
DeclareMatrix ();
BuildFromConfusionMatrix (cm);
}
void RandomSplitJobManager::GenerateFinalResultsReport ()
{
KKStr reportFileName = osGetRootName (ManagerName ()) + "_Results.html;";
ofstream f (reportFileName.Str ());
f << "Run Time Parameters" << endl
<< "Run Time" << "\t" << osGetLocalDateTime () << endl
<< "configFileName" << "\t" << configFileName << endl
<< "DataFileName" << "\t" << dataFileName << endl
<< "Format" << "\t" << format->DriverName () << endl
<< "DataIndexFileName" << "\t" << dataIndexFileName << endl
<< "NumFolds" << "\t" << numFolds << endl
<< "NumSplits" << "\t" << numSplits << endl
<< "splitFraction" << "\t" << splitFraction << endl
<< endl;
KKJobList::const_iterator idx;
ConfusionMatrix2 avgResults (*(this->MLClasses ()));
KKB::uint x = 0;
for (idx = Jobs ()->begin (); idx != Jobs ()->end (); idx++)
{
RandomSplitJobPtr j = dynamic_cast<RandomSplitJobPtr> (*idx);
if (j->RandomSplitsResults () != NULL)
{
f << endl
<< "Random Split[" << j->SplitNum () << "]" << endl;
j->RandomSplitsResults ()->PrintConfusionMatrixTabDelimited (f);
f << endl << endl;
j->PrintClassCounts (f);
f << endl << endl;
avgResults.AddIn (*(j->RandomSplitsResults ()), log);
x++;
}
}
f << endl << "Mean Average of all random Splits." << endl;
avgResults.FactorCounts (1.0 / (double)x);
avgResults.PrintConfusionMatrixTabDelimited (f);
f << endl
<< endl
<< endl
<< endl
<< "Class Counts" << endl
<< endl;
kkuint32 numClasses = (kkuint32)mlClasses->size ();
VectorFloat classAccs;
VectorDouble knownCounts;
VectorDouble predCounts;
VectorDouble adjCounts;
VectorDouble adjCountsStdError;
VectorDouble predDelta;
VectorDouble adjDelta;
KKStr l1, l2, l3;
mlClasses->ExtractThreeTitleLines (l1, l2, l3);
VectorKKStr knownCountLines;
VectorKKStr predCountLines;
VectorKKStr adjCountLines;
VectorKKStr deltaPredCountLines;
VectorKKStr deltaAdjCountLines;
VectorKKStr accLines;
ConfusionMatrix2 totalCM (*MLClasses ());
int totalCmCount = 0;
// Known Counts
for (idx = Jobs ()->begin (); idx != Jobs ()->end (); idx++)
{
RandomSplitJobPtr j = dynamic_cast<RandomSplitJobPtr> (*idx);
if (j->RandomSplitsResults () != NULL)
{
KKStr splitNumStr = StrFormatInt (j->SplitNum (), "ZZZ0");
j->GetClassCounts (classAccs, knownCounts, predCounts, adjCounts, adjCountsStdError, predDelta, adjDelta);
totalCM.AddIn (*(j->RandomSplitsResults ()), log);
totalCmCount++;
KKStr accLine = "Acc By Class\t" + splitNumStr;
KKStr knownLine = "Known\t" + splitNumStr;
KKStr predLine = "Predicted\t" + splitNumStr;
KKStr adjLine = "Adjusted\t" + splitNumStr;
KKStr deltaPredLine = "Delta Pred\t" + splitNumStr;
KKStr deltaAdjLine = "Delta Adj\t" + splitNumStr;
double totalAcc = 0.0;
double totalDeltaPred = 0.0;
double totalDeltaAdj = 0.0;
for (x = 0; x < numClasses; x++)
{
accLine << "\t" << StrFormatDouble (classAccs [x], "zz0.00") << "%";
knownLine << "\t" << StrFormatDouble (knownCounts [x], "-Z,ZZZ,ZZ0.0");
predLine << "\t" << StrFormatDouble (predCounts [x], "-Z,ZZZ,ZZ0.0");
adjLine << "\t" << StrFormatDouble (adjCounts [x], "-Z,ZZZ,ZZ0.0");
deltaPredLine << "\t" << StrFormatDouble (predDelta [x], "-Z,ZZZ,ZZ0.0");
deltaAdjLine << "\t" << StrFormatDouble (adjDelta [x], "-Z,ZZZ,ZZ0.0");
totalAcc += classAccs [x];
totalDeltaPred += fabs (predDelta[x]);
totalDeltaAdj += fabs (adjDelta[x]);
}
accLine << "\t" << StrFormatDouble ((totalAcc / (double)classAccs.size ()), "ZZ0.00") << "%";
deltaPredLine << "\t" << StrFormatDouble ((totalDeltaPred / (double)predDelta.size ()), "ZZ0.00");
deltaAdjLine << "\t" << StrFormatDouble ((totalDeltaAdj / (double)adjDelta.size ()), "ZZ0.00");
accLines.push_back (accLine);
knownCountLines.push_back (knownLine);
predCountLines.push_back (predLine);
adjCountLines.push_back (adjLine);
deltaPredCountLines.push_back (deltaPredLine);
deltaAdjCountLines.push_back (deltaAdjLine);
}
}
double factor = 0.0;
if (totalCmCount > 0)
factor = 1.0 / (double)totalCmCount;
totalCM.FactorCounts (factor);
f << endl << endl
<< "Average Confusion Matrix" << endl
<< endl;
totalCM.PrintConfusionMatrixTabDelimited (f);
f << "" << "\t" << "" << "\t" << l1 << endl
<< "" << "\t" << "Split" << "\t" << l2 << endl
<< "Description" << "\t" << "Num" << "\t" << l3 << endl;
f << endl << endl;
for (x = 0; x < knownCountLines.size (); x++)
f << knownCountLines[x] << endl;
f << endl << endl;
for (x = 0; x < predCountLines.size (); x++)
f << predCountLines[x] << endl;
f << endl << endl;
for (x = 0; x < adjCountLines.size (); x++)
f << adjCountLines[x] << endl;
f << endl << endl;
for (x = 0; x < deltaPredCountLines.size (); x++)
f << deltaPredCountLines[x] << endl;
f << endl << endl;
for (x = 0; x < deltaAdjCountLines.size (); x++)
f << deltaAdjCountLines[x] << endl;
f << endl << endl;
for (x = 0; x < knownCountLines.size (); x++)
f << accLines[x] << endl;
VectorFloat avgAccuracies = totalCM.AccuracyByClass ();
f << "Avg-Accuracies";
for (x = 0; x < avgAccuracies.size (); x++)
f << "\t" << StrFormatDouble (avgAccuracies[x], "zz0.00") << "%";
f << "\t" << StrFormatDouble (totalCM.Accuracy (), "zz0.00") << "%";
f << endl;
f << endl << endl;
f.close ();
} /* GenerateFinalResultsReport */
void GradeClassification::GradeExamplesAgainstGroundTruth (FeatureVectorListPtr examplesToGrade,
FeatureVectorListPtr groundTruth
)
{
log.Level (10) << "GradeClassification::GradeExamplesAgainstGroundTruth" << endl;
groundTruth->SortByRootName ();
MLClassConstPtr unknownClass = mlClasses->GetUnKnownClass ();
MLClassConstListPtr classes = NULL;
{
MLClassConstListPtr examplesToGradeClasses = examplesToGrade->ExtractMLClassConstList ();
MLClassConstListPtr groundTruthClasses = groundTruth->ExtractMLClassConstList ();
classes = MLClassConstList::MergeClassList (*examplesToGradeClasses, *groundTruthClasses);
delete examplesToGradeClasses;
delete groundTruthClasses;
}
uint16 maxHierarchialLevel = 0;
{
MLClassConstList::iterator idx;
for (idx = classes->begin (); idx != classes->end (); idx++)
{
MLClassConstPtr c = *idx;
maxHierarchialLevel = Max (maxHierarchialLevel, c->NumHierarchialLevels ());
}
}
// Create ConfusionMatrix objects for each posible level of Hierarchy. The 'resultsSummary' vector will
// end up owning the instances of 'ConfusionMatrix2' and th edestructr will be responable for deleting them.
uint curLevel = 0;
vector<ConfusionMatrix2Ptr> cmByLevel;
for (curLevel = 0; curLevel < maxHierarchialLevel; curLevel++)
{
MLClassConstListPtr classesThisLevel = classes->ExtractListOfClassesForAGivenHierarchialLevel (curLevel);
ConfusionMatrix2Ptr cm = new ConfusionMatrix2 (*classesThisLevel);
cmByLevel.push_back (cm);
}
ConfusionMatrix2 cm (*classes);
ImageFeaturesList::iterator idx;
for (idx = examplesToGrade->begin (); idx != examplesToGrade->end (); idx++)
{
ImageFeaturesPtr exampleToGrade = *idx;
MLClassConstPtr predictedClass = exampleToGrade->MLClass ();
float origSize = exampleToGrade->OrigSize ();
float probability = exampleToGrade->Probability ();
KKStr rootName = osGetRootName (exampleToGrade->ImageFileName ());
FeatureVectorPtr groundTruthExample = groundTruth->LookUpByRootName (rootName);
MLClassConstPtr groundTruthClass = unknownClass;
if (groundTruthExample)
groundTruthClass = groundTruthExample->MLClass ();
cm.Increment (groundTruthClass, predictedClass, (int)origSize, probability, log);
for (curLevel = 0; curLevel < maxHierarchialLevel; curLevel++)
{
MLClassConstPtr groundTruthClasssThisLevel = groundTruthClass->MLClassForGivenHierarchialLevel (curLevel);
MLClassConstPtr predictedClassThisLevel = predictedClass->MLClassForGivenHierarchialLevel (curLevel);
cmByLevel[curLevel]->Increment (groundTruthClasssThisLevel, predictedClassThisLevel, (int)origSize, probability, log);
}
}
//cm.PrintTrueFalsePositivesTabDelimited (*report);
{
// report Hierarchial results
for (curLevel = 0; curLevel < maxHierarchialLevel; curLevel++)
{
log.Level (10) << "GradeClassification::GradeExamplesAgainstGroundTruth Printing Level[" << curLevel << "]" << endl;
*report << endl << endl << endl
<< "Confusion Matrix Training Level[" << maxHierarchialLevel << "] Preduction Level[" << (curLevel + 1) << "]" << endl
<< endl;
cmByLevel[curLevel]->PrintConfusionMatrixTabDelimited (*report);
resultsSummary.push_back (SummaryRec (maxHierarchialLevel, curLevel + 1, cmByLevel[curLevel]));
}
*report << endl << endl << endl;
}
log.Level (10) << "GradeClassification::GradeExamplesAgainstGroundTruth Exiting" << endl;
} /* GradeExamplesAgainstGroundTruth */
