void AbundanceCorrectionStatsBuilder::GenerateReportAndStats () { AbundanceCorrectionMatrix abundanceCorrMatrix (*trainLibDataClasses, otherClass, log); ConfusionMatrix2 confusionMatrix (*allClasses); TrainTestThreadList::iterator idx; for (idx = queueDone->begin (); idx != queueDone->end (); ++idx) { TrainTestThreadPtr thread = *idx; const AbundanceCorrectionMatrixPtr acm = thread->AbundanceCorMatrix (); const ConfusionMatrix2Ptr cm = thread->ConfusionMatrix (); if (acm) abundanceCorrMatrix.AddIn (*acm, log); if (cm) confusionMatrix.AddIn (*cm, log); } *report << endl << endl << endl; abundanceCorrMatrix.PrintConfusionMatrixTabDelimited (*report); *report << endl << endl << endl << endl; confusionMatrix.PrintConfusionMatrixTabDelimited (*report); *report << endl << endl << endl; abundanceCorrMatrix.SaveForConfigFileName (configFileName); } /* GenerateReportAndStats */
void ClassificationBiasMatrix::BuildFromConfusionMatrix (const ConfusionMatrix2& cm) { kkint32 classesRowIdx = 0; kkint32 classesColIdx = 0; for (classesRowIdx = 0; classesRowIdx < numClasses; classesRowIdx++) { double knownCount = cm.CountsByKnownClass (classesRowIdx); for (classesColIdx = 0; classesColIdx < numClasses; classesColIdx++) { double predCount = cm.PredictedCountsCM (classesRowIdx, classesColIdx); double prob = 0.0; if (knownCount != 0.0) prob = predCount / knownCount; (*counts) [classesRowIdx][classesColIdx] = predCount; (*probabilities)[classesRowIdx][classesColIdx] = prob; } } } /* BuildFromConfusionMatrix*/
ClassificationBiasMatrix::ClassificationBiasMatrix (const ConfusionMatrix2& cm, RunLog& _runLog ): biasFileName (), classes (NULL), configFileName (), configFileNameFromMatrixBiasFile (), configDateTime (), counts (NULL), dateTimeFileWritten (), errMsgs (), numClasses (0), probabilities (NULL), runLog (_runLog), valid (true) { classes = new MLClassList (cm.MLClasses ()); numClasses = classes->QueueSize (); DeclareMatrix (); BuildFromConfusionMatrix (cm); }
void RandomSplitJobManager::GenerateFinalResultsReport () { KKStr reportFileName = osGetRootName (ManagerName ()) + "_Results.html;"; ofstream f (reportFileName.Str ()); f << "Run Time Parameters" << endl << "Run Time" << "\t" << osGetLocalDateTime () << endl << "configFileName" << "\t" << configFileName << endl << "DataFileName" << "\t" << dataFileName << endl << "Format" << "\t" << format->DriverName () << endl << "DataIndexFileName" << "\t" << dataIndexFileName << endl << "NumFolds" << "\t" << numFolds << endl << "NumSplits" << "\t" << numSplits << endl << "splitFraction" << "\t" << splitFraction << endl << endl; KKJobList::const_iterator idx; ConfusionMatrix2 avgResults (*(this->MLClasses ())); KKB::uint x = 0; for (idx = Jobs ()->begin (); idx != Jobs ()->end (); idx++) { RandomSplitJobPtr j = dynamic_cast<RandomSplitJobPtr> (*idx); if (j->RandomSplitsResults () != NULL) { f << endl << "Random Split[" << j->SplitNum () << "]" << endl; j->RandomSplitsResults ()->PrintConfusionMatrixTabDelimited (f); f << endl << endl; j->PrintClassCounts (f); f << endl << endl; avgResults.AddIn (*(j->RandomSplitsResults ()), log); x++; } } f << endl << "Mean Average of all random Splits." << endl; avgResults.FactorCounts (1.0 / (double)x); avgResults.PrintConfusionMatrixTabDelimited (f); f << endl << endl << endl << endl << "Class Counts" << endl << endl; kkuint32 numClasses = (kkuint32)mlClasses->size (); VectorFloat classAccs; VectorDouble knownCounts; VectorDouble predCounts; VectorDouble adjCounts; VectorDouble adjCountsStdError; VectorDouble predDelta; VectorDouble adjDelta; KKStr l1, l2, l3; mlClasses->ExtractThreeTitleLines (l1, l2, l3); VectorKKStr knownCountLines; VectorKKStr predCountLines; VectorKKStr adjCountLines; VectorKKStr deltaPredCountLines; VectorKKStr deltaAdjCountLines; VectorKKStr accLines; ConfusionMatrix2 totalCM (*MLClasses ()); int totalCmCount = 0; // Known Counts for (idx = Jobs ()->begin (); idx != Jobs ()->end (); idx++) { RandomSplitJobPtr j = dynamic_cast<RandomSplitJobPtr> (*idx); if (j->RandomSplitsResults () != NULL) { KKStr splitNumStr = StrFormatInt (j->SplitNum (), "ZZZ0"); j->GetClassCounts (classAccs, knownCounts, predCounts, adjCounts, adjCountsStdError, predDelta, adjDelta); totalCM.AddIn (*(j->RandomSplitsResults ()), log); totalCmCount++; KKStr accLine = "Acc By Class\t" + splitNumStr; KKStr knownLine = "Known\t" + splitNumStr; KKStr predLine = "Predicted\t" + splitNumStr; KKStr adjLine = "Adjusted\t" + splitNumStr; KKStr deltaPredLine = "Delta Pred\t" + splitNumStr; KKStr deltaAdjLine = "Delta Adj\t" + splitNumStr; double totalAcc = 0.0; double totalDeltaPred = 0.0; double totalDeltaAdj = 0.0; for (x = 0; x < numClasses; x++) { accLine << "\t" << StrFormatDouble (classAccs [x], "zz0.00") << "%"; knownLine << "\t" << StrFormatDouble (knownCounts [x], "-Z,ZZZ,ZZ0.0"); predLine << "\t" << StrFormatDouble (predCounts [x], "-Z,ZZZ,ZZ0.0"); adjLine << "\t" << StrFormatDouble (adjCounts [x], "-Z,ZZZ,ZZ0.0"); deltaPredLine << "\t" << StrFormatDouble (predDelta [x], "-Z,ZZZ,ZZ0.0"); deltaAdjLine << "\t" << StrFormatDouble (adjDelta [x], "-Z,ZZZ,ZZ0.0"); totalAcc += classAccs [x]; totalDeltaPred += fabs (predDelta[x]); totalDeltaAdj += fabs (adjDelta[x]); } accLine << "\t" << StrFormatDouble ((totalAcc / (double)classAccs.size ()), "ZZ0.00") << "%"; deltaPredLine << "\t" << StrFormatDouble ((totalDeltaPred / (double)predDelta.size ()), "ZZ0.00"); deltaAdjLine << "\t" << StrFormatDouble ((totalDeltaAdj / (double)adjDelta.size ()), "ZZ0.00"); accLines.push_back (accLine); knownCountLines.push_back (knownLine); predCountLines.push_back (predLine); adjCountLines.push_back (adjLine); deltaPredCountLines.push_back (deltaPredLine); deltaAdjCountLines.push_back (deltaAdjLine); } } double factor = 0.0; if (totalCmCount > 0) factor = 1.0 / (double)totalCmCount; totalCM.FactorCounts (factor); f << endl << endl << "Average Confusion Matrix" << endl << endl; totalCM.PrintConfusionMatrixTabDelimited (f); f << "" << "\t" << "" << "\t" << l1 << endl << "" << "\t" << "Split" << "\t" << l2 << endl << "Description" << "\t" << "Num" << "\t" << l3 << endl; f << endl << endl; for (x = 0; x < knownCountLines.size (); x++) f << knownCountLines[x] << endl; f << endl << endl; for (x = 0; x < predCountLines.size (); x++) f << predCountLines[x] << endl; f << endl << endl; for (x = 0; x < adjCountLines.size (); x++) f << adjCountLines[x] << endl; f << endl << endl; for (x = 0; x < deltaPredCountLines.size (); x++) f << deltaPredCountLines[x] << endl; f << endl << endl; for (x = 0; x < deltaAdjCountLines.size (); x++) f << deltaAdjCountLines[x] << endl; f << endl << endl; for (x = 0; x < knownCountLines.size (); x++) f << accLines[x] << endl; VectorFloat avgAccuracies = totalCM.AccuracyByClass (); f << "Avg-Accuracies"; for (x = 0; x < avgAccuracies.size (); x++) f << "\t" << StrFormatDouble (avgAccuracies[x], "zz0.00") << "%"; f << "\t" << StrFormatDouble (totalCM.Accuracy (), "zz0.00") << "%"; f << endl; f << endl << endl; f.close (); } /* GenerateFinalResultsReport */
void GradeClassification::GradeExamplesAgainstGroundTruth (FeatureVectorListPtr examplesToGrade, FeatureVectorListPtr groundTruth ) { log.Level (10) << "GradeClassification::GradeExamplesAgainstGroundTruth" << endl; groundTruth->SortByRootName (); MLClassConstPtr unknownClass = mlClasses->GetUnKnownClass (); MLClassConstListPtr classes = NULL; { MLClassConstListPtr examplesToGradeClasses = examplesToGrade->ExtractMLClassConstList (); MLClassConstListPtr groundTruthClasses = groundTruth->ExtractMLClassConstList (); classes = MLClassConstList::MergeClassList (*examplesToGradeClasses, *groundTruthClasses); delete examplesToGradeClasses; delete groundTruthClasses; } uint16 maxHierarchialLevel = 0; { MLClassConstList::iterator idx; for (idx = classes->begin (); idx != classes->end (); idx++) { MLClassConstPtr c = *idx; maxHierarchialLevel = Max (maxHierarchialLevel, c->NumHierarchialLevels ()); } } // Create ConfusionMatrix objects for each posible level of Hierarchy. The 'resultsSummary' vector will // end up owning the instances of 'ConfusionMatrix2' and th edestructr will be responable for deleting them. uint curLevel = 0; vector<ConfusionMatrix2Ptr> cmByLevel; for (curLevel = 0; curLevel < maxHierarchialLevel; curLevel++) { MLClassConstListPtr classesThisLevel = classes->ExtractListOfClassesForAGivenHierarchialLevel (curLevel); ConfusionMatrix2Ptr cm = new ConfusionMatrix2 (*classesThisLevel); cmByLevel.push_back (cm); } ConfusionMatrix2 cm (*classes); ImageFeaturesList::iterator idx; for (idx = examplesToGrade->begin (); idx != examplesToGrade->end (); idx++) { ImageFeaturesPtr exampleToGrade = *idx; MLClassConstPtr predictedClass = exampleToGrade->MLClass (); float origSize = exampleToGrade->OrigSize (); float probability = exampleToGrade->Probability (); KKStr rootName = osGetRootName (exampleToGrade->ImageFileName ()); FeatureVectorPtr groundTruthExample = groundTruth->LookUpByRootName (rootName); MLClassConstPtr groundTruthClass = unknownClass; if (groundTruthExample) groundTruthClass = groundTruthExample->MLClass (); cm.Increment (groundTruthClass, predictedClass, (int)origSize, probability, log); for (curLevel = 0; curLevel < maxHierarchialLevel; curLevel++) { MLClassConstPtr groundTruthClasssThisLevel = groundTruthClass->MLClassForGivenHierarchialLevel (curLevel); MLClassConstPtr predictedClassThisLevel = predictedClass->MLClassForGivenHierarchialLevel (curLevel); cmByLevel[curLevel]->Increment (groundTruthClasssThisLevel, predictedClassThisLevel, (int)origSize, probability, log); } } //cm.PrintTrueFalsePositivesTabDelimited (*report); { // report Hierarchial results for (curLevel = 0; curLevel < maxHierarchialLevel; curLevel++) { log.Level (10) << "GradeClassification::GradeExamplesAgainstGroundTruth Printing Level[" << curLevel << "]" << endl; *report << endl << endl << endl << "Confusion Matrix Training Level[" << maxHierarchialLevel << "] Preduction Level[" << (curLevel + 1) << "]" << endl << endl; cmByLevel[curLevel]->PrintConfusionMatrixTabDelimited (*report); resultsSummary.push_back (SummaryRec (maxHierarchialLevel, curLevel + 1, cmByLevel[curLevel])); } *report << endl << endl << endl; } log.Level (10) << "GradeClassification::GradeExamplesAgainstGroundTruth Exiting" << endl; } /* GradeExamplesAgainstGroundTruth */