std::vector<std::vector<int> > annTrain::getConfusionMatrix()
{
cv::Mat testOutput;
mlp->predict(testSamples, testOutput);
// we now have 5 classes
std::vector<std::vector<int> > confusionMatrix(5, std::vector<int>(5));
for (int i = 0; i < testOutput.rows; i++)
{
int predictedClass = getPredictedClass(testOutput.row(i));
int expectedClass = testOutputExpected.at(i);
cout << expectedClass << " " << predictedClass;
confusionMatrix[expectedClass][predictedClass]++;
}
return confusionMatrix;
}
void printIndices(int k, const int classLabelLength,
const std::shared_ptr<std::vector<util::FuzzyCluster> >& bestClusters,
int classlabels[]) {
util::ConfusionMatrix confusionMatrix(k, 3);
for (int i = 0; i < classLabelLength; i++) {
confusionMatrix.putObject(i,
clustering::FWRDCA::getBestClusterIndex(bestClusters, i),
classlabels[i]);
}
std::cout << ">>>>>>>>>>>> The F-Measure is: ";
std::cout << confusionMatrix.fMeasureGlobal() << std::endl;
std::cout << ">>>>>>>>>>>> The CR-Index is: ";
std::cout << confusionMatrix.CRIndex() << std::endl;
std::cout << ">>>>>>>>>>>> OERC Index is: ";
std::cout << confusionMatrix.OERCIndex() << std::endl;
std::cout << ">>>>>>>>>>>> NMI Index is: ";
std::cout << confusionMatrix.nMIIndex() << std::endl;
util::FuzzyIndices fuzzyIndices(bestClusters);
for (int i = 0; i < classLabelLength; i++) {
fuzzyIndices.insertPertinenceDegree(std::make_pair(i, i / 50), 1.0);
}
util::FuzzyIndices::CampelloIndices campIndices(fuzzyIndices);
std::cout << ">>>>>>>>>>>> Campello's Rand Index is: ";
std::cout << campIndices.randIndex() << std::endl;
std::cout << ">>>>>>>>>>>> Campello's Adjusted Rand Index is: ";
std::cout << campIndices.adjustedRandIndex() << std::endl;
std::cout << ">>>>>>>>>>>> Campello's Jaccard Coef is: ";
std::cout << campIndices.jaccardCoef() << std::endl;
std::cout << ">>>>>>>>>>>> Campello's Fowlkes-Mallows Index is: ";
std::cout << campIndices.fowlkesMallowsIndex() << std::endl;
std::cout << ">>>>>>>>>>>> Campello's Minkowski Measure is: ";
std::cout << campIndices.minkowskiMeasure() << std::endl;
util::FuzzyIndices::AndersonIndices andIndices(fuzzyIndices);
std::cout << ">>>>>>>>>>>> (Anderson)Fuz Rand Index Rifqi is: ";
std::cout << andIndices.sFRHRIndex() << std::endl;
std::cout << ">>>>>>>>>>>> Anderson's Rand Index (4a) is: ";
std::cout << andIndices.randIndex_4a() << std::endl;
std::cout << ">>>>>>>>>>>> Anderson's Johnson Index (4b) is: ";
std::cout << andIndices.jmcabIndex_4b() << std::endl;
std::cout << ">>>>>>>>>>>> Anderson's Hubert Index (4c) is: ";
std::cout << andIndices.hubertIndex_4c() << std::endl;
std::cout << ">>>>>>>>>>>> Anderson's Wallace Index (4d1) is: ";
std::cout << andIndices.wallaceIndex_4d1() << std::endl;
std::cout << ">>>>>>>>>>>> Anderson's Wallace Index (4d2) is: ";
std::cout << andIndices.wallaceIndex_4d2() << std::endl;
std::cout << ">>>>>>>>>>>> Anderson's FowlkesM Index (4e) is: ";
std::cout << andIndices.fowlkesMallow_4e() << std::endl;
std::cout << ">>>>>>>>>>>> Anderson's Jacard Index (4f) is: ";
std::cout << andIndices.jacard_4f() << std::endl;
std::cout << ">>>>>>>>>>>> Anderson's Wallace Index (4g) is: ";
std::cout << andIndices.adjRandHA_4g() << std::endl;
std::cout << ">>>>>>>>>>>> Anderson's Adj Rand Campello(4h) is: ";
std::cout << andIndices.adjRandCampello_4h() << std::endl;
std::cout << ">>>>>>>>>>>> Anderson's Adj Rand Brower (4i) is: ";
std::cout << andIndices.adjRandBrouwer_4i() << std::endl;
std::cout << ">>>>>>>>>>>> Anderson's Minkowski (4j) is: ";
std::cout << andIndices.minkowski_4j() << std::endl;
std::cout << ">>>>>>>>>>>> Anderson's Hubert's Gamma (4k) is: ";
std::cout << andIndices.hGamma_4k() << std::endl;
std::cout << ">>>>>>>>>>>> Anderson's Yule (4l) is: ";
std::cout << andIndices.yule_4l() << std::endl;
std::cout << ">>>>>>>>>>>> Anderson's Chi-Squared (4m) is: ";
std::cout << andIndices.chisq_4m() << std::endl;
std::cout << ">>>>>>>>>>>> Anderson's Goodman-Kruskal (4n) is: ";
std::cout << andIndices.goodmnKrkl_4n() << std::endl;
}
void test(RandomForestImage& randomForest, const std::string& folderTesting,
const std::string& folderPrediction, const bool useDepthFilling,
const bool writeProbabilityImages) {
auto filenames = listImageFilenames(folderTesting);
if (filenames.empty()) {
throw std::runtime_error(std::string("found no files in ") + folderTesting);
}
CURFIL_INFO("got " << filenames.size() << " files for prediction");
CURFIL_INFO("label/color map:");
const auto labelColorMap = randomForest.getLabelColorMap();
for (const auto& labelColor : labelColorMap) {
const auto color = LabelImage::decodeLabel(labelColor.first);
CURFIL_INFO("label: " << static_cast<int>(labelColor.first) << ", color: RGB(" << color << ")");
}
tbb::mutex totalMutex;
utils::Average averageAccuracy;
utils::Average averageAccuracyWithoutVoid;
const LabelType numClasses = randomForest.getNumClasses();
ConfusionMatrix totalConfusionMatrix(numClasses);
size_t i = 0;
const bool useCIELab = randomForest.getConfiguration().isUseCIELab();
CURFIL_INFO("CIELab: " << useCIELab);
CURFIL_INFO("DepthFilling: " << useDepthFilling);
bool onGPU = randomForest.getConfiguration().getAccelerationMode() == GPU_ONLY;
size_t grainSize = 1;
if (!onGPU) {
grainSize = filenames.size();
}
bool writeImages = true;
if (folderPrediction.empty()) {
CURFIL_WARNING("no prediction folder given. will not write images");
writeImages = false;
}
tbb::parallel_for(tbb::blocked_range<size_t>(0, filenames.size(), grainSize),
[&](const tbb::blocked_range<size_t>& range) {
for(size_t fileNr = range.begin(); fileNr != range.end(); fileNr++) {
const std::string& filename = filenames[fileNr];
const auto imageLabelPair = loadImagePair(filename, useCIELab, useDepthFilling);
const RGBDImage& testImage = imageLabelPair.getRGBDImage();
const LabelImage& groundTruth = imageLabelPair.getLabelImage();
LabelImage prediction(testImage.getWidth(), testImage.getHeight());
for(int y = 0; y < groundTruth.getHeight(); y++) {
for(int x = 0; x < groundTruth.getWidth(); x++) {
const LabelType label = groundTruth.getLabel(x, y);
if (label >= numClasses) {
const auto msg = (boost::format("illegal label in ground truth image '%s' at pixel (%d,%d): %d RGB(%3d,%3d,%3d) (numClasses: %d)")
% filename
% x % y
% static_cast<int>(label)
% LabelImage::decodeLabel(label)[0]
% LabelImage::decodeLabel(label)[1]
% LabelImage::decodeLabel(label)[2]
% static_cast<int>(numClasses)
).str();
throw std::runtime_error(msg);
}
}
}
boost::filesystem::path fn(testImage.getFilename());
const std::string basepath = folderPrediction + "/" + boost::filesystem::basename(fn);
cuv::ndarray<float, cuv::host_memory_space> probabilities;
prediction = randomForest.predict(testImage, &probabilities, onGPU);
#ifndef NDEBUG
for(LabelType label = 0; label < randomForest.getNumClasses(); label++) {
if (!randomForest.shouldIgnoreLabel(label)) {
continue;
}
// ignored classes must not be predicted as we did not sample them
for(size_t y = 0; y < probabilities.shape(0); y++) {
for(size_t x = 0; x < probabilities.shape(1); x++) {
const float& probability = probabilities(label, y, x);
assert(probability == 0.0);
}
}
}
#endif
if (writeImages && writeProbabilityImages) {
utils::Profile profile("writeProbabilityImages");
RGBDImage probabilityImage(testImage.getWidth(), testImage.getHeight());
for(LabelType label = 0; label< randomForest.getNumClasses(); label++) {
if (randomForest.shouldIgnoreLabel(label)) {
continue;
}
for(int y = 0; y < probabilityImage.getHeight(); y++) {
for(int x = 0; x < probabilityImage.getWidth(); x++) {
const float& probability = probabilities(label, y, x);
for(int c=0; c<3; c++) {
probabilityImage.setColor(x, y, c, probability);
}
}
}
const std::string filename = (boost::format("%s_label_%d.png") % basepath % static_cast<int>(label)).str();
probabilityImage.saveColor(filename);
}
}
int thisNumber;
{
tbb::mutex::scoped_lock total(totalMutex);
thisNumber = i++;
}
if (writeImages) {
utils::Profile profile("writeImages");
testImage.saveColor(basepath + ".png");
testImage.saveDepth(basepath + "_depth.png");
groundTruth.save(basepath + "_ground_truth.png");
prediction.save(basepath + "_prediction.png");
}
ConfusionMatrix confusionMatrix(numClasses);
double accuracy = calculatePixelAccuracy(prediction, groundTruth, true, &confusionMatrix);
double accuracyWithoutVoid = calculatePixelAccuracy(prediction, groundTruth, false);
tbb::mutex::scoped_lock lock(totalMutex);
CURFIL_INFO("prediction " << (thisNumber + 1) << "/" << filenames.size()
<< " (" << testImage.getFilename() << "): pixel accuracy (without void): " << 100 * accuracy
<< " (" << 100 * accuracyWithoutVoid << ")");
averageAccuracy.addValue(accuracy);
averageAccuracyWithoutVoid.addValue(accuracyWithoutVoid);
totalConfusionMatrix += confusionMatrix;
}
});
tbb::mutex::scoped_lock lock(totalMutex);
double accuracy = averageAccuracy.getAverage();
double accuracyWithoutVoid = averageAccuracyWithoutVoid.getAverage();
totalConfusionMatrix.normalize();
CURFIL_INFO(totalConfusionMatrix);
CURFIL_INFO("pixel accuracy: " << 100 * accuracy);
CURFIL_INFO("pixel accuracy without void: " << 100 * accuracyWithoutVoid);
}
