void Saliency::Evaluate(const string gtImgsW, const string &salDir, const string &resName) { vector<vecD> prec(SAL_TYPE_NUM), recall(SAL_TYPE_NUM); static const int SHOW_COLOR_NUM = 7; static const char* colorShow[SHOW_COLOR_NUM] = {"'k'", "'b'", "'g'", "'r'", "'c'", "'m'", "'y'"}; FILE* f = fopen(resName.c_str(), "w"); CV_Assert(f != NULL); fprintf(f, "clear;\nclose all;\nclc;\nhold on;\nfigure(1);\n\n"); #pragma omp parallel for for (int i = 0; i < SAL_TYPE_NUM; i++) Evaluate(salDir + "*" + SAL_TYPE_DES[i] + ".png", gtImgsW, prec[i], recall[i]); string leglendStr("legend("); for (int i = 0; i < SAL_TYPE_NUM; i++) { string strPre = format("Precision%s", SAL_TYPE_DES[i]); string strRecal = format("Recall%s", SAL_TYPE_DES[i]); int dim = PrintVector(f, recall[i], strRecal); PrintVector(f, prec[i], strPre, dim); fprintf(f, "plot(%s, %s, %s, 'linewidth', 2);\n\n", strRecal.c_str(), strPre.c_str(), colorShow[i % SHOW_COLOR_NUM]); leglendStr += format("'%s', ", SAL_TYPE_DES[i] + 1); } leglendStr.resize(leglendStr.size() - 2); leglendStr += ");"; fprintf(f, "hold off;\nxlabel('Recall');\nylabel('Precision');\n\n%s\ngrid on;\n", leglendStr.c_str()); fprintf(f, "\n\nfigure(2);hold on;\n"); for (int i = 0; i < SAL_TYPE_NUM; i++) fprintf(f, "plot(Recall%s, %s, 'linewidth', 2);\n", SAL_TYPE_DES[i], colorShow[i % SHOW_COLOR_NUM]); fprintf(f, "%s\nhold off;\nxlabel('Threshold');\nylabel('Recall');\ngrid on;", leglendStr.c_str()); fclose(f); CmLog::LogProgress("Evaluation finished%-40s\n", ""); }
void CmEvaluation::Evaluate(CStr gtW, CStr &salDir, CStr &resName, vecS &des) { int NumMethod = des.size(); // Number of different methods vector<vecD> precision(NumMethod), recall(NumMethod), tpr(NumMethod), fpr(NumMethod); static const int CN = 21; // Color Number static const char* c[CN] = {"'k'", "'b'", "'g'", "'r'", "'c'", "'m'", "'y'", "':k'", "':b'", "':g'", "':r'", "':c'", "':m'", "':y'", "'--k'", "'--b'", "'--g'", "'--r'", "'--c'", "'--m'", "'--y'" }; FILE* f = fopen(_S(resName), "w"); CV_Assert(f != NULL); fprintf(f, "clear;\nclose all;\nclc;\n\n\n%%%%\nfigure(1);\nhold on;\n"); vecD thr(NUM_THRESHOLD); for (int i = 0; i < NUM_THRESHOLD; i++) thr[i] = i * STEP; PrintVector(f, thr, "Threshold"); fprintf(f, "\n"); vecD mae(NumMethod); for (int i = 0; i < NumMethod; i++) mae[i] = Evaluate_(gtW, salDir, "_" + des[i] + ".png", precision[i], recall[i], tpr[i], fpr[i]); //Evaluate(salDir + "*" + des[i] + ".png", gtW, val[i], recall[i], t); string leglendStr("legend("); vecS strPre(NumMethod), strRecall(NumMethod), strTpr(NumMethod), strFpr(NumMethod); for (int i = 0; i < NumMethod; i++){ strPre[i] = format("Precision_%s", _S(des[i])); strRecall[i] = format("Recall_%s", _S(des[i])); strTpr[i] = format("TPR_%s", _S(des[i])); strFpr[i] = format("FPR_%s", _S(des[i])); PrintVector(f, recall[i], strRecall[i]); PrintVector(f, precision[i], strPre[i]); PrintVector(f, tpr[i], strTpr[i]); PrintVector(f, fpr[i], strFpr[i]); fprintf(f, "plot(%s, %s, %s, 'linewidth', %d);\n", _S(strRecall[i]), _S(strPre[i]), c[i % CN], i < CN ? 2 : 1); leglendStr += format("'%s', ", _S(des[i])); } leglendStr.resize(leglendStr.size() - 2); leglendStr += ");"; string xLabel = "label('Recall');\n"; string yLabel = "label('Precision')\n"; fprintf(f, "hold off;\nx%sy%s\n%s\ngrid on;\naxis([0 1 0 1]);\ntitle('Precision recall curve');\n", _S(xLabel), _S(yLabel), _S(leglendStr)); fprintf(f, "\n\n\n%%%%\nfigure(2);\nhold on;\n"); for (int i = 0; i < NumMethod; i++) fprintf(f, "plot(%s, %s, %s, 'linewidth', %d);\n", _S(strFpr[i]), _S(strTpr[i]), c[i % CN], i < CN ? 2 : 1); xLabel = "label('False positive rate');\n"; yLabel = "label('True positive rate')\n"; fprintf(f, "hold off;\nx%sy%s\n%s\ngrid on;\naxis([0 1 0 1]);\n\n\n%%%%\nfigure(3);\ntitle('ROC curve');\n", _S(xLabel), _S(yLabel), _S(leglendStr)); double betaSqr = 0.3; // As suggested by most papers for salient object detection vecD areaROC(NumMethod, 0), avgFMeasure(NumMethod, 0), maxFMeasure(NumMethod, 0); for (int i = 0; i < NumMethod; i++){ CV_Assert(fpr[i].size() == tpr[i].size() && precision[i].size() == recall[i].size() && fpr[i].size() == precision[i].size()); for (size_t t = 0; t < fpr[i].size(); t++){ double fMeasure = (1+betaSqr) * precision[i][t] * recall[i][t] / (betaSqr * precision[i][t] + recall[i][t]); avgFMeasure[i] += fMeasure/fpr[i].size(); // Doing average like this might have strange effect as in: maxFMeasure[i] = max(maxFMeasure[i], fMeasure); if (t > 0){ areaROC[i] += (tpr[i][t] + tpr[i][t - 1]) * (fpr[i][t - 1] - fpr[i][t]) / 2.0; } } fprintf(f, "%%%5s: AUC = %5.3f, MeanF = %5.3f, MaxF = %5.3f, MAE = %5.3f\n", _S(des[i]), areaROC[i], avgFMeasure[i], maxFMeasure[i], mae[i]); } PrintVector(f, areaROC, "AUC"); PrintVector(f, avgFMeasure, "MeanFMeasure"); PrintVector(f, maxFMeasure, "MaxFMeasure"); PrintVector(f, mae, "MAE"); // methodLabels = {'AC', 'SR', 'DRFI', 'GU', 'GB'}; fprintf(f, "methodLabels = {'%s'", _S(des[0])); for (int i = 1; i < NumMethod; i++) fprintf(f, ", '%s'", _S(des[i])); fprintf(f, "};\n\nbar([MeanFMeasure; MaxFMeasure; AUC]');\nlegend('Mean F_\\beta', 'Max F_\\beta', 'AUC');xlim([0 %d]);\ngrid on;\n", NumMethod+1); fprintf(f, "xticklabel_rotate([1:%d],90, methodLabels,'interpreter','none');\n", NumMethod); fprintf(f, "\n\nfigure(4);\nbar(MAE);\ntitle('MAE');\ngrid on;\nxlim([0 %d]);", NumMethod+1); fprintf(f, "xticklabel_rotate([1:%d],90, methodLabels,'interpreter','none');\n", NumMethod); fclose(f); printf("%-70s\r", ""); }
void Objectness::evaluatePerClassRecall(vector<vector<Vec4i> > &boxesTests, CStr &saveName, const int WIN_NUM) { const int TEST_NUM = _voc.testSet.size(), CLS_NUM = _voc.classNames.size(); if (boxesTests.size() != TEST_NUM) { boxesTests.resize(TEST_NUM); for (int i = 0; i < TEST_NUM; i++) { Mat boxes; matRead(_voc.localDir + _voc.testSet[i] + ".dat", boxes); Vec4i* d = (Vec4i*)boxes.data; boxesTests[i].resize(boxes.rows, WIN_NUM); memcpy(&boxesTests[i][0], boxes.data, sizeof(Vec4i)*boxes.rows); } } for (int i = 0; i < TEST_NUM; i++) if ((int)boxesTests[i].size() < WIN_NUM) { printf("%s.dat: %d, %d\n", _S(_voc.testSet[i]), boxesTests[i].size(), WIN_NUM); boxesTests[i].resize(WIN_NUM); } // #class by #win matrix for saving correct detection number and gt number Mat crNum1i = Mat::zeros(CLS_NUM, WIN_NUM, CV_32S); vecD gtNums(CLS_NUM); { for (int i = 0; i < TEST_NUM; i++) { const vector<Vec4i> &boxes = boxesTests[i]; const vector<Vec4i> &boxesGT = _voc.gtTestBoxes[i]; const vecI &clsGT = _voc.gtTestClsIdx[i]; CV_Assert((int)boxes.size() >= WIN_NUM); const int gtNumCrnt = boxesGT.size(); for (int j = 0; j < gtNumCrnt; j++) { gtNums[clsGT[j]]++; double maxIntUni = 0; int* crNum = crNum1i.ptr<int>(clsGT[j]); for (int k = 0; k < WIN_NUM; k++) { double val = DataSetVOC::interUnio(boxes[k], boxesGT[j]); maxIntUni = max(maxIntUni, val); crNum[k] += maxIntUni >= 0.5 ? 1 : 0; } } } } FILE* f = fopen(_S(_voc.resDir + saveName), "w"); { CV_Assert(f != NULL); fprintf(f, "figure(1);\nhold on;\n\n\n"); vecD val(WIN_NUM), recallObjs(WIN_NUM), recallClss(WIN_NUM); for (int i = 0; i < WIN_NUM; i++) val[i] = i; PrintVector(f, gtNums, "GtNum"); PrintVector(f, val, "WinNum"); fprintf(f, "\n"); string leglendStr("legend("); double sumObjs = 0; for (int c = 0; c < CLS_NUM; c++) { sumObjs += gtNums[c]; memset(&val[0], 0, sizeof(double)*WIN_NUM); int* crNum = crNum1i.ptr<int>(c); for (int i = 0; i < WIN_NUM; i++) { val[i] = crNum[i]/(gtNums[c] + 1e-200); recallClss[i] += val[i]; recallObjs[i] += crNum[i]; } CStr className = _voc.classNames[c]; PrintVector(f, val, className); fprintf(f, "plot(WinNum, %s, %s, 'linewidth', 2);\n", _S(className), COLORs[c % CN]); leglendStr += format("'%s', ", _S(className)); } for (int i = 0; i < WIN_NUM; i++) { recallClss[i] /= CLS_NUM; recallObjs[i] /= sumObjs; } PrintVector(f, recallClss, "class"); fprintf(f, "plot(WinNum, %s, %s, 'linewidth', 2);\n", "class", COLORs[CLS_NUM % CN]); leglendStr += format("'%s', ", "class"); PrintVector(f, recallObjs, "objects"); fprintf(f, "plot(WinNum, %s, %s, 'linewidth', 2);\n", "objects", COLORs[(CLS_NUM+1) % CN]); leglendStr += format("'%s', ", "objects"); leglendStr.resize(leglendStr.size() - 2); leglendStr += ");"; fprintf(f, "%s\nhold off;\nxlabel('#WIN');\nylabel('Recall');\ngrid on;\naxis([0 %d 0 1]);\n", _S(leglendStr), WIN_NUM); fprintf(f, "[class([1,10,100,1000]);objects([1,10,100,1000])]\ntitle('%s')\n", _S(saveName)); fclose(f); printf("%-70s\r", ""); } evaluatePerImgRecall(boxesTests, CmFile::GetNameNE(saveName) + "_PerI.m", WIN_NUM); }