void Saliency::Evaluate(const string gtImgsW, const string &salDir, const string &resName)
{
vector<vecD> prec(SAL_TYPE_NUM), recall(SAL_TYPE_NUM);
static const int SHOW_COLOR_NUM = 7;
static const char* colorShow[SHOW_COLOR_NUM] = {"'k'", "'b'", "'g'", "'r'", "'c'", "'m'", "'y'"};
FILE* f = fopen(resName.c_str(), "w");
CV_Assert(f != NULL);
fprintf(f, "clear;\nclose all;\nclc;\nhold on;\nfigure(1);\n\n");
#pragma omp parallel for
for (int i = 0; i < SAL_TYPE_NUM; i++)
Evaluate(salDir + "*" + SAL_TYPE_DES[i] + ".png", gtImgsW, prec[i], recall[i]);
string leglendStr("legend(");
for (int i = 0; i < SAL_TYPE_NUM; i++)
{
string strPre = format("Precision%s", SAL_TYPE_DES[i]);
string strRecal = format("Recall%s", SAL_TYPE_DES[i]);
int dim = PrintVector(f, recall[i], strRecal);
PrintVector(f, prec[i], strPre, dim);
fprintf(f, "plot(%s, %s, %s, 'linewidth', 2);\n\n", strRecal.c_str(), strPre.c_str(), colorShow[i % SHOW_COLOR_NUM]);
leglendStr += format("'%s', ", SAL_TYPE_DES[i] + 1);
}
leglendStr.resize(leglendStr.size() - 2);
leglendStr += ");";
fprintf(f, "hold off;\nxlabel('Recall');\nylabel('Precision');\n\n%s\ngrid on;\n", leglendStr.c_str());
fprintf(f, "\n\nfigure(2);hold on;\n");
for (int i = 0; i < SAL_TYPE_NUM; i++)
fprintf(f, "plot(Recall%s, %s, 'linewidth', 2);\n", SAL_TYPE_DES[i], colorShow[i % SHOW_COLOR_NUM]);
fprintf(f, "%s\nhold off;\nxlabel('Threshold');\nylabel('Recall');\ngrid on;", leglendStr.c_str());
fclose(f);
CmLog::LogProgress("Evaluation finished%-40s\n", "");
}
void CmEvaluation::Evaluate(CStr gtW, CStr &salDir, CStr &resName, vecS &des)
{
int NumMethod = des.size(); // Number of different methods
vector<vecD> precision(NumMethod), recall(NumMethod), tpr(NumMethod), fpr(NumMethod);
static const int CN = 21; // Color Number
static const char* c[CN] = {"'k'", "'b'", "'g'", "'r'", "'c'", "'m'", "'y'",
"':k'", "':b'", "':g'", "':r'", "':c'", "':m'", "':y'",
"'--k'", "'--b'", "'--g'", "'--r'", "'--c'", "'--m'", "'--y'"
};
FILE* f = fopen(_S(resName), "w");
CV_Assert(f != NULL);
fprintf(f, "clear;\nclose all;\nclc;\n\n\n%%%%\nfigure(1);\nhold on;\n");
vecD thr(NUM_THRESHOLD);
for (int i = 0; i < NUM_THRESHOLD; i++)
thr[i] = i * STEP;
PrintVector(f, thr, "Threshold");
fprintf(f, "\n");
vecD mae(NumMethod);
for (int i = 0; i < NumMethod; i++)
mae[i] = Evaluate_(gtW, salDir, "_" + des[i] + ".png", precision[i], recall[i], tpr[i], fpr[i]); //Evaluate(salDir + "*" + des[i] + ".png", gtW, val[i], recall[i], t);
string leglendStr("legend(");
vecS strPre(NumMethod), strRecall(NumMethod), strTpr(NumMethod), strFpr(NumMethod);
for (int i = 0; i < NumMethod; i++){
strPre[i] = format("Precision_%s", _S(des[i]));
strRecall[i] = format("Recall_%s", _S(des[i]));
strTpr[i] = format("TPR_%s", _S(des[i]));
strFpr[i] = format("FPR_%s", _S(des[i]));
PrintVector(f, recall[i], strRecall[i]);
PrintVector(f, precision[i], strPre[i]);
PrintVector(f, tpr[i], strTpr[i]);
PrintVector(f, fpr[i], strFpr[i]);
fprintf(f, "plot(%s, %s, %s, 'linewidth', %d);\n", _S(strRecall[i]), _S(strPre[i]), c[i % CN], i < CN ? 2 : 1);
leglendStr += format("'%s', ", _S(des[i]));
}
leglendStr.resize(leglendStr.size() - 2);
leglendStr += ");";
string xLabel = "label('Recall');\n";
string yLabel = "label('Precision')\n";
fprintf(f, "hold off;\nx%sy%s\n%s\ngrid on;\naxis([0 1 0 1]);\ntitle('Precision recall curve');\n", _S(xLabel), _S(yLabel), _S(leglendStr));
fprintf(f, "\n\n\n%%%%\nfigure(2);\nhold on;\n");
for (int i = 0; i < NumMethod; i++)
fprintf(f, "plot(%s, %s, %s, 'linewidth', %d);\n", _S(strFpr[i]), _S(strTpr[i]), c[i % CN], i < CN ? 2 : 1);
xLabel = "label('False positive rate');\n";
yLabel = "label('True positive rate')\n";
fprintf(f, "hold off;\nx%sy%s\n%s\ngrid on;\naxis([0 1 0 1]);\n\n\n%%%%\nfigure(3);\ntitle('ROC curve');\n", _S(xLabel), _S(yLabel), _S(leglendStr));
double betaSqr = 0.3; // As suggested by most papers for salient object detection
vecD areaROC(NumMethod, 0), avgFMeasure(NumMethod, 0), maxFMeasure(NumMethod, 0);
for (int i = 0; i < NumMethod; i++){
CV_Assert(fpr[i].size() == tpr[i].size() && precision[i].size() == recall[i].size() && fpr[i].size() == precision[i].size());
for (size_t t = 0; t < fpr[i].size(); t++){
double fMeasure = (1+betaSqr) * precision[i][t] * recall[i][t] / (betaSqr * precision[i][t] + recall[i][t]);
avgFMeasure[i] += fMeasure/fpr[i].size(); // Doing average like this might have strange effect as in:
maxFMeasure[i] = max(maxFMeasure[i], fMeasure);
if (t > 0){
areaROC[i] += (tpr[i][t] + tpr[i][t - 1]) * (fpr[i][t - 1] - fpr[i][t]) / 2.0;
}
}
fprintf(f, "%%%5s: AUC = %5.3f, MeanF = %5.3f, MaxF = %5.3f, MAE = %5.3f\n", _S(des[i]), areaROC[i], avgFMeasure[i], maxFMeasure[i], mae[i]);
}
PrintVector(f, areaROC, "AUC");
PrintVector(f, avgFMeasure, "MeanFMeasure");
PrintVector(f, maxFMeasure, "MaxFMeasure");
PrintVector(f, mae, "MAE");
// methodLabels = {'AC', 'SR', 'DRFI', 'GU', 'GB'};
fprintf(f, "methodLabels = {'%s'", _S(des[0]));
for (int i = 1; i < NumMethod; i++)
fprintf(f, ", '%s'", _S(des[i]));
fprintf(f, "};\n\nbar([MeanFMeasure; MaxFMeasure; AUC]');\nlegend('Mean F_\\beta', 'Max F_\\beta', 'AUC');xlim([0 %d]);\ngrid on;\n", NumMethod+1);
fprintf(f, "xticklabel_rotate([1:%d],90, methodLabels,'interpreter','none');\n", NumMethod);
fprintf(f, "\n\nfigure(4);\nbar(MAE);\ntitle('MAE');\ngrid on;\nxlim([0 %d]);", NumMethod+1);
fprintf(f, "xticklabel_rotate([1:%d],90, methodLabels,'interpreter','none');\n", NumMethod);
fclose(f);
printf("%-70s\r", "");
}
void Objectness::evaluatePerClassRecall(vector<vector<Vec4i> > &boxesTests, CStr &saveName, const int WIN_NUM)
{
const int TEST_NUM = _voc.testSet.size(), CLS_NUM = _voc.classNames.size();
if (boxesTests.size() != TEST_NUM) {
boxesTests.resize(TEST_NUM);
for (int i = 0; i < TEST_NUM; i++) {
Mat boxes;
matRead(_voc.localDir + _voc.testSet[i] + ".dat", boxes);
Vec4i* d = (Vec4i*)boxes.data;
boxesTests[i].resize(boxes.rows, WIN_NUM);
memcpy(&boxesTests[i][0], boxes.data, sizeof(Vec4i)*boxes.rows);
}
}
for (int i = 0; i < TEST_NUM; i++)
if ((int)boxesTests[i].size() < WIN_NUM) {
printf("%s.dat: %d, %d\n", _S(_voc.testSet[i]), boxesTests[i].size(), WIN_NUM);
boxesTests[i].resize(WIN_NUM);
}
// #class by #win matrix for saving correct detection number and gt number
Mat crNum1i = Mat::zeros(CLS_NUM, WIN_NUM, CV_32S);
vecD gtNums(CLS_NUM);
{
for (int i = 0; i < TEST_NUM; i++) {
const vector<Vec4i> &boxes = boxesTests[i];
const vector<Vec4i> &boxesGT = _voc.gtTestBoxes[i];
const vecI &clsGT = _voc.gtTestClsIdx[i];
CV_Assert((int)boxes.size() >= WIN_NUM);
const int gtNumCrnt = boxesGT.size();
for (int j = 0; j < gtNumCrnt; j++) {
gtNums[clsGT[j]]++;
double maxIntUni = 0;
int* crNum = crNum1i.ptr<int>(clsGT[j]);
for (int k = 0; k < WIN_NUM; k++) {
double val = DataSetVOC::interUnio(boxes[k], boxesGT[j]);
maxIntUni = max(maxIntUni, val);
crNum[k] += maxIntUni >= 0.5 ? 1 : 0;
}
}
}
}
FILE* f = fopen(_S(_voc.resDir + saveName), "w");
{
CV_Assert(f != NULL);
fprintf(f, "figure(1);\nhold on;\n\n\n");
vecD val(WIN_NUM), recallObjs(WIN_NUM), recallClss(WIN_NUM);
for (int i = 0; i < WIN_NUM; i++)
val[i] = i;
PrintVector(f, gtNums, "GtNum");
PrintVector(f, val, "WinNum");
fprintf(f, "\n");
string leglendStr("legend(");
double sumObjs = 0;
for (int c = 0; c < CLS_NUM; c++) {
sumObjs += gtNums[c];
memset(&val[0], 0, sizeof(double)*WIN_NUM);
int* crNum = crNum1i.ptr<int>(c);
for (int i = 0; i < WIN_NUM; i++) {
val[i] = crNum[i]/(gtNums[c] + 1e-200);
recallClss[i] += val[i];
recallObjs[i] += crNum[i];
}
CStr className = _voc.classNames[c];
PrintVector(f, val, className);
fprintf(f, "plot(WinNum, %s, %s, 'linewidth', 2);\n", _S(className), COLORs[c % CN]);
leglendStr += format("'%s', ", _S(className));
}
for (int i = 0; i < WIN_NUM; i++) {
recallClss[i] /= CLS_NUM;
recallObjs[i] /= sumObjs;
}
PrintVector(f, recallClss, "class");
fprintf(f, "plot(WinNum, %s, %s, 'linewidth', 2);\n", "class", COLORs[CLS_NUM % CN]);
leglendStr += format("'%s', ", "class");
PrintVector(f, recallObjs, "objects");
fprintf(f, "plot(WinNum, %s, %s, 'linewidth', 2);\n", "objects", COLORs[(CLS_NUM+1) % CN]);
leglendStr += format("'%s', ", "objects");
leglendStr.resize(leglendStr.size() - 2);
leglendStr += ");";
fprintf(f, "%s\nhold off;\nxlabel('#WIN');\nylabel('Recall');\ngrid on;\naxis([0 %d 0 1]);\n", _S(leglendStr), WIN_NUM);
fprintf(f, "[class([1,10,100,1000]);objects([1,10,100,1000])]\ntitle('%s')\n", _S(saveName));
fclose(f);
printf("%-70s\r", "");
}
evaluatePerImgRecall(boxesTests, CmFile::GetNameNE(saveName) + "_PerI.m", WIN_NUM);
}
