int CmFile::GetSubFolders(CStr& folder, vecS& subFolders)
{
subFolders.clear();
DIR *dp = opendir(_S(folder));
if (dp == NULL){
cout << folder << endl;
perror("Cannot open folder");
return EXIT_FAILURE;
}
struct dirent *dirContent;
while ((dirContent = readdir(dp)) != NULL){
if (string(dirContent->d_name)[0] == '.')
continue;
struct stat st;
lstat(dirContent->d_name,&st);
if(S_ISDIR(st.st_mode)){
subFolders.push_back(string(dirContent->d_name));
}
}
closedir(dp);
return (int)subFolders.size();
}
void CmIllustr::WriteFigure(const vecS &names, FILE *f, const vecI &sHeights, const ImgsOptions &opts)
{
static int idx = -1;
printf("Output %dth big image\n", ++idx);
{//* Produce a big figure
Size sz(opts._w * opts._exts.size() + space * (opts._exts.size() - 1), 40 - space);
for (size_t i = 0; i < names.size(); i++)
sz.height += space + sHeights[i];
Mat bImg(sz, CV_8UC3);
memset(bImg.data, -1, bImg.step * sz.height);
Rect reg = Rect(0, 0, opts._w, 0);
for (size_t i = 0; i < names.size(); i++) {
reg.x = 0;
reg.height = sHeights[i];
for (size_t j = 0; j < opts._exts.size(); j++) {
Mat subImg = bImg(reg);
Mat crntImg = imread(opts._inDir + names[i] + opts._exts[j]);
if (crntImg.data != NULL)
resize(crntImg, subImg, subImg.size(), INTER_AREA);
reg.x += space + opts._w;
}
reg.y += space + sHeights[i];
}
imwrite(opts._outDir + format("%d.jpg", idx), bImg);
}//*/
fprintf(f, "\\begin{figure*}[t!]\n\t\\centering\n");
fprintf(f, "\t\\begin{overpic}[width=\\textwidth]{%d.jpg} \n", idx); // \\footnotesize
fprintf(f, "\t\\PutDes\n\t\\end{overpic}\n");
fprintf(f, "\t\\PutCap\n\\end{figure*}\n\\clearpage\n\n"); // \\clearpage
}
int CmFile::GetNames(CStr& rootFolder, CStr &fileW, vecS &names)
{
GetNames(rootFolder + fileW, names);
vecS subFolders, tmpNames;
int subNum = CmFile::GetSubFolders(rootFolder, subFolders);
for (int i = 0; i < subNum; i++){
subFolders[i] += "/";
int subNum = GetNames(rootFolder + subFolders[i], fileW, tmpNames);
for (int j = 0; j < subNum; j++)
names.push_back(subFolders[i] + tmpNames[j]);
}
return (int)names.size();
}
void CmEvaluation::EvalueMask(CStr gtW, CStr &maskDir, vecS &des, CStr resFile, double betaSqr, bool alertNul, CStr suffix)
{
vecS namesNS;
string gtDir, gtExt;
int imgNum = CmFile::GetNamesNE(gtW, namesNS, gtDir, gtExt);
int methodNum = (int)des.size();
vecD pr(methodNum), rec(methodNum), count(methodNum), fm(methodNum);
for (int i = 0; i < imgNum; i++){
Mat truM = imread(gtDir + namesNS[i] + gtExt, CV_LOAD_IMAGE_GRAYSCALE);
for (int m = 0; m < methodNum; m++) {
string mapName = maskDir + namesNS[i] + "_" + des[m];
mapName += suffix.empty() ? ".png" : "_" + suffix + ".png";
Mat res = imread(mapName, CV_LOAD_IMAGE_GRAYSCALE);
if (truM.data == NULL || res.data == NULL || truM.size != res.size){
if (alertNul)
printf("Truth(%d, %d), Res(%d, %d): %s\n", truM.cols, truM.rows, res.cols, res.rows, _S(mapName));
continue;
}
compare(truM, 128, truM, CMP_GE);
compare(res, 128, res, CMP_GE);
Mat commMat;
bitwise_and(truM, res, commMat);
double commV = sum(commMat).val[0];
double p = commV/(sum(res).val[0] + EPS);
double r = commV/(sum(truM).val[0] + EPS);
pr[m] += p;
rec[m] += r;
count[m]++;
}
}
for (int m = 0; m < methodNum; m++){
pr[m] /= count[m], rec[m] /= count[m];
fm[m] = (1 + betaSqr) * pr[m] * rec[m] / (betaSqr * pr[m] + rec[m] + EPS);
}
FILE *f;
fopen_s(&f, _S(resFile), "a");
CV_Assert(f != NULL);
CmEvaluation::PrintVector(f, pr, "PrecisionMask" + suffix);
CmEvaluation::PrintVector(f, rec, "RecallMask" + suffix);
CmEvaluation::PrintVector(f, fm, "FMeasureMask" + suffix);
fprintf(f, "bar([%s]');\ntitle('%s');\ngrid on\n", _S("PrecisionMask" + suffix + "; RecallMask" + suffix + "; FMeasureMask" + suffix), _S("Segmentation" + suffix));
fclose(f);
if (des.size() == 1)
printf("Precision = %g, recall = %g, F-Measure = %g\n", pr[0], rec[0], fm[0]);
}
int CmFile::GetSubFolders(CStr& folder, vecS& subFolders)
{
subFolders.clear();
WIN32_FIND_DATAA fileFindData;
string nameWC = folder + "\\*";
HANDLE hFind = ::FindFirstFileA(nameWC.c_str(), &fileFindData);
if (hFind == INVALID_HANDLE_VALUE)
return 0;
do {
if (fileFindData.cFileName[0] == '.')
continue; // filter the '..' and '.' in the path
if (fileFindData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
subFolders.push_back(fileFindData.cFileName);
} while (::FindNextFileA(hFind, &fileFindData));
FindClose(hFind);
return (int)subFolders.size();
}
bool CmFile::writeStrList(CStr &fName, const vecS &strs)
{
FILE *f = fopen(_S(fName), "w");
if (f == NULL)
return false;
for (size_t i = 0; i < strs.size(); i++)
fprintf(f, "%s\n", _S(strs[i]));
fclose(f);
return true;
}
// Get image names from a wildcard. Eg: GetNames("D:\\*.jpg", imgNames);
int CmFile::GetNames(CStr &nameW, vecS &names, string &dir)
{
dir = GetFolder(nameW);
names.clear();
names.reserve(10000);
WIN32_FIND_DATAA fileFindData;
HANDLE hFind = ::FindFirstFileA(_S(nameW), &fileFindData);
if (hFind == INVALID_HANDLE_VALUE)
return 0;
do{
if (fileFindData.cFileName[0] == '.')
continue; // filter the '..' and '.' in the path
if (fileFindData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
continue; // Ignore sub-folders
names.push_back(fileFindData.cFileName);
} while (::FindNextFileA(hFind, &fileFindData));
FindClose(hFind);
return (int)names.size();
}
void CmEvaluation::MeanAbsoluteError(CStr inDir, CStr salDir, vecS des, bool zeroMapIfMissing)
{
vecS namesNE;
int imgNum = CmFile::GetNamesNE(inDir + "*.jpg", namesNE), count = 0, methodNum = des.size();
vecD costs(des.size());
for (int i = 0; i < imgNum; i++){
Mat gt = imread(inDir + namesNE[i] + ".png", CV_LOAD_IMAGE_GRAYSCALE);
if (gt.empty()) {
if (zeroMapIfMissing){
Mat img = imread(inDir + namesNE[i] + ".jpg");
gt = Mat::zeros(img.size(), CV_8U);
}
else
continue;
}
//printf("%s.jpg ", _S(namesNE[i]));
gt.convertTo(gt, CV_32F, 1.0/255);
#pragma omp parallel for
for (int j = 0; j < methodNum; j++){
Mat res = imread(salDir + namesNE[i] + "_" + des[j] + ".png", CV_LOAD_IMAGE_GRAYSCALE);
CV_Assert_(res.data != NULL, ("Can't load file %s\n", _S(namesNE[i] + "_" + des[j] + ".png")));
if (res.size != gt.size){
printf("size don't match %s\n", _S(namesNE[i] + "_" + des[j] + ".png"));
resize(res, res, gt.size());
CmFile::MkDir(salDir + "Out/");
imwrite(salDir + "Out/" + namesNE[i] + "_" + des[j] + ".png", res);
}
res.convertTo(res, CV_32F, 1.0/255);
cv::absdiff(gt, res, res);
costs[j] += sum(res).val[0] / (gt.rows * gt.cols);
}
count++;
}
for (size_t j = 0; j < des.size(); j++)
printf("%4s:%5.3f ", _S(des[j]), costs[j]/count);
printf("\n");
}
void BenchMarkLatex::bestWostCases(CStr &rootDir, const vecS &dbNames, CStr &outDirRoot)
{
const int NUM_M = 2;
const char* methodNames[NUM_M] = {"DRFI", "MC"};
for (int d = 0; d < dbNames.size(); d++){
CStr inDir = rootDir + dbNames[d] + "/";
vecS namesNE;
CmValStructVec<double, string> scoreNames[NUM_M];
int imgNum = CmFile::GetNamesNE(inDir + "Imgs/*.png", namesNE);
for (int m = 0; m < NUM_M; m++)
scoreNames[m].reserve(imgNum);
#pragma omp parallel for
for (int i = 0; i < imgNum; i++){ // For each image
Mat gtMap = imread(inDir + "Imgs/" + namesNE[i] + ".png", CV_LOAD_IMAGE_GRAYSCALE);
for (int m = 0; m < NUM_M; m++) { // For each method
Mat salMap = imread(inDir + "Saliency/" + namesNE[i] + "_" + methodNames[m] + ".png", CV_LOAD_IMAGE_GRAYSCALE), dif1f;
normalize(salMap, salMap, 0, 255, CV_MINMAX);
double score = avergeFMeare(salMap, gtMap);
scoreNames[m].pushBack(score, namesNE[i]);
}
}
for (int m = 0; m < NUM_M; m++) {
CStr outDir = outDirRoot + dbNames[d] + "_" + methodNames[m] + "/";
CmFile::MkDir(outDir);
CmFile::CleanFolder(outDir);
scoreNames[m].sort(true); // Worst first
for (int i = 0; i < 3; i++) {//
copySampleWithGt(inDir, scoreNames[m][i], methodNames[m], outDir + format("B%d_", i));
int idx = imgNum - 1 - i;
copySampleWithGt(inDir, scoreNames[m][idx], methodNames[m], outDir + format("W%d_", i));
}
CmIluImgs::Demo(outDir);
}
}
}
void CmEvaluation::Evaluate(CStr gtW, CStr &salDir, CStr &resName, vecS &des)
{
int NumMethod = des.size(); // Number of different methods
vector<vecD> precision(NumMethod), recall(NumMethod), tpr(NumMethod), fpr(NumMethod);
static const int CN = 21; // Color Number
static const char* c[CN] = {"'k'", "'b'", "'g'", "'r'", "'c'", "'m'", "'y'",
"':k'", "':b'", "':g'", "':r'", "':c'", "':m'", "':y'",
"'--k'", "'--b'", "'--g'", "'--r'", "'--c'", "'--m'", "'--y'"
};
FILE* f = fopen(_S(resName), "w");
CV_Assert(f != NULL);
fprintf(f, "clear;\nclose all;\nclc;\n\n\n%%%%\nfigure(1);\nhold on;\n");
vecD thr(NUM_THRESHOLD);
for (int i = 0; i < NUM_THRESHOLD; i++)
thr[i] = i * STEP;
PrintVector(f, thr, "Threshold");
fprintf(f, "\n");
vecD mae(NumMethod);
for (int i = 0; i < NumMethod; i++)
mae[i] = Evaluate_(gtW, salDir, "_" + des[i] + ".png", precision[i], recall[i], tpr[i], fpr[i]); //Evaluate(salDir + "*" + des[i] + ".png", gtW, val[i], recall[i], t);
string leglendStr("legend(");
vecS strPre(NumMethod), strRecall(NumMethod), strTpr(NumMethod), strFpr(NumMethod);
for (int i = 0; i < NumMethod; i++){
strPre[i] = format("Precision_%s", _S(des[i]));
strRecall[i] = format("Recall_%s", _S(des[i]));
strTpr[i] = format("TPR_%s", _S(des[i]));
strFpr[i] = format("FPR_%s", _S(des[i]));
PrintVector(f, recall[i], strRecall[i]);
PrintVector(f, precision[i], strPre[i]);
PrintVector(f, tpr[i], strTpr[i]);
PrintVector(f, fpr[i], strFpr[i]);
fprintf(f, "plot(%s, %s, %s, 'linewidth', %d);\n", _S(strRecall[i]), _S(strPre[i]), c[i % CN], i < CN ? 2 : 1);
leglendStr += format("'%s', ", _S(des[i]));
}
leglendStr.resize(leglendStr.size() - 2);
leglendStr += ");";
string xLabel = "label('Recall');\n";
string yLabel = "label('Precision')\n";
fprintf(f, "hold off;\nx%sy%s\n%s\ngrid on;\naxis([0 1 0 1]);\ntitle('Precision recall curve');\n", _S(xLabel), _S(yLabel), _S(leglendStr));
fprintf(f, "\n\n\n%%%%\nfigure(2);\nhold on;\n");
for (int i = 0; i < NumMethod; i++)
fprintf(f, "plot(%s, %s, %s, 'linewidth', %d);\n", _S(strFpr[i]), _S(strTpr[i]), c[i % CN], i < CN ? 2 : 1);
xLabel = "label('False positive rate');\n";
yLabel = "label('True positive rate')\n";
fprintf(f, "hold off;\nx%sy%s\n%s\ngrid on;\naxis([0 1 0 1]);\n\n\n%%%%\nfigure(3);\ntitle('ROC curve');\n", _S(xLabel), _S(yLabel), _S(leglendStr));
double betaSqr = 0.3; // As suggested by most papers for salient object detection
vecD areaROC(NumMethod, 0), avgFMeasure(NumMethod, 0), maxFMeasure(NumMethod, 0);
for (int i = 0; i < NumMethod; i++){
CV_Assert(fpr[i].size() == tpr[i].size() && precision[i].size() == recall[i].size() && fpr[i].size() == precision[i].size());
for (size_t t = 0; t < fpr[i].size(); t++){
double fMeasure = (1+betaSqr) * precision[i][t] * recall[i][t] / (betaSqr * precision[i][t] + recall[i][t]);
avgFMeasure[i] += fMeasure/fpr[i].size(); // Doing average like this might have strange effect as in:
maxFMeasure[i] = max(maxFMeasure[i], fMeasure);
if (t > 0){
areaROC[i] += (tpr[i][t] + tpr[i][t - 1]) * (fpr[i][t - 1] - fpr[i][t]) / 2.0;
}
}
fprintf(f, "%%%5s: AUC = %5.3f, MeanF = %5.3f, MaxF = %5.3f, MAE = %5.3f\n", _S(des[i]), areaROC[i], avgFMeasure[i], maxFMeasure[i], mae[i]);
}
PrintVector(f, areaROC, "AUC");
PrintVector(f, avgFMeasure, "MeanFMeasure");
PrintVector(f, maxFMeasure, "MaxFMeasure");
PrintVector(f, mae, "MAE");
// methodLabels = {'AC', 'SR', 'DRFI', 'GU', 'GB'};
fprintf(f, "methodLabels = {'%s'", _S(des[0]));
for (int i = 1; i < NumMethod; i++)
fprintf(f, ", '%s'", _S(des[i]));
fprintf(f, "};\n\nbar([MeanFMeasure; MaxFMeasure; AUC]');\nlegend('Mean F_\\beta', 'Max F_\\beta', 'AUC');xlim([0 %d]);\ngrid on;\n", NumMethod+1);
fprintf(f, "xticklabel_rotate([1:%d],90, methodLabels,'interpreter','none');\n", NumMethod);
fprintf(f, "\n\nfigure(4);\nbar(MAE);\ntitle('MAE');\ngrid on;\nxlim([0 %d]);", NumMethod+1);
fprintf(f, "xticklabel_rotate([1:%d],90, methodLabels,'interpreter','none');\n", NumMethod);
fclose(f);
printf("%-70s\r", "");
}
void CmEvaluation::EvalueMask(CStr gtW, CStr &maskDir, vecS &des, CStr resFile, double betaSqr, bool alertNul, CStr suffix, CStr title)
{
vecS namesNS;
string gtDir, gtExt;
int imgNum = CmFile::GetNamesNE(gtW, namesNS, gtDir, gtExt);
int methodNum = (int)des.size();
vecD pr(methodNum), rec(methodNum), count(methodNum), fm(methodNum);
vecD intUnio(methodNum), mae(methodNum);
for (int i = 0; i < imgNum; i++){
Mat truM = imread(gtDir + namesNS[i] + gtExt, CV_LOAD_IMAGE_GRAYSCALE);
for (int m = 0; m < methodNum; m++) {
string mapName = maskDir + namesNS[i] + "_" + des[m];
mapName += suffix.empty() ? ".png" : "_" + suffix + ".png";
Mat res = imread(mapName, CV_LOAD_IMAGE_GRAYSCALE);
if (truM.data == NULL || res.data == NULL || truM.size != res.size){
if (alertNul)
printf("Truth(%d, %d), Res(%d, %d): %s\n", truM.cols, truM.rows, res.cols, res.rows, _S(mapName));
continue;
}
compare(truM, 128, truM, CMP_GE);
compare(res, 128, res, CMP_GE);
Mat commMat, unionMat, diff1f;
bitwise_and(truM, res, commMat);
bitwise_or(truM, res, unionMat);
double commV = sum(commMat).val[0];
double p = commV/(sum(res).val[0] + EPS);
double r = commV/(sum(truM).val[0] + EPS);
pr[m] += p;
rec[m] += r;
intUnio[m] += commV / (sum(unionMat).val[0] + EPS);
absdiff(truM, res, diff1f);
mae[m] += sum(diff1f).val[0]/(diff1f.rows * diff1f.cols * 255);
count[m]++;
}
}
for (int m = 0; m < methodNum; m++){
pr[m] /= count[m], rec[m] /= count[m];
fm[m] = (1 + betaSqr) * pr[m] * rec[m] / (betaSqr * pr[m] + rec[m] + EPS);
intUnio[m] /= count[m];
mae[m] /= count[m];
}
#ifndef fopen_s
FILE *f = fopen(_S(resFile), "a");
#else
FILE *f;
fopen_s(&f, _S(resFile), "a");
#endif
if (f != NULL){
fprintf(f, "\n%%%%\n");
CmEvaluation::PrintVector(f, pr, "PrecisionMask" + suffix);
CmEvaluation::PrintVector(f, rec, "RecallMask" + suffix);
CmEvaluation::PrintVector(f, fm, "FMeasureMask" + suffix);
CmEvaluation::PrintVector(f, intUnio, "IntUnion" + suffix);
CmEvaluation::PrintVector(f, intUnio, "MAE" + suffix);
fprintf(f, "bar([%s]');\ngrid on\n", _S("PrecisionMask" + suffix + "; RecallMask" + suffix + "; FMeasureMask" + suffix + "; IntUnion" + suffix));
fprintf(f, "title('%s');\naxis([0 %d 0.8 1]);\nmethodLabels = { '%s'", _S(title), des.size() + 1, _S(des[0]));
for (size_t i = 1; i < des.size(); i++)
fprintf(f, ", '%s'", _S(des[i]));
fprintf(f, " };\nlegend('Precision', 'Recall', 'FMeasure', 'IntUnion');\n");
fprintf(f, "xticklabel_rotate([1:%d], 90, methodLabels, 'interpreter', 'none');\n", des.size());
fclose(f);
}
if (des.size() == 1)
printf("Precision = %g, recall = %g, F-Measure = %g, intUnion = %g, mae = %g\n", pr[0], rec[0], fm[0], intUnio[0], mae[0]);
}
