void Objectness::predictBBoxSII(ValStructVec<float, Vec4i> &valBoxes, const vecI &sz)
{
int numI = valBoxes.size();
for (int i = 0; i < numI; i++) {
const float* svmIIw = _svmReW1f.ptr<float>(sz[i]);
valBoxes(i) = valBoxes(i) * svmIIw[0] + svmIIw[1];
}
valBoxes.sort();
}
void RunObjectness(CStr &resName, double base, int W, int NSS, int numPerSz)
{
srand((unsigned int) time(NULL));
// DataSetVOC voc2007("/Datasets/VOC2007/");
// voc2007.loadAnnotations();
//voc2007.loadDataGenericOverCls();
// printf("Dataset:`%s' with %d training and %d testing\n", _S(voc2007.wkDir), voc2007.trainNum, voc2007.testNum);
// printf("%s Base = %g, W = %d, NSS = %d, perSz = %d\n", _S(resName), base, W, NSS, numPerSz);
Objectness objNess(base, W, NSS);
objNess.loadTrainedModel("/Datasets/VOC2007/model");
// printf("Model\n");
for(int i = 1; i < 10; i++)
{
string filename = format("/Datasets/VOC2007/JPEGImages/00001%d.jpg", i);
printf("%s\n", filename.c_str());
Mat img3u = imread(filename);
ValStructVec<float, Vec4i> boxesTests;
boxesTests.reserve(10000);
objNess.getObjBndBoxes(img3u, boxesTests, numPerSz);
int xmin, ymin, xmax, ymax;
int num = boxesTests.size();
printf(" %d\n", num);
for (int j = 0; j < num; j++){
Vec4i bb = boxesTests[j];
xmin = bb[0];
ymin = bb[1];
xmax = bb[2];
ymax = bb[3];
// printf(" (%d, %d) -> (%d, %d)\n", xmin, ymin, xmax, ymax);
rectangle(img3u, cvPoint(xmin, ymin), cvPoint(xmax, ymax), cvScalar(0, 0, 255), 3);
}
imshow(filename, img3u);
waitKey(0);
}
// Objectness objNess(voc2007, base, W, NSS);
// vector<vector<Vec4i> > boxesTests;
// objNess.getObjBndBoxesForTests(boxesTests, 250);
// objNess.getObjBndBoxesForTestsFast(boxesTests, numPerSz);
//objNess.getRandomBoxes(boxesTests);
//objNess.evaluatePerClassRecall(boxesTests, resName, 1000);
//objNess.illuTestReults(boxesTests);
//objNess.evaluatePAMI12();
//objNess.evaluateIJCV13();
}
void Objectness::predictBBoxSI(CMat &img3u, ValStructVec<float, Vec4i> &valBoxes, vecI &sz, int NUM_WIN_PSZ, bool fast)
{
const int numSz = _svmSzIdxs.size();
const int imgW = img3u.cols, imgH = img3u.rows;
valBoxes.reserve(10000);
sz.clear();
sz.reserve(10000);
for (int ir = numSz - 1; ir >= 0; ir--) {
int r = _svmSzIdxs[ir];
int height = cvRound(pow(_base, r/_numT + _minT)), width = cvRound(pow(_base, r%_numT + _minT));
if (height > imgH * _base || width > imgW * _base)
continue;
height = min(height, imgH), width = min(width, imgW);
Mat im3u, matchCost1f, mag1u;
resize(img3u, im3u, Size(cvRound(_W*imgW*1.0/width), cvRound(_W*imgH*1.0/height)));
gradientMag(im3u, mag1u);
matchCost1f = _tigF.matchTemplate(mag1u);
ValStructVec<float, Point> matchCost;
nonMaxSup(matchCost1f, matchCost, _NSS, NUM_WIN_PSZ, fast);
// Find true locations and match values
double ratioX = width/_W, ratioY = height/_W;
int iMax = min(matchCost.size(), NUM_WIN_PSZ);
for (int i = 0; i < iMax; i++) {
float mVal = matchCost(i);
Point pnt = matchCost[i];
Vec4i box(cvRound(pnt.x * ratioX), cvRound(pnt.y*ratioY));
box[2] = cvRound(min(box[0] + width, imgW));
box[3] = cvRound(min(box[1] + height, imgH));
box[0] ++;
box[1] ++;
valBoxes.pushBack(mVal, box);
sz.push_back(ir);
}
}
//exit(0);
}
bool GenericObjectDetector::GetObjectsFromBing(const cv::Mat& cimg, vector<ImgWin>& detWins, int winnum, bool showres)
{
if( !isBingInitialized )
{
cerr<<"Bing is not initialized."<<endl;
return false;
}
ValStructVec<float, Vec4i> boxes;
bingObjectness->getObjBndBoxes(cimg, boxes);
int validwinnum = MIN(winnum, boxes.size());
detWins.clear();
detWins.resize(validwinnum);
for (int i=0; i<validwinnum; i++)
{
detWins[i] = ImgWin( boxes[i].val[0], boxes[i].val[1], boxes[i].val[2]-boxes[i].val[0], boxes[i].val[3]-boxes[i].val[1] );
detWins[i].score = 1;
}
return true;
}
void Objectness::nonMaxSup(CMat &matchCost1f, ValStructVec<float, Point> &matchCost, int NSS, int maxPoint, bool fast)
{
const int _h = matchCost1f.rows, _w = matchCost1f.cols;
Mat isMax1u = Mat::ones(_h, _w, CV_8U), costSmooth1f;
ValStructVec<float, Point> valPnt;
matchCost.reserve(_h * _w);
valPnt.reserve(_h * _w);
if (fast) {
blur(matchCost1f, costSmooth1f, Size(3, 3));
for (int r = 0; r < _h; r++) {
const float* d = matchCost1f.ptr<float>(r);
const float* ds = costSmooth1f.ptr<float>(r);
for (int c = 0; c < _w; c++)
if (d[c] >= ds[c])
valPnt.pushBack(d[c], Point(c, r));
}
}
else {
for (int r = 0; r < _h; r++) {
const float* d = matchCost1f.ptr<float>(r);
for (int c = 0; c < _w; c++)
valPnt.pushBack(d[c], Point(c, r));
}
}
valPnt.sort();
for (int i = 0; i < valPnt.size(); i++) {
Point &pnt = valPnt[i];
if (isMax1u.at<byte>(pnt)) {
matchCost.pushBack(valPnt(i), pnt);
for (int dy = -NSS; dy <= NSS; dy++) for (int dx = -NSS; dx <= NSS; dx++) {
Point neighbor = pnt + Point(dx, dy);
if (!CHK_IND(neighbor))
continue;
isMax1u.at<byte>(neighbor) = false;
}
}
if (matchCost.size() >= maxPoint)
return;
}
}
void bingQdpmRocTest(vector<string> &dirs,
int windowLimit = -1, double timeLimitMs = -1, float ratioThreshold = -1)
{
size_t imageCount = 0;
size_t personCount = 0;
size_t matchCount = 0;
vector<ScoreTp> pScores;
TickMeter tm;
vector<std::string>::const_iterator it = dirs.begin();
char buf[512];
for (; it != dirs.end(); it++) {
string dir = *it;
DataSetVOC voc(dir, true, true);
voc.loadAnnotations();
const size_t testNum = voc.testSet.size();
const char *imgPath =_S(voc.imgPathW);
// Objectness
double base = 2;
double intUionThr = 0.5;
int W = 8;
int NSS = 2;
#ifdef WINDOW_GUESS
Objectness objNess(voc, base, intUionThr, W, NSS);
objNess.loadTrainedModel(TRAIN_MODEL);
#endif
// LSVM DPM
string dpmPersonModel = "../ExtraData/latentsvmXml/person.xml";
vector<string> models;
models.push_back(dpmPersonModel);
QUniLsvmDetector detector(models);
float overlapThreshold = 0.2f;
if (ratioThreshold > 0)
detector.setRatioThreshold(ratioThreshold);
printf("%d: \n", testNum);
for (int i = 0; i < testNum; i++) {
const vector<Vec4i> &boxesGT = voc.gtTestBoxes[i];
const size_t gtNumCrnt = boxesGT.size();
if (gtNumCrnt <= 0)
continue;
imageCount++;
personCount += gtNumCrnt;
Mat image = imread(format(imgPath, _S(voc.testSet[i])));
if (image.ptr() == NULL) {
fprintf(stderr, "No JPG Image !\n");
exit(1);
}
int numPerSz = 130;
ValStructVec<float, Vec4i> boxes;
double preObj = tm.getTimeMilli();
double objTime = 0.;
#ifdef WINDOW_GUESS // window guess
tm.start();
objNess.getObjBndBoxes(image, boxes, numPerSz);
tm.stop();
objTime = tm.getTimeMilli() - preObj;
#endif
double localTimeLimitMs = timeLimitMs;
if (timeLimitMs > 0) {
localTimeLimitMs -= objTime;
if (localTimeLimitMs < 0.)
localTimeLimitMs = 0.;
}
vector<QRect> searchBoxes;
if (windowLimit > 0) {
for (int j = 0; j < (int)boxes.size() && j < windowLimit; j++) {
const Vec4i &bb = boxes[j];
QRect rt(bb[0], bb[1], bb[2], bb[3]);
searchBoxes.push_back(rt);
}
} else {
for (int j = 0; j < (int)boxes.size(); j++) {
const Vec4i &bb = boxes[j];
QRect rt(bb[0], bb[1], bb[2], bb[3]);
searchBoxes.push_back(rt);
}
}
tm.start();
detector.setup(image, overlapThreshold, localTimeLimitMs);
tm.stop();
vector<FeatureMapCoord> ftrMapCoords;
#ifdef WINDOW_GUESS
detector.cvtBox2FtrMapCoord(&searchBoxes, &ftrMapCoords);
#else
detector.genFullFtrMapCoord(&ftrMapCoords);
preObj = tm.getTimeMilli();
tm.start();
#ifdef SHUFFLE_WINDOW
random_shuffle(ftrMapCoords.begin(), ftrMapCoords.end());
#endif
tm.stop();
double randGenTime = tm.getTimeMilli() - preObj;
if (localTimeLimitMs > 0 && localTimeLimitMs - preObj >= 0.)
detector.setTimeLimit(localTimeLimitMs - preObj);
#endif
vector<QUniLsvmDetector::ObjectDetection> detections;
vector<vector<FeatureMapCoord> *> fmcss;
fmcss.push_back(&ftrMapCoords);
tm.start();
detector.detect(detections, fmcss);
tm.stop();
vector<DetectedInfo> di(detections.size());
vector<int> gtIdx(gtNumCrnt, -1);
int detectCount = 0;
for (size_t j = 0; j < detections.size(); j++) {
const QUniLsvmDetector::ObjectDetection& od = detections[j];
if (od.score < RECOMMENDABLE_THRESHOLD)
continue;
detectCount++;
Vec4i bb(od.rect.x + 1, od.rect.y + 1, od.rect.x + od.rect.width, od.rect.y + od.rect.height);
// score matchScore for the ROC curve
double maxMatchScore = 0;
int maxMatchId = -1;
for (int k = 0; k < gtNumCrnt; k++) {
double matchScore = DataSetVOC::interUnio(bb, boxesGT[k]);
if (matchScore > maxMatchScore) {
maxMatchScore = matchScore;
maxMatchId = k;
}
}
uchar match = maxMatchScore > 0.5 ? 1 : 0;
if (match) {
int preDetectedIdx = gtIdx[maxMatchId];
if (preDetectedIdx >= 0) {
if (maxMatchScore > di[preDetectedIdx].matchScore) {
di[preDetectedIdx].matched = false;
gtIdx[maxMatchId] = int(j);
di[j].matchScore = maxMatchScore;
di[j].matched = true;
}
} else {
gtIdx[maxMatchId] = int(j);
di[j].matchScore = maxMatchScore;
di[j].matched = true;
}
}
#ifdef SAVE_IMAGE_RESULT
// save the result image
char buf[256];
sprintf(buf, "%2f", od.score);
Point pt(max((bb[2] + bb[0] - 85) / 2, 0), (bb[1] + bb[3]) / 2);
putText(image, buf, pt, FONT_HERSHEY_SIMPLEX, 0.5, Scalar::all(255), 3, CV_AA);
putText(image, buf, pt, FONT_HERSHEY_SIMPLEX, 0.5, Scalar::all(0), 1, CV_AA);
rectangle(image, od.rect, cv::Scalar(0, 255, 0), 2);
#endif
}
for (size_t j = 0; j < detectCount; j++) { // detections are sorted in descending order
const QUniLsvmDetector::ObjectDetection& od = detections[j];
if (di[j].matched)
matchCount++;
pScores.push_back(ScoreTp(od.score, di[j].matched));
}
#ifdef SAVE_IMAGE_RESULT
imwrite((voc.testSet[i] + "_DpmResult.png").c_str(), image);
#endif
printf("%d ", i + 1);
}
printf("\n");
}
printf("BingQdpmRocTest time = %f sec\n", tm.getTimeSec());
printf("GT %d, Matched %d/%d \n", personCount, matchCount, pScores.size());
// Calculate log-average miss rate
stable_sort(begin(pScores), end(pScores),
[](const ScoreTp &p1, const ScoreTp &p2) { return p1.first > p2.first; });
vector<float> fp(pScores.size());
for (size_t i = 0; i < fp.size(); i++)
fp[i] = !pScores[i].second;
vector<float> tp(pScores.size());
tp[0] = pScores[0].second;
for (size_t i = 1; i < tp.size(); i++)
tp[i] = tp[i - 1] + pScores[i].second;
for (size_t i = 0; i < tp.size(); i++)
tp[i] /= personCount;
for (size_t i = 1; i < fp.size(); i++)
fp[i] += fp[i - 1];
for (size_t i = 0; i < fp.size(); i++)
fp[i] /= imageCount;
sprintf(buf, "%s%03d_%03.fms_6137gt_%04dtp_%04ddt_%.0fs.m",
METHOD_NAME, max(windowLimit, 0), timeLimitMs, matchCount, pScores.size(), tm.getTimeSec());
FILE *matlabFile = fopen(buf, "w");
printVector(matlabFile, tp, "tp");
printVector(matlabFile, fp, "fp");
char *matlabContent = "tp = tp'; fp = fp';\n"
"addpath(genpath('d:/81_dataset/03_Caltech/piotr_toolbox/')); savepath;\n"
"xs1=[-inf; fp];\n"
"ys1=[0; tp];\n"
"ref=10.^(-2:.25:0);\n"
"lims=[3.1e-4 1e1 .5 1];\n"
"m=length(ref);\n"
"for i=1:m\n"
"\tj=find(xs1<=ref(i));\n"
"\tmiss(i)=ys1(j(end));\n"
"end\n"
"miss=exp(mean(log(max(1e-10,1-miss))));\n"
"show=figure();\n"
"plotRoc([fp tp],'logx',1,'logy',1,'xLbl','fppi',...\n"
"\t'lims',lims,'color','g','smooth',1,'fpTarget',ref);\n"
"title(sprintf('log-average miss rate = %.2f%%',miss*100));\n"
"savefig(['MORU' 'Roc'],show,'png');\n";
fwrite(matlabContent, strlen(matlabContent), 1, matlabFile);
fclose(matlabFile);
}
// BING test code
void RunFaceProposal(int W, int NSS, int numPerSz)
{
// configuration
string imgPath = "Z:\\User\\wuxiang\\data\\face_detection\\FDDB\\originalPics";
string listPath = "Z:\\User\\wuxiang\\data\\face_detection\\FDDB\\FDDB_list.txt";
string frPath = "Z:\\Temp\\CNN_Face_Detection\\fr\\man";
string modelPath = "D:\\svn\\Algorithm\\wuxiang\\Code\\C\\BING\\model\\ObjNessB2W8MAXBGR.wS1";
string savePath = "D:\\BING\\Proposal\\fr";
#if ILLUSTRATE == 1
string saveErrorPath = "D:\\BING\\Proposal\\error";
string saveImgPath = "D:\\BING\\Proposal\\img";
FILE *list = fopen((saveErrorPath+"\\"+"list.txt").c_str(), "wt");
#endif
vector<vector<Vec4i>> frsImgs;
char fr[_MAX_PATH];
// load image
DataSet dataSet(imgPath, listPath, frPath);
dataSet.loadAnnotations();
// predict
ObjectnessTest objectNessTest(dataSet, modelPath, W, NSS);
objectNessTest.loadTrainedModel(modelPath);
objectNessTest.getFaceProposalsForImgsFast(frsImgs, numPerSz);
// save
for (int i = 0; i < frsImgs.size(); i++)
{
//cout << dataSet.imgPathFr[i].c_str() << "is saving" <<endl;
sprintf(fr, "%s/%s", savePath.c_str(), dataSet.imgPathFr[i].c_str());
create_directory_from_filename(fr);
FILE *fp = fopen(fr, "wt");
fprintf(fp, "%d\n", frsImgs[i].size());
for (int j = 0; j < frsImgs[i].size(); j++)
{
Vec4i box = frsImgs[i][j];
fprintf(fp, "%d\t%d\t%d\t%d\n", box[0], box[1], box[2], box[3]);
}
fclose(fp);
// illustrate
#if ILLUSTRATE == 1
ValStructVec<float, Vec4i> box;
box.reserve(frsImgs[i].size());
string imgSavePath = saveImgPath + "\\" + dataSet.imgPathName[i] + "_Match.jpg";
create_directory_from_filename(imgSavePath.c_str());
objectNessTest.illuTestReults(imgPath + "\\" + dataSet.imgPathName[i], imgSavePath, dataSet.imgFr[i], frsImgs[i], box);
if (box.size() == 0)
continue;
char error[_MAX_PATH];
sprintf(error, "%s/%s", saveErrorPath.c_str(), dataSet.imgPathFr[i].c_str());
create_directory_from_filename(error);
FILE *errorFp = fopen(error, "wt");
fprintf(errorFp, "%d\n", box.size());
for (int j = 0; j < box.size(); j++)
{
Vec4i out = box[j];
fprintf(errorFp, "%d\t%d\t%d\t%d\t%f\n", out[0], out[1], out[2], out[3], box(j));
}
fclose(errorFp);
fprintf(list, "%s\n", dataSet.imgPathName[i].c_str());
#endif
}
#if ILLUSTRATE == 1
fclose(list);
#endif
}
// cnn face detection test
void cnnFaceDetectionTest()
{
// configuration
string imgPath = "Z:\\User\\wuxiang\\data\\face_detection\\FDDB\\originalPics";
string listPath = "Z:\\User\\wuxiang\\data\\face_detection\\FDDB\\FDDB_list.txt";
string frPath = "Z:\\Temp\\CNN_Face_Detection\\fr\\man";
string bingModelPath = "D:\\svn\\Algorithm\\wuxiang\\Code\\C\\BING\\model\\ObjNessB2W8MAXBGR.wS1";
string cnnModelPath = "D:\\svn\\Algorithm\\wuxiang\\Code\\C\\BING\\model\\36_detection.bin";
string savePath = "D:\\BING\\36_net\\fr";
const int W = 8, NSS = 2, numPerSz = 150;
const int netSize = 36, netProbLayer = 7;
const float thr = 0.4;
vector<Vec4i> boxTestStageI;
ValStructVec<float, Vec4i> boxTestStageII;
ValStructVec<float, Vec4i> box;
char fr[_MAX_PATH];
// load image
DataSet dataSet(imgPath, listPath, frPath);
dataSet.loadAnnotations();
// predict
ObjectnessTest objectNessTest(dataSet, bingModelPath, W, NSS);
objectNessTest.loadTrainedModel(bingModelPath);
CnnFace cnn(cnnModelPath, netSize, netProbLayer);
cnn.loadTrainedModel();
#pragma openmp parallel for
for(int i = 0; i < dataSet.testNum; i++)
{
// face detection Stage I: get face region proposal
boxTestStageI.clear();
printf("Process %d images..\n", i);
CmTimer tm("Predict");
tm.Start();
Mat img = imread(dataSet.imgPath + "\\" + dataSet.imgPathName[i]);
boxTestStageI.reserve(10000);
objectNessTest.getFaceProposaksForPerImgFast(img, boxTestStageI, numPerSz);
// cnn
cnn.getFaceDetectionPerImg(img, boxTestStageI, boxTestStageII, thr);
// nms
cnn.nonMaxSup(boxTestStageII, box, 0.2);
tm.Stop();
printf("Predicting an image is %gs\n", tm.TimeInSeconds());
// save
sprintf(fr, "%s/%s", savePath.c_str(), dataSet.imgPathFr[i].c_str());
create_directory_from_filename(fr);
FILE *fp = fopen(fr, "wt");
fprintf(fp, "%d\n", box.size());
for (int j = 0; j < box.size(); j++)
{
Vec4i &out = box[j];
fprintf(fp, "%d\t%d\t%d\t%d\t%f\n", out[0], out[1], out[2], out[3], box(j));
}
fclose(fp);
img.~Mat();
}
}
void Objectness::trainStateII(int numPerSz)
{
loadTrainedModel();
const int NUM_TRAIN = _voc.trainNum;
vector<vecI> SZ(NUM_TRAIN), Y(NUM_TRAIN);
vector<vecF> VAL(NUM_TRAIN);
#pragma omp parallel for
for (int i = 0; i < _voc.trainNum; i++) {
const vector<Vec4i> &bbgts = _voc.gtTrainBoxes[i];
ValStructVec<float, Vec4i> valBoxes;
vecI &sz = SZ[i], &y = Y[i];
vecF &val = VAL[i];
CStr imgPath = format(_S(_voc.imgPathW), _S(_voc.trainSet[i]));
predictBBoxSI(imread(imgPath), valBoxes, sz, numPerSz, false);
const int num = valBoxes.size();
CV_Assert(sz.size() == num);
y.resize(num), val.resize(num);
for (int j = 0; j < num; j++) {
Vec4i bb = valBoxes[j];
val[j] = valBoxes(j);
y[j] = maxIntUnion(bb, bbgts) >= 0.5 ? 1 : -1;
}
}
const int NUM_SZ = _svmSzIdxs.size();
const int maxTrainNum = 100000;
vector<vecM> rXP(NUM_SZ), rXN(NUM_SZ);
for (int r = 0; r < NUM_SZ; r++) {
rXP[r].reserve(maxTrainNum);
rXN[r].reserve(1000000);
}
for (int i = 0; i < NUM_TRAIN; i++) {
const vecI &sz = SZ[i], &y = Y[i];
vecF &val = VAL[i];
int num = sz.size();
for (int j = 0; j < num; j++) {
int r = sz[j];
CV_Assert(r >= 0 && r < NUM_SZ);
if (y[j] == 1)
rXP[r].push_back(Mat(1, 1, CV_32F, &val[j]));
else
rXN[r].push_back(Mat(1, 1, CV_32F, &val[j]));
}
}
Mat wMat(NUM_SZ, 2, CV_32F);
for (int i = 0; i < NUM_SZ; i++) {
const vecM &xP = rXP[i], &xN = rXN[i];
if (xP.size() < 10 || xN.size() < 10)
printf("Warning %s:%d not enough training sample for r[%d] = %d. P = %d, N = %d\n", __FILE__, __LINE__, i, _svmSzIdxs[i], xP.size(), xN.size());
for (size_t k = 0; k < xP.size(); k++)
CV_Assert(xP[k].size() == Size(1, 1) && xP[k].type() == CV_32F);
Mat wr = trainSVM(xP, xN, L1R_L2LOSS_SVC, 100, 1);
CV_Assert(wr.size() == Size(2, 1));
wr.copyTo(wMat.row(i));
}
matWrite(_modelName + ".wS2", wMat);
_svmReW1f = wMat;
}
