int TrackerTLDImpl::Pexpert::additionalExamples(std::vector<Mat_<uchar> >& examplesForModel, std::vector<Mat_<uchar> >& examplesForEnsemble)
{
examplesForModel.clear();
examplesForEnsemble.clear();
examplesForModel.reserve(100);
examplesForEnsemble.reserve(100);
std::vector<Rect2d> closest, scanGrid;
Mat scaledImg, blurredImg;
double scale = scaleAndBlur(img_, cvRound(log(1.0 * resultBox_.width / (initSize_.width)) / log(SCALE_STEP)),
scaledImg, blurredImg, GaussBlurKernelSize, SCALE_STEP);
TLDDetector::generateScanGrid(img_.rows, img_.cols, initSize_, scanGrid);
getClosestN(scanGrid, Rect2d(resultBox_.x / scale, resultBox_.y / scale, resultBox_.width / scale, resultBox_.height / scale), 10, closest);
for( int i = 0; i < (int)closest.size(); i++ )
{
for( int j = 0; j < 10; j++ )
{
Point2f center;
Size2f size;
Mat_<uchar> standardPatch(STANDARD_PATCH_SIZE, STANDARD_PATCH_SIZE), blurredPatch(initSize_);
center.x = (float)(closest[i].x + closest[i].width * (0.5 + rng.uniform(-0.01, 0.01)));
center.y = (float)(closest[i].y + closest[i].height * (0.5 + rng.uniform(-0.01, 0.01)));
size.width = (float)(closest[i].width * rng.uniform((double)0.99, (double)1.01));
size.height = (float)(closest[i].height * rng.uniform((double)0.99, (double)1.01));
float angle = (float)rng.uniform(-5.0, 5.0);
for( int y = 0; y < standardPatch.rows; y++ )
{
for( int x = 0; x < standardPatch.cols; x++ )
{
standardPatch(x, y) += (uchar)rng.gaussian(5.0);
}
}
#ifdef BLUR_AS_VADIM
GaussianBlur(standardPatch, blurredPatch, GaussBlurKernelSize, 0.0);
resize(blurredPatch, blurredPatch, initSize_);
#else
resample(blurredImg, RotatedRect(center, size, angle), blurredPatch);
#endif
resample(scaledImg, RotatedRect(center, size, angle), standardPatch);
examplesForModel.push_back(standardPatch);
examplesForEnsemble.push_back(blurredPatch);
}
}
return 0;
}
bool TrackerTLDImpl::updateImpl(const Mat& image, Rect2d& boundingBox)
{
Mat image_gray, image_blurred, imageForDetector;
cvtColor( image, image_gray, COLOR_BGR2GRAY );
double scale = data->getScale();
if( scale > 1.0 )
resize(image_gray, imageForDetector, Size(cvRound(image.cols*scale), cvRound(image.rows*scale)), 0, 0, DOWNSCALE_MODE);
else
imageForDetector = image_gray;
GaussianBlur(imageForDetector, image_blurred, GaussBlurKernelSize, 0.0);
TrackerTLDModel* tldModel = ((TrackerTLDModel*)static_cast<TrackerModel*>(model));
data->frameNum++;
Mat_<uchar> standardPatch(STANDARD_PATCH_SIZE, STANDARD_PATCH_SIZE);
std::vector<TLDDetector::LabeledPatch> detectorResults;
//best overlap around 92%
std::vector<Rect2d> candidates;
std::vector<double> candidatesRes;
bool trackerNeedsReInit = false;
bool DETECT_FLG = false;
for( int i = 0; i < 2; i++ )
{
Rect2d tmpCandid = boundingBox;
if (i == 1)
{
#ifdef HAVE_OPENCL
if (ocl::haveOpenCL())
DETECT_FLG = tldModel->detector->ocl_detect(imageForDetector, image_blurred, tmpCandid, detectorResults, tldModel->getMinSize());
else
#endif
DETECT_FLG = tldModel->detector->detect(imageForDetector, image_blurred, tmpCandid, detectorResults, tldModel->getMinSize());
}
if( ( (i == 0) && !data->failedLastTime && trackerProxy->update(image, tmpCandid) ) || ( DETECT_FLG))
{
candidates.push_back(tmpCandid);
if( i == 0 )
resample(image_gray, tmpCandid, standardPatch);
else
resample(imageForDetector, tmpCandid, standardPatch);
candidatesRes.push_back(tldModel->detector->Sc(standardPatch));
}
else
{
if( i == 0 )
trackerNeedsReInit = true;
}
}
std::vector<double>::iterator it = std::max_element(candidatesRes.begin(), candidatesRes.end());
if( it == candidatesRes.end() )
{
data->confident = false;
data->failedLastTime = true;
return false;
}
else
{
boundingBox = candidates[it - candidatesRes.begin()];
data->failedLastTime = false;
if( trackerNeedsReInit || it != candidatesRes.begin() )
trackerProxy->init(image, boundingBox);
}
#if 1
if( it != candidatesRes.end() )
resample(imageForDetector, candidates[it - candidatesRes.begin()], standardPatch);
#endif
if( *it > CORE_THRESHOLD )
data->confident = true;
if( data->confident )
{
Pexpert pExpert(imageForDetector, image_blurred, boundingBox, tldModel->detector, params, data->getMinSize());
Nexpert nExpert(imageForDetector, boundingBox, tldModel->detector, params);
std::vector<Mat_<uchar> > examplesForModel, examplesForEnsemble;
examplesForModel.reserve(100);
examplesForEnsemble.reserve(100);
int negRelabeled = 0;
for( int i = 0; i < (int)detectorResults.size(); i++ )
{
bool expertResult;
if( detectorResults[i].isObject )
{
expertResult = nExpert(detectorResults[i].rect);
if( expertResult != detectorResults[i].isObject )
negRelabeled++;
}
else
{
expertResult = pExpert(detectorResults[i].rect);
}
detectorResults[i].shouldBeIntegrated = detectorResults[i].shouldBeIntegrated || (detectorResults[i].isObject != expertResult);
detectorResults[i].isObject = expertResult;
}
tldModel->integrateRelabeled(imageForDetector, image_blurred, detectorResults);
pExpert.additionalExamples(examplesForModel, examplesForEnsemble);
#ifdef HAVE_OPENCL
if (ocl::haveOpenCL())
tldModel->ocl_integrateAdditional(examplesForModel, examplesForEnsemble, true);
else
#endif
tldModel->integrateAdditional(examplesForModel, examplesForEnsemble, true);
examplesForModel.clear();
examplesForEnsemble.clear();
nExpert.additionalExamples(examplesForModel, examplesForEnsemble);
#ifdef HAVE_OPENCL
if (ocl::haveOpenCL())
tldModel->ocl_integrateAdditional(examplesForModel, examplesForEnsemble, false);
else
#endif
tldModel->integrateAdditional(examplesForModel, examplesForEnsemble, false);
}
else
{
#ifdef CLOSED_LOOP
tldModel->integrateRelabeled(imageForDetector, image_blurred, detectorResults);
#endif
}
return true;
}
bool TLDDetector::ocl_detect(const Mat& img, const Mat& imgBlurred, Rect2d& res, std::vector<LabeledPatch>& patches, Size initSize)
{
patches.clear();
Mat_<uchar> standardPatch(STANDARD_PATCH_SIZE, STANDARD_PATCH_SIZE);
Mat tmp;
int dx = initSize.width / 10, dy = initSize.height / 10;
Size2d size = img.size();
double scale = 1.0;
int npos = 0, nneg = 0;
double maxSc = -5.0;
Rect2d maxScRect;
int scaleID;
std::vector <Mat> resized_imgs, blurred_imgs;
std::vector <Point> varBuffer, ensBuffer;
std::vector <int> varScaleIDs, ensScaleIDs;
//Detection part
//Generate windows and filter by variance
scaleID = 0;
resized_imgs.push_back(img);
blurred_imgs.push_back(imgBlurred);
do
{
Mat_<double> intImgP, intImgP2;
computeIntegralImages(resized_imgs[scaleID], intImgP, intImgP2);
for (int i = 0, imax = cvFloor((0.0 + resized_imgs[scaleID].cols - initSize.width) / dx); i < imax; i++)
{
for (int j = 0, jmax = cvFloor((0.0 + resized_imgs[scaleID].rows - initSize.height) / dy); j < jmax; j++)
{
if (!patchVariance(intImgP, intImgP2, originalVariancePtr, Point(dx * i, dy * j), initSize))
continue;
varBuffer.push_back(Point(dx * i, dy * j));
varScaleIDs.push_back(scaleID);
}
}
scaleID++;
size.width /= SCALE_STEP;
size.height /= SCALE_STEP;
scale *= SCALE_STEP;
resize(img, tmp, size, 0, 0, DOWNSCALE_MODE);
resized_imgs.push_back(tmp);
GaussianBlur(resized_imgs[scaleID], tmp, GaussBlurKernelSize, 0.0f);
blurred_imgs.push_back(tmp);
} while (size.width >= initSize.width && size.height >= initSize.height);
//Encsemble classification
for (int i = 0; i < (int)varBuffer.size(); i++)
{
prepareClassifiers((int)blurred_imgs[varScaleIDs[i]].step[0]);
if (ensembleClassifierNum(&blurred_imgs[varScaleIDs[i]].at<uchar>(varBuffer[i].y, varBuffer[i].x)) <= ENSEMBLE_THRESHOLD)
continue;
ensBuffer.push_back(varBuffer[i]);
ensScaleIDs.push_back(varScaleIDs[i]);
}
//NN classification
//Prepare batch of patches
int numOfPatches = (int)ensBuffer.size();
Mat_<uchar> stdPatches(numOfPatches, 225);
double *resultSr = new double[numOfPatches];
double *resultSc = new double[numOfPatches];
uchar *patchesData = stdPatches.data;
for (int i = 0; i < (int)ensBuffer.size(); i++)
{
resample(resized_imgs[ensScaleIDs[i]], Rect2d(ensBuffer[i], initSize), standardPatch);
uchar *stdPatchData = standardPatch.data;
for (int j = 0; j < 225; j++)
patchesData[225*i+j] = stdPatchData[j];
}
//Calculate Sr and Sc batches
ocl_batchSrSc(stdPatches, resultSr, resultSc, numOfPatches);
for (int i = 0; i < (int)ensBuffer.size(); i++)
{
LabeledPatch labPatch;
standardPatch.data = &stdPatches.data[225 * i];
double curScale = pow(SCALE_STEP, ensScaleIDs[i]);
labPatch.rect = Rect2d(ensBuffer[i].x*curScale, ensBuffer[i].y*curScale, initSize.width * curScale, initSize.height * curScale);
double srValue, scValue;
srValue = resultSr[i];
////To fix: Check the paper, probably this cause wrong learning
//
labPatch.isObject = srValue > THETA_NN;
labPatch.shouldBeIntegrated = abs(srValue - THETA_NN) < 0.1;
patches.push_back(labPatch);
//
if (!labPatch.isObject)
{
nneg++;
continue;
}
else
{
npos++;
}
scValue = resultSc[i];
if (scValue > maxSc)
{
maxSc = scValue;
maxScRect = labPatch.rect;
}
}
if (maxSc < 0)
return false;
res = maxScRect;
return true;
}
