int TrackerTLDImpl::Pexpert::additionalExamples(std::vector<Mat_<uchar> >& examplesForModel, std::vector<Mat_<uchar> >& examplesForEnsemble) { examplesForModel.clear(); examplesForEnsemble.clear(); examplesForModel.reserve(100); examplesForEnsemble.reserve(100); std::vector<Rect2d> closest, scanGrid; Mat scaledImg, blurredImg; double scale = scaleAndBlur(img_, cvRound(log(1.0 * resultBox_.width / (initSize_.width)) / log(SCALE_STEP)), scaledImg, blurredImg, GaussBlurKernelSize, SCALE_STEP); TLDDetector::generateScanGrid(img_.rows, img_.cols, initSize_, scanGrid); getClosestN(scanGrid, Rect2d(resultBox_.x / scale, resultBox_.y / scale, resultBox_.width / scale, resultBox_.height / scale), 10, closest); for( int i = 0; i < (int)closest.size(); i++ ) { for( int j = 0; j < 10; j++ ) { Point2f center; Size2f size; Mat_<uchar> standardPatch(STANDARD_PATCH_SIZE, STANDARD_PATCH_SIZE), blurredPatch(initSize_); center.x = (float)(closest[i].x + closest[i].width * (0.5 + rng.uniform(-0.01, 0.01))); center.y = (float)(closest[i].y + closest[i].height * (0.5 + rng.uniform(-0.01, 0.01))); size.width = (float)(closest[i].width * rng.uniform((double)0.99, (double)1.01)); size.height = (float)(closest[i].height * rng.uniform((double)0.99, (double)1.01)); float angle = (float)rng.uniform(-5.0, 5.0); for( int y = 0; y < standardPatch.rows; y++ ) { for( int x = 0; x < standardPatch.cols; x++ ) { standardPatch(x, y) += (uchar)rng.gaussian(5.0); } } #ifdef BLUR_AS_VADIM GaussianBlur(standardPatch, blurredPatch, GaussBlurKernelSize, 0.0); resize(blurredPatch, blurredPatch, initSize_); #else resample(blurredImg, RotatedRect(center, size, angle), blurredPatch); #endif resample(scaledImg, RotatedRect(center, size, angle), standardPatch); examplesForModel.push_back(standardPatch); examplesForEnsemble.push_back(blurredPatch); } } return 0; }
bool TrackerTLDImpl::updateImpl(const Mat& image, Rect2d& boundingBox) { Mat image_gray, image_blurred, imageForDetector; cvtColor( image, image_gray, COLOR_BGR2GRAY ); double scale = data->getScale(); if( scale > 1.0 ) resize(image_gray, imageForDetector, Size(cvRound(image.cols*scale), cvRound(image.rows*scale)), 0, 0, DOWNSCALE_MODE); else imageForDetector = image_gray; GaussianBlur(imageForDetector, image_blurred, GaussBlurKernelSize, 0.0); TrackerTLDModel* tldModel = ((TrackerTLDModel*)static_cast<TrackerModel*>(model)); data->frameNum++; Mat_<uchar> standardPatch(STANDARD_PATCH_SIZE, STANDARD_PATCH_SIZE); std::vector<TLDDetector::LabeledPatch> detectorResults; //best overlap around 92% std::vector<Rect2d> candidates; std::vector<double> candidatesRes; bool trackerNeedsReInit = false; bool DETECT_FLG = false; for( int i = 0; i < 2; i++ ) { Rect2d tmpCandid = boundingBox; if (i == 1) { #ifdef HAVE_OPENCL if (ocl::haveOpenCL()) DETECT_FLG = tldModel->detector->ocl_detect(imageForDetector, image_blurred, tmpCandid, detectorResults, tldModel->getMinSize()); else #endif DETECT_FLG = tldModel->detector->detect(imageForDetector, image_blurred, tmpCandid, detectorResults, tldModel->getMinSize()); } if( ( (i == 0) && !data->failedLastTime && trackerProxy->update(image, tmpCandid) ) || ( DETECT_FLG)) { candidates.push_back(tmpCandid); if( i == 0 ) resample(image_gray, tmpCandid, standardPatch); else resample(imageForDetector, tmpCandid, standardPatch); candidatesRes.push_back(tldModel->detector->Sc(standardPatch)); } else { if( i == 0 ) trackerNeedsReInit = true; } } std::vector<double>::iterator it = std::max_element(candidatesRes.begin(), candidatesRes.end()); if( it == candidatesRes.end() ) { data->confident = false; data->failedLastTime = true; return false; } else { boundingBox = candidates[it - candidatesRes.begin()]; data->failedLastTime = false; if( trackerNeedsReInit || it != candidatesRes.begin() ) trackerProxy->init(image, boundingBox); } #if 1 if( it != candidatesRes.end() ) resample(imageForDetector, candidates[it - candidatesRes.begin()], standardPatch); #endif if( *it > CORE_THRESHOLD ) data->confident = true; if( data->confident ) { Pexpert pExpert(imageForDetector, image_blurred, boundingBox, tldModel->detector, params, data->getMinSize()); Nexpert nExpert(imageForDetector, boundingBox, tldModel->detector, params); std::vector<Mat_<uchar> > examplesForModel, examplesForEnsemble; examplesForModel.reserve(100); examplesForEnsemble.reserve(100); int negRelabeled = 0; for( int i = 0; i < (int)detectorResults.size(); i++ ) { bool expertResult; if( detectorResults[i].isObject ) { expertResult = nExpert(detectorResults[i].rect); if( expertResult != detectorResults[i].isObject ) negRelabeled++; } else { expertResult = pExpert(detectorResults[i].rect); } detectorResults[i].shouldBeIntegrated = detectorResults[i].shouldBeIntegrated || (detectorResults[i].isObject != expertResult); detectorResults[i].isObject = expertResult; } tldModel->integrateRelabeled(imageForDetector, image_blurred, detectorResults); pExpert.additionalExamples(examplesForModel, examplesForEnsemble); #ifdef HAVE_OPENCL if (ocl::haveOpenCL()) tldModel->ocl_integrateAdditional(examplesForModel, examplesForEnsemble, true); else #endif tldModel->integrateAdditional(examplesForModel, examplesForEnsemble, true); examplesForModel.clear(); examplesForEnsemble.clear(); nExpert.additionalExamples(examplesForModel, examplesForEnsemble); #ifdef HAVE_OPENCL if (ocl::haveOpenCL()) tldModel->ocl_integrateAdditional(examplesForModel, examplesForEnsemble, false); else #endif tldModel->integrateAdditional(examplesForModel, examplesForEnsemble, false); } else { #ifdef CLOSED_LOOP tldModel->integrateRelabeled(imageForDetector, image_blurred, detectorResults); #endif } return true; }
bool TLDDetector::ocl_detect(const Mat& img, const Mat& imgBlurred, Rect2d& res, std::vector<LabeledPatch>& patches, Size initSize) { patches.clear(); Mat_<uchar> standardPatch(STANDARD_PATCH_SIZE, STANDARD_PATCH_SIZE); Mat tmp; int dx = initSize.width / 10, dy = initSize.height / 10; Size2d size = img.size(); double scale = 1.0; int npos = 0, nneg = 0; double maxSc = -5.0; Rect2d maxScRect; int scaleID; std::vector <Mat> resized_imgs, blurred_imgs; std::vector <Point> varBuffer, ensBuffer; std::vector <int> varScaleIDs, ensScaleIDs; //Detection part //Generate windows and filter by variance scaleID = 0; resized_imgs.push_back(img); blurred_imgs.push_back(imgBlurred); do { Mat_<double> intImgP, intImgP2; computeIntegralImages(resized_imgs[scaleID], intImgP, intImgP2); for (int i = 0, imax = cvFloor((0.0 + resized_imgs[scaleID].cols - initSize.width) / dx); i < imax; i++) { for (int j = 0, jmax = cvFloor((0.0 + resized_imgs[scaleID].rows - initSize.height) / dy); j < jmax; j++) { if (!patchVariance(intImgP, intImgP2, originalVariancePtr, Point(dx * i, dy * j), initSize)) continue; varBuffer.push_back(Point(dx * i, dy * j)); varScaleIDs.push_back(scaleID); } } scaleID++; size.width /= SCALE_STEP; size.height /= SCALE_STEP; scale *= SCALE_STEP; resize(img, tmp, size, 0, 0, DOWNSCALE_MODE); resized_imgs.push_back(tmp); GaussianBlur(resized_imgs[scaleID], tmp, GaussBlurKernelSize, 0.0f); blurred_imgs.push_back(tmp); } while (size.width >= initSize.width && size.height >= initSize.height); //Encsemble classification for (int i = 0; i < (int)varBuffer.size(); i++) { prepareClassifiers((int)blurred_imgs[varScaleIDs[i]].step[0]); if (ensembleClassifierNum(&blurred_imgs[varScaleIDs[i]].at<uchar>(varBuffer[i].y, varBuffer[i].x)) <= ENSEMBLE_THRESHOLD) continue; ensBuffer.push_back(varBuffer[i]); ensScaleIDs.push_back(varScaleIDs[i]); } //NN classification //Prepare batch of patches int numOfPatches = (int)ensBuffer.size(); Mat_<uchar> stdPatches(numOfPatches, 225); double *resultSr = new double[numOfPatches]; double *resultSc = new double[numOfPatches]; uchar *patchesData = stdPatches.data; for (int i = 0; i < (int)ensBuffer.size(); i++) { resample(resized_imgs[ensScaleIDs[i]], Rect2d(ensBuffer[i], initSize), standardPatch); uchar *stdPatchData = standardPatch.data; for (int j = 0; j < 225; j++) patchesData[225*i+j] = stdPatchData[j]; } //Calculate Sr and Sc batches ocl_batchSrSc(stdPatches, resultSr, resultSc, numOfPatches); for (int i = 0; i < (int)ensBuffer.size(); i++) { LabeledPatch labPatch; standardPatch.data = &stdPatches.data[225 * i]; double curScale = pow(SCALE_STEP, ensScaleIDs[i]); labPatch.rect = Rect2d(ensBuffer[i].x*curScale, ensBuffer[i].y*curScale, initSize.width * curScale, initSize.height * curScale); double srValue, scValue; srValue = resultSr[i]; ////To fix: Check the paper, probably this cause wrong learning // labPatch.isObject = srValue > THETA_NN; labPatch.shouldBeIntegrated = abs(srValue - THETA_NN) < 0.1; patches.push_back(labPatch); // if (!labPatch.isObject) { nneg++; continue; } else { npos++; } scValue = resultSc[i]; if (scValue > maxSc) { maxSc = scValue; maxScRect = labPatch.rect; } } if (maxSc < 0) return false; res = maxScRect; return true; }