//Do this when current trajectory is valid
void TLD::learn()
{
if(!learningEnabled || !valid || !detectorEnabled)
{
learning = false;
return;
}
learning = true;
DetectionResult *detectionResult = detectorCascade->detectionResult;
if(!detectionResult->containsValidData)
{
#ifndef USE_HTLD
detectorCascade->detect(currImg);
#else
detectorCascade->detect(currImg,
hTLDMaster->getTLDObject(tldObjId)->getFastDetStr(),
false);
#endif
}
//This is the positive patch
NormalizedPatch patch;
tldExtractNormalizedPatchRect(currImg, currBB, patch.values);
float *overlap = new float[detectorCascade->numWindows];
tldOverlapRect(detectorCascade->windows, detectorCascade->numWindows, currBB, overlap);
//Add all bounding boxes with high overlap
vector<std::pair<int, float> > positiveIndices;
vector<int> negativeIndices;
vector<int> negativeIndicesForNN;
//First: Find overlapping positive and negative patches
for(int i = 0; i < detectorCascade->numWindows; i++)
{
if(overlap[i] > 0.6)
{
positiveIndices.push_back(std::pair<int, float>(i, overlap[i]));
}
if(overlap[i] < 0.2)
{
if(!detectorCascade->ensembleClassifier->enabled || detectionResult->posteriors[i] > 0.5) //Should be 0.5 according to the paper
{
negativeIndices.push_back(i);
negativeIndicesForNN.push_back(i);
}
}
}
sort(positiveIndices.begin(), positiveIndices.end(), tldSortByOverlapDesc);
vector<NormalizedPatch> patches;
patch.positive = 1;
patches.push_back(patch);
//TODO: Flip
int numIterations = std::min<size_t>(positiveIndices.size(), 10); //Take at most 10 bounding boxes (sorted by overlap)
for(size_t i = 0; i < negativeIndices.size(); i++)
{
int idx = negativeIndices.at(i);
//TODO: Somewhere here image warping might be possible
detectorCascade->ensembleClassifier->learn(&detectorCascade->windows[TLD_WINDOW_SIZE * idx], false, &detectionResult->featureVectors[detectorCascade->numTrees * idx]);
}
//***********************************************************************************
//**************Warping Positive Patches & Traininng the Classifier...***************
//***********************************************************************************
int bbW;
int bbH;
std::vector<int> indices(numIterations);
for(int i = 0; i<numIterations; i++)
indices[i] = positiveIndices.at(i).first;
bbHull(detectorCascade->windowOffsets,
indices,
currImg.cols,
hull);
bbW = hull[2] - hull[0] + 1;
bbH = hull[3] - hull[1] + 1;
cv::Rect roi(hull[0], hull[1], bbW, bbH);
#ifdef USE_HTLD
//Move Blurred Image to CPU...
cudaMemcpy((void*)imBlurred->data,
(void*)memMgr->getDevBlurredCurFrame(),
sizeof(Npp8u) * currImg.rows * currImg.cols,
cudaMemcpyDeviceToHost);
#else
gaussianFilter->apply(currImg,
*imBlurred);
#endif
cv::Mat noise(bbH, bbW, CV_64FC1);
cv::Mat result(bbH, bbW, CV_8UC1);
//TODO: Make All Patch Related Params Configurable...
for(int i = 0; i < 20; i++) { //Here 20 is equal to # of Warped-Images...
if(i > 0) {
//TODO: GPU is a Better Prospect for This Sort of Operations(Next Step for Parallelization)???!!!!
extractPatch(*imBlurred,
hull,
bbW,
bbH,
(unsigned char)0,
rng,
5.0,
20.0,
0.02,
0.02,
noise,
result);
result.copyTo((*ppHolder)(roi));
}//End of if-Block...
for(int j = 0; j<numIterations; j++) {
int idx = positiveIndices.at(j).first;
detectorCascade->ensembleClassifier->calcFeatureVectorPatch(ppHolder->data,
idx,
&detectionResult->featureVectors[detectorCascade->numTrees * idx]);
detectorCascade->ensembleClassifier->learn(&detectorCascade->windows[TLD_WINDOW_SIZE * idx],
true,
&detectionResult->featureVectors[detectorCascade->numTrees * idx]);
}
}
for(size_t i = 0; i < negativeIndicesForNN.size(); i++)
{
int idx = negativeIndicesForNN.at(i);
NormalizedPatch patch;
tldExtractNormalizedPatchBB(currImg, &detectorCascade->windows[TLD_WINDOW_SIZE * idx], patch.values);
patch.positive = 0;
patches.push_back(patch);
}
detectorCascade->nnClassifier->learn(patches);
//cout << "NN has now " << detectorCascade->nnClassifier->truePositives->size() << " positives and " << detectorCascade->nnClassifier->falsePositives->size() << " negatives.\n";
delete[] overlap;
}
void Learner::learn(const Mat &img, const Mat &imgB, const Mat &img32F, const TYPE_BBOX &ret)
{
TYPE_TRAIN_DATA_SET &nnTrainDataset = detector->trainDataSetNN;
nnTrainDataset.clear();
TYPE_TRAIN_DATA_SET &rfTrainDataset = detector->trainDataSetRF;
rfTrainDataset.clear();
auto &scanBBs = detector->scanBBs;
detector->sortByOverlap(ret, true);
int count = 0;
// P-expert - NN
nnTrainDataset.push_back(make_pair(img32F(scanBBs[0]), CLASS_POS));
// P-expert - RF
int tlx = img.cols, tly = img.rows, brx = 0, bry = 0;
for(int i = 0; i < LEARNER_N_GOOD_BB && scanBBs[i].overlap >= GOODBB_OL; i++)
{
tlx = min(tlx, scanBBs[i].tl().x);
tly = min(tly, scanBBs[i].tl().y);
brx = max(brx, scanBBs[i].br().x);
bry = max(bry, scanBBs[i].br().y);
}
Point tl(tlx, tly), br(brx, bry);
Rect bbHull(tl, br);
int cx, cy;
cx = round((double)(tlx + brx) / 2);
cy = round((double)(tly + bry) / 2);
for(int j = 0; j < LEARNER_N_WARPED; j++)
{
Mat warped;
if(j != 0)
{
patchGenerator(imgB, Point(cx, cy), warped, bbHull.size(), theRNG());
// for optimum in RF::getcode()
Mat tmp(imgB.size() ,CV_8U, Scalar::all(0));
warped.copyTo(tmp(Rect(0, 0, bbHull.size().width, bbHull.size().height)));
warped = tmp;
}
else
{
warped = imgB(bbHull);
}
for(int i = 0; i < LEARNER_N_GOOD_BB && scanBBs[i].overlap >= GOODBB_OL; i++)
{
Rect rect(scanBBs[i].tl() - tl, scanBBs[i].br() - tl);
TYPE_TRAIN_DATA trainData(make_pair(warped(rect), CLASS_POS));
rfTrainDataset.push_back(trainData);
count++;
}
}
// N-expert - NN
int nCountNN = 0;
for(int i = 0; i < detector->scanBBs.size(); i++)
{
TYPE_DETECTOR_SCANBB &sbb = detector->scanBBs[i];
if(sbb.status != DETECTOR_REJECT_VAR)
{
if(sbb.overlap < BADBB_OL)
{
nCountNN++;
TYPE_TRAIN_DATA trainData(make_pair(img32F(sbb), CLASS_NEG));
nnTrainDataset.push_back(trainData);
}
}
if(nCountNN == LEARNER_N_NN_NEG) break;
}
// N-expert - RF
int nCountRF = 0;
for(int i = 0; i < detector->scanBBs.size(); i++)
{
TYPE_DETECTOR_SCANBB &sbb = detector->scanBBs[i];
if(sbb.status != DETECTOR_REJECT_VAR)
{
if(sbb.overlap < BADBB_OL && sbb.posterior >= 0.1)
{
nCountRF++;
TYPE_TRAIN_DATA trainData(make_pair(imgB(sbb), CLASS_NEG));
rfTrainDataset.push_back(trainData);
}
}
}
stringstream info;
info << "Generated 1 positive example for NN, and " << count << " positive example for RF.";
outputInfo("Learner", info.str());
stringstream info2;
info2 << "Generated " << nCountNN << " NN negative sample(s), and " << nCountRF << " RF negative example(s).";
outputInfo("Learner", info2.str());
detector->update();
stringstream info3;
info3 << "Updated detector.";
outputInfo("Learner", info3.str());
detector->nNClassifier.showModel();
}
