float NNClassifier::classifyBB(const Mat& img, Rect* bb) {
NormalizedPatch patch;
tldExtractNormalizedPatchRect(img, bb, patch.values);
return classifyPatch(&patch);
}
//Do this when current trajectory is valid
void TLD::learn() {
if (!learningEnabled || !valid || !detectorEnabled) {
learning = false;
return;
}
learning = true;
DetectionResult* detectionResult = detectorCascade->detectionResult;
if (!detectionResult->containsValidData)
{ detectorCascade->detect(currImg); }
//This is the positive patch
NormalizedPatch patch;
tldExtractNormalizedPatchRect(currImg, currBB, patch.values);
float* overlap = new float[detectorCascade->numWindows] {};
tldOverlapRect(detectorCascade->windows, detectorCascade->numWindows, currBB, overlap);
//Add all bounding boxes with high overlap
vector<pair<int, float> > positiveIndices;
vector<int> negativeIndices;
vector<int> negativeIndicesForNN;
//First: Find overlapping positive and negative patches
for (int i = 0; i < detectorCascade->numWindows; i++) {
if (overlap[i] > 0.7) {
positiveIndices.push_back(pair<int, float>(i, overlap[i]));
}
if (overlap[i] < 0.2) {
if (!detectorCascade->ensembleClassifier->enabled || detectionResult->posteriors[i] > 0.5) { //Should be 0.5 according to the paper
negativeIndices.push_back(i);
}
if (!detectorCascade->ensembleClassifier->enabled || detectionResult->posteriors[i] > 0.5) {
negativeIndicesForNN.push_back(i);
}
}
}
sort(positiveIndices.begin(), positiveIndices.end(), tldSortByOverlapDesc);
vector<NormalizedPatch> patches;
patch.positive = 1;
patches.push_back(patch);
//TODO: Flip
size_t numIterations = std::min<size_t>(positiveIndices.size(), 10); //Take at most 10 bounding boxes (sorted by overlap)
for (size_t i = 0; i < negativeIndices.size(); i++) {
int idx = negativeIndices.at(i);
//TODO: Somewhere here image warping might be possible
detectorCascade->ensembleClassifier->learn(&detectorCascade->windows[TLD_WINDOW_SIZE * idx], false, &detectionResult->featureVectors[detectorCascade->numTrees * idx]);
}
//TODO: Randomization might be a good idea
for (int i = 0; i < numIterations; i++) {
int idx = positiveIndices.at(i).first;
//TODO: Somewhere here image warping might be possible
detectorCascade->ensembleClassifier->learn(&detectorCascade->windows[TLD_WINDOW_SIZE * idx], true, &detectionResult->featureVectors[detectorCascade->numTrees * idx]);
}
for (size_t i = 0; i < negativeIndicesForNN.size(); i++) {
int idx = negativeIndicesForNN.at(i);
NormalizedPatch patch;
tldExtractNormalizedPatchBB(currImg, &detectorCascade->windows[TLD_WINDOW_SIZE * idx], patch.values);
patch.positive = 0;
patches.push_back(patch);
}
detectorCascade->nnClassifier->learn(patches);
delete[] overlap;
}
void TLD::initialLearning() {
learning = true; //This is just for display purposes
DetectionResult* detectionResult = detectorCascade->detectionResult;
detectorCascade->detect(currImg);
//This is the positive patch
NormalizedPatch initPatch;
tldExtractNormalizedPatchRect(currImg, currBB, initPatch.values);
initPatch.positive = 1;
float initVar = tldCalcVariance(initPatch.values, TLD_PATCH_SIZE * TLD_PATCH_SIZE);
detectorCascade->varianceFilter->minVar = initVar / 2;
float* overlap = new float[detectorCascade->numWindows] {};
tldOverlapRect(detectorCascade->windows, detectorCascade->numWindows, currBB, overlap);
//Add all bounding boxes with high overlap
vector< pair<int, float> > positiveIndices;
vector<int> negativeIndices;
//First: Find overlapping positive and negative patches
for (int i = 0; i < detectorCascade->numWindows; i++) {
if (overlap[i] > 0.7) {
positiveIndices.push_back(pair<int, float>(i, overlap[i]));
}
if (overlap[i] < 0.2) {
float variance = detectionResult->variances[i];
if (!detectorCascade->varianceFilter->enabled || variance > detectorCascade->varianceFilter->minVar) { //TODO: This check is unnecessary if minVar would be set before calling detect.
negativeIndices.push_back(i);
}
}
}
sort(positiveIndices.begin(), positiveIndices.end(), tldSortByOverlapDesc);
vector<NormalizedPatch> patches;
patches.push_back(initPatch); //Add first patch to patch list
size_t numIterations = std::min<size_t>(positiveIndices.size(), 10); //Take at most 10 bounding boxes (sorted by overlap)
for (int i = 0; i < numIterations; i++) {
int idx = positiveIndices.at(i).first;
//Learn this bounding box
//TODO: Somewhere here image warping might be possible
detectorCascade->ensembleClassifier->learn(&detectorCascade->windows[TLD_WINDOW_SIZE * idx], true, &detectionResult->featureVectors[detectorCascade->numTrees * idx]);
}
std::shuffle(negativeIndices.begin(), negativeIndices.end(), *rng);
//Choose 100 random patches for negative examples
for (size_t i = 0; i < std::min<size_t>(100, negativeIndices.size()); i++) {
int idx = negativeIndices.at(i);
NormalizedPatch patch;
tldExtractNormalizedPatchBB(currImg, &detectorCascade->windows[TLD_WINDOW_SIZE * idx], patch.values);
patch.positive = 0;
patches.push_back(patch);
}
detectorCascade->nnClassifier->learn(patches);
delete[] overlap;
}
//Do this when current trajectory is valid
void TLD::learn()
{
if(!learningEnabled || !valid || !detectorEnabled)
{
learning = false;
return;
}
learning = true;
DetectionResult *detectionResult = detectorCascade->detectionResult;
if(!detectionResult->containsValidData)
{
#ifndef USE_HTLD
detectorCascade->detect(currImg);
#else
detectorCascade->detect(currImg,
hTLDMaster->getTLDObject(tldObjId)->getFastDetStr(),
false);
#endif
}
//This is the positive patch
NormalizedPatch patch;
tldExtractNormalizedPatchRect(currImg, currBB, patch.values);
float *overlap = new float[detectorCascade->numWindows];
tldOverlapRect(detectorCascade->windows, detectorCascade->numWindows, currBB, overlap);
//Add all bounding boxes with high overlap
vector<std::pair<int, float> > positiveIndices;
vector<int> negativeIndices;
vector<int> negativeIndicesForNN;
//First: Find overlapping positive and negative patches
for(int i = 0; i < detectorCascade->numWindows; i++)
{
if(overlap[i] > 0.6)
{
positiveIndices.push_back(std::pair<int, float>(i, overlap[i]));
}
if(overlap[i] < 0.2)
{
if(!detectorCascade->ensembleClassifier->enabled || detectionResult->posteriors[i] > 0.5) //Should be 0.5 according to the paper
{
negativeIndices.push_back(i);
negativeIndicesForNN.push_back(i);
}
}
}
sort(positiveIndices.begin(), positiveIndices.end(), tldSortByOverlapDesc);
vector<NormalizedPatch> patches;
patch.positive = 1;
patches.push_back(patch);
//TODO: Flip
int numIterations = std::min<size_t>(positiveIndices.size(), 10); //Take at most 10 bounding boxes (sorted by overlap)
for(size_t i = 0; i < negativeIndices.size(); i++)
{
int idx = negativeIndices.at(i);
//TODO: Somewhere here image warping might be possible
detectorCascade->ensembleClassifier->learn(&detectorCascade->windows[TLD_WINDOW_SIZE * idx], false, &detectionResult->featureVectors[detectorCascade->numTrees * idx]);
}
//***********************************************************************************
//**************Warping Positive Patches & Traininng the Classifier...***************
//***********************************************************************************
int bbW;
int bbH;
std::vector<int> indices(numIterations);
for(int i = 0; i<numIterations; i++)
indices[i] = positiveIndices.at(i).first;
bbHull(detectorCascade->windowOffsets,
indices,
currImg.cols,
hull);
bbW = hull[2] - hull[0] + 1;
bbH = hull[3] - hull[1] + 1;
cv::Rect roi(hull[0], hull[1], bbW, bbH);
#ifdef USE_HTLD
//Move Blurred Image to CPU...
cudaMemcpy((void*)imBlurred->data,
(void*)memMgr->getDevBlurredCurFrame(),
sizeof(Npp8u) * currImg.rows * currImg.cols,
cudaMemcpyDeviceToHost);
#else
gaussianFilter->apply(currImg,
*imBlurred);
#endif
cv::Mat noise(bbH, bbW, CV_64FC1);
cv::Mat result(bbH, bbW, CV_8UC1);
//TODO: Make All Patch Related Params Configurable...
for(int i = 0; i < 20; i++) { //Here 20 is equal to # of Warped-Images...
if(i > 0) {
//TODO: GPU is a Better Prospect for This Sort of Operations(Next Step for Parallelization)???!!!!
extractPatch(*imBlurred,
hull,
bbW,
bbH,
(unsigned char)0,
rng,
5.0,
20.0,
0.02,
0.02,
noise,
result);
result.copyTo((*ppHolder)(roi));
}//End of if-Block...
for(int j = 0; j<numIterations; j++) {
int idx = positiveIndices.at(j).first;
detectorCascade->ensembleClassifier->calcFeatureVectorPatch(ppHolder->data,
idx,
&detectionResult->featureVectors[detectorCascade->numTrees * idx]);
detectorCascade->ensembleClassifier->learn(&detectorCascade->windows[TLD_WINDOW_SIZE * idx],
true,
&detectionResult->featureVectors[detectorCascade->numTrees * idx]);
}
}
for(size_t i = 0; i < negativeIndicesForNN.size(); i++)
{
int idx = negativeIndicesForNN.at(i);
NormalizedPatch patch;
tldExtractNormalizedPatchBB(currImg, &detectorCascade->windows[TLD_WINDOW_SIZE * idx], patch.values);
patch.positive = 0;
patches.push_back(patch);
}
detectorCascade->nnClassifier->learn(patches);
//cout << "NN has now " << detectorCascade->nnClassifier->truePositives->size() << " positives and " << detectorCascade->nnClassifier->falsePositives->size() << " negatives.\n";
delete[] overlap;
}