//Do this when current trajectory is valid void TLD::learn() { if(!learningEnabled || !valid || !detectorEnabled) { learning = false; return; } learning = true; DetectionResult *detectionResult = detectorCascade->detectionResult; if(!detectionResult->containsValidData) { #ifndef USE_HTLD detectorCascade->detect(currImg); #else detectorCascade->detect(currImg, hTLDMaster->getTLDObject(tldObjId)->getFastDetStr(), false); #endif } //This is the positive patch NormalizedPatch patch; tldExtractNormalizedPatchRect(currImg, currBB, patch.values); float *overlap = new float[detectorCascade->numWindows]; tldOverlapRect(detectorCascade->windows, detectorCascade->numWindows, currBB, overlap); //Add all bounding boxes with high overlap vector<std::pair<int, float> > positiveIndices; vector<int> negativeIndices; vector<int> negativeIndicesForNN; //First: Find overlapping positive and negative patches for(int i = 0; i < detectorCascade->numWindows; i++) { if(overlap[i] > 0.6) { positiveIndices.push_back(std::pair<int, float>(i, overlap[i])); } if(overlap[i] < 0.2) { if(!detectorCascade->ensembleClassifier->enabled || detectionResult->posteriors[i] > 0.5) //Should be 0.5 according to the paper { negativeIndices.push_back(i); negativeIndicesForNN.push_back(i); } } } sort(positiveIndices.begin(), positiveIndices.end(), tldSortByOverlapDesc); vector<NormalizedPatch> patches; patch.positive = 1; patches.push_back(patch); //TODO: Flip int numIterations = std::min<size_t>(positiveIndices.size(), 10); //Take at most 10 bounding boxes (sorted by overlap) for(size_t i = 0; i < negativeIndices.size(); i++) { int idx = negativeIndices.at(i); //TODO: Somewhere here image warping might be possible detectorCascade->ensembleClassifier->learn(&detectorCascade->windows[TLD_WINDOW_SIZE * idx], false, &detectionResult->featureVectors[detectorCascade->numTrees * idx]); } //*********************************************************************************** //**************Warping Positive Patches & Traininng the Classifier...*************** //*********************************************************************************** int bbW; int bbH; std::vector<int> indices(numIterations); for(int i = 0; i<numIterations; i++) indices[i] = positiveIndices.at(i).first; bbHull(detectorCascade->windowOffsets, indices, currImg.cols, hull); bbW = hull[2] - hull[0] + 1; bbH = hull[3] - hull[1] + 1; cv::Rect roi(hull[0], hull[1], bbW, bbH); #ifdef USE_HTLD //Move Blurred Image to CPU... cudaMemcpy((void*)imBlurred->data, (void*)memMgr->getDevBlurredCurFrame(), sizeof(Npp8u) * currImg.rows * currImg.cols, cudaMemcpyDeviceToHost); #else gaussianFilter->apply(currImg, *imBlurred); #endif cv::Mat noise(bbH, bbW, CV_64FC1); cv::Mat result(bbH, bbW, CV_8UC1); //TODO: Make All Patch Related Params Configurable... for(int i = 0; i < 20; i++) { //Here 20 is equal to # of Warped-Images... if(i > 0) { //TODO: GPU is a Better Prospect for This Sort of Operations(Next Step for Parallelization)???!!!! extractPatch(*imBlurred, hull, bbW, bbH, (unsigned char)0, rng, 5.0, 20.0, 0.02, 0.02, noise, result); result.copyTo((*ppHolder)(roi)); }//End of if-Block... for(int j = 0; j<numIterations; j++) { int idx = positiveIndices.at(j).first; detectorCascade->ensembleClassifier->calcFeatureVectorPatch(ppHolder->data, idx, &detectionResult->featureVectors[detectorCascade->numTrees * idx]); detectorCascade->ensembleClassifier->learn(&detectorCascade->windows[TLD_WINDOW_SIZE * idx], true, &detectionResult->featureVectors[detectorCascade->numTrees * idx]); } } for(size_t i = 0; i < negativeIndicesForNN.size(); i++) { int idx = negativeIndicesForNN.at(i); NormalizedPatch patch; tldExtractNormalizedPatchBB(currImg, &detectorCascade->windows[TLD_WINDOW_SIZE * idx], patch.values); patch.positive = 0; patches.push_back(patch); } detectorCascade->nnClassifier->learn(patches); //cout << "NN has now " << detectorCascade->nnClassifier->truePositives->size() << " positives and " << detectorCascade->nnClassifier->falsePositives->size() << " negatives.\n"; delete[] overlap; }
void Learner::learn(const Mat &img, const Mat &imgB, const Mat &img32F, const TYPE_BBOX &ret) { TYPE_TRAIN_DATA_SET &nnTrainDataset = detector->trainDataSetNN; nnTrainDataset.clear(); TYPE_TRAIN_DATA_SET &rfTrainDataset = detector->trainDataSetRF; rfTrainDataset.clear(); auto &scanBBs = detector->scanBBs; detector->sortByOverlap(ret, true); int count = 0; // P-expert - NN nnTrainDataset.push_back(make_pair(img32F(scanBBs[0]), CLASS_POS)); // P-expert - RF int tlx = img.cols, tly = img.rows, brx = 0, bry = 0; for(int i = 0; i < LEARNER_N_GOOD_BB && scanBBs[i].overlap >= GOODBB_OL; i++) { tlx = min(tlx, scanBBs[i].tl().x); tly = min(tly, scanBBs[i].tl().y); brx = max(brx, scanBBs[i].br().x); bry = max(bry, scanBBs[i].br().y); } Point tl(tlx, tly), br(brx, bry); Rect bbHull(tl, br); int cx, cy; cx = round((double)(tlx + brx) / 2); cy = round((double)(tly + bry) / 2); for(int j = 0; j < LEARNER_N_WARPED; j++) { Mat warped; if(j != 0) { patchGenerator(imgB, Point(cx, cy), warped, bbHull.size(), theRNG()); // for optimum in RF::getcode() Mat tmp(imgB.size() ,CV_8U, Scalar::all(0)); warped.copyTo(tmp(Rect(0, 0, bbHull.size().width, bbHull.size().height))); warped = tmp; } else { warped = imgB(bbHull); } for(int i = 0; i < LEARNER_N_GOOD_BB && scanBBs[i].overlap >= GOODBB_OL; i++) { Rect rect(scanBBs[i].tl() - tl, scanBBs[i].br() - tl); TYPE_TRAIN_DATA trainData(make_pair(warped(rect), CLASS_POS)); rfTrainDataset.push_back(trainData); count++; } } // N-expert - NN int nCountNN = 0; for(int i = 0; i < detector->scanBBs.size(); i++) { TYPE_DETECTOR_SCANBB &sbb = detector->scanBBs[i]; if(sbb.status != DETECTOR_REJECT_VAR) { if(sbb.overlap < BADBB_OL) { nCountNN++; TYPE_TRAIN_DATA trainData(make_pair(img32F(sbb), CLASS_NEG)); nnTrainDataset.push_back(trainData); } } if(nCountNN == LEARNER_N_NN_NEG) break; } // N-expert - RF int nCountRF = 0; for(int i = 0; i < detector->scanBBs.size(); i++) { TYPE_DETECTOR_SCANBB &sbb = detector->scanBBs[i]; if(sbb.status != DETECTOR_REJECT_VAR) { if(sbb.overlap < BADBB_OL && sbb.posterior >= 0.1) { nCountRF++; TYPE_TRAIN_DATA trainData(make_pair(imgB(sbb), CLASS_NEG)); rfTrainDataset.push_back(trainData); } } } stringstream info; info << "Generated 1 positive example for NN, and " << count << " positive example for RF."; outputInfo("Learner", info.str()); stringstream info2; info2 << "Generated " << nCountNN << " NN negative sample(s), and " << nCountRF << " RF negative example(s)."; outputInfo("Learner", info2.str()); detector->update(); stringstream info3; info3 << "Updated detector."; outputInfo("Learner", info3.str()); detector->nNClassifier.showModel(); }