void Noiser::EstimateNoise()
{
cout<<"Estimating noise..."<<endl;
Mat desv_estandar(h,w,CV_32F,Scalar(0));
for(int i =0;i<total;i++)
{
//desviacion estandar
Mat aux(h,w,CV_32F,Scalar(0));
pow(cleanImage-images[i],2,aux);
desv_estandar += aux;
}
desv_estandar/=total;
sqrt(desv_estandar,desv_estandar);
double ruido_promedio = mean(desv_estandar).val[0];
double ruido_peor;
minMaxLoc(desv_estandar,NULL,&ruido_peor);
Mat aux;
meanStdDev(cleanImage,noArray(),aux);
double signal = aux.at<double>(0);
meanStdDev(desv_estandar,noArray(),aux);
double noise = aux.at<double>(0);
double SNR = 10*log10(signal/noise);
cout<<"Ruido Promedio: "<<ruido_promedio<<endl;
cout<<"Ruido Peor: "<<ruido_peor<<endl;
cout<<"SNR: "<<SNR<<" db"<<endl;
this->ready = false;
}
std::vector<int> visionUtils::updateHSVAdaptiveSkin(std::vector<Mat> pixelPlanes, bool displayFaces)//( int, char** argv )
{
if (displayFaces)
{
drawHist(pixelPlanes, 35);
}
Scalar b_mean, b_stdDev;
meanStdDev(pixelPlanes[0], b_mean, b_stdDev);
int b_min=b_mean[0] - b_stdDev[0]*3;
int b_max=b_mean[0] + b_stdDev[0]*3;
Scalar g_mean,g_stdDev;
meanStdDev(pixelPlanes[1], g_mean,g_stdDev);
int g_min=g_mean[0] - g_stdDev[0]*3;
int g_max=g_mean[0] + g_stdDev[0]*3;
Scalar r_mean, r_stdDev;
meanStdDev(pixelPlanes[2], r_mean, r_stdDev);
int r_min=r_mean[0] - r_stdDev[0]*3;
int r_max=r_mean[0] + r_stdDev[0]*3;
std::vector<int> hsvAdaptiveUpdated;
hsvAdaptiveUpdated.push_back(b_min);
hsvAdaptiveUpdated.push_back(g_min);
hsvAdaptiveUpdated.push_back(r_min);
hsvAdaptiveUpdated.push_back(b_max);
hsvAdaptiveUpdated.push_back(g_max);
hsvAdaptiveUpdated.push_back(r_max);
return hsvAdaptiveUpdated;
}
void Objectness::illustrate()
{
Mat xP1f, xN1f;
CV_Assert(matRead(_modelName + ".xP", xP1f) && matRead(_modelName + ".xN", xN1f));
CV_Assert(xP1f.cols == xN1f.cols && xP1f.cols == _W*_W && xP1f.type() == CV_32F && xN1f.type() == CV_32F);
Mat meanP, meanN, stdDevP, stdDevN;
meanStdDev(xP1f, meanP, stdDevP);
meanStdDev(xN1f, meanN, stdDevN);
Mat meanV(_W, _W*2, CV_32F), stdDev(_W, _W*2, CV_32F);
meanP.reshape(1, _W).copyTo(meanV.colRange(0, _W));
meanN.reshape(1, _W).copyTo(meanV.colRange(_W, _W*2));
stdDevP.reshape(1, _W).copyTo(stdDev.colRange(0, _W));
stdDevN.reshape(1, _W).copyTo(stdDev.colRange(_W, _W*2));
normalize(meanV, meanV, 0, 255, NORM_MINMAX, CV_8U);
CmShow::showTinyMat(_voc.resDir + "PosNeg.png", meanV);
FILE* f = fopen(_S(_voc.resDir + "PosNeg.m"), "w");
CV_Assert(f != NULL);
fprintf(f, "figure(1);\n\n");
PrintVector(f, getVector(meanP), "MeanP");
PrintVector(f, getVector(meanN), "MeanN");
PrintVector(f, getVector(stdDevP), "StdDevP");
PrintVector(f, getVector(stdDevN), "StdDevN");
PrintVector(f, getVector(_svmFilter), "Filter");
fprintf(f, "hold on;\nerrorbar(MeanP, StdDevP, 'r');\nerrorbar(MeanN, StdDevN, 'g');\nhold off;");
fclose(f);
}
//Descriptor media y desviacion RGB
vector<float> RGB::extract_RGB_des(Mat img)
{
vector<float> V_des_RGB;
V_des_RGB.clear();
Mat img_rgb, RGB_m;
Scalar RGBm, RGBstdv, img_med, img_stdv;
//----------------------------------------------------
//V_des_RGB interprete
//V_des_RGB[0] = media canal B
//V_des_RGB[1] = media canal G
//V_des_RGB[2] = media canal R
//V_des_RGB[3] = media todos los 3 canales
//V_des_RGB[4] = desviacion estandar de los 3 canales
//V_des_RGB[5] = brillo //pendiente
//----------------------------------------------------
if(!img.empty())
{
cvtColor(img,img_rgb,CV_BGR2RGB,0);
// Calculamos la media de cada canal (H, S, V) y la
// desviacion tipica de cada canal
meanStdDev(img_rgb, RGBm, RGBstdv);
// RED
V_des_RGB.push_back(RGBm[2]);
// GREEN
V_des_RGB.push_back(RGBm[1]);
// BLUE
V_des_RGB.push_back(RGBm[0]);
RGB_m = (Mat_<double>(3,1) << RGBm[0], RGBm[1], RGBm[2]);
// Calculamos la media de la media de los tres canales,
// y la desviacion tipica de la media de cada canal
meanStdDev(RGB_m, img_med, img_stdv);
#ifdef DEBUG
// cout << "+++++++++++++++++++++++++++++++++" << endl;
// cout << RGBm << endl;
// cout << RGBstdv << endl;
cout << "Media RGB: " << img_med[0] << endl;
cout << "Desv. RGB: " << img_stdv[0] << endl;
// cout << "+++++++++++++++++++++++++++++++++" << endl;
#endif
V_des_RGB.push_back(img_med[0]);
V_des_RGB.push_back(img_stdv[0]);
}
return V_des_RGB;
} // end extract_RGB_des
double compare_float(const Mat& one, const Mat& two) {
Scalar oMean;
Scalar tMean;
Scalar oSdv;
Scalar tSdv;
meanStdDev(one, oMean, oSdv);
meanStdDev(two, tMean, tSdv);
double mean = fabs(oMean.val[0] - tMean.val[0]);
double sdv = fabs(oSdv.val[0] - tSdv.val[0]);
return 1.0 - ((mean + sdv) / 2.0);
}
Vec2f DepthMapJudge::CalculateNeighbours(Vec3f center, int cx, int cy, Mat xyz)
{
int count = 0;
const double max_z = 10000;
for(int y = max(0,cy-ws); y < xyz.rows && y<=cy+ws; y++)
{
for(int x = max(0,cx-ws); x < xyz.cols && x<=cx+ws; x++)
{
Vec3f point = xyz.at<Vec3f>(y, x);
if(fabs(point[2] - max_z) < FLT_EPSILON || fabs(point[2]) > max_z) continue;
if(norm(point-center)<=radius)
{
points.at<float>(0,count) = point[2];
count++;
}
}
}
//create a mask that selects the non zero vlues in points
Mat mask(points.rows,points.cols,CV_8U,Scalar::all(0));
for(int i=0;i<count;i++){ mask.at<uchar>(0,i)=255; }
//calculate standard deviation
Mat aux;
meanStdDev(points,noArray(),aux,mask);
double stdev = aux.at<double>(0);
//clean points
bitwise_and(points,0,points);
return Vec2f(count,stdev);
}
void Feature::calcColorFeature()
{
// TODO: optimize this part, reduce extra work
Mat hsv;
cvtColor(mROI, hsv, CV_BGR2HSV_FULL);
Mat temp(mROI.size(), CV_8UC3), mixed;
Mat src[] = { mROI, mGray, hsv };
int fromTo[] = { 2,0, 3,1, 5,2 };
mixChannels(src, 3, &temp, 1, fromTo, 3);
temp.convertTo(mixed, CV_64F);
Scalar avg, stdDev;
meanStdDev(mixed, avg, stdDev, mMask);
Scalar var = stdDev.mul(stdDev);
Mat temp1 = mixed - avg;
Mat temp2 = temp1.mul(temp1);
Scalar sk = mean(temp1.mul(temp2), mMask) / (var.mul(stdDev));
Scalar ku = mean(temp2.mul(temp2), mMask) / (var.mul(var));
Scalar stat[] = { avg, stdDev, sk, ku };
for (int i = 0; i < 4; i++) {
red[i] = stat[i][0];
gray[i] = stat[i][1];
saturation[i] = stat[i][2];
}
}
int InterframeRegister::cvutStdDevStretch( Mat src, Mat dst)
{
// Contrast Stretching:
// Stretching by Standard deviation
// A two standard deviation linear contrast stretch is applied
Scalar mean;
Scalar std_deviation;
double min, max, scale, shift;
// Computes mean and std dev of image src
meanStdDev( src, mean, std_deviation);
// Computes the stretch min and max
min = mean.val[0] - 2.0 * std_deviation.val[0];
max = mean.val[0] + 2.0 * std_deviation.val[0];
if ( min != max )
{
scale = 255.0/(max - min);
shift = -min * scale;
}
else
{
scale = 1.0;
shift = 0.0;
}
// Computes:
// image = ((image-mind)/(maxd-mind))*255;
convertScaleAbs( src, dst, scale, shift );
return true;
}
//RETURNS THE TOP-LEFT CORNER OF THE REFLECTION OF sourceTemplate ON image
cv::Rect findBestMatchLocation( const cv::Mat& image, const cv::Rect& source_rect,
double* nsigma, const cv::Mat& mask )
{
cv::Mat image_gray;
cvtColor( image, image_gray, CV_RGB2GRAY, 1 );
cv::Mat image_copy = image_gray.clone();
// Create template.
cv::Mat image_template_copy = image_gray.clone();
cv::Mat sourceTemplate = image_template_copy( source_rect );
flip( sourceTemplate, sourceTemplate, 0 );
// Creates results matrix where the top left corner of the
// template is slid across each pixel of the source
int result_cols = image.cols-sourceTemplate.cols+1;
int result_rows = image.rows-sourceTemplate.rows+1;
cv::Mat result;
result.create( result_cols,result_rows, CV_32FC1 );
// Mask image to match only in selected ROI.
if( !mask.empty() )
{
cv::Mat tmp;
image_copy.copyTo( tmp, mask );
image_copy = tmp;
}
//0:CV_TM_SQDIFF
//1:CV_TM_SQDIFF_NORMED
//2:CV_TM_CORR
//3:CV_TM_CCOR_NORMED
//4:CV_TM_CCOEFF
//5:CV_TM_CCOEFF_NORMED <----Most succesful at finding reflections
int match_method = CV_TM_CCOEFF_NORMED; // 4 seemed good for stddev thresholding.
//matchTemplate( masked_scene, sourceTemplate, result, match_method );
matchTemplate( image_gray, sourceTemplate, result, match_method );
double minVal, maxVal;
cv::Point minLoc, maxLoc, matchLoc;
minMaxLoc( result, &minVal, &maxVal, &minLoc, &maxLoc, cv::Mat() );
if( match_method == CV_TM_SQDIFF || match_method == CV_TM_SQDIFF_NORMED )
{
matchLoc = minLoc;
}
else
{
matchLoc = maxLoc;
}
cv::Scalar mean, stddev;
meanStdDev( result, mean, stddev, cv::Mat() );
*nsigma = ( maxVal-mean[0] )/ stddev[0];
// matchLoc is the location of the top left corner of the reflection
// that matchTemplate found
return cv::Rect( matchLoc, source_rect.size() );
}
virtual int filter(const Mat &src, Mat &dest) const
{
Scalar m,s;
meanStdDev(src, m, s);
dest = src - m[0];
dest /= s[0];
return 0;
}
PatchMatcher::PatchMatcher(const Image<Vec3b> *patch, const Image<Vec3b> *output, const Image<uchar> *outputMask) :
OffsetChooser(patch, outputMask), output(output) {
Scalar mean, stddev;
meanStdDev(*patch, mean, stddev);
factor = - 1.0 / (K * pow(norm(stddev), 2) * patch->width() * patch->height());
}
double getContraste(const Mat& src)
{
Mat gray;
cvtColor(src, gray, CV_BGR2HLS);
Scalar mean;
Scalar stddev;
meanStdDev(src, mean, stddev);
return stddev[1];
}
double GroupClassifier::operator()(vector<int> *group, vector<Region> *regions)
{
assert(group != NULL);
assert(group->size() > 1);
assert(regions != NULL);
Mat votes ( group->size(), 1, CV_32F, 1 );
Mat strokes ( group->size(), 1, CV_32F, 1 );
Mat aspect_ratios ( group->size(), 1, CV_32F, 1 );
Mat compactnesses ( group->size(), 1, CV_32F, 1 );
Mat nums_holes ( group->size(), 1, CV_32F, 1 );
Mat holeareas_area ( group->size(), 1, CV_32F, 1 );
for (int i=group->size()-1; i>=0; i--)
{
// TODO check first if regions->at(group->at(i)).votes_ has already been calculated !!!
regions->at(group->at(i)).classifier_votes_ = character_classifier_->get_votes(®ions->at(group->at(i)));
votes.at<float>(i,0) = regions->at(group->at(i)).classifier_votes_;
strokes.at<float>(i,0) = (float)regions->at(group->at(i)).stroke_mean_;
aspect_ratios.at<float>(i,0) = (float)min( regions->at(group->at(i)).rect_.size.width, regions->at(group->at(i)).rect_.size.height)/max( regions->at(group->at(i)).rect_.size.width, regions->at(group->at(i)).rect_.size.height);
compactnesses.at<float>(i,0) = sqrt(regions->at(group->at(i)).area_)/regions->at(group->at(i)).perimeter_;
nums_holes.at<float>(i,0) = (float)regions->at(group->at(i)).num_holes_;
holeareas_area.at<float>(i,0) = (float)regions->at(group->at(i)).holes_area_/regions->at(group->at(i)).area_;
}
vector<float> sample;
sample.push_back(0);
Scalar mean,std;
meanStdDev( votes, mean, std );
sample.push_back( mean[0]);
sample.push_back( std[0]);
sample.push_back( std[0]/mean[0] );
meanStdDev( strokes, mean, std );
sample.push_back( mean[0]);
sample.push_back( std[0]);
sample.push_back( std[0]/mean[0] );
meanStdDev( aspect_ratios, mean, std );
sample.push_back( mean[0]);
sample.push_back( std[0]);
sample.push_back( std[0]/mean[0] );
meanStdDev( compactnesses, mean, std );
sample.push_back( mean[0]);
sample.push_back( std[0]);
sample.push_back( std[0]/mean[0] );
meanStdDev( nums_holes, mean, std );
sample.push_back( mean[0]);
sample.push_back( std[0]);
sample.push_back( std[0]/mean[0] );
meanStdDev( holeareas_area, mean, std );
sample.push_back(mean[0]);
sample.push_back( std[0]);
sample.push_back( std[0]/mean[0] );
float votes_group = boost_.predict( Mat(sample), Mat(), Range::all(), false, true );
return (double)1-(double)1/(1+exp(-2*votes_group));
}
Mat ZeroMeanUnitVarianceFilter::applyTo(const Mat& image, Mat& filtered) const {
if (image.channels() > 1)
throw invalid_argument("ZeroMeanUnitVarianceFilter: The image must have exactly one channel.");
Scalar mean, deviation;
meanStdDev(image, mean, deviation);
image.convertTo(filtered, CV_32F);
if (deviation[0] == 0)
filtered = Scalar(0);
else
filtered = (filtered - mean) / deviation[0];
return filtered;
}
Mat NNClassifier::getPatch(const Mat &img32F)
{
Mat patch;
resize(img32F, patch, Size(NN_PATCH_SIZE, NN_PATCH_SIZE));
Scalar mean, stddev;
meanStdDev(patch, mean, stddev);
patch -= mean.val[0]; // to reduce brightness influence
return patch;
}
// normalized value concatenation
// gray patch has problem when contrast of object patch is flipped, gradient is more robust
bool ObjPatchMatcher::ComputePatchFeat(MatFeatureSet& patches, Mat& feat) {
feat.release();
feat.create(0, 0, CV_32F);
if(patches.find("gray") != patches.end()) {
Scalar mean_, std_;
//normalize(patches["gray"], patches["gray"], 1, 0, NORM_MINMAX);
meanStdDev(patches["gray"], mean_, std_);
Mat patch_ = (patches["gray"]-0);///std_.val[0];
if(feat.empty()) patch_.reshape(1,1).copyTo(feat);
else hconcat(patch_.reshape(1, 1), feat, feat);
}
if(patches.find("gradient") != patches.end()) {
normalize(patches["gradient"], patches["gradient"], 1, 0, NORM_MINMAX);
Scalar mean_, std_;
meanStdDev(patches["gradient"], mean_, std_);
Mat patch_ = (patches["gradient"]-0);///std_.val[0];
if(feat.empty()) patch_.reshape(1,1).copyTo(feat);
else hconcat(patch_.reshape(1, 1), feat, feat);
}
if(patches.find("normal") != patches.end()) {
Scalar mean_, std_;
meanStdDev(patches["normal"], mean_, std_);
//Mat patch_ = (patches["normal"]-mean_.val);
if(feat.empty()) patches["normal"].reshape(1,1).copyTo(feat);
else hconcat(patches["normal"].reshape(1,1), feat, feat);
}
if(patches.find("depth") != patches.end()) {
//if(feat.empty()) patches["normal"].reshape(1,1).copyTo(feat);
//else
Scalar mean_, std_;
//normalize(patches["depth"], patches["depth"], 1, 0, NORM_MINMAX);
meanStdDev(patches["depth"], mean_, std_);
Mat patch_ = (patches["depth"]-mean_.val[0]);///std_.val[0];
hconcat(patch_.reshape(1,1), feat, feat);
}
return true;
}
void COpenCVInterfaceDlg::OnMorphologyConvexhull()
{
if(mainImage.cols)
{
ConvexHull c;
Mat rez=mainImage.clone();
mainImage=Filters::gaussianFilter(mainImage,1);
Scalar meanVal,stdval;
meanStdDev(mainImage,meanVal,stdval);
Canny(mainImage,mainImage,meanVal.val[0]*1/3.,meanVal.val[0]);
//imshow("contours",mainImage);
vector<std::vector<cv::Point>> contours;
findContours(mainImage,contours,CV_RETR_LIST,CV_CHAIN_APPROX_NONE,Point(2,2));
vector<Point> Points;
//Points.resize(contours.size()*contours[0].size());
for(int i=0;i<contours.size();i++)
{
for(int j=0;j<contours[i].size();j++)
{
Points.push_back(contours[i][j]);
}
}
//for(int i=2;i<mainImage.rows-2;i++)
//{
// for(int j=2;j<mainImage.cols-2;j++)
// {
// if(mainImage.at<uchar>(i,j)==255)
// Points.push_back(Point(i,j));
// }
//}
std::vector<Point> chull;
c.graham(Points, chull);
for(int i=0;i<chull.size()-1;i++)
{
line(rez,chull[i],chull[i+1],Scalar(255,255,255),2);
}
line(rez,chull[chull.size()-1],chull[0],Scalar(255,255,255),2);
prelImage=rez.clone();
ShowResult(prelImage);
}
else
MessageBox("No image loaded");
}
void Feature::calcAreaVar()
{
// TODO: optimize this part
if (mAreaVec.size() < MAX_AREA_VEC_SIZE) {
areaVar = -1;
return;
}
Scalar m, s;
meanStdDev(mAreaVec, m, s);
areaVar = s[0] / m[0];
#ifdef DEBUG_OUTPUT
cout << "areaVar: " << areaVar << endl;
#endif
}
/*
img : input image
degradedImg : degraded output image
filterDev : standard deviation of kernel for gaussian blur
snr : signal to noise ratio for additive gaussian noise
return : the used gaussian kernel
*/
Mat Dip4::degradeImage(Mat& img, Mat& degradedImg, double filterDev, double snr){
int kSize = round(filterDev*3)*2 - 1;
Mat gaussKernel = getGaussianKernel(kSize, filterDev, CV_32FC1);
gaussKernel = gaussKernel * gaussKernel.t();
filter2D(img, degradedImg, -1, gaussKernel);
Mat mean, stddev;
meanStdDev(img, mean, stddev);
Mat noise = Mat::zeros(img.rows, img.cols, CV_32FC1);
randn(noise, 0, stddev.at<double>(0)/snr);
degradedImg = degradedImg + noise;
threshold(degradedImg, degradedImg, 255, 255, CV_THRESH_TRUNC);
threshold(degradedImg, degradedImg, 0, 0, CV_THRESH_TOZERO);
return gaussKernel;
}
// Gets the hue/sat/val for areas that we believe are license plate characters
// Then uses that to filter the whole image and provide a mask.
void ColorFilter::findCharColors()
{
int MINIMUM_SATURATION = 45;
if (this->debug)
cout << "ColorFilter::findCharColors" << endl;
//charMask.copyTo(this->colorMask);
this->colorMask = Mat::zeros(charMask.size(), CV_8U);
bitwise_not(this->colorMask, this->colorMask);
Mat erodedCharMask(charMask.size(), CV_8U);
Mat element = getStructuringElement( 1,
Size( 2 + 1, 2+1 ),
Point( 1, 1 ) );
erode(charMask, erodedCharMask, element);
vector<vector<Point> > contours;
vector<Vec4i> hierarchy;
findContours(erodedCharMask, contours, hierarchy, CV_RETR_TREE, CV_CHAIN_APPROX_SIMPLE);
vector<float> hMeans, sMeans, vMeans;
vector<float> hStdDevs, sStdDevs, vStdDevs;
for (unsigned int i = 0; i < contours.size(); i++)
{
if (hierarchy[i][3] != -1)
continue;
Mat singleCharMask = Mat::zeros(hsv.size(), CV_8U);
drawContours(singleCharMask, contours,
i, // draw this contour
cv::Scalar(255,255,255), // in
CV_FILLED,
8,
hierarchy
);
// get rid of the outline by drawing a 1 pixel width black line
drawContours(singleCharMask, contours,
i, // draw this contour
cv::Scalar(0,0,0), // in
1,
8,
hierarchy
);
//drawAndWait(&singleCharMask);
Scalar mean;
Scalar stddev;
meanStdDev(hsv, mean, stddev, singleCharMask);
if (this->debug)
{
cout << "ColorFilter " << setw(3) << i << ". Mean: h: " << setw(7) << mean[0] << " s: " << setw(7) <<mean[1] << " v: " << setw(7) << mean[2]
<< " | Std: h: " << setw(7) <<stddev[0] << " s: " << setw(7) <<stddev[1] << " v: " << stddev[2] << endl;
}
if (mean[0] == 0 && mean[1] == 0 && mean[2] == 0)
continue;
hMeans.push_back(mean[0]);
sMeans.push_back(mean[1]);
vMeans.push_back(mean[2]);
hStdDevs.push_back(stddev[0]);
sStdDevs.push_back(stddev[1]);
vStdDevs.push_back(stddev[2]);
}
if (hMeans.size() == 0)
return;
int bestHueIndex = this->getMajorityOpinion(hMeans, .65, 30);
int bestSatIndex = this->getMajorityOpinion(sMeans, .65, 35);
int bestValIndex = this->getMajorityOpinion(vMeans, .65, 30);
if (sMeans[bestSatIndex] < MINIMUM_SATURATION)
return;
bool doHueFilter = false, doSatFilter = false, doValFilter = false;
float hueMin, hueMax;
float satMin, satMax;
float valMin, valMax;
if (this->debug)
cout << "ColorFilter Winning indices:" << endl;
if (bestHueIndex != -1)
{
doHueFilter = true;
hueMin = hMeans[bestHueIndex] - (2 * hStdDevs[bestHueIndex]);
hueMax = hMeans[bestHueIndex] + (2 * hStdDevs[bestHueIndex]);
if (abs(hueMin - hueMax) < 20)
{
hueMin = hMeans[bestHueIndex] - 20;
hueMax = hMeans[bestHueIndex] + 20;
}
if (hueMin < 0)
hueMin = 0;
if (hueMax > 180)
hueMax = 180;
if (this->debug)
cout << "ColorFilter Hue: " << bestHueIndex << " : " << setw(7) << hMeans[bestHueIndex] << " -- " << hueMin << "-" << hueMax << endl;
}
if (bestSatIndex != -1)
{
doSatFilter = true;
satMin = sMeans[bestSatIndex] - (2 * sStdDevs[bestSatIndex]);
satMax = sMeans[bestSatIndex] + (2 * sStdDevs[bestSatIndex]);
if (abs(satMin - satMax) < 20)
{
satMin = sMeans[bestSatIndex] - 20;
satMax = sMeans[bestSatIndex] + 20;
}
if (satMin < 0)
satMin = 0;
if (satMax > 255)
satMax = 255;
if (this->debug)
cout << "ColorFilter Sat: " << bestSatIndex << " : " << setw(7) << sMeans[bestSatIndex] << " -- " << satMin << "-" << satMax << endl;
}
if (bestValIndex != -1)
{
doValFilter = true;
valMin = vMeans[bestValIndex] - (1.5 * vStdDevs[bestValIndex]);
valMax = vMeans[bestValIndex] + (1.5 * vStdDevs[bestValIndex]);
if (abs(valMin - valMax) < 20)
{
valMin = vMeans[bestValIndex] - 20;
valMax = vMeans[bestValIndex] + 20;
}
if (valMin < 0)
valMin = 0;
if (valMax > 255)
valMax = 255;
if (this->debug)
cout << "ColorFilter Val: " << bestValIndex << " : " << setw(7) << vMeans[bestValIndex] << " -- " << valMin << "-" << valMax << endl;
}
Mat imgDebugHueOnly = Mat::zeros(hsv.size(), hsv.type());
Mat imgDebug = Mat::zeros(hsv.size(), hsv.type());
Mat imgDistanceFromCenter = Mat::zeros(hsv.size(), CV_8U);
Mat debugMask = Mat::zeros(hsv.size(), CV_8U);
bitwise_not(debugMask, debugMask);
for (int row = 0; row < charMask.rows; row++)
{
for (int col = 0; col < charMask.cols; col++)
{
int h = (int) hsv.at<Vec3b>(row, col)[0];
int s = (int) hsv.at<Vec3b>(row, col)[1];
int v = (int) hsv.at<Vec3b>(row, col)[2];
bool hPasses = true;
bool sPasses = true;
bool vPasses = true;
int vDistance = abs(v - vMeans[bestValIndex]);
imgDebugHueOnly.at<Vec3b>(row, col)[0] = h;
imgDebugHueOnly.at<Vec3b>(row, col)[1] = 255;
imgDebugHueOnly.at<Vec3b>(row, col)[2] = 255;
imgDebug.at<Vec3b>(row, col)[0] = 255;
imgDebug.at<Vec3b>(row, col)[1] = 255;
imgDebug.at<Vec3b>(row, col)[2] = 255;
if (doHueFilter && (h < hueMin || h > hueMax))
{
hPasses = false;
imgDebug.at<Vec3b>(row, col)[0] = 0;
debugMask.at<uchar>(row, col) = 0;
}
if (doSatFilter && (s < satMin || s > satMax))
{
sPasses = false;
imgDebug.at<Vec3b>(row, col)[1] = 0;
}
if (doValFilter && (v < valMin || v > valMax))
{
vPasses = false;
imgDebug.at<Vec3b>(row, col)[2] = 0;
}
//if (pixelPasses)
// colorMask.at<uchar>(row, col) = 255;
//else
//imgDebug.at<Vec3b>(row, col)[0] = hPasses & 255;
//imgDebug.at<Vec3b>(row, col)[1] = sPasses & 255;
//imgDebug.at<Vec3b>(row, col)[2] = vPasses & 255;
if ((hPasses) || (hPasses && sPasses))//(hPasses && vPasses) || (sPasses && vPasses) ||
this->colorMask.at<uchar>(row, col) = 255;
else
this->colorMask.at<uchar>(row, col) = 0;
if ((hPasses && sPasses) || (hPasses && vPasses) || (sPasses && vPasses))
{
vDistance = pow(vDistance, 0.9);
}
else
{
vDistance = pow(vDistance, 1.1);
}
if (vDistance > 255)
vDistance = 255;
imgDistanceFromCenter.at<uchar>(row, col) = vDistance;
}
}
vector<Mat> debugImagesSet;
if (this->debug)
{
debugImagesSet.push_back(addLabel(charMask, "Charecter mask"));
//debugImagesSet1.push_back(erodedCharMask);
Mat maskCopy(colorMask.size(), colorMask.type());
colorMask.copyTo(maskCopy);
debugImagesSet.push_back(addLabel(maskCopy, "color Mask Before"));
}
Mat bigElement = getStructuringElement( 1,
Size( 3 + 1, 3+1 ),
Point( 1, 1 ) );
Mat smallElement = getStructuringElement( 1,
Size( 1 + 1, 1+1 ),
Point( 1, 1 ) );
morphologyEx(this->colorMask, this->colorMask, MORPH_CLOSE, bigElement);
//dilate(this->colorMask, this->colorMask, bigElement);
Mat combined(charMask.size(), charMask.type());
bitwise_and(charMask, colorMask, combined);
if (this->debug)
{
debugImagesSet.push_back(addLabel(colorMask, "Color Mask After"));
debugImagesSet.push_back(addLabel(combined, "Combined"));
//displayImage(config, "COLOR filter Mask", colorMask);
debugImagesSet.push_back(addLabel(imgDebug, "Color filter Debug"));
cvtColor(imgDebugHueOnly, imgDebugHueOnly, CV_HSV2BGR);
debugImagesSet.push_back(addLabel(imgDebugHueOnly, "Color Filter Hue"));
equalizeHist(imgDistanceFromCenter, imgDistanceFromCenter);
debugImagesSet.push_back(addLabel(imgDistanceFromCenter, "COLOR filter Distance"));
debugImagesSet.push_back(addLabel(debugMask, "COLOR Hues off"));
Mat dashboard = drawImageDashboard(debugImagesSet, imgDebugHueOnly.type(), 3);
displayImage(config, "Color Filter Images", dashboard);
}
}
int main(int argc, char **argv) {
bool die(false);
string filename("snapshot");
string suffix(".png");
int i_snap(0),iter(0);
Mat depthMat(Size(640,480),CV_16UC1);
Mat depthf (Size(640,480),CV_8UC1);
Mat rgbMat(Size(640,480),CV_8UC3,Scalar(0));
Mat ownMat(Size(640,480),CV_8UC3,Scalar(0));
Freenect::Freenect<MyFreenectDevice> freenect;
MyFreenectDevice& device = freenect.createDevice(0);
device.setTiltDegrees(10.0);
bool registered = false;
Mat blobMaskOutput = Mat::zeros(Size(640,480),CV_8UC1),outC;
Point midBlob;
int startX = 200, sizeX = 180, num_x_reps = 18, num_y_reps = 48;
double height_over_num_y_reps = 480/num_y_reps,
width_over_num_x_reps = sizeX/num_x_reps;
vector<double> _d(num_x_reps * num_y_reps); //the descriptor
Mat descriptorMat(_d);
// CvNormalBayesClassifier classifier; //doesnt work
CvKNearest classifier;
// CvSVM classifier; //doesnt work
// CvBoost classifier; //only good for 2 classes
// CvDTree classifier;
vector<vector<double> > training_data;
vector<int> label_data;
PCA pca;
Mat labelMat, dataMat;
vector<float> label_counts(4);
bool trained = false, loaded = false;
device.startVideo();
device.startDepth();
while (!die) {
device.getVideo(rgbMat);
device.getDepth(depthMat);
// cv::imshow("rgb", rgbMat);
depthMat.convertTo(depthf, CV_8UC1, 255.0/2048.0);
cv::imshow("depth",depthf);
//interpolation & inpainting
{
Mat _tmp,_tmp1; // = (depthMat - 400.0); //minimum observed value is ~440. so shift a bit
Mat(depthMat - 400.0).convertTo(_tmp1,CV_64FC1);
_tmp.setTo(Scalar(2048), depthMat > 750.0); //cut off at 600 to create a "box" where the user interacts
// _tmp.convertTo(depthf, CV_8UC1, 255.0/1648.0); //values are 0-2048 (11bit), account for -400 = 1648
//quadratic interpolation
// cv::pow(_tmp,2.0,_tmp1);
// _tmp1 = _tmp1 * 4.0;
// try {
// cv:log(_tmp,_tmp1);
// }
// catch (cv::Exception e) {
// cerr << e.what() << endl;
// exit(0);
// }
Point minLoc; double minval,maxval;
minMaxLoc(_tmp1, &minval, &maxval, NULL, NULL);
_tmp1.convertTo(depthf, CV_8UC1, 255.0/maxval);
Mat small_depthf; resize(depthf,small_depthf,Size(),0.2,0.2);
cv::inpaint(small_depthf,(small_depthf == 255),_tmp1,5.0,INPAINT_TELEA);
resize(_tmp1, _tmp, depthf.size());
_tmp.copyTo(depthf, (depthf == 255));
}
{
// Mat smallDepth = depthf; //cv::resize(depthf,smallDepth,Size(),0.5,0.5);
// Mat edges; //Laplacian(smallDepth, edges, -1, 7, 1.0);
// Sobel(smallDepth, edges, -1, 1, 1, 7);
//medianBlur(edges, edges, 11);
// for (int x=0; x < edges.cols; x+=20) {
// for (int y=0; y < edges.rows; y+=20) {
// //int nz = countNonZero(edges(Range(y,MIN(y+20,edges.rows-1)),Range(x,MIN(x+20,edges.cols-1))));
// Mat _i = edges(Range(y,MIN(y+20,edges.rows-1)),Range(x,MIN(x+20,edges.cols-1)));
// medianBlur(_i, _i, 7);
// //rectangle(edges, Point(x,y), Point(x+20,y+20), Scalar(nz), CV_FILLED);
// }
// }
// imshow("edges", edges);
}
cvtColor(depthf, outC, CV_GRAY2BGR);
Mat blobMaskInput = depthf < 120; //anything not white is "real" depth, TODO: inpainting invalid data
vector<Point> ctr,ctr2;
//closest point to the camera
Point minLoc; double minval,maxval;
minMaxLoc(depthf, &minval, &maxval, &minLoc, NULL, blobMaskInput);
circle(outC, minLoc, 5, Scalar(0,255,0), 3);
blobMaskInput = depthf < (minval + 18);
Scalar blb = refineSegments(Mat(),blobMaskInput,blobMaskOutput,ctr,ctr2,midBlob); //find contours in the foreground, choose biggest
// if (blobMaskOutput.data != NULL) {
// imshow("first", blobMaskOutput);
// }
/////// blb :
//blb[0] = x, blb[1] = y, blb[2] = 1st blob size, blb[3] = 2nd blob size.
if(blb[0]>=0 && blb[2] > 500) { //1st blob detected, and is big enough
//cvtColor(depthf, outC, CV_GRAY2BGR);
Scalar mn,stdv;
meanStdDev(depthf,mn,stdv,blobMaskInput);
//cout << "min: " << minval << ", max: " << maxval << ", mean: " << mn[0] << endl;
//now refining blob by looking at the mean depth value it has...
blobMaskInput = depthf < (mn[0] + stdv[0]);
//(very simple) bias with hand color
{
Mat hsv; cvtColor(rgbMat, hsv, CV_RGB2HSV);
Mat _col_p(hsv.size(),CV_32FC1);
int jump = 5;
for (int x=0; x < hsv.cols; x+=jump) {
for (int y=0; y < hsv.rows; y+=jump) {
Mat _i = hsv(Range(y,MIN(y+jump,hsv.rows-1)),Range(x,MIN(x+jump,hsv.cols-1)));
Scalar hsv_mean = mean(_i);
Vec2i u; u[0] = hsv_mean[0]; u[1] = hsv_mean[1];
Vec2i v; v[0] = 120; v[1] = 110;
rectangle(_col_p, Point(x,y), Point(x+jump,y+jump), Scalar(1.0-MIN(norm(u-v)/125.0,1.0)), CV_FILLED);
}
}
// hsv = hsv - Scalar(0,0,255);
Mat _t = (Mat_<double>(2,3) << 1, 0, 15, 0, 1, -20);
Mat col_p(_col_p.size(),CV_32FC1);
warpAffine(_col_p, col_p, _t, col_p.size());
GaussianBlur(col_p, col_p, Size(11.0,11.0), 2.5);
imshow("hand color",col_p);
// imshow("rgb",rgbMat);
Mat blobMaskInput_32FC1; blobMaskInput.convertTo(blobMaskInput_32FC1, CV_32FC1, 1.0/255.0);
blobMaskInput_32FC1 = blobMaskInput_32FC1.mul(col_p, 1.0);
blobMaskInput_32FC1.convertTo(blobMaskInput, CV_8UC1, 255.0);
blobMaskInput = blobMaskInput > 128;
imshow("blob bias", blobMaskInput);
}
blb = refineSegments(Mat(),blobMaskInput,blobMaskOutput,ctr,ctr2,midBlob);
imshow("blob", blobMaskOutput);
if(blb[0] >= 0 && blb[2] > 300) {
//draw contour
Scalar color(0,0,255);
for (int idx=0; idx<ctr.size()-1; idx++)
line(outC, ctr[idx], ctr[idx+1], color, 1);
line(outC, ctr[ctr.size()-1], ctr[0], color, 1);
if(ctr2.size() > 0) { //second blob detected
Scalar color2(255,0,255);
for (int idx=0; idx<ctr2.size()-1; idx++)
line(outC, ctr2[idx], ctr2[idx+1], color2, 2);
line(outC, ctr2[ctr2.size()-1], ctr2[0], color2, 2);
}
//blob center
circle(outC, Point(blb[0],blb[1]), 50, Scalar(255,0,0), 3);
{
Mat hsv; cvtColor(rgbMat, hsv, CV_RGB2HSV);
Scalar hsv_mean,hsv_stddev; meanStdDev(hsv, hsv_mean, hsv_stddev, blobMaskOutput);
stringstream ss; ss << hsv_mean[0] << "," << hsv_mean[1] << "," << hsv_mean[2];
putText(outC, ss.str(), Point(blb[0],blb[1]), CV_FONT_HERSHEY_PLAIN, 1.0, Scalar(0,255,255));
}
Mat blobDepth,blobEdge;
depthf.copyTo(blobDepth,blobMaskOutput);
Laplacian(blobDepth, blobEdge, 8);
// equalizeHist(blobEdge, blobEdge);//just for visualization
Mat logPolar(depthf.size(),CV_8UC1);
cvLogPolar(&((IplImage)blobEdge), &((IplImage)logPolar), Point2f(blb[0],blb[1]), 80.0);
// for (int i=0; i<num_x_reps+1; i++) {
// //verical lines
// line(logPolar, Point(startX+i*width_over_num_x_reps, 0), Point(startX+i*width_over_num_x_reps,479), Scalar(255), 1);
// }
// for(int i=0; i<num_y_reps+1; i++) {
// //horizontal
// line(logPolar, Point(startX, i*height_over_num_y_reps), Point(startX+sizeX,i*height_over_num_y_reps), Scalar(255), 1);
// }
double total = 0.0;
//histogram
for (int i=0; i<num_x_reps; i++) {
for(int j=0; j<num_y_reps; j++) {
Mat part = logPolar(
Range(j*height_over_num_y_reps,(j+1)*height_over_num_y_reps),
Range(startX+i*width_over_num_x_reps,startX+(i+1)*width_over_num_x_reps)
);
// int count = countNonZero(part); //TODO: use calcHist
// _d[i*num_x_reps + j] = count;
Scalar mn = mean(part);
// part.setTo(Scalar(mn[0])); //for debug: show the value in the image
_d[i*num_x_reps + j] = mn[0];
total += mn[0];
}
}
descriptorMat = descriptorMat / total;
/*
Mat images[1] = {logPolar(Range(0,30),Range(0,30))};
int nimages = 1;
int channels[1] = {0};
int dims = 1;
float range_0[]={0,256};
float* ranges[] = { range_0 };
int histSize[1] = { 5 };
calcHist(, <#int nimages#>, <#const int *channels#>, <#const Mat mask#>, <#MatND hist#>, <#int dims#>, <#const int *histSize#>, <#const float **ranges#>, <#bool uniform#>, <#bool accumulate#>)
*/
// Mat _tmp(logPolar.size(),CV_8UC1);
// cvLogPolar(&((IplImage)logPolar), &((IplImage)_tmp),Point2f(blb[0],blb[1]), 80.0, CV_WARP_INVERSE_MAP);
// imshow("descriptor", _tmp);
// imshow("logpolar", logPolar);
}
}
if(trained) {
Mat results(1,1,CV_32FC1);
Mat samples; Mat(Mat(_d).t()).convertTo(samples,CV_32FC1);
Mat samplesAfterPCA = samples; //pca.project(samples);
classifier.find_nearest(&((CvMat)samplesAfterPCA), 1, &((CvMat)results));
// ((float*)results.data)[0] = classifier.predict(&((CvMat)samples))->value;
Mat lc(label_counts); lc *= 0.9;
// label_counts[(int)((float*)results.data)[0]] *= 0.9;
label_counts[(int)((float*)results.data)[0]] += 0.1;
Point maxLoc;
minMaxLoc(lc, NULL, NULL, NULL, &maxLoc);
int res = maxLoc.y;
stringstream ss; ss << "prediction: ";
if (res == LABEL_OPEN) {
ss << "Open hand";
}
if (res == LABEL_FIST) {
ss << "Fist";
}
if (res == LABEL_THUMB) {
ss << "Thumb";
}
if (res == LABEL_GARBAGE) {
ss << "Garbage";
}
putText(outC, ss.str(), Point(20,50), CV_FONT_HERSHEY_PLAIN, 3.0, Scalar(0,0,255), 2);
}
stringstream ss; ss << "samples: " << training_data.size();
putText(outC, ss.str(), Point(30,outC.rows - 30), CV_FONT_HERSHEY_PLAIN, 2.0, Scalar(0,0,255), 1);
imshow("blobs", outC);
char k = cvWaitKey(5);
if( k == 27 ){
break;
}
if( k == 8 ) {
std::ostringstream file;
file << filename << i_snap << suffix;
cv::imwrite(file.str(),rgbMat);
i_snap++;
}
if (k == 'g') {
//put into training as 'garbage'
training_data.push_back(_d);
label_data.push_back(LABEL_GARBAGE);
cout << "learn grabage" << endl;
}
if(k == 'o') {
//put into training as 'open'
training_data.push_back(_d);
label_data.push_back(LABEL_OPEN);
cout << "learn open" << endl;
}
if(k == 'f') {
//put into training as 'fist'
training_data.push_back(_d);
label_data.push_back(LABEL_FIST);
cout << "learn fist" << endl;
}
if(k == 'h') {
//put into training as 'thumb'
training_data.push_back(_d);
label_data.push_back(LABEL_THUMB);
cout << "learn thumb" << endl;
}
if (k=='t') {
//train model
cout << "train model" << endl;
if(loaded != true) {
dataMat = Mat(training_data.size(),_d.size(),CV_32FC1); //descriptors as matrix rows
for (uint i=0; i<training_data.size(); i++) {
Mat v = dataMat(Range(i,i+1),Range::all());
Mat(Mat(training_data[i]).t()).convertTo(v,CV_32FC1,1.0);
}
Mat(label_data).convertTo(labelMat,CV_32FC1);
}
// pca = pca(dataMat,Mat(),CV_PCA_DATA_AS_ROW,15);
Mat dataAfterPCA = dataMat;
// pca.project(dataMat,dataAfterPCA);
classifier.train(&((CvMat)dataAfterPCA), &((CvMat)labelMat));
trained = true;
}
// if(k=='p' && trained) {
// //predict
// Mat results(1,1,CV_32FC1);
// Mat samples(1,64,CV_32FC1); Mat(Mat(_d).t()).convertTo(samples,CV_32FC1);
// classifier.find_nearest(&((CvMat)samples), 1, &((CvMat)results));
// cout << "prediction: " << ((float*)results.data)[0] << endl;
// }
if(k=='s') {
cout << "save training data" << endl;
// classifier.save("knn-classifier-open-fist-thumb.yaml"); //not implemented
dataMat = Mat(training_data.size(),_d.size(),CV_32FC1); //descriptors as matrix rows
for (uint i=0; i<training_data.size(); i++) {
Mat v = dataMat(Range(i,i+1),Range::all());
Mat(Mat(training_data[i]).t()).convertTo(v,CV_32FC1,1.0);
}
Mat(label_data).convertTo(labelMat,CV_32FC1);
FileStorage fs;
fs.open("data-samples-labels.yaml", CV_STORAGE_WRITE);
if (fs.isOpened()) {
fs << "samples" << dataMat;
fs << "labels" << labelMat;
fs << "startX" << startX;
fs << "sizeX" << sizeX;
fs << "num_x_reps" << num_x_reps;
fs << "num_y_reps" << num_y_reps;
loaded = true;
fs.release();
} else {
cerr << "can't open saved data" << endl;
}
}
if(k=='l') {
cout << "try to load training data" << endl;
FileStorage fs;
fs.open("data-samples-labels.yaml", CV_STORAGE_READ);
if (fs.isOpened()) {
fs["samples"] >> dataMat;
fs["labels"] >> labelMat;
fs["startX"] >> startX;
fs["sizeX"] >> sizeX;
fs["num_x_reps"] >> num_x_reps;
fs["num_y_reps"] >> num_y_reps;
height_over_num_y_reps = 480/num_y_reps;
width_over_num_x_reps = sizeX/num_x_reps;
loaded = true;
fs.release();
} else {
int Judgement::JudgementYON(Mat &image)
{
int success = 0;
MatND dstHist;
Mat histoImg = image.clone();
calcHist(&histoImg, 1, &channels, Mat(), dstHist, 1, &size, ranges);
Mat dstImg(256, 256, CV_8U, Scalar(0));//画直方图
double minValue = 0;
double maxValue = 0;
Point maxloc;
minMaxLoc(dstHist, &minValue, &maxValue, NULL, &maxloc);
//cout << " " << n << "." << m << " " << maxValue << endl;
int hpt = saturate_cast<int>(0.9 * 256);
vector<int> Boundnum;
for (int j = 0; j < 256; j++)
{
float binValue = dstHist.at<float>(j);
int realValue = saturate_cast<int>(binValue * hpt / maxValue);
if (realValue != 0)
{
rectangle(dstImg, Point(j, 255), Point(j, 256 - realValue), Scalar(255));
Boundnum.push_back(j);
}
}
int maxdata = *max_element(Boundnum.begin(), Boundnum.end());
int mindata = *min_element(Boundnum.begin(), Boundnum.end());//寻找直方图动态范围
Rect recttemp;
recttemp.x = maxloc.x;
recttemp.y = maxloc.y - int((maxdata - mindata)*0.15);
recttemp.width = 1;
recttemp.height = int((maxdata - mindata)*0.3);
rectangle(dstHist, recttemp, Scalar(0), -1);
minMaxLoc(dstHist, &minValue, &maxValue, NULL, &maxloc);
int anoThres = maxloc.y;//寻找次峰值
Scalar avgnum;
Mat StdDevImg;
meanStdDev(histoImg, avgnum, StdDevImg);
double Stdnum = StdDevImg.at<double>(Point(0, 0));
int ThreStep = maxdata - mindata;
int StepNum = 30;
int OrStep = mindata + int(ThreStep / 10);
int Dstep = int(ThreStep / 30.0 + 0.5);
if (Dstep == 0)
{
Dstep = 1;
StepNum = ThreStep;
}
Mat TempImg;
histoImg.copyTo(TempImg);
vector<vector<Point>> contours;
vector<Vec4i> hierarchy;
Point pointSN, maxPoint = Point(0, 0);
int Marknumone = 0;
int Marknumtwo = 0;
int Marknumthree = 0;
for (int i = 0; i < StepNum; i++)
{
vector<Point> SN;
OrStep = OrStep + Dstep;
threshold(histoImg, TempImg, OrStep, 255, CV_THRESH_BINARY);
/*Mat element = getStructuringElement(MORPH_RECT,Size(2,2));
erode(TempImg, TempImg, cv::Mat());
dilate(TempImg, TempImg, cv::Mat());*/
TempImg = ~TempImg;
/*stringstream strstrone;
strstrone << "水渍动态图" << i << ".jpg";
imwrite(strstrone.str(), TempImg);*/
Mat BoundImg(TempImg.rows, TempImg.cols, CV_8UC1, Scalar(255));
Rect Wrect;
Wrect.x = 1;
Wrect.y = 1;
Wrect.width = BoundImg.cols - 2;
Wrect.height = BoundImg.rows - 2;
rectangle(BoundImg, Wrect, Scalar(0), -1);
Mat PlusImg(TempImg.rows + 2, TempImg.cols + 2, CV_8UC1, Scalar(255));
Mat PlusROI = PlusImg(Rect(1, 1, TempImg.cols, TempImg.rows));
TempImg.copyTo(PlusROI);
Mat ContoursImg = PlusImg.clone();
findContours(ContoursImg, contours, hierarchy, RETR_TREE, CV_CHAIN_APPROX_SIMPLE);
for (size_t j = 0; j < contours.size(); j++)
{
double area = cv::contourArea(contours[j]);
pointSN.x = int(area);
pointSN.y = j;
SN.push_back(pointSN);
}
if (contours.size() != 0)
{
sort(SN.begin(), SN.end(), SortByM2);
maxPoint = SN.back();
if (OrStep > anoThres - 5 && OrStep<anoThres + 20)
Dstep = 1;
else
{
Dstep = int(ThreStep / 30.0 + 0.5);
}
if (Dstep == 0)
Dstep = 1;
int k = maxPoint.y;
Mat MarkImg(TempImg.rows, TempImg.cols, CV_8UC1, Scalar(0));
drawContours(MarkImg, contours, k, Scalar(255), -1);
bitwise_and(BoundImg, MarkImg, MarkImg);
int Mbound = 0;//判断轮廓是否到边界
Mbound = countNonZero(MarkImg);
if (Mbound>0.5*(histoImg.cols))
break;
if (contours[k].size() <= 4)
continue;
int son = hierarchy[k][2];
Point gravitycore = barycenter(contours[k]);//寻找轮廓重心
Rect maxcontours = boundingRect(contours[k]);
int wValue = maxcontours.width / 12;
gravitycore = gravitycore + Point(wValue - 1, wValue - 1);
Mat gravityImg(TempImg.rows + 2 * wValue, TempImg.cols + 2 * wValue, CV_8UC1, Scalar(0));
Mat gravityImgROI = gravityImg(Rect(wValue, wValue, TempImg.cols, TempImg.rows));
TempImg.copyTo(gravityImgROI);
Rect gravityrect = Rect(gravitycore - Point(1, 1), gravitycore + Point(2 * wValue, 2 * wValue) - Point(2, 2));//画出重心周围(2 * wValue)*(2 * wValue)的矩形区域
if (gravityrect.x < 0 || gravityrect.y < 0)
continue;
int avnum = countNonZero(gravityImg(Rect(gravityrect)));
vector<Point> hull;
convexHull(contours[k], hull, false);
double promark = (contourArea(contours[k])) / (contourArea(hull));
if (son >= 0)//判断是否为父轮廓
{
int sonarea = 0;
for (size_t j = 0; j < contours.size(); j++)
{
if (hierarchy[j][3] == k&&contourArea(contours[j])>4.0)
sonarea = sonarea + contourArea(contours[j]);
}
if (50 * sonarea>maxPoint.x)//此处忽略一些偶然出现的中空点
Marknumone++;
}
if (avnum < double(0.5 * gravityrect.width*gravityrect.width))//在重心区域中的白色点的数量是否过半
Marknumtwo++;
if (promark < 0.6)
Marknumthree++;
}
}
if (Marknumone > 2 || Marknumtwo >= 2 || Marknumthree > 3)//缺陷点也可能偶然出现包含
{
/*cout << "该点是水渍2" << endl;*/
}
else
{
/*cout << "该点是缺陷2" << endl;*/
success++;
}
return success;
}
int main(int argc, char **argv)
{
int res;
try {
socket.bind ("tcp://*:14444");
s_sendmore (socket, "event");
s_send (socket, "{type:\"up\"}");
}
catch (zmq::error_t e) {
cerr << "Cannot bind to socket: " <<e.what() << endl;
return -1;
}
// printf("Kinect camera test\n");
//
// int i;
// for (i=0; i<2048; i++) {
// float v = i/2048.0;
// v = powf(v, 3)* 6;
// t_gamma[i] = v*6*256;
// }
//
// g_argc = argc;
// g_argv = argv;
//
// //setup Freenect...
// if (freenect_init(&f_ctx, NULL) < 0) {
// printf("freenect_init() failed\n");
// return 1;
// }
//
// freenect_set_log_level(f_ctx, FREENECT_LOG_ERROR);
//
// int nr_devices = freenect_num_devices (f_ctx);
// printf ("Number of devices found: %d\n", nr_devices);
//
// int user_device_number = 0;
// if (argc > 1)
// user_device_number = atoi(argv[1]);
//
// if (nr_devices < 1)
// return 1;
//
// if (freenect_open_device(f_ctx, &f_dev, user_device_number) < 0) {
// printf("Could not open device\n");
// return 1;
// }
//
// freenect_set_tilt_degs(f_dev,freenect_angle);
// freenect_set_led(f_dev,LED_RED);
// freenect_set_depth_callback(f_dev, depth_cb);
// freenect_set_video_callback(f_dev, rgb_cb);
// freenect_set_video_format(f_dev, FREENECT_VIDEO_RGB);
// freenect_set_depth_format(f_dev, FREENECT_DEPTH_11BIT);
//
// freenect_start_depth(f_dev);
// freenect_start_video(f_dev);
initFreenect();
//start the freenect thread to poll for events
res = pthread_create(&ocv_thread, NULL, freenect_threadfunc, NULL);
if (res) {
printf("pthread_create failed\n");
return 1;
}
Mat depthf;
Mat frameMat(rgbMat);
Mat blobMaskOutput(frameMat.size(),CV_8UC1),
outC(frameMat.size(),CV_8UC3);
Mat prevImg(frameMat.size(),CV_8UC1),
nextImg(frameMat.size(),CV_8UC1),
prevDepth(depthMat.size(),CV_8UC1);
vector<Point2f> prevPts,nextPts;
vector<uchar> statusv;
vector<float> errv;
Rect cursor(frameMat.cols/2,frameMat.rows/2,10,10);
bool update_bg_model = true;
int fr = 1;
int register_ctr = 0,register_secondbloc_ctr = 0;
bool registered = false;
Point2i appear(-1,-1); double appearTS = -1;
Point2i midBlob(-1,-1);
Point2i lastMove(-1,-1);
int hcr_ctr = -1;
vector<int> hc_stack(20); int hc_stack_ptr = 0;
while (!die) {
fr++;
// imshow("rgb", rgbMat);
pthread_mutex_lock(&buf_mutex);
//Linear interpolation
{
Mat _tmp = (depthMat - 400.0); //minimum observed value is ~440. so shift a bit
_tmp.setTo(Scalar(2048), depthMat > ((!registered) ? 700.0 : 750.0)); //cut off at 600 to create a "box" where the user interacts
_tmp.convertTo(depthf, CV_8UC1, 255.0/1648.0); //values are 0-2048 (11bit), account for -400 = 1648
}
{
Mat _tmp;
depthMat.convertTo(_tmp, CV_8UC1, 255.0/2048.0);
cvtColor(_tmp, outC, CV_GRAY2BGR);
}
pthread_mutex_unlock(&buf_mutex);
// { //saving the frames to files for debug
// stringstream ss; ss << "depth_"<<fr<<".png";
// imwrite(ss.str(), depthf);
// }
//Logarithm interpolation - try it!, It should be more "sensitive" for closer depths
// {
// Mat tmp,tmp1;
// depthMat.convertTo(tmp, CV_32FC1);
// log(tmp,tmp1);
// tmp1.convertTo(depthf, CV_8UC1, 255.0/7.6246189861593985);
// }
// imshow("depth",depthf);
Mat blobMaskInput = depthf < 255; //anything not white is "real" depth
vector<Point> ctr,ctr2;
Scalar blb = refineSegments(Mat(),blobMaskInput,blobMaskOutput,ctr,ctr2,midBlob); //find contours in the foreground, choose biggest
imshow("first", blobMaskOutput);
/////// blb :
//blb[0] = x, blb[1] = y, blb[2] = 1st blob size, blb[3] = 2nd blob size.
// uint mode_counters[3] = {0};
if(blb[0]>=0 && blb[2] > 500) { //1st blob detected, and is big enough
//cvtColor(depthf, outC, CV_GRAY2BGR);
//closest point to the camera
Point minLoc; double minval,maxval;
minMaxLoc(depthf, &minval, &maxval, &minLoc, NULL, blobMaskInput);
circle(outC, minLoc, 5, Scalar(0,255,0), 3);
Scalar mn,stdv;
meanStdDev(depthf,mn,stdv,blobMaskInput);
//cout << "min: " << minval << ", max: " << maxval << ", mean: " << mn[0] << endl;
blobMaskInput = depthf < (mn[0] + stdv[0]*.5);
blb = refineSegments(Mat(),blobMaskInput,blobMaskOutput,ctr,ctr2,midBlob);
imshow("second", blobMaskOutput);
if(blb[0] >= 0 && blb[2] > 300) {
//draw contour
Scalar color(0,0,255);
for (int idx=0; idx<ctr.size()-1; idx++)
line(outC, ctr[idx], ctr[idx+1], color, 1);
line(outC, ctr[ctr.size()-1], ctr[0], color, 1);
if(ctr2.size() > 0) {
Scalar color2(255,0,255);
for (int idx=0; idx<ctr2.size()-1; idx++)
line(outC, ctr2[idx], ctr2[idx+1], color2, 2);
line(outC, ctr2[ctr2.size()-1], ctr2[0], color2, 2);
}
//draw "major axis"
// Vec4f _line;
Mat curve(ctr);
// fitLine(curve, _line, CV_DIST_L2, 0, 0.01, 0.01);
// line(outC, Point(blb[0]-_line[0]*70,blb[1]-_line[1]*70),
// Point(blb[0]+_line[0]*70,blb[1]+_line[1]*70),
// Scalar(255,255,0), 1);
//blob center
circle(outC, Point(blb[0],blb[1]), 50, Scalar(255,0,0), 3);
// cout << "min depth " << minval << endl;
register_ctr = MIN((register_ctr + 1),60);
if(blb[3] > 5000)
register_secondbloc_ctr = MIN((register_secondbloc_ctr + 1),60);
if (register_ctr > 30 && !registered) {
registered = true;
appear.x = -1;
update_bg_model = false;
lastMove.x = blb[0]; lastMove.y = blb[1];
cout << "blob size " << blb[2] << endl;
if(register_secondbloc_ctr < 30) {
if(blb[2] > 10000) {
cout << "register panner" << endl;
send_event("Register", "\"mode\":\"openhand\"");
} else {
cout << "register pointer" << endl;
send_event("Register", "\"mode\":\"theforce\"");
}
} else {
cout << "register tab swithcer" << endl;
send_event("Register", "\"mode\":\"twohands\"");
}
}
if(registered) {
stringstream ss;
ss << "\"x\":" << (int)floor(blb[0]*100.0/640.0)
<< ",\"y\":" << (int)floor(blb[1]*100.0/480.0)
<< ",\"z\":" << (int)(mn[0] * 2.0);
//cout << "move: " << ss.str() << endl;
send_event("Move", ss.str());
//---------------------- fist detection ---------------------
//calc laplacian of curve
vector<Point> approxCurve; //approximate curve
approxPolyDP(curve, approxCurve, 10.0, true);
Mat approxCurveM(approxCurve);
Mat curve_lap;
calc_laplacian(approxCurveM, curve_lap); //calc laplacian
hcr_ctr = 0;
for (int i=0; i<approxCurve.size(); i++) {
double n = norm(((Point2d*)(curve_lap.data))[i]);
if (n > 10.0) {
//high curvature point
circle(outC, approxCurve[i], 3, Scalar(50,155,255), 2);
hcr_ctr++;
}
}
hc_stack.at(hc_stack_ptr) = hcr_ctr;
hc_stack_ptr = (hc_stack_ptr + 1) % hc_stack.size();
Scalar _avg = mean(Mat(hc_stack));
if (abs(_avg[0] - (double)hcr_ctr) > 5.0) { //a big change in curvature = hand fisted/opened?
cout << "Hand click!" << endl;
send_event("HandClick", "");
}
if (mode_state == MODE_NONE) {
}
// imshow("out",out);
//doHist(depthf,out);
{ //some debug on screen..
stringstream ss; ss << "high curve pts " << hcr_ctr << ", avg " << _avg[0];
putText(outC, ss.str(), Point(50,50), CV_FONT_HERSHEY_PLAIN, 2.0,Scalar(0,0,255), 2);
}
} else {
//not registered, look for gestures
if(appear.x<0) {
//first appearence of blob
appear = midBlob;
// update_bg_model = false;
appearTS = getTickCount();
cout << "appear ("<<appearTS<<") " << appear.x << "," << appear.y << endl;
} else {
//blob was seen before, how much time passed
double timediff = ((double)getTickCount()-appearTS)/getTickFrequency();
if (timediff > .2 && timediff < 1.0) {
//enough time passed from appearence
line(outC, appear, Point(blb[0],blb[1]), Scalar(0,0,255), 3);
if (appear.x - blb[0] > 100) {
cout << "right"<<endl; appear.x = -1;
send_event("SwipeRight", "");
update_bg_model = true;
register_ctr = 0;
} else if (appear.x - blb[0] < -100) {
cout << "left" <<endl; appear.x = -1;
send_event("SwipeLeft", "");
update_bg_model = true;
register_ctr = 0;
} else if (appear.y - blb[1] > 100) {
cout << "up" << endl; appear.x = -1;
send_event("SwipeUp", "");
update_bg_model = true;
register_ctr = 0;
} else if (appear.y - blb[1] < -100) {
cout << "down" << endl; appear.x = -1;
send_event("SwipeDown", "");
update_bg_model = true;
register_ctr = 0;
}
}
if(timediff >= 1.0) {
cout << "a ghost..."<<endl;
update_bg_model = true;
//a second passed from appearence - reset 1st appear
appear.x = -1;
appearTS = -1;
midBlob.x = midBlob.y = -1;
}
}
}
send_image(outC);
}
} else {
send_image(depthf);
register_ctr = MAX((register_ctr - 1),0);
register_secondbloc_ctr = MAX((register_secondbloc_ctr - 1),0);
}
imshow("blob",outC);
if (register_ctr <= 15 && registered) {
midBlob.x = midBlob.y = -1;
registered = false;
mode_state = MODE_NONE;
update_bg_model = true;
cout << "unregister" << endl;
send_event("Unregister", "");
}
char k = cvWaitKey(5);
if( k == 27 ) break;
if( k == ' ' )
update_bg_model = !update_bg_model;
if (k=='s') {
cout << "send test event" << endl;
send_event("TestEvent", "");
}
}
printf("-- done!\n");
pthread_join(ocv_thread, NULL);
pthread_exit(NULL);
return 0;
}
void LaneDetection::AlgoFilterLanes(ntuple_list line_out)
{
//fout_2.open("pairs.txt");
unsigned int dim = line_out->dim;
int n_size = line_out->size;
Mat lines_candidats = Mat::zeros(processGrayImage.size(), CV_32SC1);
for (int i = 0; i < n_size; i++)
{
const Point2d p1(line_out->values[i * dim + 0], line_out->values[i * dim + 1]);
const Point2d p2(line_out->values[i * dim + 2], line_out->values[i * dim + 3]);
Segment2d p12 = Segment2d(p1, p2);//p1.y < p2.y
for (int j = 0; j < n_size; j++)
{
if (j == i) continue;
Point2d seg1(line_out->values[j * dim + 0], line_out->values[j * dim + 1]);
Point2d seg2(line_out->values[j * dim + 2], line_out->values[j * dim + 3]);
Segment2d seg(seg1, seg2);
if (seg.getLength() > p12.getLength())
{
continue;
}
//simple check
if (!p12.isNeighbor(seg))
{
continue;
}
//slope difference?
if (abs(atan(p12.getSlope()) - atan(seg.getSlope())) > 15 * CV_PI / 180)
{
//fout_2 << p1 << p2 << seg1 << seg2 << p12.getSlope() << "," << seg.getSlope() << endl;
//fout_2 << "continue in abs(atan(p12.getSlope()) - atan(seg.getSlope())) > 10 * CV_PI / 180" << endl;
continue;
}
vector<Segment2d> v = p12.getValidPoly(seg, processGrayImage.rows);
//valid?
if (v.empty())
{
//fout_2 << p1 << p2 << seg1 << seg2 << endl;
//fout_2 << "continue in v.empty()" << endl;
continue;
}
//color compare
Point2d translation[3];
translation[0] = Point2d(0, 0);
translation[1] = v[0].p1 - v[1].p1;
translation[2] = -translation[1];
vector<uchar> vec_color[3];
double mean_color[3];
double stdDev_color[3];
for (int _trans = 0; _trans < 3; _trans++)
{
Rect box = getBoundingBox(v[0].p1 + translation[_trans], v[0].p2 + translation[_trans],
v[1].p1 + translation[_trans], v[1].p2 + translation[_trans]);
for (int _y = box.y; _y < box.y + box.height; _y++)
{
if (_y >= processImage.rows || _y < 0) continue;
uchar* ptr_row_processImage = processGrayImage.ptr<uchar>(_y);
for (int _x = box.x; _x < box.x + box.width; _x++)
{
if (_x >= processImage.cols || _x < 0) continue;
Point2d p(_x, _y);
int direc = cross(p, v[0].p1 + translation[_trans], v[0].p2 + translation[_trans]) > -0.0 ? 1 : -1;
if (direc != (cross(p, v[0].p2 + translation[_trans], v[1].p2 + translation[_trans]) > -0.0 ? 1 : -1))
continue;
if (direc != (cross(p, v[1].p2 + translation[_trans], v[1].p1 + translation[_trans]) > -0.0 ? 1 : -1))
continue;
if (direc != (cross(p, v[1].p1 + translation[_trans], v[0].p1 + translation[_trans]) > -0.0 ? 1 : -1))
continue;
vec_color[_trans].push_back(ptr_row_processImage[_x]);
}
}
if (vec_color[_trans].empty())
;
else
{
Mat mean, stdDev;
meanStdDev(vec_color[_trans], mean, stdDev);
mean_color[_trans] = mean.at<double>(0);
stdDev_color[_trans] = stdDev.at<double>(0);
}
}
bool color_matched = (mean_color[0] > mean_color[1] + 10) && (mean_color[0] > mean_color[2] + 10)
&& stdDev_color[0] < 50;
if (!color_matched)
{
//fout_2 << p1 << p2 << seg1 << seg2 << mean_color[0] << "," << mean_color[1] << "," << mean_color[2];
//fout_2 << "," << stdDev_color[0] << endl;
//fout_2 << "continue in color_matched" << endl;
continue;
}
//fout_2 << p1 << p2 << seg1 << seg2 << endl;
//fout_2 << "Paired!!!!!!" << endl;
line(processImage, p1, p2, Scalar(0, 0, 255));
line(processImage, seg1, seg2, Scalar(0, 255, 0));
//imshow("step1", processImage);
//waitKey();
//break;
}//end for j
}
//fout_2 << "-------------------" << endl;
//fout_2.close();
}
void RGB::obtainDescriptors (int nGroups)
{
// Grupos en los que se dividen las imagenes
int grup10 = nGroups;
int ix_vec = -1;
int sizepng = _fileList.size();
Mat img;
QString namePng;
string namePngPath;
// RGB variables
vector<float> V_des_RGB;
vector<float> V_des_mRGB_10;
vector<float> V_des_mB_10;
vector<float> V_des_mG_10;
vector<float> V_des_mR_10;
vector<float> V_des_sdv_10;
vector<float> V_des_RGB_10;
Scalar med_medBlue;
Scalar med_medGreen;
Scalar med_medRed;
Scalar med_medRGB;
Scalar med_svd;
Scalar desvi_med_medBlue;
Scalar desvi_med_medGreen;
Scalar desvi_med_medRed;
Scalar desvi_med_medRGB;
Scalar desvi_med_svd;
for(int cont = 0; cont < sizepng; ++cont)
{
namePng = _fileList.at(cont);
namePngPath = this->getImagesPath() + namePng.toStdString();
#ifdef DEBUG
cout << "name image: " << namePng.toStdString() << endl;
cout << "path image: " << namePngPath << endl;
#endif
img = imread(namePngPath);
if(img.empty())
{
cout<<"Imagen no leida"<< endl;
}
if(!img.empty())
{
#ifdef DEBUG
char key;
imshow("img_in",img);
key = waitKey(10);
if(key == 'q' || key == 'c' || key == 'Q' || key == 'C')
{
break;
}
#endif
grup10 = grup10 - 1;
// Se calcula el descriptor de la media y la desviacion estandar del RGB
V_des_RGB = this->extract_RGB_des(img);
// Almacenamos todos los valores de media y desviacion estandar de cada
// imagen en el vector _VV_des_RGB
_VV_des_RGB.push_back(V_des_RGB);
//Agrupando en grupos de 10
if(grup10 <= nGroups)
{
ix_vec = ix_vec + 1;
_orden << (Mat)_VV_des_RGB[ix_vec] << " ----> " << namePngPath << "\n";
// ALMACENAMOS LOS DESCRIPTORES: MEDIA B, G, R, RGB Y DESVIACION ESTANDAR RGB
V_des_mB_10.push_back(V_des_RGB[0]);
V_des_mG_10.push_back(V_des_RGB[1]);
V_des_mR_10.push_back(V_des_RGB[2]);
V_des_mRGB_10.push_back(V_des_RGB[3]);
V_des_sdv_10.push_back(V_des_RGB[4]);
}
if(grup10 == 0)
{
grup10 = nGroups;
// Calculamos la media R, G, B Y RGB,y la desv tipica de la media RGB para grupos de 10
meanStdDev((Mat)V_des_mB_10, med_medBlue, desvi_med_medBlue);
_orden << "med_medBlue : " << med_medBlue[0] << " \t desv : " << desvi_med_medBlue[0] << "\n";
meanStdDev((Mat)V_des_mG_10, med_medGreen, desvi_med_medGreen);
_orden << "med_medGreen : " << med_medGreen[0] << " \t desv : " << desvi_med_medGreen[0] << "\n";
meanStdDev((Mat)V_des_mR_10, med_medRed, desvi_med_medRed);
_orden << "med_medRed : " << med_medRed[0] << " \t desv : " << desvi_med_medRed[0] << "\n";
meanStdDev((Mat)V_des_mRGB_10, med_medRGB, desvi_med_medRGB);
_orden << "med_medRGB : " << med_medRGB[0] << " \t desv : " << desvi_med_medRGB[0] << "\n";
meanStdDev((Mat)V_des_sdv_10, med_svd, desvi_med_svd);
_orden << "med_svd : " << med_svd[0] << " \t desv : " << desvi_med_svd[0] << "\n";
//Almacenando los descriptores
_medBlue << med_medBlue[0] << " " << desvi_med_medBlue[0] << "\n";
_medGreen << med_medGreen[0] << " " << desvi_med_medGreen[0] << "\n";
_medRed << med_medRed[0] << " " << desvi_med_medRed[0] << "\n";
_medRGB << med_medRGB[0] << " " << desvi_med_medRGB[0] << "\n";
_svd << med_svd[0] << " " << desvi_med_svd[0] << "\n";
// Creamos dos Mat con 1 columna y nfilas de imagenes
_V_des_RGB_10_SVDmedBlue.push_back(desvi_med_medBlue[0]);
_V_des_RGB_10_SVDmedGreen.push_back(desvi_med_medGreen[0]);
_V_des_RGB_10_SVDmedRed.push_back(desvi_med_medRed[0]);
_V_des_RGB_10_SVDmedRGB.push_back(desvi_med_medRGB[0]);
_V_des_RGB_10_SVDmedSVD.push_back(desvi_med_svd[0]);
_V_des_RGB_10_MEDmedBlue.push_back(med_medBlue[0]);
_V_des_RGB_10_MEDmedGreen.push_back(med_medGreen[0]);
_V_des_RGB_10_MEDmedRed.push_back(med_medRed[0]);
_V_des_RGB_10_MEDmedRGB.push_back(med_medRGB[0]);
_V_des_RGB_10_MEDmedSVD.push_back(med_svd[0]);
// Almacenamos todos los valores de media y desviacion estandar de cada
// grupo de imagenes en el vector _VV_des_RGB_10
V_des_RGB_10.push_back(med_medBlue[0]);
V_des_RGB_10.push_back(med_medGreen[0]);
V_des_RGB_10.push_back(med_medRed[0]);
V_des_RGB_10.push_back(med_medRGB[0]);
V_des_RGB_10.push_back(med_svd[0]);
_VV_des_RGB_10.push_back(V_des_RGB_10);
V_des_RGB_10.clear();
V_des_mB_10.clear();
V_des_mG_10.clear();
V_des_mR_10.clear();
V_des_mRGB_10.clear();
V_des_sdv_10.clear();
_orden << "======================================================" <<"\n" ;
}
} // si img no esta vacia
} // del for cont
// Cerramos la escritura de los ficheros
_medBlue.close();
_medGreen.close();
_medRed.close();
_medRGB.close();
_svd.close();
_orden.close();
}
void TargetExtractor::regionGrow(int threshold)
{
Mat gray;
cvtColor(mFrame, gray, CV_BGR2GRAY);
Mat temp;
mMask.copyTo(temp);
vector<vector<Point> > contours;
findContours(temp, contours, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_NONE);
int maxQueueSize = mFrame.rows * mFrame.cols / 4;
static int direction[8][2] = {
{ 0, 1 }, { 1, 1 }, { 1, 0 }, { 1, -1 },
{ 0, -1 }, { -1, -1 }, { -1, 0 }, { -1, 1 }
};
for (int i = 0; i < contours.size(); i++) {
Rect rect = boundingRect(Mat(contours[i]));
Mat mask = Mat::zeros(gray.size(), CV_8U);
drawContours(mask, contours, i, Scalar::all(255), CV_FILLED);
int size = sum(mask(rect))[0] / 255;
Scalar m, s;
meanStdDev(gray(rect), m, s, mask(rect));
double mean = m[0], stdDev = s[0];
Mat temp;
mMask.copyTo(temp);
int origSize = size;
queue<Point> pointQueue;
for (int j = 0; j < contours[i].size(); j++) {
uchar pixel = gray.at<uchar>(contours[i][j]);
if (abs(pixel - mean) < 1.0 * stdDev) {
pointQueue.push(contours[i][j]);
}
}
Point cur, pop;
while (!pointQueue.empty() && pointQueue.size() < maxQueueSize) {
pop = pointQueue.front();
pointQueue.pop();
uchar pixel = gray.at<uchar>(pop);
for (int k = 0; k < 8; k++) {
cur.x = pop.x + direction[k][0];
cur.y = pop.y + direction[k][1];
if (cur.x < 0 || cur.x > gray.cols - 1 || cur.y < 0 || cur.y > gray.rows - 1) {
continue;
}
if (temp.at<uchar>(cur) != 255) {
uchar curPixel = gray.at<uchar>(cur);
if (abs(curPixel - pixel) < threshold &&
abs(curPixel - mean) < 1.0 * stdDev) {
temp.at<uchar>(cur) = 255;
double diff = curPixel - mean;
double learningRate = 1.0 / (++size);
mean = (1 - learningRate) * mean + learningRate * curPixel;
stdDev = sqrt((1 - learningRate) * stdDev * stdDev + learningRate * diff * diff);
pointQueue.push(cur);
}
}
}
}
if (pointQueue.empty()) {
int incSize = size - origSize;
if (incSize < mFrame.rows * mFrame.cols / 6 && incSize / origSize < 5) {
mMask = temp;
}
}
}
}
//bool Depth_and_Disparity::calc_disperity(int desiredPhase, Mat in_left_clr, Mat in_right_clr,
bool Depth_and_Disparity::calc_disperity(int desiredPhase, Mat left_im_gray, Mat right_im_gray,
Mat BgMask , Target *previousTarget,
Mat *disperity_out, double *min_depth_of_ROI)
{
double max_disperity;
const double allowed_disp_delta_between_cycles = 0.1 ; //10[%]
if ( ! (desiredPhase==2) ) //calculate the new disparity for new inputs
{
// delivers new input , when the process is waiting (not in calculation process)
set_disparity_input(right_im_gray,left_im_gray, /*myStereoCams.GetFrameCycleCounter()*/ 1 );
// waiting trial :
while (! get_rectified_and_disparity(last_result_of_disparity, last_result_of_disparity_struct) )
{
}
if ((desiredPhase==1))
{
*disperity_out = last_result_of_disparity.clone() ;
return true;
}
}
// continue to give the filtered disparity (for the new or the last calculated)
/* if output is ready from disparity calculation , it returns true */
//if ( localDisp.get_rectified_and_disparity(disp_temporary, disperity_struct) )
{
/* calculate average depth for the ROI of the target */
Mat tmpma = last_result_of_disparity;
if ( ! BgMask.empty() )
{
filtered_disparity = Mat();
last_result_of_disparity.copyTo(filtered_disparity , BgMask);
threshold (filtered_disparity , filtered_disparity , minDisparityToCut , 255,THRESH_TOZERO);
}
else
/* no additional mask */
if ((*previousTarget).target_object_prop.relevant_disparity > -999)
{
threshold (last_result_of_disparity , filtered_disparity ,
(*previousTarget).target_object_prop.relevant_disparity * (1-allowed_disp_delta_between_cycles) ,
(*previousTarget).target_object_prop.relevant_disparity * (1+allowed_disp_delta_between_cycles),
THRESH_TOZERO); //.15??
}
else
/* loose the far away objects (small disparities) */
threshold (last_result_of_disparity , filtered_disparity , minDisparityToCut , 255,THRESH_TOZERO);
int an=1; //an=1->kernel of 3
Mat element = getStructuringElement(MORPH_RECT, Size(an*2+1, an*2+1), Point(an, an) );
medianBlur (filtered_disparity, filtered_disparity, an*3);
erode (filtered_disparity , filtered_disparity, element);
dilate (filtered_disparity, filtered_disparity, element);
//max disperity into avg_disp var
// minMaxLoc(filtered_disparity, 0, &max_disperity );
// convert_disperity_value_to_depth(max_disperity , *min_depth_of_ROI);
// last_disparity_min_depth = *min_depth_of_ROI;
Scalar mean;
Scalar stddev;
Rect tmp = boundingRect(filtered_disparity);
Mat tmpMask = filtered_disparity;
///meanStdDev ( filtered_disparity, mean, stddev );
///meanStdDev ( filtered_disparity(tmp), mean, stddev );
meanStdDev ( filtered_disparity, mean, stddev , tmpMask);
//uchar mean_pxl = mean.val[0];
int mean_val = mean.val[0];
//uchar stddev_pxl = stddev.val[0];
int stddev_val = stddev.val[0];
int minDispToTake = mean_val - stddev_val * 1 ;
if (minDispToTake > mean_val * (1-allowed_disp_delta_between_cycles/2.) )
minDispToTake = mean_val * (1-allowed_disp_delta_between_cycles/2.) ; //minimum for case of ~zero std ; // furthest object
//int maxDispToTake = max_disperity ; // closest object
convert_disperity_value_to_depth(mean_val , *min_depth_of_ROI);
last_disparity_depth = *min_depth_of_ROI;
/* filter far or close objects then the target itself (mean) */
threshold (filtered_disparity , filtered_disparity , minDispToTake , 255/*max_disperity*/,THRESH_TOZERO);
/* set partial data for current target */
(*previousTarget).target_object_prop.relevant_disparity = mean_val;
}
*disperity_out = filtered_disparity.clone() ;
return true; // TODO: set as SUCCESS system enum
}
/*
* 1. CALCULATE RANGE FROM MEAN AND STANDARD DEVIATION
* 2. CREATE A MASK FROM THE RANGE
* 3. SMOOTH STUFF USING MORPHOLOGY
* 4. DETECT THE CIRCLES
*/
vector<ILAC_Sphere>
ILAC_SphereFinder::findSpheres ( ILAC_Square &square, Mat &img,
const size_t pixSphDiam )
{
/* 1. CALCULATE RANGE FROM MEAN AND STANDARD DEVIATION */
Mat mean, stddev;
{/* Isolate the Hue */
Mat tmpImg;
vector<Mat> tmp_dim;
cvtColor ( square.getImg(), tmpImg, CV_BGR2HSV_FULL );
split( tmpImg, tmp_dim );
tmpImg = tmp_dim[0];
meanStdDev ( tmpImg, mean, stddev );
}
/*
* Range will be -+ 1 standard deviation. This has aprox 68% of the data
* (http://en.wikipedia.org/wiki/Standard_deviation)
*/
Mat lowerb = mean - stddev;
Mat upperb = mean + stddev;
/* 2. CREATE A MASK FROM THE RANGE */
Mat himg;
{
Mat tmpImg;
vector<Mat> tmp_dim;
cvtColor ( img, tmpImg, CV_BGR2HSV_FULL );
split ( tmpImg, tmp_dim );
himg = tmp_dim[0];
}
Mat mask = Mat::ones(img.rows, img.cols, CV_8UC1);
inRange(himg, lowerb, upperb, mask);
/* 3. SMOOTH STUFF USING MORPHOLOGY */
{
/*
* Morphological open is 1.Erode and 2.Dilate. We use 1/4 of the sphere
* diameter in the hope that its big enough to clean the noise, but not big
* enough to remove the big sphere blob.
*/
int openSize = pixSphDiam/4;
Mat se = getStructuringElement ( MORPH_ELLIPSE, Size(openSize,openSize) );
morphologyEx ( mask, mask, MORPH_OPEN, se );
/*
* We dilate with half of the sphere diameter and hope for a blob
* that is approx double the radius of the original blob. The edges are more
* roundy this way.
*/
int dilateSize = pixSphDiam/2;
se = getStructuringElement ( MORPH_ELLIPSE,
Size(dilateSize,dilateSize) );
dilate ( mask, mask, se );
}
/* 4. DETECT THE CIRCLES */
/* Play with the arguments for HoughCircles. */
vector<Vec3f> circles;
vector<ILAC_Sphere> spheres;
int minCircDist = 3*pixSphDiam/2;
GaussianBlur ( mask, mask, Size(15, 15), 2, 2 );
HoughCircles ( mask, circles, CV_HOUGH_GRADIENT, 2, minCircDist, 100, 40);
for( size_t i = 0; i < circles.size(); i++ )
{
Point center(cvRound(circles[i][0]), cvRound(circles[i][1]));
int radius = cvRound(circles[i][2]);
ILAC_Sphere temp ( &img, center, radius );
spheres.push_back(temp);
for ( int j = i ;
j > 0 && spheres[j].getRadius() < spheres[j-1].getRadius() ; j-- )
std::swap( spheres[j], spheres[j-1] );
}
if ( spheres.size() < 3 )
throw ILACExLessThanThreeSpheres ();
return spheres;
}
void run( Mat& image,
string& out_sequence,
vector<Rect>* component_rects,
vector<string>* component_texts,
vector<float>* component_confidences,
int component_level)
{
CV_Assert( (image.type() == CV_8UC1) || (image.type() == CV_8UC3) );
CV_Assert( (image.cols > 0) && (image.rows > 0) );
CV_Assert( component_level == OCR_LEVEL_WORD );
out_sequence.clear();
if (component_rects != NULL)
component_rects->clear();
if (component_texts != NULL)
component_texts->clear();
if (component_confidences != NULL)
component_confidences->clear();
// First we split a line into words
vector<Mat> words_mask;
vector<Rect> words_rect;
/// Find contours
vector<vector<Point> > contours;
vector<Vec4i> hierarchy;
Mat tmp;
image.copyTo(tmp);
findContours( tmp, contours, hierarchy, RETR_EXTERNAL, CHAIN_APPROX_SIMPLE, Point(0, 0) );
if (contours.size() < 6)
{
//do not split lines with less than 6 characters
words_mask.push_back(image);
words_rect.push_back(Rect(0,0,image.cols,image.rows));
}
else
{
Mat_<float> vector_w((int)image.cols,1);
reduce(image, vector_w, 0, REDUCE_SUM, -1);
vector<int> spaces;
vector<int> spaces_start;
vector<int> spaces_end;
int space_count=0;
int last_one_idx;
int s_init = 0, s_end=vector_w.cols;
for (int s=0; s<vector_w.cols; s++)
{
if (vector_w.at<float>(0,s) == 0)
s_init = s+1;
else
break;
}
for (int s=vector_w.cols-1; s>=0; s--)
{
if (vector_w.at<float>(0,s) == 0)
s_end = s;
else
break;
}
for (int s=s_init; s<s_end; s++)
{
if (vector_w.at<float>(0,s) == 0)
{
space_count++;
} else {
if (space_count!=0)
{
spaces.push_back(space_count);
spaces_start.push_back(last_one_idx);
spaces_end.push_back(s-1);
}
space_count = 0;
last_one_idx = s;
}
}
Scalar mean_space,std_space;
meanStdDev(Mat(spaces),mean_space,std_space);
int num_word_spaces = 0;
int last_word_space_end = 0;
for (int s=0; s<(int)spaces.size(); s++)
{
if (spaces_end.at(s)-spaces_start.at(s) > mean_space[0]+(mean_space[0]*1.1)) //this 1.1 is a param?
{
if (num_word_spaces == 0)
{
//cout << " we have a word from 0 to " << spaces_start.at(s) << endl;
Mat word_mask;
Rect word_rect = Rect(0,0,spaces_start.at(s),image.rows);
image(word_rect).copyTo(word_mask);
words_mask.push_back(word_mask);
words_rect.push_back(word_rect);
}
else
{
//cout << " we have a word from " << last_word_space_end << " to " << spaces_start.at(s) << endl;
Mat word_mask;
Rect word_rect = Rect(last_word_space_end,0,spaces_start.at(s)-last_word_space_end,image.rows);
image(word_rect).copyTo(word_mask);
words_mask.push_back(word_mask);
words_rect.push_back(word_rect);
}
num_word_spaces++;
last_word_space_end = spaces_end.at(s);
}
}
//cout << " we have a word from " << last_word_space_end << " to " << vector_w.cols << endl << endl << endl;
Mat word_mask;
Rect word_rect = Rect(last_word_space_end,0,vector_w.cols-last_word_space_end,image.rows);
image(word_rect).copyTo(word_mask);
words_mask.push_back(word_mask);
words_rect.push_back(word_rect);
}
for (int w=0; w<(int)words_mask.size(); w++)
{
vector< vector<int> > observations;
vector< vector<double> > confidences;
vector<int> obs;
// First find contours and sort by x coordinate of bbox
words_mask[w].copyTo(tmp);
if (tmp.empty())
continue;
contours.clear();
hierarchy.clear();
/// Find contours
findContours( tmp, contours, hierarchy, RETR_EXTERNAL, CHAIN_APPROX_SIMPLE, Point(0, 0) );
vector<Rect> contours_rect;
for (int i=0; i<(int)contours.size(); i++)
{
contours_rect.push_back(boundingRect(contours[i]));
}
sort(contours_rect.begin(), contours_rect.end(), sort_rect_horiz);
// Do character recognition foreach contour
for (int i=0; i<(int)contours.size(); i++)
{
Mat tmp_mask;
words_mask[w](contours_rect.at(i)).copyTo(tmp_mask);
vector<int> out_class;
vector<double> out_conf;
classifier->eval(tmp_mask,out_class,out_conf);
if (!out_class.empty())
obs.push_back(out_class[0]);
observations.push_back(out_class);
confidences.push_back(out_conf);
}
//This must be extracted from dictionary, or just assumed to be equal for all characters
vector<double> start_p(vocabulary.size());
for (int i=0; i<(int)vocabulary.size(); i++)
start_p[i] = 1.0/vocabulary.size();
Mat V = Mat::zeros((int)observations.size(),(int)vocabulary.size(),CV_64FC1);
vector<string> path(vocabulary.size());
// Initialize base cases (t == 0)
for (int i=0; i<(int)vocabulary.size(); i++)
{
for (int j=0; j<(int)observations[0].size(); j++)
{
emission_p.at<double>(observations[0][j],obs[0]) = confidences[0][j];
}
V.at<double>(0,i) = start_p[i] * emission_p.at<double>(i,obs[0]);
path[i] = vocabulary.at(i);
}
// Run Viterbi for t > 0
for (int t=1; t<(int)obs.size(); t++)
{
//Dude this has to be done each time!!
emission_p = Mat::eye(62,62,CV_64FC1);
for (int e=0; e<(int)observations[t].size(); e++)
{
emission_p.at<double>(observations[t][e],obs[t]) = confidences[t][e];
}
vector<string> newpath(vocabulary.size());
for (int i=0; i<(int)vocabulary.size(); i++)
{
double max_prob = 0;
int best_idx = 0;
for (int j=0; j<(int)vocabulary.size(); j++)
{
double prob = V.at<double>(t-1,j) * transition_p.at<double>(j,i) * emission_p.at<double>(i,obs[t]);
if ( prob > max_prob)
{
max_prob = prob;
best_idx = j;
}
}
V.at<double>(t,i) = max_prob;
newpath[i] = path[best_idx] + vocabulary.at(i);
}
// Don't need to remember the old paths
path.swap(newpath);
}
double max_prob = 0;
int best_idx = 0;
for (int i=0; i<(int)vocabulary.size(); i++)
{
double prob = V.at<double>((int)obs.size()-1,i);
if ( prob > max_prob)
{
max_prob = prob;
best_idx = i;
}
}
//cout << path[best_idx] << endl;
out_sequence = out_sequence+" "+path[best_idx];
if (component_rects != NULL)
component_rects->push_back(words_rect[w]);
if (component_texts != NULL)
component_texts->push_back(path[best_idx]);
if (component_confidences != NULL)
component_confidences->push_back((float)max_prob);
}
return;
}
double score_segmentation( vector<int> &segmentation, string& outstring )
{
// Score Heuristics:
// No need to use Viterbi to know a given segmentation is bad
// e.g.: in some cases we discard a segmentation because it includes a very large character
// in other cases we do it because the overlapping between two chars is too large
// TODO Add more heuristics (e.g. penalize large inter-character variance)
Mat interdist ((int)segmentation.size()-1, 1, CV_32F, 1);
for (size_t i=0; i<segmentation.size()-1; i++)
{
interdist.at<float>((int)i,0) = (float)oversegmentation[segmentation[(int)i+1]]*step_size
- (float)oversegmentation[segmentation[(int)i]]*step_size;
if ((float)interdist.at<float>((int)i,0)/win_size > 2.25) // TODO explain how did you set this thrs
{
return -DBL_MAX;
}
if ((float)interdist.at<float>((int)i,0)/win_size < 0.15) // TODO explain how did you set this thrs
{
return -DBL_MAX;
}
}
Scalar m, std;
meanStdDev(interdist, m, std);
//double interdist_std = std[0];
//TODO Extracting start probs from lexicon (if we have it) may boost accuracy!
vector<double> start_p(vocabulary.size());
for (int i=0; i<(int)vocabulary.size(); i++)
start_p[i] = log(1.0/vocabulary.size());
Mat V = Mat::ones((int)segmentation.size(),(int)vocabulary.size(),CV_64FC1);
V = V * -DBL_MAX;
vector<string> path(vocabulary.size());
// Initialize base cases (t == 0)
for (int i=0; i<(int)vocabulary.size(); i++)
{
V.at<double>(0,i) = start_p[i] + recognition_probabilities[segmentation[0]][i];
path[i] = vocabulary.at(i);
}
// Run Viterbi for t > 0
for (int t=1; t<(int)segmentation.size(); t++)
{
vector<string> newpath(vocabulary.size());
for (int i=0; i<(int)vocabulary.size(); i++)
{
double max_prob = -DBL_MAX;
int best_idx = 0;
for (int j=0; j<(int)vocabulary.size(); j++)
{
double prob = V.at<double>(t-1,j) + transition_p.at<double>(j,i) + recognition_probabilities[segmentation[t]][i];
if ( prob > max_prob)
{
max_prob = prob;
best_idx = j;
}
}
V.at<double>(t,i) = max_prob;
newpath[i] = path[best_idx] + vocabulary.at(i);
}
// Don't need to remember the old paths
path.swap(newpath);
}
double max_prob = -DBL_MAX;
int best_idx = 0;
for (int i=0; i<(int)vocabulary.size(); i++)
{
double prob = V.at<double>((int)segmentation.size()-1,i);
if ( prob > max_prob)
{
max_prob = prob;
best_idx = i;
}
}
outstring = path[best_idx];
return (max_prob / (segmentation.size()-1));
}