void ImageManipulator::findShapes(Mat &contourImage,vector<RotatedRect> &ROIrects,Mat &src)
{
//cv::Mat contourImage = cv::imread("polygon.png");
// cv::Mat contourImage = cv::imread("c:/scripts/stop.jpg");
// cv::GaussianBlur(contourImage,contourImage,cv::Size(5,5),0,0);
// cv::threshold(contourImage,contourImage,100,255,0);
if (contourImage.empty())
return ;
// Convert to grayscale
cv::Mat gray;
cv::cvtColor(contourImage, gray, CV_BGR2GRAY);
// Use Canny instead of threshold to catch squares with gradient shading
cv::Mat bw;
cv::Canny(gray, bw, 0, 10, 5);
// namedWindow("detect",CV_WINDOW_FREERATIO);
// imshow("detect",bw);
// Find contours
std::vector<std::vector<cv::Point> > contours;
cv::findContours(bw.clone(), contours, CV_RETR_CCOMP, CV_CHAIN_APPROX_SIMPLE);
RNG rng(12345);
Mat canny_output;
// vector<vector<Point> > con1;
vector<Vec4i> hierarchy;
/// Detect edges using canny
// Canny( contourImage_gray, canny_output, thresh, thresh*2, 3 );
/// Find con1
// findcon( canny_output, con1, hierarchy, CV_RETR_TREE, CV_CHAIN_APPROX_SIMPLE, Point(0, 0) );
/// Draw con1
Mat shapes = Mat::zeros( bw.size(), CV_8UC3 );
for( int i = 0; i< contours.size(); i++ )
{
Scalar color = Scalar( rng.uniform(0, 255), rng.uniform(0,255), rng.uniform(0,255) );
drawContours(shapes, contours, i, color, 2, 8, hierarchy, 0, Point() );
}
std::vector<cv::Point> approx;
Mat idShapes = contourImage.clone();
Mat final=contourImage.clone();
ROIrects.clear();
for (int i = 0; i < contours.size(); i++)
{
// Approximate contour with accuracy proportional
// to the contour perimeter
cv::approxPolyDP(cv::Mat(contours[i]), approx, cv::arcLength(cv::Mat(contours[i]), true)*0.02, true);
// Skip small or non-convex objects
if (std::fabs(cv::contourArea(contours[i])) < 100 || !cv::isContourConvex(approx))
continue;
if (approx.size() == 3)
{
setLabel(idShapes, "TRI", contours[i]); // Triangles
RotatedRect R=minAreaRect(contours[i]); // Get bounding box for contour i
R.size.height=R.size.height+R.size.height/3.;
R.size.width=R.size.width+R.size.width/3.;
ROIrects.push_back(R);
// Mat ROI=contourImage(R); //Set ROI on source image
// imshow("roi",ROI);
}
else if (approx.size() >= 4 && approx.size() <= 6)
{
// Number of vertices of polygonal curve
int vtc = approx.size();
// Get the cosines of all corners
std::vector<double> cos;
for (int j = 2; j < vtc+1; j++)
cos.push_back(angle(approx[j%vtc], approx[j-2], approx[j-1]));
// Sort ascending the cosine values
std::sort(cos.begin(), cos.end());
// Get the lowest and the highest cosine
double mincos = cos.front();
double maxcos = cos.back();
// Use the degrees obtained above and the number of vertices
// to determine the shape of the contour
if (vtc == 4 && mincos >= -0.2 && maxcos <= 0.3){
setLabel(idShapes, "RECT", contours[i]);
RotatedRect R=minAreaRect(contours[i]); // Get bounding box for contour i
if ((R.size.height/R.size.width)<10 && (R.size.width/R.size.height)<10)
ROIrects.push_back(R);}
else if (vtc == 5 && mincos >= -0.4 && maxcos <= -0.20)
setLabel(idShapes, "PENTA", contours[i]);
else if ( vtc <= 8 && vtc >= 6 && mincos >= -0.8 && maxcos <= -0.30)
setLabel(idShapes, "HEXA", contours[i]);
}
else
{
// Detect and label circles
double area = cv::contourArea(contours[i]);
cv::Rect r = cv::boundingRect(contours[i]);
int radius = r.width / 2;
if (std::abs(1 - ((double)r.width / r.height)) <= 0.2 &&
std::abs(1 - (area / (CV_PI * std::pow(radius, 2)))) <= 0.2)
setLabel(idShapes, "CIR", contours[i]);
RotatedRect R=minAreaRect(contours[i]); // Get bounding box for contour i
R.size.height=R.size.height+R.size.height/3.;
R.size.width=R.size.width+R.size.width/3.;
ROIrects.push_back(R);
}
}
// sort(ROIrects.begin(),ROIrects.end(),&MainWindow::checkcenter);
std::sort(ROIrects.begin(), ROIrects.end(), top_to_bottom()); //sort the vector based on height
int k = 0;
// vector<RotatedRect> ROIrectsTemp;
// for (std::vector<RotatedRect>::iterator it = ROIrects.begin() ; it != ROIrects.end(); ++it){
// cout<<(*it).size.height<<endl;
// if (k>0){
// if (!(((*--it).center.x)/((*++it).center.x) <1.1 && ((*--it).center.x)/((*++it).center.x) >-1.1)){
// ROIrects.erase(ROIrects.begin()+k+1);
//// it--;
// }
// }
// k+=1;
// }
namedWindow("contourImage",CV_WINDOW_FREERATIO);
// namedWindow("idShapes",CV_WINDOW_FREERATIO);
// Mat src1=src.clone();
// cv::imshow("idShapes", idShapes);
// drawRectangles(ROIrects,src); //sets image for 4th video feed in GUI
for( int i = 0; i< ROIrects.size(); i++ ){
Point2f rect_points[4]; ROIrects[i].points( rect_points );
for( int j = 0; j < 4; j++ )
line( src, rect_points[j], rect_points[(j+1)%4], Scalar(255,0,0), 1, 8 );}
// cvtColor(src1,src1,CV_RGB2BGR);
Mat drawing = Mat::zeros( src.size(), CV_8UC3 );
// Mat drawing1 = Mat::zeros( SrcRoi.size(), CV_8UC3 );
// Mat drawing2 = Mat::zeros( SrcRoi.size(), CV_8UC3 );
// /// Show in a window
// namedWindow( "Contours", CV_WINDOW_FREERATIO );
// imshow( "Contours", drawing );
// selectROI(drawing,src,1);
/// Show in a window
// namedWindow( "Contours2", CV_WINDOW_FREERATIO );
// imshow( "Contours2", SrcRoi );
// selectROI(drawing1,drawing2);
// /// Show in a window
// namedWindow( "Contours3", CV_WINDOW_FREERATIO );
// imshow( "Contours3", drawing2 );
// cv::imshow("contourImage", src);
// cv::waitKey(0);
return ;
}
/*----------------------------////?????
* 功能 : 生成近距物体信息序列
*----------------------------
* 函数 : PointCloudAnalyzer::parseCandidates
* 访问 : private
* 返回 : void
*
* 参数 : objects [in] 深度阈值化后的二值图像，显示了近距物体的分布
* 参数 : depthMap [in] 从三维点云矩阵中抽取的深度数据矩阵
* 参数 : objectInfos [out] 目标信息序列
*/
void PointCloudAnalyzer::parseCandidates(cv::Mat& objects, cv::Mat& depthMap, vector<ObjectInfo>& objectInfos)
{
// 提取物体轮廓
// Mat canny_output;
vector<vector<cv::Point> > contours; // 物体轮廓点链
// vector<Vec4i>hierarchy;
cv::Mat tempobjects=objects;
/// 用Canny算子检测边缘
// Canny( tempobjects, canny_output, 80, 160, 3 );
findContours(tempobjects, contours, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_SIMPLE);
/* Mat drawing = Mat::zeros( canny_output.size(), CV_8UC3 );
for( int i = 0; i< contours.size(); i++ )
{
Scalar color = Scalar(0, 233, 0);
drawContours( drawing, contours, i, color, 2, 8);
}
/// 在窗体中显示结果
namedWindow( "Contours", CV_WINDOW_AUTOSIZE );
imshow( "Contours", drawing );*/
// findContours(objects, contours, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_NONE);
// findContours(objects,contours,hierarchy,CV_RETR_TREE, CV_CHAIN_APPROX_SIMPLE);
/*
findContours后的轮廓信息contours可能过于复杂不平滑，可以用approxPolyDP函数对该多边形曲线做适当近似
contourArea函数可以得到当前轮廓包含区域的大小，方便轮廓的筛选
findContours经常与drawContours配合使用，用来将轮廓绘制出来。
其中第一个参数image表示目标图像，第二个参数contours表示输入的轮廓组，每一组轮廓由点vector构成，
第三个参数contourIdx指明画第几个轮廓，如果该参数为负值，则画全部轮廓，第四个参数color为轮廓的颜色，
第五个参数thickness为轮廓的线宽，如果为负值或CV_FILLED表示填充轮廓内部，第六个参数lineType为线型，
第七个参数为轮廓结构信息，第八个参数为maxLevel*/
// 分析轮廓
double areaThresh = 0.005 * depthMap.rows * depthMap.cols;
cv::Mat mask = cv::Mat::zeros(objects.size(), CV_8UC1);////1
bool useMeanDepth = false;
// CString str;
// str.Format(_T("%d"),contours.size());
// AfxMessageBox(str);
for( UINT objID = 0; objID < contours.size(); objID++ )//得出每个轮廓的信息
{
cv::Mat contour = cv::Mat( contours[objID] );
double area = contourArea( contour );//计算轮廓面积
if(area>0)
{
ObjectInfo object;
// 填充物体内部轮廓作为掩码区域
mask = cv::Scalar(0);
drawContours(mask,contours,objID,cv::Scalar(255),-1);
/*
用来将轮廓绘制出来。其中第一个参数image表示目标图像，
第二个参数contours表示输入的轮廓组，每一组轮廓由点vector构成，
第三个参数contourIdx指明画第几个轮廓，如果该参数为负值，
则画全部轮廓，第四个参数color为轮廓的颜色，
第五个参数thickness为轮廓的线宽，
如果为负值或CV_FILLED表示填充轮廓内部，
第六个参数lineType为线型，第七个参数为轮廓结构信息，第八个参数为maxLevel
*/
double minVal = 0, maxVal = 0;
cv::Point minPos;
cv::minMaxLoc(depthMap, &minVal, &maxVal, &minPos, NULL, mask);
object.distance = depthMap.at<float>(minPos.y, minPos.x);
// }
// 计算轮廓矩形
object.boundRect = boundingRect( contour );//计算点集的最外面（up-right）矩形边界
object.minRect = minAreaRect( contour );
object.center = object.minRect.center;
// 保存物体轮廓信息
objectInfos.push_back( object );
}
}
// 按物体距离重新排序
//std::sort( objectInfos.begin(), objectInfos.end(), std::greater<ObjectInfo>() );
}
// Thought: simply use the bounding box of the feature points as the head.
void HeartFeatureTracker::track(Mat &colorImage, Mat &depthImage) {
Mat gray;
vector<uchar> status;
vector<float> err;
vector<Point2f> points;
cvtColor(depthImage, gray, COLOR_BGR2GRAY);
Mat roiColor = depthImage(bbox);
Mat roi = gray(bbox);
if(prevPoints.empty()) {
return;
}
calcOpticalFlowPyrLK(prevGray, roi, prevPoints, points, status, err, Size(10,10),
3, TermCriteria(CV_TERMCRIT_ITER|CV_TERMCRIT_EPS, 20, 0.03), 0, 0.001);
Mat transform = estimateRigidTransform(prevPoints, points, false);
//applyTransformToPoints(boundBox, transform);
//applyTransformToPoints(patchOfInterest, transform);
size_t i, k;
bool brokeOut = false;
for( i = k = 0; i < points.size(); i++ ) {
if( !status[i] ) {
continue;
}
points[k++] = points[i];
circle( roiColor, points[i], 3, Scalar(0,255,0), -1, 8);
}
points.resize(k);
Rect dumbRect = boundingRect(points);
convertRectToMats(boundBox, dumbRect);
rectangle(roiColor, dumbRect, Scalar(0,255,0));
convertRectToMats(patchOfInterest, getForeheadFromBbox(dumbRect));
RotatedRect dumbRotated = minAreaRect(points);
Point2f rect_points[4];
dumbRotated.points(rect_points);
Scalar color = Scalar( 255, 0, 0 );
for( int j = 0; j < 4; j++ )
line( roiColor, rect_points[j], rect_points[(j+1)%4], color, 1, 8 );
DrawBoxFromPoints(boundBox, roiColor);
DrawBoxFromPoints(patchOfInterest, roiColor);
prevGray = roi;
prevPoints = points;
}
vector<PlateRegion> DetectorMorph::detect(Mat frame, std::vector<cv::Rect> regionsOfInterest) {
Mat frame_gray,frame_gray_cp;
if (frame.channels() > 2)
{
cvtColor( frame, frame_gray, CV_BGR2GRAY );
}
else
{
frame.copyTo(frame_gray);
}
frame_gray.copyTo(frame_gray_cp);
blur(frame_gray, frame_gray, Size(5, 5));
vector<PlateRegion> detectedRegions;
for (int i = 0; i < regionsOfInterest.size(); i++) {
Mat img_open, img_result;
Mat element = getStructuringElement(MORPH_RECT, Size(30, 4));
morphologyEx(frame_gray, img_open, CV_MOP_OPEN, element, cv::Point(-1, -1));
img_result = frame_gray - img_open;
if (config->debugDetector && config->debugShowImages) {
imshow("Opening", img_result);
}
//threshold image using otsu thresholding
Mat img_threshold, img_open2;
threshold(img_result, img_threshold, 0, 255, CV_THRESH_OTSU + CV_THRESH_BINARY);
if (config->debugDetector && config->debugShowImages) {
imshow("Threshold Detector", img_threshold);
}
Mat diamond(5, 5, CV_8U, cv::Scalar(1));
diamond.at<uchar>(0, 0) = 0;
diamond.at<uchar>(0, 1) = 0;
diamond.at<uchar>(1, 0) = 0;
diamond.at<uchar>(4, 4) = 0;
diamond.at<uchar>(3, 4) = 0;
diamond.at<uchar>(4, 3) = 0;
diamond.at<uchar>(4, 0) = 0;
diamond.at<uchar>(4, 1) = 0;
diamond.at<uchar>(3, 0) = 0;
diamond.at<uchar>(0, 4) = 0;
diamond.at<uchar>(0, 3) = 0;
diamond.at<uchar>(1, 4) = 0;
morphologyEx(img_threshold, img_open2, CV_MOP_OPEN, diamond, cv::Point(-1, -1));
Mat rectElement = getStructuringElement(cv::MORPH_RECT, Size(13, 4));
morphologyEx(img_open2, img_threshold, CV_MOP_CLOSE, rectElement, cv::Point(-1, -1));
if (config->debugDetector && config->debugShowImages) {
imshow("Close", img_threshold);
waitKey(0);
}
//Find contours of possibles plates
vector< vector< Point> > contours;
findContours(img_threshold,
contours, // a vector of contours
CV_RETR_EXTERNAL, // retrieve the external contours
CV_CHAIN_APPROX_NONE); // all pixels of each contours
//Start to iterate to each contour founded
vector<vector<Point> >::iterator itc = contours.begin();
vector<RotatedRect> rects;
//Remove patch that are no inside limits of aspect ratio and area.
while (itc != contours.end()) {
//Create bounding rect of object
RotatedRect mr = minAreaRect(Mat(*itc));
if (mr.angle < -45.) {
mr.angle += 90.0;
swap(mr.size.width, mr.size.height);
}
if (!CheckSizes(mr))
itc = contours.erase(itc);
else {
++itc;
rects.push_back(mr);
}
}
//Now prunning based on checking all candidate plates for a min/max number of blobsc
Mat img_crop, img_crop_b, img_crop_th, img_crop_th_inv;
vector< vector< Point> > plateBlobs;
vector< vector< Point> > plateBlobsInv;
double thresholds[] = { 10, 40, 80, 120, 160, 200, 240 };
const int num_thresholds = 7;
int numValidChars = 0;
Mat rotated;
for (int i = 0; i < rects.size(); i++) {
numValidChars = 0;
RotatedRect PlateRect = rects[i];
Size rect_size = PlateRect.size;
// get the rotation matrix
Mat M = getRotationMatrix2D(PlateRect.center, PlateRect.angle, 1.0);
// perform the affine transformation
warpAffine(frame_gray_cp, rotated, M, frame_gray_cp.size(), INTER_CUBIC);
//Crop area around candidate plate
getRectSubPix(rotated, rect_size, PlateRect.center, img_crop);
if (config->debugDetector && config->debugShowImages) {
imshow("Tilt Correction", img_crop);
waitKey(0);
}
for (int z = 0; z < num_thresholds; z++) {
cv::threshold(img_crop, img_crop_th, thresholds[z], 255, cv::THRESH_BINARY);
cv::threshold(img_crop, img_crop_th_inv, thresholds[z], 255, cv::THRESH_BINARY_INV);
findContours(img_crop_th,
plateBlobs, // a vector of contours
CV_RETR_LIST, // retrieve the contour list
CV_CHAIN_APPROX_NONE); // all pixels of each contours
findContours(img_crop_th_inv,
plateBlobsInv, // a vector of contours
CV_RETR_LIST, // retrieve the contour list
CV_CHAIN_APPROX_NONE); // all pixels of each contours
int numBlobs = plateBlobs.size();
int numBlobsInv = plateBlobsInv.size();
float idealAspect = config->avgCharWidthMM / config->avgCharHeightMM;
for (int j = 0; j < numBlobs; j++) {
cv::Rect r0 = cv::boundingRect(cv::Mat(plateBlobs[j]));
if (ValidateCharAspect(r0, idealAspect))
numValidChars++;
}
for (int j = 0; j < numBlobsInv; j++) {
cv::Rect r0 = cv::boundingRect(cv::Mat(plateBlobsInv[j]));
if (ValidateCharAspect(r0, idealAspect))
numValidChars++;
}
}
//If too much or too lcittle might not be a true plate
//if (numBlobs < 3 || numBlobs > 50) continue;
if (numValidChars < 4 || numValidChars > 50) continue;
PlateRegion PlateReg;
// Ensure that the rectangle isn't < 0 or > maxWidth/Height
Rect bounding_rect = PlateRect.boundingRect();
PlateReg.rect = expandRect(bounding_rect, 0, 0, frame.cols, frame.rows);
detectedRegions.push_back(PlateReg);
}
}
return detectedRegions;
}
cv::RotatedRect minAreaRect(const ofPolyline& polyline) {
return minAreaRect(Mat(toCv(polyline)));
}
void Extractor::extract(MatSet& srcSet, Region& result) {
//いったん手続き型でアルゴリズムを作成する
//TODO : メソッド分割すべし
ChannelSet channelSet(srcSet);
//エッジ画像を取得する
vector<Mat> rawEdges;
_edgeFactory.createEdges(srcSet, rawEdges, channelSet);
Mat dstEdgeImg(srcSet.size().height,srcSet.size().width, CV_8UC1, 255);
revMergeEdges(rawEdges, _previousRegion.expectedRoi(), dstEdgeImg);
erode(dstEdgeImg, dstEdgeImg, cv::Mat(), Point(-1,-1), 1);
imshow("revEdge", dstEdgeImg);
Mat mat = Mat::zeros(srcSet.size(), CV_8UC1);
if(_indexOfMaxArea >=0){
int yBegin = _previousRegion.expectedRoi().y;
int yEnd = _previousRegion.expectedRoi().y+_previousRegion.expectedRoi().height;
int xBegin = _previousRegion.expectedRoi().x;
int xEnd = _previousRegion.expectedRoi().x+_previousRegion.expectedRoi().width;
for(int y=yBegin; y<yEnd; y++) {
for(int x=xBegin; x<xEnd; x++) {
// if(_featureReference.isWithinThreshold(srcSet, Point(x,y)) ) {
// L(mat,x,y) = 255;
// }
if(L(channelSet.crMat(),x,y) >= _binarizationThreshold[7] && L(channelSet.gMat(),x,y) <=_binarizationThreshold[1] && L(channelSet.rMat(),x,y) >=_binarizationThreshold[2]) {
L(mat,x,y) = 255;
} else if(L(channelSet.sMat(),x,y) >=_binarizationThreshold[4] && L(channelSet.crMat(),x,y) >= 154 && L(channelSet.gMat(),x,y) <= 40 && L(channelSet.bMat(),x,y) <= 41 ) {
L(mat,x,y) = 255;
}
}
}
} else {
for(int y=0; y<srcSet.size().height; y++) {
for(int x=0; x<srcSet.size().width; x++) {
// if(_featureReference.isWithinThreshold(srcSet, Point(x,y)) ) {
// L(mat,x,y) = 255;
// }
if(L(channelSet.crMat(),x,y) >= _binarizationThreshold[7] && L(channelSet.gMat(),x,y) <=_binarizationThreshold[1] && L(channelSet.rMat(),x,y) >=_binarizationThreshold[2]) {
L(mat,x,y) = 255;
} else if(L(channelSet.sMat(),x,y) >=_binarizationThreshold[4] && L(channelSet.crMat(),x,y) >= 154 && L(channelSet.gMat(),x,y) <= 40 && L(channelSet.bMat(),x,y) <= 41 ) {
L(mat,x,y) = 255;
}
}
}
}
dilate(mat, mat, cv::Mat(), Point(-1,-1), _extractionManager.dilateCount());
erode(mat, mat, cv::Mat(), Point(-1,-1), _extractionManager.erodeCount());
// imshow("colorExtract", mat);
//最大面積
vPs contours;
findContours(mat, contours, CV_RETR_LIST, CV_CHAIN_APPROX_NONE);
vector<int> indexiesOfTop3Area = calcIndexiesOfTop3Area(contours);
// _indexOfMaxArea = calcIndexOfMaxArea(contours);
_indexOfMaxArea = indexiesOfTop3Area[0];
Mat mat2 = Mat::zeros(srcSet.size(), CV_8UC1);
if(indexiesOfTop3Area[1] >= 0) {
Point2f vertices[4];
_previousRegion.rotatedRect().points(vertices);
vector<Point> allPoints = contours[indexiesOfTop3Area[0]];
if(indexiesOfTop3Area[2] >= 0) {
drawContours(mat2, contours, indexiesOfTop3Area[1], Scalar(255, 255, 255), CV_FILLED, LINK_EIGHT);
drawContours(mat2, contours, indexiesOfTop3Area[2], Scalar(255, 255, 255), CV_FILLED, LINK_EIGHT);
drawContours(mat2, contours, _indexOfMaxArea, Scalar(255, 255, 255), CV_FILLED, LINK_EIGHT);
for(int i=1; i<3; i++) {
for(Point point :contours[indexiesOfTop3Area[i]]) {
if(isInROI(point, vertices)) {
allPoints.push_back(point);
}
}
}
} else {
drawContours(mat2, contours, indexiesOfTop3Area[1], Scalar(255, 255, 255), CV_FILLED, LINK_EIGHT);
drawContours(mat2, contours, _indexOfMaxArea, Scalar(255, 255, 255), CV_FILLED, LINK_EIGHT);
for(int i=1; i<2; i++) {
for(Point point :contours[indexiesOfTop3Area[i]]) {
if(isInROI(point, vertices)) {
allPoints.push_back(point);
}
}
}
}
RotatedRect rect = minAreaRect(Mat(allPoints));
// Point2f vertices[4];
// rect.points(vertices);
// for (int i = 0; i < 4; i++) line(mat2, vertices[i], vertices[(i+1)%4], Scalar(255,255,255), 4, 8, 0);
result.setContour(contours[_indexOfMaxArea]);//TODO:本来は結合した輪郭を入れるか別にいれる必要がある
result.setMaskImg(mat2);
result.setRotatedRect(rect);
result.calcRoiWithRotatedRect();
result.calcExpectedRoiConsideringMoveWithRotatedRect(_previousRegion);
_previousRegion = result;
// imshow("merge", mat2);
} else {
drawContours(mat2, contours, _indexOfMaxArea, Scalar(255, 255, 255), CV_FILLED, LINK_EIGHT);
result.setMaskImg(mat2);
if(_indexOfMaxArea>=0) {
result.setContour(contours[_indexOfMaxArea]);
result.calcRotatedRect();
result.calcRoi();
result.calcExpectedRoiConsideringMove(_previousRegion);
}
_previousRegion = result;
// imshow("merge", mat2);
}
//エッジ画像を取得する
// // _edgeService.extractEdge(rawEdges, areamaxRegion.rois()[0], dstEdgeImg);
// // imshow("edge", dstEdgeImg);
// //残ったエッジ画像と色による抽出画像を合成する
// bitwise_or(areamaxRegion.maskImg(), dstEdgeImg, dstEdgeImg);
// drawContours(dstEdgeImg, areamaxRegion.contours(), 0, Scalar(255, 255, 255), CV_FILLED, LINK_EIGHT);
// int minSize = 200;
// _contourService.fillContours(dstEdgeImg, minSize);
}
int DetectorBarcode::Detect() {
std::cout << "DetectorBarcode::Detect()" << std::endl;
assert(this->image_.data != NULL);
bool debugstripecode = true; //@TODO MAKE SURE TO SET ME TO FALSE IN PRODUCTION
bool useAdaptiveThersholding = true;
int dpi = 400; //this works well for all scales and sizes..
Mat matImageK;
cvtColor(this->image_, matImageK, cv::COLOR_BGR2GRAY);
cv::Mat matThres;
// VARIABLES //
double bar_height_mm_min = 3.7; //[7.5mm=our NMNH c39] [10.7mm=NMNH cover c39]
double bar_height_mm_max = 20;
double bar_ar_min = 4;
double bar_ar_max = 110;
int min_characters = 5; //minimum characters in barcode string
double bar_dist_group_mm_max = 9.0; //Maximum distance between any grouped bar to be part of the bar group
// COMPUTE //
double bar_height_px_min = bar_height_mm_min/25.4*dpi;
double bar_height_px_max = bar_height_mm_max/25.4*dpi;
double bar_area_px_min = bar_height_px_min*(bar_height_px_min*1.0/bar_ar_max);
//Dont allow the area to be less than 1px row
bar_area_px_min = bar_area_px_min < bar_height_px_min ? bar_height_px_min : bar_area_px_min;
double bar_area_px_max = bar_height_px_max*(bar_height_px_max*1.0/bar_ar_min);
double bar_dist_group_px_max = bar_dist_group_mm_max/25.4*dpi;
if (useAdaptiveThersholding) {
//int AT_blocksize = dpi*0.05;
int AT_blocksize = bar_height_px_min*0.5;
int AT_iseven=AT_blocksize%2;
AT_blocksize += 1+AT_iseven; //Makes sure the blocksize is an even number
//cout << "AT_blocksize=" << AT_blocksize << endl;
adaptiveThreshold(matImageK, matThres, 255, ADAPTIVE_THRESH_MEAN_C, THRESH_BINARY, AT_blocksize, 20);
} else {
threshold(matImageK, matThres, 127, 255, THRESH_BINARY_INV);
}
if (debugstripecode) {
//cout << "dpi=" << dpi << endl;
imwrite("/Users/tzaman/Desktop/bc/matImage.tif", this->image_);
imwrite("/Users/tzaman/Desktop/bc/matThres.tif", matThres);
}
vector< vector<Point> > contours;
vector<Vec4i> hierarchy;
findContours( matThres, contours, hierarchy, CV_RETR_LIST, CV_CHAIN_APPROX_SIMPLE );
//cout << "contours.size()=" << contours.size() << endl;
if (contours.size() == 0) {
string strErr = "No contours found.";
cout << strErr << endl;
return RET_NONE_FOUND;
}
//RANSAC vars
int min_inliers = (min_characters+2)*5*0.75; //+2 (start&stop), *5 (stripes per char), *0.x (margin)
double max_px_dist = (bar_height_px_min+bar_height_px_max)*0.5*0.05; //Maximum distance from RANSAC line to a point
vector<RotatedRect> stripeCandidates;
for(int i = 0; i >= 0; i = hierarchy[i][0] ) {
double cArea = contourArea( contours[i],false);
if (cArea < bar_area_px_min*0.5){
continue;
}
if (cArea > bar_area_px_max){
continue;
}
//cout << "[" << i << "]" << " cArea=" << cArea << endl;
RotatedRect rotRect= minAreaRect(contours[i]);
double ar = max(double(rotRect.size.width),double(rotRect.size.height)) / min(double(rotRect.size.width),double(rotRect.size.height));
if (ar < bar_ar_min){
continue;
}
if (ar > bar_ar_max){
continue;
}
double width = std::min(rotRect.size.width, rotRect.size.height);
double height = std::max(rotRect.size.width, rotRect.size.height);
//Check the length
if (height < bar_height_px_min){
//Stripe too small
continue;
}
if (height > bar_height_px_max ){
//Stripe too long
continue;
}
//cout << i << " rotRect: sz=" << rotRect.size << " cp=" << rotRect.center << " a=" << rotRect.angle << " ar=" << ar << endl;
Rect rCrop = boundingRect(contours[i]);
//Below parameter is dynamic, plz note
double min_area_fill=0.15;// = 0.25 ;// 0.4 means 40% of the bounding rectangle of the contour needs to be filled
//The min_area_fill threshold should be dependent on the width in pixels, because there's more noise in thinner ones
if (width<3){
min_area_fill = 0.05;
} else if (width <5){
min_area_fill = 0.10;
}
//Check if the rectangle is actually filled well
int fullarea = rCrop.area();
if ( (double(cArea)/double(fullarea)) < min_area_fill){
continue;
}
//cout << i << " fullarea=" << fullarea << " carea=" << cArea << endl;
if (debugstripecode){
imwrite("/Users/tzaman/Desktop/seg/" + std::to_string(i) + ".tif", matImageK(rCrop));
}
stripeCandidates.push_back(rotRect);
}
if (debugstripecode){
Mat matBarcodeFull = this->image_.clone();
for (int j=0; j<stripeCandidates.size(); j++){
util::rectangle(matBarcodeFull, stripeCandidates[j], cv::Scalar(255,0,0), 2);
}
imwrite("/Users/tzaman/Desktop/bc/_candidates.tif", matBarcodeFull);
}
//cout << "stripeCandidates.size()=" << stripeCandidates.size() << endl;
if (stripeCandidates.size() < min_inliers){
string strErr = "Code 39 did not find enough bars to accurately make a code.";
cout << strErr << endl;
return RET_NONE_FOUND;
}
std::vector<Point> vecPtRectCenter = util::vecrotrect2vecpt(stripeCandidates);
std::vector<std::vector<int> > vecGroupIdxs = util::groupPoints(vecPtRectCenter, bar_dist_group_px_max, min_inliers);
//std::vector<std::vector<cv::Point> > vecGroupPts(vecGroupIdxs.size());
std::vector<std::vector<cv::RotatedRect> > vecGroupRects(vecGroupIdxs.size());
//Relate indexes to points and add to group vector
for (int i=0; i<vecGroupIdxs.size(); i++){
//vecGroupPts[i].resize(vecGroupIdxs[i].size());
vecGroupRects[i].resize(vecGroupIdxs[i].size());
for (int j=0; j<vecGroupIdxs[i].size(); j++){
//cout << i << "," << j << endl;
//vecGroupPts[i][j] = vecPtRectCenter[vecGroupIdxs[i][j]];
vecGroupRects[i][j] = stripeCandidates[vecGroupIdxs[i][j]];
}
}
//Draw all groups
//if(debugstripecode){
// for (int i=0; i<vecGroupPts.size(); i++){
// Mat matGroup = matImage.clone();
// for (int j=0; j<vecGroupPts[i].size(); j++){
// circle(matGroup, vecGroupPts[i][j], 5, Scalar(255,0,255), 1, CV_AA,0);
// }
// imwrite("/Users/tzaman/Desktop/bc/_group_" + std::to_string(i) + ".tif", matGroup);
// }
//}
//exit(-1);
//cout << "vecGroupPts.size()=" << vecGroupPts.size() << endl;
//Erase small groups
//for (int i=vecGroupPts.size()-1; i>=0; i--){
// if (vecGroupPts[i].size() < min_inliers){
// //Skipping group, too small.
// vecGroupIdxs.erase(vecGroupIdxs.begin()+i);
// vecGroupPts.erase(vecGroupPts.begin()+i);
// }
//}
//cout << "vecGroupPts.size()=" << vecGroupPts.size() << endl;
if (vecGroupIdxs.size() == 0) {
string strErr = "Code 39 failed to ransac bars in a line.";
cout << strErr << endl;
return RET_NONE_FOUND;
}
//Now cycle over the groups
vector<vector<int> > vecVecInlierIdx;
vector<Vec4f> vecLines;
vector<int> vecFromGroup; //Keeps track of which group the vecvecInlierIdx belongs to
for (int i = 0; i < vecGroupRects.size(); i++) {
Ransac(vecGroupRects[i], min_inliers, max_px_dist, vecVecInlierIdx, vecLines, this->image_);
vecFromGroup.resize(vecVecInlierIdx.size(), i);
}
if (vecLines.size() == 0) {
string strErr = "Code 39 failed to ransac bars in a line.";
cout << strErr << endl;
return RET_NONE_FOUND;
} else {
//cout << "Code39 ransac succesfull" << endl;
}
//for (int i=0; i<vecGroupIdxs.size(); i++){
// cout << "Group " << i << " (" << vecGroupIdxs[i].size() << ") : ";
// for (int j=0; j<vecGroupIdxs[i].size(); j++){
// cout << vecGroupIdxs[i][j] << " ";
// }
// cout << endl;
//}
//Convert back vecVecInlierIdx to original indices
for (int i=0; i<vecVecInlierIdx.size(); i++){
//cout << "vecVecInlierIdx[" << i << "] is from group " << vecFromGroup[i] << endl;
for (int j=0; j<vecVecInlierIdx[i].size(); j++){
//cout << " " << vecVecInlierIdx[i][j] << " -> " << vecGroupIdxs[vecFromGroup[i]][vecVecInlierIdx[i][j]] << endl;
vecVecInlierIdx[i][j] = vecGroupIdxs[vecFromGroup[i]][vecVecInlierIdx[i][j]];
}
}
for (int i=0; i < vecLines.size(); i++){
int numpts = vecVecInlierIdx[i].size();
cout << "Potential barcode #" << i << " with " << numpts << " points." << endl;
//double angle=atan2(vecLines[i][1],vecLines[i][0])*180/M_PI; //For some reason it clips from [-90,90]
double angle_rad = atan2(vecLines[i][1],vecLines[i][0]); //For some reason it clips from [-90,90]
double angle_deg = angle_rad*180.0/M_PI;
//cout << " angle_deg=" << angle_deg << endl;
vector<double> bar_heights(numpts);
vector<double> bar_widths(numpts);
vector<double> coords_x(numpts);
//Loop over all found and ransac-verified stripes in this barcode
vector<cv::RotatedRect> stripesVerified(numpts);
for (int j=0; j < numpts; j++){
//cout << vecVecInlierIdx[i][j] << endl;
//cout << "checking out stripecandidate[" << vecVecInlierIdx[i][j] << "] #" << vecVecInlierIdx[i][j] << endl;
stripesVerified[j] = stripeCandidates[vecVecInlierIdx[i][j]];
double dim_smallest = min(stripesVerified[j].size.width, stripesVerified[j].size.height); //For rotation invariance
double dim_tallest = max(stripesVerified[j].size.width, stripesVerified[j].size.height); //For rotation invariance
bar_heights[j] = dim_tallest;
bar_widths[j] = dim_smallest;
//Rotate the points straight
Point2f ptRot = util::rotatePoint(stripesVerified[j].center, Point(matImageK.cols, matImageK.rows), angle_rad);
//cout << ptRot << endl;
coords_x[j] = ptRot.x;
}
double height_median = util::calcMedian(bar_heights);
double width_mean = util::calcMean(bar_widths);
//cout << "height_median=" << height_median <<" width_mean=" << width_mean << endl;
//Find the start and end position for reading
vector<size_t> coords_sorted_index;
vector<double> coords_x_sorted;
sort(coords_x, coords_x_sorted, coords_sorted_index);
//cout << coords_x_sorted[0] << " -> " << coords_x_sorted[coords_x_sorted.size()-1] << endl;
//Get extrema-stripes
Point2f pt_stripe_left = stripeCandidates[vecVecInlierIdx[i][coords_sorted_index[0]]].center;
Point2f pt_stripe_right = stripeCandidates[vecVecInlierIdx[i][coords_sorted_index[coords_sorted_index.size() - 1]]].center;
//cout << "pt_stripe_left=" << pt_stripe_left << endl;
//cout << "pt_stripe_right=" << pt_stripe_right << endl;
Point2f pt_barcode_center = (pt_stripe_left + pt_stripe_right) * 0.5;
//cout << "pt_barcode_center=" << pt_barcode_center << endl;
//Calculate width of the barcode
double barcode_width = util::pointDist(pt_stripe_left, pt_stripe_right);
//cout << "barcode_width=" << barcode_width << endl;
//Make the rotated rectangle around the barcode
cv::RotatedRect rotrect_candidate(pt_barcode_center, Size2f(barcode_width, height_median), angle_deg);
const double add_width_on_sides_before_decoding = 7.0; // this number will be multiplied by average bar width
//Add margin (of a few median widths)
rotrect_candidate.size += Size2f(width_mean * add_width_on_sides_before_decoding, 0);
const double height_retrainer_for_collapse = 0.25;
//Extract the barcode itself
cv::RotatedRect rotrect_candidate_thin = rotrect_candidate;
//Crop off some margin in thickness because we dont want to collapse the entire barcode.
if (rotrect_candidate_thin.size.width < rotrect_candidate_thin.size.height) {
rotrect_candidate_thin.size.width *= height_retrainer_for_collapse;
} else {
rotrect_candidate_thin.size.height *= height_retrainer_for_collapse;
}
openbarcode::code code_candidate;
code_candidate.rotrect = rotrect_candidate_thin;
this->code_candidates_.push_back(code_candidate);
}
return RET_SUCCESS;
}
/* Find cell soma */
bool findCellSoma( std::vector<cv::Point> nucleus_contour,
cv::Mat cell_mask,
cv::Mat *intersection,
std::vector<cv::Point> *soma_contour ) {
bool status = false;
// Calculate the min bounding rectangle
cv::RotatedRect min_area_rect = minAreaRect(cv::Mat(nucleus_contour));
cv::RotatedRect scaled_rect = minAreaRect(cv::Mat(nucleus_contour));
// Nucleus' region of influence
cv::Mat roi_mask = cv::Mat::zeros(cell_mask.size(), CV_8UC1);
scaled_rect.size.width = (float)(SOMA_FACTOR * scaled_rect.size.width);
scaled_rect.size.height = (float)(SOMA_FACTOR * scaled_rect.size.height);
ellipse(roi_mask, scaled_rect, 255, -1, 8);
ellipse(roi_mask, min_area_rect, 0, -1, 8);
int mask_score = countNonZero(roi_mask);
// Soma present in ROI
bitwise_and(roi_mask, cell_mask, *intersection);
int intersection_score = countNonZero(*intersection);
// Add the nucleus contour to intersection region
ellipse(*intersection, min_area_rect, 255, -1, 8);
// Add to the soma mask if coverage area exceeds a certain threshold
float ratio = ((float) intersection_score) / mask_score;
if (ratio >= SOMA_COVERAGE_RATIO) {
// Segment
cv::Mat soma_segmented;
std::vector<std::vector<cv::Point>> contours_soma;
std::vector<cv::Vec4i> hierarchy_soma;
std::vector<HierarchyType> soma_contour_mask;
std::vector<double> soma_contour_area;
contourCalc( *intersection,
1.0,
&soma_segmented,
&contours_soma,
&hierarchy_soma,
&soma_contour_mask,
&soma_contour_area
);
double max_area = 0.0;
for (size_t i = 0; i < contours_soma.size(); i++) {
if (soma_contour_mask[i] != HierarchyType::PARENT_CNTR) continue;
if (contours_soma[i].size() < 5) continue;
if (soma_contour_area[i] < MIN_SOMA_SIZE) continue;
// Find the largest permissible contour
if (soma_contour_area[i] > max_area) {
max_area = soma_contour_area[i];
*soma_contour = contours_soma[i];
status = true;
}
}
}
return status;
}
int ConvexityClassifier::Convexity_Computing(Mat &segmentedHand) {
Mat out;
vector<Point> contours,polygon;
vector<Vec4i> hierarchy;
vector<vector<Point> > contours_points;
Scalar color(rand()&255, rand()&255, rand()&255);
//cout << "FIND_CONTOURS_POINTS" << endl;
/*Looking for Contours Points*/
findContours( segmentedHand, contours_points, hierarchy, CV_RETR_TREE, CV_CHAIN_APPROX_SIMPLE );
//cout << "BIGGEST_CONTOURS" << endl;
/*Convert vector<vector<Point>> to vector<Point> and find the biggest contours*/
contours = BiggestContour (contours_points);
/*Approximation of Hands Contours by Polygon*/
//cout << "POLY_APPROX" << endl;
approxPolyDP(contours,polygon,15,true);
contours = polygon;
/*Finding the center of palm*/
//cout << "MIN_AREA_RECT" << endl;
RotatedRect Palm = minAreaRect(contours);
float Palm_Radius;
if( Palm.size.height <= Palm.size.width )
Palm_Radius = Palm.size.height / 2;
else
Palm_Radius = Palm.size.width / 2;
vector<int> index_hull_points(contours_points.size());
vector<Point> convexityDefects(contours_points.size());
vector<Point> Concave_points;
vector<int> Convex_points;
//cout << "CONVEX_HULL" << endl;
convexHull(contours,index_hull_points,false,false); //Find the index of Convex points
/*Convexity Adapt from OpenCV [C versions]*/
vector<Point>& contour = contours;
vector<Point>& convexDefects = convexityDefects;
vector<int>& hull = index_hull_points;
//cout << "FIND_CONVEXITY_DEFECTS" << endl;
findConvexityDefects(contour,hull,convexDefects);
/*Controling Result*/
//cout << "ALL Concave points: " << convexDefects.size() << endl;
//cout << "ALL Convex points: " << hull.size() << endl;
/*Filtering Concave points*/
//cout << "FILTERING_CONCAVE_POINTS" << endl;
Concave_points = Filtering_Concave_Point( convexDefects , Palm );
/*Filtering Convex points*/
//cout << "FILTERING_CONVEX_POINTS" << endl;
Convex_points = Filtering_Convex_Point( hull , contour , Palm );
//cout << "First Filter Convex points: " << Convex_points.size() << endl;
vector<int> tmp;
/*Isolating the interesting convex points*/
//cout << "ISOLATING_CONVEX_POINTS" << endl;
tmp = Isolating_Convex_Point( Convex_points , contour );
//cout << "Second Filter Convex points: " << tmp.size() << endl;
vector<int> result;
float min_distance = Palm.center.y - Palm_Radius;
/*Isolating convex_points by the Average Radius of the palm**/
//cout << "ISOLATING_BY_AVERAGE" << endl;
result = Isolating_Convex_Point_byAverage( contour , Concave_points , min_distance , tmp );
//cout << "Convex points: " << result.size() << endl;
//cout << "Concave points: " << Concave_points.size() << endl;
float min_distance2 = Palm.center.y - (Palm_Radius * 2);
/*Compute result*/
float result_digital_numbers;
//cout << "COMPUTE_RESULT" << endl;
result_digital_numbers = Compute_Result( contour , Concave_points , result , min_distance2 );
//cout<< "********************************" << endl;
//cout<< "SIZE: " << segmentedHand.size() << endl;
//cout<< "********************************" << endl;
/*Drawing Convex of polygon*/
for(int i = 0; i < contours_points.size() ; i++)
{
drawContours( segmentedHand, contours_points, i, color, 1, 8, vector<Vec4i>(), 0, Point() );
}
/*Affichage*/
// imshow("contour",segmentedHand);
// waitKey(0);
return result_digital_numbers;
}
vector<Plate> DetectRegions::segment(Mat input){
vector<Plate> output;
//convert image to gray
Mat img_gray; //= *new Mat(input.size().width,input.size().height, CV_8UC1);
cvtColor(input, img_gray, CV_BGR2GRAY);
blur(img_gray, img_gray, Size(5,5));
//Finde vertical lines. Car plates have high density of vertical lines
Mat img_sobel;
Sobel(img_gray, img_sobel, CV_8U, 1, 0, 3, 1, 0, BORDER_DEFAULT);
if(showSteps)
imshow("Sobel", img_sobel);
//threshold image
Mat img_threshold;
threshold(img_sobel, img_threshold, 0, 255, CV_THRESH_OTSU+CV_THRESH_BINARY);
if(showSteps)
imshow("Threshold", img_threshold);
//Morphplogic operation close
Mat element = getStructuringElement(MORPH_RECT, Size(17, 3) );
morphologyEx(img_threshold, img_threshold, CV_MOP_CLOSE, element);
if(showSteps)
imshow("Close", img_threshold);
//Find contours of possibles plates
vector< vector< Point> > contours;
findContours(img_threshold,
contours, // a vector of contours
CV_RETR_EXTERNAL, // retrieve the external contours
CV_CHAIN_APPROX_NONE); // all pixels of each contours
//Start to iterate to each contour founded
vector<vector<Point> >::iterator itc= contours.begin();
vector<RotatedRect> rects;
//Remove patch that are no inside limits of aspect ratio and area.
while (itc!=contours.end()) {
//Create bounding rect of object
RotatedRect mr= minAreaRect(Mat(*itc));
if( !verifySizes(mr)){
itc= contours.erase(itc);
}else{
++itc;
rects.push_back(mr);
}
}
// Draw blue contours on a white image
cv::Mat result;
input.copyTo(result);
cv::drawContours(result,contours,
-1, // draw all contours
cv::Scalar(255,0,0), // in blue
1); // with a thickness of 1
for(int i=0; i< rects.size(); i++){
//For better rect cropping for each posible box
//Make floodfill algorithm because the plate has white background
//And then we can retrieve more clearly the contour box
circle(result, rects[i].center, 3, Scalar(0,255,0), -1);
//get the min size between width and height
float minSize=(rects[i].size.width < rects[i].size.height)?rects[i].size.width:rects[i].size.height;
minSize=minSize-minSize*0.5;
//initialize rand and get 5 points around center for floodfill algorithm
srand ( time(NULL) );
//Initialize floodfill parameters and variables
Mat mask;
mask.create(input.rows + 2, input.cols + 2, CV_8UC1);
mask= Scalar::all(0);
int loDiff = 30;
int upDiff = 30;
int connectivity = 4;
int newMaskVal = 255;
int NumSeeds = 10;
Rect ccomp;
int flags = connectivity + (newMaskVal << 8 ) + CV_FLOODFILL_FIXED_RANGE + CV_FLOODFILL_MASK_ONLY;
for(int j=0; j<NumSeeds; j++){
Point seed;
seed.x=rects[i].center.x+rand()%(int)minSize-(minSize/2);
seed.y=rects[i].center.y+rand()%(int)minSize-(minSize/2);
circle(result, seed, 1, Scalar(0,255,255), -1);
int area = floodFill(input, mask, seed, Scalar(255,0,0), &ccomp, Scalar(loDiff, loDiff, loDiff), Scalar(upDiff, upDiff, upDiff), flags);
}
if(showSteps)
imshow("MASK", mask);
//cvWaitKey(0);
//Check new floodfill mask match for a correct patch.
//Get all points detected for get Minimal rotated Rect
vector<Point> pointsInterest;
Mat_<uchar>::iterator itMask= mask.begin<uchar>();
Mat_<uchar>::iterator end= mask.end<uchar>();
for( ; itMask!=end; ++itMask)
if(*itMask==255)
pointsInterest.push_back(itMask.pos());
RotatedRect minRect = minAreaRect(pointsInterest);
if(verifySizes(minRect)){
// rotated rectangle drawing
Point2f rect_points[4]; minRect.points( rect_points );
for( int j = 0; j < 4; j++ )
line( result, rect_points[j], rect_points[(j+1)%4], Scalar(0,0,255), 1, 8 );
//Get rotation matrix
float r= (float)minRect.size.width / (float)minRect.size.height;
float angle=minRect.angle;
if(r<1)
angle=90+angle;
Mat rotmat= getRotationMatrix2D(minRect.center, angle,1);
//Create and rotate image
Mat img_rotated;
warpAffine(input, img_rotated, rotmat, input.size(), CV_INTER_CUBIC);
//Crop image
Size rect_size=minRect.size;
if(r < 1)
swap(rect_size.width, rect_size.height);
Mat img_crop;
getRectSubPix(img_rotated, rect_size, minRect.center, img_crop);
Mat resultResized;
resultResized.create(33,144, CV_8UC3);
resize(img_crop, resultResized, resultResized.size(), 0, 0, INTER_CUBIC);
//Equalize croped image
Mat grayResult;
cvtColor(resultResized, grayResult, CV_BGR2GRAY);
blur(grayResult, grayResult, Size(3,3));
grayResult=histeq(grayResult);
if(saveRegions){
stringstream ss(stringstream::in | stringstream::out);
ss << "tmp/" << filename << "_" << i << ".jpg";
imwrite(ss.str(), grayResult);
}
output.push_back(Plate(grayResult,minRect.boundingRect()));
}
}
if(showSteps)
imshow("Contours", result);
return output;
}
void RecognitionDemos( Mat& full_image, Mat& template1, Mat& template2, Mat& template1locations, Mat& template2locations, VideoCapture& bicycle_video, Mat& bicycle_background, Mat& bicycle_model, VideoCapture& people_video, CascadeClassifier& cascade, Mat& numbers, Mat& good_orings, Mat& bad_orings, Mat& unknown_orings )
{
Timestamper* timer = new Timestamper();
// Principal Components Analysis
PCASimpleExample();
char ch = cvWaitKey();
cvDestroyAllWindows();
PCAFaceRecognition();
ch = cvWaitKey();
cvDestroyAllWindows();
// Statistical Pattern Recognition
Mat gray_numbers,binary_numbers;
cvtColor(numbers, gray_numbers, CV_BGR2GRAY);
threshold(gray_numbers,binary_numbers,128,255,THRESH_BINARY_INV);
vector<vector<Point>> contours;
vector<Vec4i> hierarchy;
findContours(binary_numbers,contours,hierarchy,CV_RETR_TREE,CV_CHAIN_APPROX_NONE);
Mat contours_image = Mat::zeros(binary_numbers.size(), CV_8UC3);
contours_image = Scalar(255,255,255);
// Do some processing on all contours (objects and holes!)
vector<RotatedRect> min_bounding_rectangle(contours.size());
vector<vector<Point>> hulls(contours.size());
vector<vector<int>> hull_indices(contours.size());
vector<vector<Vec4i>> convexity_defects(contours.size());
vector<Moments> contour_moments(contours.size());
for (int contour_number=0; (contour_number<(int)contours.size()); contour_number++)
{
if (contours[contour_number].size() > 10)
{
min_bounding_rectangle[contour_number] = minAreaRect(contours[contour_number]);
convexHull(contours[contour_number], hulls[contour_number]);
convexHull(contours[contour_number], hull_indices[contour_number]);
convexityDefects( contours[contour_number], hull_indices[contour_number], convexity_defects[contour_number]);
contour_moments[contour_number] = moments( contours[contour_number] );
}
}
for (int contour_number=0; (contour_number>=0); contour_number=hierarchy[contour_number][0])
{
if (contours[contour_number].size() > 10)
{
Scalar colour( rand()&0x7F, rand()&0x7F, rand()&0x7F );
drawContours( contours_image, contours, contour_number, colour, CV_FILLED, 8, hierarchy );
char output[500];
double area = contourArea(contours[contour_number])+contours[contour_number].size()/2+1;
// Process any holes (removing the area from the are of the enclosing contour)
for (int hole_number=hierarchy[contour_number][2]; (hole_number>=0); hole_number=hierarchy[hole_number][0])
{
area -= (contourArea(contours[hole_number])-contours[hole_number].size()/2+1);
Scalar colour( rand()&0x7F, rand()&0x7F, rand()&0x7F );
drawContours( contours_image, contours, hole_number, colour, CV_FILLED, 8, hierarchy );
sprintf(output,"Area=%.0f", contourArea(contours[hole_number])-contours[hole_number].size()/2+1);
Point location( contours[hole_number][0].x +20, contours[hole_number][0].y +5 );
putText( contours_image, output, location, FONT_HERSHEY_SIMPLEX, 0.4, colour );
}
// Draw the minimum bounding rectangle
Point2f bounding_rect_points[4];
min_bounding_rectangle[contour_number].points(bounding_rect_points);
line( contours_image, bounding_rect_points[0], bounding_rect_points[1], Scalar(0, 0, 127));
line( contours_image, bounding_rect_points[1], bounding_rect_points[2], Scalar(0, 0, 127));
line( contours_image, bounding_rect_points[2], bounding_rect_points[3], Scalar(0, 0, 127));
line( contours_image, bounding_rect_points[3], bounding_rect_points[0], Scalar(0, 0, 127));
float bounding_rectangle_area = min_bounding_rectangle[contour_number].size.area();
// Draw the convex hull
drawContours( contours_image, hulls, contour_number, Scalar(127,0,127) );
// Highlight any convexities
int largest_convexity_depth=0;
for (int convexity_index=0; convexity_index < (int)convexity_defects[contour_number].size(); convexity_index++)
{
if (convexity_defects[contour_number][convexity_index][3] > largest_convexity_depth)
largest_convexity_depth = convexity_defects[contour_number][convexity_index][3];
if (convexity_defects[contour_number][convexity_index][3] > 256*2)
{
line( contours_image, contours[contour_number][convexity_defects[contour_number][convexity_index][0]], contours[contour_number][convexity_defects[contour_number][convexity_index][2]], Scalar(0,0, 255));
line( contours_image, contours[contour_number][convexity_defects[contour_number][convexity_index][1]], contours[contour_number][convexity_defects[contour_number][convexity_index][2]], Scalar(0,0, 255));
}
}
double hu_moments[7];
HuMoments( contour_moments[contour_number], hu_moments );
sprintf(output,"Perimeter=%d, Area=%.0f, BArea=%.0f, CArea=%.0f", contours[contour_number].size(),area,min_bounding_rectangle[contour_number].size.area(),contourArea(hulls[contour_number]));
Point location( contours[contour_number][0].x, contours[contour_number][0].y-3 );
putText( contours_image, output, location, FONT_HERSHEY_SIMPLEX, 0.4, colour );
sprintf(output,"HuMoments = %.2f, %.2f, %.2f", hu_moments[0],hu_moments[1],hu_moments[2]);
Point location2( contours[contour_number][0].x+100, contours[contour_number][0].y-3+15 );
putText( contours_image, output, location2, FONT_HERSHEY_SIMPLEX, 0.4, colour );
}
}
imshow("Shape Statistics", contours_image );
char c = cvWaitKey();
cvDestroyAllWindows();
// Support Vector Machine
imshow("Good - original",good_orings);
imshow("Defective - original",bad_orings);
imshow("Unknown - original",unknown_orings);
SupportVectorMachineDemo(good_orings,"Good",bad_orings,"Defective",unknown_orings);
c = cvWaitKey();
cvDestroyAllWindows();
// Template Matching
Mat display_image, correlation_image;
full_image.copyTo( display_image );
double min_correlation, max_correlation;
Mat matched_template_map;
int result_columns = full_image.cols - template1.cols + 1;
int result_rows = full_image.rows - template1.rows + 1;
correlation_image.create( result_columns, result_rows, CV_32FC1 );
timer->reset();
double before_tick_count = static_cast<double>(getTickCount());
matchTemplate( full_image, template1, correlation_image, CV_TM_CCORR_NORMED );
double after_tick_count = static_cast<double>(getTickCount());
double duration_in_ms = 1000.0*(after_tick_count-before_tick_count)/getTickFrequency();
minMaxLoc( correlation_image, &min_correlation, &max_correlation );
FindLocalMaxima( correlation_image, matched_template_map, max_correlation*0.99 );
timer->recordTime("Template Matching (1)");
Mat matched_template_display1;
cvtColor(matched_template_map, matched_template_display1, CV_GRAY2BGR);
Mat correlation_window1 = convert_32bit_image_for_display( correlation_image, 0.0 );
DrawMatchingTemplateRectangles( display_image, matched_template_map, template1, Scalar(0,0,255) );
double precision, recall, accuracy, specificity, f1;
Mat template1locations_gray;
cvtColor(template1locations, template1locations_gray, CV_BGR2GRAY);
CompareRecognitionResults( matched_template_map, template1locations_gray, precision, recall, accuracy, specificity, f1 );
char results[400];
Scalar colour( 255, 255, 255);
sprintf( results, "precision=%.2f", precision);
Point location( 7, 213 );
putText( display_image, "Results (1)", location, FONT_HERSHEY_SIMPLEX, 0.4, colour );
location.y += 13;
putText( display_image, results, location, FONT_HERSHEY_SIMPLEX, 0.4, colour );
sprintf( results, "recall=%.2f", recall);
location.y += 13;
putText( display_image, results, location, FONT_HERSHEY_SIMPLEX, 0.4, colour );
sprintf( results, "accuracy=%.2f", accuracy);
location.y += 13;
putText( display_image, results, location, FONT_HERSHEY_SIMPLEX, 0.4, colour );
sprintf( results, "specificity=%.2f", specificity);
location.y += 13;
putText( display_image, results, location, FONT_HERSHEY_SIMPLEX, 0.4, colour );
sprintf( results, "f1=%.2f", f1);
location.y += 13;
putText( display_image, results, location, FONT_HERSHEY_SIMPLEX, 0.4, colour );
result_columns = full_image.cols - template2.cols + 1;
result_rows = full_image.rows - template2.rows + 1;
correlation_image.create( result_columns, result_rows, CV_32FC1 );
timer->ignoreTimeSinceLastRecorded();
matchTemplate( full_image, template2, correlation_image, CV_TM_CCORR_NORMED );
minMaxLoc( correlation_image, &min_correlation, &max_correlation );
FindLocalMaxima( correlation_image, matched_template_map, max_correlation*0.99 );
timer->recordTime("Template Matching (2)");
Mat matched_template_display2;
cvtColor(matched_template_map, matched_template_display2, CV_GRAY2BGR);
Mat correlation_window2 = convert_32bit_image_for_display( correlation_image, 0.0 );
DrawMatchingTemplateRectangles( display_image, matched_template_map, template2, Scalar(0,0,255) );
timer->putTimes(display_image);
Mat template2locations_gray;
cvtColor(template2locations, template2locations_gray, CV_BGR2GRAY);
CompareRecognitionResults( matched_template_map, template2locations_gray, precision, recall, accuracy, specificity, f1 );
sprintf( results, "precision=%.2f", precision);
location.x = 123;
location.y = 213;
putText( display_image, "Results (2)", location, FONT_HERSHEY_SIMPLEX, 0.4, colour );
location.y += 13;
putText( display_image, results, location, FONT_HERSHEY_SIMPLEX, 0.4, colour );
sprintf( results, "recall=%.2f", recall);
location.y += 13;
putText( display_image, results, location, FONT_HERSHEY_SIMPLEX, 0.4, colour );
sprintf( results, "accuracy=%.2f", accuracy);
location.y += 13;
putText( display_image, results, location, FONT_HERSHEY_SIMPLEX, 0.4, colour );
sprintf( results, "specificity=%.2f", specificity);
location.y += 13;
putText( display_image, results, location, FONT_HERSHEY_SIMPLEX, 0.4, colour );
sprintf( results, "f1=%.2f", f1);
location.y += 13;
putText( display_image, results, location, FONT_HERSHEY_SIMPLEX, 0.4, colour );
Mat correlation_display1, correlation_display2;
cvtColor(correlation_window1, correlation_display1, CV_GRAY2BGR);
cvtColor(correlation_window2, correlation_display2, CV_GRAY2BGR);
Mat output1 = JoinImagesVertically(template1,"Template (1)",correlation_display1,"Correlation (1)",4);
Mat output2 = JoinImagesVertically(output1,"",matched_template_display1,"Local maxima (1)",4);
Mat output3 = JoinImagesVertically(template2,"Template (2)",correlation_display2,"Correlation (2)",4);
Mat output4 = JoinImagesVertically(output3,"",matched_template_display2,"Local maxima (2)",4);
Mat output5 = JoinImagesHorizontally( full_image, "Original Image", output2, "", 4 );
Mat output6 = JoinImagesHorizontally( output5, "", output4, "", 4 );
Mat output7 = JoinImagesHorizontally( output6, "", display_image, "", 4 );
imshow( "Template matching result", output7 );
c = cvWaitKey();
cvDestroyAllWindows();
// Chamfer Matching
Mat model_gray,model_edges,model_edges2;
cvtColor(bicycle_model, model_gray, CV_BGR2GRAY);
threshold(model_gray,model_edges,127,255,THRESH_BINARY);
Mat current_frame;
bicycle_video.set(CV_CAP_PROP_POS_FRAMES,400); // Just in case the video has already been used.
bicycle_video >> current_frame;
bicycle_background = current_frame.clone();
bicycle_video.set(CV_CAP_PROP_POS_FRAMES,500);
timer->reset();
int count = 0;
while (!current_frame.empty() && (count < 8))
{
Mat result_image = current_frame.clone();
count++;
Mat difference_frame, difference_gray, current_edges;
absdiff(current_frame,bicycle_background,difference_frame);
cvtColor(difference_frame, difference_gray, CV_BGR2GRAY);
Canny(difference_frame, current_edges, 100, 200, 3);
vector<vector<Point> > results;
vector<float> costs;
threshold(model_gray,model_edges,127,255,THRESH_BINARY);
Mat matching_image, chamfer_image, local_minima;
timer->ignoreTimeSinceLastRecorded();
threshold(current_edges,current_edges,127,255,THRESH_BINARY_INV);
distanceTransform( current_edges, chamfer_image, CV_DIST_L2 , 3);
timer->recordTime("Chamfer Image");
ChamferMatching( chamfer_image, model_edges, matching_image );
timer->recordTime("Matching");
FindLocalMinima( matching_image, local_minima, 500.0 );
timer->recordTime("Find Minima");
DrawMatchingTemplateRectangles( result_image, local_minima, model_edges, Scalar( 255, 0, 0 ) );
Mat chamfer_display_image = convert_32bit_image_for_display( chamfer_image );
Mat matching_display_image = convert_32bit_image_for_display( matching_image );
//timer->putTimes(result_image);
Mat current_edges_display, local_minima_display, model_edges_display, colour_matching_display_image, colour_chamfer_display_image;
cvtColor(current_edges, current_edges_display, CV_GRAY2BGR);
cvtColor(local_minima, local_minima_display, CV_GRAY2BGR);
cvtColor(model_edges, model_edges_display, CV_GRAY2BGR);
cvtColor(matching_display_image, colour_matching_display_image, CV_GRAY2BGR);
cvtColor(chamfer_display_image, colour_chamfer_display_image, CV_GRAY2BGR);
Mat output1 = JoinImagesVertically(current_frame,"Video Input",current_edges_display,"Edges from difference", 4);
Mat output2 = JoinImagesVertically(output1,"",model_edges_display,"Model", 4);
Mat output3 = JoinImagesVertically(bicycle_background,"Static Background",colour_chamfer_display_image,"Chamfer image", 4);
Mat output4 = JoinImagesVertically(output3,"",colour_matching_display_image,"Degree of fit", 4);
Mat output5 = JoinImagesVertically(difference_frame,"Difference",result_image,"Result", 4);
Mat output6 = JoinImagesVertically(output5,"",local_minima_display,"Local minima", 4);
Mat output7 = JoinImagesHorizontally( output2, "", output4, "", 4 );
Mat output8 = JoinImagesHorizontally( output7, "", output6, "", 4 );
imshow("Chamfer matching", output8);
c = waitKey(1000); // This makes the image appear on screen
bicycle_video >> current_frame;
}
c = cvWaitKey();
cvDestroyAllWindows();
// Cascade of Haar classifiers (most often shown for face detection).
VideoCapture camera;
camera.open(1);
camera.set(CV_CAP_PROP_FRAME_WIDTH, 320);
camera.set(CV_CAP_PROP_FRAME_HEIGHT, 240);
if( camera.isOpened() )
{
timer->reset();
Mat current_frame;
do {
camera >> current_frame;
if( current_frame.empty() )
break;
vector<Rect> faces;
timer->ignoreTimeSinceLastRecorded();
Mat gray;
cvtColor( current_frame, gray, CV_BGR2GRAY );
equalizeHist( gray, gray );
cascade.detectMultiScale( gray, faces, 1.1, 2, CV_HAAR_SCALE_IMAGE, Size(30, 30) );
timer->recordTime("Haar Classifier");
for( int count = 0; count < (int)faces.size(); count++ )
rectangle(current_frame, faces[count], cv::Scalar(255,0,0), 2);
//timer->putTimes(current_frame);
imshow( "Cascade of Haar Classifiers", current_frame );
c = waitKey(10); // This makes the image appear on screen
} while (c == -1);
}
vector<Rect> visionUtils::segmentLineBoxFit(Mat img0, int minPixelSize, int maxSegments, Mat *returnMask, std::vector<std::vector<cv::Point> > *returnContours, vector<RotatedRect> *rotatedBoundingBox, bool displayFaces)
{
// Segments items in gray image (img0)
// minPixelSize= pixels, threshold for removing smaller regions, with less than minPixelSize pixels
// 0, returns all detected segments
// maxSegments = max no segments to return, 0 = all
RNG rng(12345);
int padPixels=15;
// Rect border added at start...
Rect tempRect;
tempRect.x=padPixels;
tempRect.y=padPixels;
tempRect.width=img0.cols;
tempRect.height=img0.rows;
Mat img1 = Mat::zeros(img0.rows+(padPixels*2), img0.cols+(padPixels*2), CV_8UC1);
img0.copyTo(img1(tempRect));
// find the contours
std::vector<std::vector<cv::Point> > contours;
vector<Vec4i> hierarchy;
findContours(img1, contours, hierarchy, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_NONE);
// Mask for segmented region
Mat mask = Mat::zeros(img1.rows, img1.cols, CV_8UC3);
vector<double> areas(contours.size());
// Case for using minimum pixel size
Vec4f lines;
Scalar color;
// sort contours
std::sort(contours.begin(), contours.end(), compareContourAreas);
// grab contours
vector<Rect> boundingBox;
// LB testing
vector<RotatedRect> tempRotatedBoundingBox;
std::vector<std::vector<cv::Point> > tempReturnContours;
int maxIterations = 0;
if( contours.size() > 0 )
{
if (maxSegments==0)// return all contours..
maxIterations = contours.size();
else if((int)contours.size() >= maxSegments)
maxIterations = maxSegments;
else
maxIterations = 1; // LB: need to check this is correct!
int contourCount=0;
for (int j = 1; j < maxIterations+1; j++)
{
int i = contours.size()-j;
if (contourArea(Mat(contours[i]))>minPixelSize)
{
// Fit rotated rect to contour
tempRotatedBoundingBox.push_back(minAreaRect( Mat(contours[i]) ));
Point2f rectCentre=tempRotatedBoundingBox[contourCount].center;
rectCentre.x=rectCentre.x-padPixels;
rectCentre.y=rectCentre.y-padPixels;
tempRotatedBoundingBox[contourCount].center=rectCentre;
// Find line limits....
boundingBox.push_back(boundingRect(Mat(contours[i])));
// Remove edge padding effects....
boundingBox[contourCount].x=boundingBox[contourCount].x-padPixels;
boundingBox[contourCount].y=boundingBox[contourCount].y-padPixels;
boundingBox[contourCount]=checkRoiInImage(img0, boundingBox[contourCount]);
contourCount++;
tempReturnContours.push_back(contours[i]);
}
}
// Return contours
returnContours->resize(tempReturnContours.size());
*returnContours = tempReturnContours;
// Return rotated rects
rotatedBoundingBox->resize(tempRotatedBoundingBox.size());
*rotatedBoundingBox = tempRotatedBoundingBox;
// normalize so imwrite(...)/imshow(...) shows the mask correctly!
cv::normalize(mask.clone(), mask, 0.0, 255.0, CV_MINMAX, CV_8UC1);
// To Remove border added at start...
*returnMask=mask(tempRect);
// show the images
if (displayFaces) imshow("Seg line utils: Img in", img0);
if (displayFaces) imshow("Seg line utils: Mask", *returnMask);
if (displayFaces) imshow("Seg line utils: Output", img1);
}
return boundingBox;
}
/**
@brief Finds rotated bounding boxes from blob outlines.
The blob's area must be larger than
Parameters::bounding_box_min_area_threshold.
The blob and its bounding rectangle must be inside the image's limits.
@param[in] inImage [const cv::Mat&] The input image
@param[in] blobsOutlineVector
[const std::vector<std::vector<cv::Point2f> >&]
The outline points of the blobs
@param[in] blobsArea [const std::vector<float>&] The blobs' area
@param[out] outRectangles [std::vector< std::vector<cv::Point2f> >*]
The rectangles of the bounding boxes
@return void
**/
void BoundingBoxDetection::findRotatedBoundingBoxesFromOutline(
const cv::Mat& inImage,
const std::vector<std::vector<cv::Point2f> >& blobsOutlineVector,
const std::vector<float>& blobsArea,
std::vector<std::vector<cv::Point2f> >* outRectangles)
{
#ifdef DEBUG_TIME
Timer::start("findRotatedBoundingBoxesFromOutline", "validateBlobs");
#endif
// Find the rotated rectangles for each blob based on its outline
std::vector<cv::RotatedRect> minRect;
for (unsigned int i = 0; i < blobsOutlineVector.size(); i++)
{
// The area of the blob should be greater than a threshold,
// so that tiny formations of pixels are not identified as blobs
if (blobsArea[i] >= Parameters::Blob::min_area)
{
minRect.push_back(minAreaRect(cv::Mat(blobsOutlineVector[i])));
}
}
// For each rotated rectangle whose corresponding blob exceeds the minimum
// area threshold, if its vertices reside within the image's boundaries,
// store its vertices
for (unsigned int i = 0; i < minRect.size(); i++)
{
// The for vertices of the rotated rectangle
cv::Point2f rect_points[4];
minRect[i].points(rect_points);
// Check if the vertices reside in the image's boundaries
int numVerticesWithinImageLimits = 0;
for (int j = 0; j < 4; j++)
{
if (rect_points[j].x < inImage.cols &&
rect_points[j].x >= 0 &&
rect_points[j].y < inImage.rows &&
rect_points[j].y >= 0)
{
numVerticesWithinImageLimits++;
}
}
// If the rotated rectangle's edges reside outside the image's edges,
// discard this rotated rectangle
if (numVerticesWithinImageLimits < 4)
{
continue;
}
// If all four vertices reside within the image's boundaries,
// store them in their respective position
// Same as rect_points array, but vector
std::vector<cv::Point2f> rect_points_vector;
for (int j = 0; j < 4; j++)
{
rect_points_vector.push_back(rect_points[j]);
}
// Push back the 4 vertices of rectangle i
outRectangles->push_back(rect_points_vector);
}
#ifdef DEBUG_TIME
Timer::tick("findRotatedBoundingBoxesFromOutline");
#endif
}
Skeleton::Skeleton(cv::Mat skeletonizedImage, cv::Mat normalImage){
//QList<LabeledPoint> startList;
QList<cv::Point2i> dummyJunctionList;
QList<LabeledPoint> removedPoints;
QList<int> survivors;
int currentLabel = 1;
// SEARCH FOR ALL CURRENT BRANCHES
for (int x = 0; x < skeletonizedImage.cols; x++){
for (int y = 0; y < skeletonizedImage.rows; y++){
if (skeletonizedImage.at<uchar>(y, x) == 255){
int count = 0;
QVector<cv::Point2i> list;
for (int i = -1; i <= 1; i++){
for (int j = -1; j <= 1; j++){
if (i != 0 || j != 0){
if (y+j >= 0 && y+j < skeletonizedImage.rows && x+i >= 0 && x+i < skeletonizedImage.cols && skeletonizedImage.at<uchar>(y+j, x+i) > 0){
bool neighbourg = false;
cv::Point2i point(x+i,y+j);
for (int k = 0; k < list.size(); k++){
if (cv::norm(point - list[k]) == 1){
neighbourg = true;
}
}
if (!neighbourg){
count++;
list.append(point);
}
}
}
}
}
if (count == 1){
cv::Point2i point(x,y);
LabeledPoint lp;
lp.label = currentLabel;
lp.point = point;
startList.append(lp);
currentLabel++;
}
else if (count > 2){
cv::Point2i point(x,y);
dummyJunctionList.append(point);
}
}
}
}
// GO THROUGH THE BRANCH AND REMOVE ONE PIXEL AT A TIME
for (int i = 0; i < startList.size(); i++){
cv::Point2i current = startList[i].point;
int label = startList[i].label;
int round = 0;
bool done = false;
do {
LabeledPoint lp;
lp.point = current;
lp.label = label;
removedPoints.append(lp);
skeletonizedImage.at<uchar>(current.y, current.x) = 0;
int count = 0;
int x = current.x;
int y = current.y;
QVector<cv::Point2i> list;
for (int i = -1; i <= 1; i++){
for (int j = -1; j <= 1; j++){
if (i != 0 || j != 0){
if (y+j >= 0 && y+j < skeletonizedImage.rows && x+i >= 0 && x+i < skeletonizedImage.cols && skeletonizedImage.at<uchar>(y+j, x+i) == 255){
bool junction = false;
cv::Point2i point(x+i,y+j);
for (int k = 0; k < dummyJunctionList.size(); k++){
if (cv::norm(point - dummyJunctionList[k]) <= 1){
junction = true;
}
}
if (!junction){
count++;
list.append(point);
}
}
}
}
}
if (count >= 1){
current = list[0];
}
round ++;
done = (count < 1);
if (round == 12){
survivors.append(label);
}
} while(round < 12 && !done);
}
// IF THE BRANCH IS STILL LONG ENOUGH, REDRAW IT
for (int i = 0; i < removedPoints.size(); i++){
cv::Point2i point = removedPoints[i].point;
int label = removedPoints[i].label;
if (survivors.contains(label)){
skeletonizedImage.at<uchar>(point) = 255;
}
}
// LOOPS
std::vector<std::vector<cv::Point> > contours;
std::vector<cv::Vec4i> hierarchy;
cv::Mat imageClone = normalImage.clone();
cv::Mat dilateElement = cv::getStructuringElement(cv::MORPH_RECT, cv::Size(3, 3), cv::Point(1, 1));
cv::dilate(imageClone, imageClone, dilateElement);
findContours(imageClone.clone(), contours, hierarchy, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_SIMPLE);
cv::Mat invertedImage = normalImage.clone();
if (contours.size() < 1){
qDebug() << "Error, 0 connected components detected";
}
else {
for (int x = 0; x < normalImage.cols; x++){
for (int y = 0; y < normalImage.rows; y++){
if (imageClone.at<uchar>(y, x) == 255){
invertedImage.at<uchar>(y, x) = 0;
}
else if (pointPolygonTest(contours[0], cv::Point2f(x,y), true) >= 0){
invertedImage.at<uchar>(y, x) = 255;
}
else {
invertedImage.at<uchar>(y, x) = 0;
}
}
}
}
std::vector<std::vector<cv::Point> > invertedContours;
std::vector<cv::Vec4i> invertedHierarchy;
findContours(invertedImage, invertedContours, invertedHierarchy, CV_RETR_CCOMP, CV_CHAIN_APPROX_SIMPLE);
for (unsigned int i = 0; i < invertedContours.size(); i++){
cv::Rect rect = minAreaRect(invertedContours[i]).boundingRect();
double xNorm = ((double) rect.x + rect.width/2) / skeletonizedImage.cols;
double yNorm = ((double) rect.y + rect.height/2) / skeletonizedImage.rows;
cv::Point2d point(xNorm, yNorm);
if (rect.width > 2 && rect.height > 2){
listHoles.push_back(point);
}
else {
listFakeHoles.push_back(point);
}
}
// COUNTING JUNCTIONS AND LINE ENDS
for (int x = 0; x < skeletonizedImage.cols; x++){
for (int y = 0; y < skeletonizedImage.rows; y++){
if (skeletonizedImage.at<uchar>(y, x) == 255){
int count = 0;
QVector<cv::Point2i> list;
for (int i = -1; i <= 1; i++){
for (int j = -1; j <= 1; j++){
if (i != 0 || j != 0){
if (y+j >= 0 && y+j < skeletonizedImage.rows && x+i >= 0 && x+i < skeletonizedImage.cols && skeletonizedImage.at<uchar>(y+j, x+i) == 255){
bool neighbourg = false;
cv::Point2i point(x+i,y+j);
for (int k = 0; k < list.size(); k++){
if (cv::norm(point - list[k]) == 1){
neighbourg = true;
}
}
if (!neighbourg){
count++;
list.append(point);
}
}
}
}
}
if (count == 1 || count > 2){
double xNorm = ((double) x) / skeletonizedImage.cols;
double yNorm = ((double) y) / skeletonizedImage.rows;
cv::Point2d point(xNorm,yNorm);
if (count == 1){
listLineEnds.push_back(point);
}
else {
listJunctions.push_back(point);
}
}
}
}
}
// MERGING CLOSE JUNCTIONS
bool done = true;
do {
done = true;
int keepIndexJunction = -1;
int removeIndexJunction = -1;
for (int i = 0; i < listJunctions.size(); i++){
for (int j = 0; j < listJunctions.size(); j++){
if (i != j && norm(listJunctions[i] - listJunctions[j]) < MERGE_DISTANCE){
keepIndexJunction = i;
removeIndexJunction = j;
}
}
}
if (keepIndexJunction != -1 && removeIndexJunction != -1){
done = false;
listJunctions[keepIndexJunction].x = (listJunctions[keepIndexJunction].x + listJunctions[removeIndexJunction].x) / 2;
listJunctions[keepIndexJunction].y = (listJunctions[keepIndexJunction].y + listJunctions[removeIndexJunction].y) / 2;
listJunctions.removeAt(removeIndexJunction);
}
} while (!done);
// DELETING CLOSE JUNCTIONS AND LINE ENDS, ALWAYS WRONG DATA
done = true;
do {
done = true;
int removeIndexLineEnds = -1;
int removeIndexJunctions = -1;
for (int i = 0; i < listLineEnds.size(); i++){
for (int j = 0; j < listJunctions.size(); j++){
if (norm(listLineEnds[i] - listJunctions[j]) < DELETE_DISTANCE){
removeIndexLineEnds = i;
removeIndexJunctions = j;
}
}
}
if (removeIndexLineEnds != -1 && removeIndexJunctions != -1){
done = false;
listLineEnds.removeAt(removeIndexLineEnds);
listJunctions.removeAt(removeIndexJunctions);
}
} while (!done);
// DELETING JUNCTIONS CLOSE TO FAKE LOOPS
done = true;
do {
done = true;
int removeIndexJunction = -1;
for (int i = 0; i < listJunctions.size(); i++){
for (int j = 0; j < listFakeHoles.size(); j++){
if (norm(listJunctions[i] - listFakeHoles[j]) < FAKE_LOOPS_DISTANCE){
removeIndexJunction = i;
}
}
}
if (removeIndexJunction != -1){
done = false;
listJunctions.removeAt(removeIndexJunction);
}
} while (!done);
// DELETING LINE ENDS CLOSE TO FAKE LOOPS
done = true;
do {
done = true;
int removeIndexEnd = -1;
for (int i = 0; i < listLineEnds.size(); i++){
for (int j = 0; j < listFakeHoles.size(); j++){
if (norm(listLineEnds[i] - listFakeHoles[j]) < FAKE_LOOPS_DISTANCE){
removeIndexEnd = i;
}
}
}
if (removeIndexEnd != -1){
done = false;
listLineEnds.removeAt(removeIndexEnd);
}
} while (!done);
// DELETING JUNCTIONS CLOSE TO BORDERS, ALWAYS WRONG DATA
done = true;
do {
done = true;
int removeIndexJunctions = -1;
for (int i = 0; i < listJunctions.size(); i++){
if (listJunctions[i].x < JUNCTION_MARGIN || listJunctions[i].y < JUNCTION_MARGIN || listJunctions[i].x > 1 - JUNCTION_MARGIN || listJunctions[i].y > 1 - JUNCTION_MARGIN){
removeIndexJunctions = i;
}
}
if (removeIndexJunctions != -1){
done = false;
listJunctions.removeAt(removeIndexJunctions);
}
} while (!done);
massCenter = getMassCenter(normalImage);
total = getCount(normalImage);
setParts(normalImage);
listLineEnds = sort(listLineEnds);
listHoles = sort(listHoles);
listJunctions = sort(listJunctions);
}
int process(VideoCapture& capture) {
long captureTime;
cout << "Press q or escape to quit!" << endl;
CvFont infoFont;
cvInitFont(&infoFont, CV_FONT_HERSHEY_SIMPLEX, 1, 1);
namedWindow(VIDEO_WINDOW_NAME, CV_WINDOW_AUTOSIZE);
namedWindow(ERODE_PREVIEW_WIN_NAME, CV_WINDOW_NORMAL);
resizeWindow(ERODE_PREVIEW_WIN_NAME, 320, 240);
ControlsWindow* controlsWindow = new ControlsWindow();
if(fileExists(preferenceFileName)) {
loadSettings(controlsWindow, (char*)preferenceFileName);
}
Mat frame;
while (true) {
capture >> frame;
captureTime = (int)(getTickCount()/getTickFrequency())*1000;
if (frame.empty())
break;
int target_width = 320;
int height = (target_width/capture.get(3 /*width*/)) * capture.get(4 /*height*/);
resize(frame, frame, Size(target_width, height));
if (controlsWindow->getBlurDeviation() > 0) {
GaussianBlur(frame, frame, Size(GAUSSIAN_KERNEL, GAUSSIAN_KERNEL), controlsWindow->getBlurDeviation());
}
//Apply brightness and contrast
frame.convertTo(frame, -1, controlsWindow->getContrast(), controlsWindow->getBrightness());
Mat maskedImage = thresholdImage(controlsWindow, frame);
Mat erodedImage = erodeDilate(maskedImage, controlsWindow);
Mat erodedImageBinary;
cvtColor(erodedImage, erodedImageBinary, COLOR_BGR2GRAY);
threshold(erodedImageBinary, erodedImageBinary, 0, 255, CV_THRESH_BINARY);
if(controlsWindow->getInvert()) {
erodedImageBinary = 255 - erodedImageBinary;
}
cv::SimpleBlobDetector::Params params;
params.minDistBetweenBlobs = 50.0f;
params.filterByInertia = false;
params.filterByConvexity = false;
params.filterByColor = true;
params.filterByCircularity = false;
params.filterByArea = true;
params.minArea = 1000.0f;
params.maxArea = 100000.0f;
params.blobColor = 255;
vector<KeyPoint> centers;
vector<vector<Point>> contours;
ModBlobDetector* blobDetector = new ModBlobDetector(params);
vector<vector<Point>> contourHulls;
vector<RotatedRect> contourRects;
blobDetector->findBlobs(erodedImageBinary, erodedImageBinary, centers, contours);
for(vector<Point> ctpts : contours) {
vector<Point> hull;
convexHull(ctpts, hull);
contourHulls.push_back(hull);
contourRects.push_back(minAreaRect(hull));
}
#ifdef DEBUG_BLOBS
drawContours(frame, contours, -1, Scalar(128,255,128), 2, CV_AA);
drawContours(frame, contourHulls, -1, Scalar(255, 128,0), 2, CV_AA);
int ptnum;
for(KeyPoint pt : centers) {
Scalar color(255, 0, 255);
circle(frame, pt.pt, 5
, color, -1 /*filled*/, CV_AA);
circle(frame, pt.pt, pt.size, color, 1, CV_AA);
ptnum++;
}
#endif
for(RotatedRect rr : contourRects) {
Point2f points[4];
rr.points(points);
float side1 = distance(points[0], points[1]);
float side2 = distance(points[1], points[2]);
float shortestSide = min(side1, side2);
float longestSide = max(side1, side2);
float aspectRatio = longestSide/shortestSide;
int b = 0;
bool isTape = objInfo.aspectRatio == 0 ? false :
abs(objInfo.aspectRatio - aspectRatio) < 0.2*objInfo.aspectRatio;
/*
* TODO
* Make a list of possible tape candidates
* Use tape candidate with smallest difference in ratio to the real ratio as the tape
*/
if(isTape) {
b = 255;
string widthText = "Width (px): ";
widthText.append(toString(longestSide));
string heightText = "Height (px): ";
heightText.append(toString(shortestSide));
string rotText = "Rotation (deg): ";
rotText.append(toString(abs((int)rr.angle)));
string distText;
if(camSettings.focalLength == -1) {
distText = "Focal length not defined";
} else {
float dist = objInfo.width * camSettings.focalLength / longestSide;
distText = "Distance (cm): ";
distText.append(toString(dist));
}
putText(frame, widthText, Point(0, 20), CV_FONT_HERSHEY_SIMPLEX, 0.5f, Scalar(0, 255, 255));
putText(frame, heightText, Point(0, 40), CV_FONT_HERSHEY_SIMPLEX, 0.5f, Scalar(0, 255, 255));
putText(frame, rotText, Point(0, 60), CV_FONT_HERSHEY_SIMPLEX, 0.5f, Scalar(0, 255, 255));
putText(frame, distText, Point(0, 80), CV_FONT_HERSHEY_SIMPLEX, 0.5f, Scalar(0, 255, 255));
}
rotated_rect(frame, rr, Scalar(b, 0, 255));
if(isTape)break;
}
if(objInfo.aspectRatio == 0) {
putText(frame, "Invalid object info (object.xml)", Point(0, 20), CV_FONT_HERSHEY_SIMPLEX, 0.5f, Scalar(0, 255, 255));
}
delete blobDetector;
imshow(ERODE_PREVIEW_WIN_NAME, erodedImageBinary);
imshow(VIDEO_WINDOW_NAME, frame);
//int waitTime = max((int)(((1.0/framerate)*1000)
// - ((int)(getTickCount()/getTickFrequency())*1000 - captureTime))
// , 1);
char key = (char)waitKey(1);
switch (key) {
case 'q':
case 'Q':
case 27: //escape
saveSettings(controlsWindow, (char*)preferenceFileName);
return 0;
default:
break;
}
std::this_thread::yield();
}
saveSettings(controlsWindow, (char*)preferenceFileName);
delete(controlsWindow);
destroyAllWindows();
return 0;
}
Mat CameraInteraction::Testmm(Mat frame){
vector<vector<Point> > contours;
//Update the current background model and get the foreground
if(backgroundFrame>0)
{bg.operator ()(frame,fore);backgroundFrame--;}
else
{bg.operator()(frame,fore,0);}
//Get background image to display it
bg.getBackgroundImage(back);
//Enhance edges in the foreground by applying erosion and dilation
erode(fore,fore,Mat());
dilate(fore,fore,Mat());
//Find the contours in the foreground
findContours(fore,contours,CV_RETR_EXTERNAL,CV_CHAIN_APPROX_NONE);
for(int i=0;i<contours.size();i++)
//Ignore all small insignificant areas
if(contourArea(contours[i])>=5000)
{
//Draw contour
vector<vector<Point> > tcontours;
tcontours.push_back(contours[i]);
drawContours(frame,tcontours,-1,cv::Scalar(0,0,255),2);
//Detect Hull in current contour
vector<vector<Point> > hulls(1);
vector<vector<int> > hullsI(1);
convexHull(Mat(tcontours[0]),hulls[0],false);
convexHull(Mat(tcontours[0]),hullsI[0],false);
drawContours(frame,hulls,-1,cv::Scalar(0,255,0),2);
//Find minimum area rectangle to enclose hand
RotatedRect rect=minAreaRect(Mat(tcontours[0]));
//Find Convex Defects
vector<Vec4i> defects;
if(hullsI[0].size()>0)
{
Point2f rect_points[4]; rect.points( rect_points );
for( int j = 0; j < 4; j++ )
line( frame, rect_points[j], rect_points[(j+1)%4], Scalar(255,0,0), 1, 8 );
Point rough_palm_center;
convexityDefects(tcontours[0], hullsI[0], defects);
if(defects.size()>=3)
{
vector<Point> palm_points;
for(int j=0;j<defects.size();j++)
{
int startidx=defects[j][0]; Point ptStart( tcontours[0][startidx] );
int endidx=defects[j][1]; Point ptEnd( tcontours[0][endidx] );
int faridx=defects[j][2]; Point ptFar( tcontours[0][faridx] );
//Sum up all the hull and defect points to compute average
rough_palm_center+=ptFar+ptStart+ptEnd;
palm_points.push_back(ptFar);
palm_points.push_back(ptStart);
palm_points.push_back(ptEnd);
}
//Get palm center by 1st getting the average of all defect points, this is the rough palm center,
//Then U chose the closest 3 points ang get the circle radius and center formed from them which is the palm center.
rough_palm_center.x/=defects.size()*3;
rough_palm_center.y/=defects.size()*3;
Point closest_pt=palm_points[0];
vector<pair<double,int> > distvec;
for(int i=0;i<palm_points.size();i++)
distvec.push_back(make_pair(dist(rough_palm_center,palm_points[i]),i));
sort(distvec.begin(),distvec.end());
//Keep choosing 3 points till you find a circle with a valid radius
//As there is a high chance that the closes points might be in a linear line or too close that it forms a very large circle
pair<Point,double> soln_circle;
for(int i=0;i+2<distvec.size();i++)
{
Point p1=palm_points[distvec[i+0].second];
Point p2=palm_points[distvec[i+1].second];
Point p3=palm_points[distvec[i+2].second];
soln_circle=circleFromPoints(p1,p2,p3);//Final palm center,radius
if(soln_circle.second!=0)
break;
}
//Find avg palm centers for the last few frames to stabilize its centers, also find the avg radius
palm_centers.push_back(soln_circle);
if(palm_centers.size()>10)
palm_centers.erase(palm_centers.begin());
Point palm_center;
double radius=0;
for(int i=0;i<palm_centers.size();i++)
{
palm_center+=palm_centers[i].first;
radius+=palm_centers[i].second;
}
palm_center.x/=palm_centers.size();
palm_center.y/=palm_centers.size();
radius/=palm_centers.size();
//Draw the palm center and the palm circle
//The size of the palm gives the depth of the hand
circle(frame,palm_center,5,Scalar(144,144,255),3);
circle(frame,palm_center,radius,Scalar(144,144,255),2);
//Detect fingers by finding points that form an almost isosceles triangle with certain thesholds
int no_of_fingers=0;
for(int j=0;j<defects.size();j++)
{
int startidx=defects[j][0]; Point ptStart( tcontours[0][startidx] );
int endidx=defects[j][1]; Point ptEnd( tcontours[0][endidx] );
int faridx=defects[j][2]; Point ptFar( tcontours[0][faridx] );
//X o--------------------------o Y
double Xdist=sqrt(dist(palm_center,ptFar));
double Ydist=sqrt(dist(palm_center,ptStart));
double length=sqrt(dist(ptFar,ptStart));
double retLength=sqrt(dist(ptEnd,ptFar));
//Play with these thresholds to improve performance
if(length<=3*radius&&Ydist>=0.4*radius&&length>=10&&retLength>=10&&max(length,retLength)/min(length,retLength)>=0.8)
if(min(Xdist,Ydist)/max(Xdist,Ydist)<=0.8)
{
if((Xdist>=0.1*radius&&Xdist<=1.3*radius&&Xdist<Ydist)||(Ydist>=0.1*radius&&Ydist<=1.3*radius&&Xdist>Ydist))
line( frame, ptEnd, ptFar, Scalar(0,255,0), 1 ),no_of_fingers++;
}
}
no_of_fingers=min(5,no_of_fingers);
qDebug()<<"NO OF FINGERS: "<<no_of_fingers;
//mouseTo(palm_center.x,palm_center.y);//Move the cursor corresponding to the palm
if(no_of_fingers<4)//If no of fingers is <4 , click , else release
// mouseClick();
qDebug()<<"Test";
else
// mouseRelease();
qDebug()<<"Hola";
}
}
}
if(backgroundFrame>0)
putText(frame, "Recording Background", cvPoint(30,30), FONT_HERSHEY_COMPLEX_SMALL, 0.8, cvScalar(200,200,250), 1, CV_AA);
// imshow("Framekj",frame);
// imshow("Background",back);
return frame;
}
///////////////////////////////////////////////////////////////////
// Panel::CannyDetection()
// Description: This function is called by DetectEdges() and it
// is the Canny edge detection function which does not contain
// debugging statements. We run the image through several different
// image processing functions to prepare the image before edge
// detection. After detection the edges we run Hough lines which
// approximates lines of minimum length as specified in
// Settings.xml. We find all intersections of the Hough lines
// then make a minimum area rectangle around the intersections to
// approximate the edges of the panel which we are trying to
// measure. From there we use the unit conversion calculated
// in DetectFeatures() to find a length and width of the current
// panel. We report the length and width with a message box.
///////////////////////////////////////////////////////////////////
Mat Panel::CannyDetection(Mat image, bool showImg)
{
Mat greyImage;
cvtColor(image, greyImage, CV_BGR2GRAY);
Mat eroded, dilated, thresh, blurredThresh, edges, edgesGray;
vector<Vec2f> lines;
threshold(greyImage, thresh, m_lowCannyThreshold, 255, THRESH_BINARY);
erode(thresh, eroded, Mat());
dilate(eroded, dilated, Mat());
GaussianBlur(thresh, blurredThresh, Size(7, 7), m_sigmaX, m_sigmaY);
Canny(blurredThresh, edges, m_cannyLow, m_cannyLow*m_ratio, 3);
HoughLines(edges, lines, 1, CV_PI / 180, m_houghLength, 0, 0);
cvtColor(edges, edgesGray, CV_GRAY2BGR);
for (size_t i = 0; i < lines.size(); i++)
{
float rho = lines[i][0], theta = lines[i][1];
Point pt1, pt2;
double a = cos(theta), b = sin(theta);
double x0 = a*rho, y0 = b*rho;
pt1.x = cvRound(x0 + 1000 * (-b));
pt1.y = cvRound(y0 + 1000 * (a));
pt2.x = cvRound(x0 - 1000 * (-b));
pt2.y = cvRound(y0 - 1000 * (a));
line(edgesGray, pt1, pt2, Scalar(0, 0, 255), 3, CV_AA);
}
////////////////////////////////////////////////////////
// Compute the intersection from the lines detected
////////////////////////////////////////////////////////
vector<Point2f> intersections;
for (size_t i = 0; i < lines.size(); i++)
{
for (size_t j = 0; j < lines.size(); j++)
{
Vec2f line1 = lines[i];
Vec2f line2 = lines[j];
if (acceptLinePair(line1, line2, (float)CV_PI / 32))
{
Point2f intersection = computeIntersect(line1, line2);
if (intersection.x >= 0 && intersection.y >= 0)
intersections.push_back(intersection);
}
}
}
if (intersections.size() > 0)
{
vector<Point2f>::iterator i;
for (i = intersections.begin(); i != intersections.end(); ++i)
{
cout << "Intersection is " << i->x << ", " << i->y << endl;
circle(image, *i, 2, Scalar(0, 255, 0), 3);
}
// Find the minimum bounding rectangle
RotatedRect rect;
Point2f rectPoints[4];
Scalar color = Scalar(255, 0, 0);
if (intersections.size() == 4)
{
// TODO
}
rect = minAreaRect(intersections);
rect.points(rectPoints);
int j = 0;
for (j; j < 4; j++)
line(image, rectPoints[j], rectPoints[(j + 1) % 4], color, 5, 8);
float topLength = (float)norm(rectPoints[1] - rectPoints[0]);
float botLength = (float)norm(rectPoints[3] - rectPoints[2]);
float panelWidthPixels = topLength < botLength ? topLength : botLength;
float leftHeight = (float)norm(rectPoints[3] - rectPoints[0]);
float rightHeight = (float)norm(rectPoints[2] - rectPoints[1]);
float panelHeightPixels = leftHeight < rightHeight ? leftHeight : rightHeight;
string dimensionDisplayPixels = "Pixels:\nWidth: " + to_string(panelWidthPixels) + " pixels\nHeight: " + to_string(panelHeightPixels) + " pixels";
// ShowMessage(dimensionDisplayPixels);
if (m_conversionRate)
{
float panelWidthReal = panelWidthPixels / m_conversionRate;
float panelHeightReal = panelHeightPixels / m_conversionRate;
string dimensionDisplayActual = "Actual:\nWidth: " + to_string(panelWidthReal) + " cm\nHeight: " + to_string(panelHeightReal) + " cm";
ShowMessage(dimensionDisplayActual);
}
}
if (showImg){
namedWindow("Intersections", CV_WINDOW_KEEPRATIO);
imshow("Intersections", image);
}
/////////////////////////////////////////////////////////////
// End of Computing the intersection from the lines detected
/////////////////////////////////////////////////////////////
return edges;
}
