void KeyPointsDeliverer::extractMouthCornerKeyPoints(Mat &mouthImg, int thresholdDifferenceToAvg, int totalLineCheck, int kp1Break, int kp5Break)
{
QList<PossibleKeyPoint> possibleKeyPoints;
PossibleKeyPoint possibleKeyPoint;
for (int i = 0; i < rMidFinal.cols; ++i) {
for (int j = rMidFinal.rows-totalLineCheck; j > totalLineCheck/2; --j) {
int currentDiffToAvg = 0;
for (int k = 1; k < totalLineCheck/2 + 1; ++k) {
currentDiffToAvg += rMidFinal.at<uchar>(j-k,i) + rMidFinal.at<uchar>(j+k,i);
}
currentDiffToAvg = currentDiffToAvg / totalLineCheck;
if(currentDiffToAvg > 0){
currentDiffToAvg = 100 - (rMidFinal.at<uchar>(j,i) * 100 / currentDiffToAvg);
}
if(currentDiffToAvg > thresholdDifferenceToAvg){
possibleKeyPoint.differenceToAvg = currentDiffToAvg;
possibleKeyPoint.keyPoint.x = i;
possibleKeyPoint.keyPoint.y = j;
possibleKeyPoints.append(possibleKeyPoint);
}
}
}
Mat contourImg(rMidFinal.rows, rMidFinal.cols, CV_8UC1, Scalar(0,0,0));
Point p;
for (int i = 0; i < possibleKeyPoints.size(); ++i) {
p = possibleKeyPoints.at(i).keyPoint;
if(i % 3 == 0)
circle(rMidFinal, p, 1, Scalar(255,255,255));
contourImg.at<uchar>(p.y, p.x) = 255;
}
Mat _img;
double otsu_thresh_val = cv::threshold(
contourImg, _img, 0, 255, CV_THRESH_BINARY | CV_THRESH_OTSU
);
Canny(contourImg, contourImg, otsu_thresh_val*0.5, otsu_thresh_val, 3, true);
vector<vector<Point> > contours;
vector<Vec4i> hierarchy;
findContours( contourImg, contours, hierarchy, CV_RETR_TREE, CV_CHAIN_APPROX_SIMPLE, Point(0, 0) );
for( uint i = 0; i< contours.size(); i++ )
{
drawContours( contourImg, contours, i, Scalar(255,255,255), 1, 8, hierarchy, 1, Point() );
}
double luminanceMean = 0.0;
double pseudoHueMean = 0.0;
double pseudoHuePxl = 0.0;
int luminancePxl = 0;
for (int i = 0; i < mouthImg.cols/2; ++i) {
for (int j = 0; j < mouthImg.rows; ++j) {
pseudoHuePxl = imageProcessing.pseudoHuePxl(mouthImg, j, i);
luminancePxl = imageProcessing.luminancePxl(mouthImg, j, i);
luminanceMean += luminancePxl;
pseudoHueMean += pseudoHuePxl;
}
}
luminanceMean /= (mouthImg.cols/2*mouthImg.rows);
pseudoHueMean /= (mouthImg.cols/2*mouthImg.rows);
QList<PossibleKeyPoint> pKPoints;
PossibleKeyPoint pKPoint;
for (int i = mouthImg.cols/2-(mouthImg.cols/2*0.4); i > 0; --i) {
for (int j = mouthImg.rows-(mouthImg.rows/2*0.4); j > 0; --j) {
pseudoHuePxl = imageProcessing.pseudoHuePxl(mouthImg, j, i);
luminancePxl = imageProcessing.luminancePxl(mouthImg, j, i);
if(contourImg.at<uchar>(j,i) == 255
){
pKPoint.keyPoint.x = i;
pKPoint.keyPoint.y = j;
pKPoints.append(pKPoint);
break;
}
}
}
keyPoint1.x = 1000;
for (int i = 0; i < pKPoints.size(); ++i) {
int diffY = 0;
if(i > 0){
diffY = abs(pKPoints.at(i).keyPoint.y - pKPoints.at(i-1).keyPoint.y);
//ROS_INFO("diff: %d", abs(pKPoints.at(i).keyPoint.y - pKPoints.at(i-1).keyPoint.y));
}
if(diffY > kp1Break){
break;
}
if(keyPoint1.x > pKPoints.at(i).keyPoint.x){
keyPoint1.x = pKPoints.at(i).keyPoint.x;
keyPoint1.y = pKPoints.at(i).keyPoint.y;
}
//circle(rMidFinal, pKPoints.at(i).keyPoint, 2, Scalar(255,255,255));
}
luminanceMean = 0.0;
pseudoHueMean = 0.0;
for (int i = mouthImg.cols/2; i < mouthImg.cols; ++i) {
for (int j = 0; j < mouthImg.rows-(mouthImg.rows/2*0.4); ++j) {
pseudoHuePxl = imageProcessing.pseudoHuePxl(mouthImg, j, i);
luminancePxl = imageProcessing.luminancePxl(mouthImg, j, i);
luminanceMean += luminancePxl;
pseudoHueMean += pseudoHuePxl;
}
}
luminanceMean /= (mouthImg.cols/2*mouthImg.rows);
pseudoHueMean /= (mouthImg.cols/2*mouthImg.rows);
pKPoints.clear();
for (int i = mouthImg.cols/2+(mouthImg.cols/2*0.3); i < mouthImg.cols; ++i) {
for (int j = mouthImg.rows; j > 0; --j) {
pseudoHuePxl = imageProcessing.pseudoHuePxl(mouthImg, j, i);
luminancePxl = imageProcessing.luminancePxl(mouthImg, j, i);
if(contourImg.at<uchar>(j,i) == 255
){
pKPoint.keyPoint.x = i;
pKPoint.keyPoint.y = j;
pKPoints.append(pKPoint);
break;
}
}
}
// for (int i = 0; i < pKPoints.size(); ++i) {
// circle(rMidFinal, pKPoints.at(i).keyPoint, 1, Scalar(255,255,255));
// }
// for (int i = 0; i < contourImg.cols; ++i) {
// for (int j = 0; j < contourImg.rows; ++j) {
// if(contourImg.at<uchar>(j,i) == 255)
// circle(rMidFinal, Point(i,j), 1, Scalar(255,255,255));
// }
// }
Point kpTmp;
kpTmp.x = 0;
kpTmp.y = 0;
//ROS_INFO("pkPoint size %d", pKPoints.size());
for (int i = 0; i < pKPoints.size(); ++i) {
int diffY = 0;
if(i > 0){
diffY = abs(pKPoints.at(i).keyPoint.y - pKPoints.at(i-1).keyPoint.y);
//ROS_INFO("diff: %d", abs(pKPoints.at(i).keyPoint.y - pKPoints.at(i-1).keyPoint.y));
}
if(diffY > kp5Break){
break;
}
if(kpTmp.x < pKPoints.at(i).keyPoint.x){
kpTmp.x = pKPoints.at(i).keyPoint.x;
keyPoint5.x = pKPoints.at(i).keyPoint.x;
keyPoint5.y = pKPoints.at(i).keyPoint.y;
}
//circle(rMidFinal, pKPoints.at(i).keyPoint, 2, Scalar(255,255,255));
}
}
void KeyPointsDeliverer::extractCupidsBowKeyPoints(int thresholdDifferenceToAvg, int totalLineCheck)
{
QList<PossibleKeyPoint> possibleKeyPoints;
PossibleKeyPoint possibleKeyPoint;
for (int i = 0; i < rTopFinal.cols; ++i) {
for (int j = rTopFinal.rows/2; j > totalLineCheck/2; --j) {
int currentDiffToAvg = 0;
for (int k = 1; k < totalLineCheck/2 + 1; ++k) {
currentDiffToAvg += rTopFinal.at<uchar>(j-k,i) + rTopFinal.at<uchar>(j+k,i);
}
currentDiffToAvg = currentDiffToAvg / totalLineCheck;
if(currentDiffToAvg > 0){
currentDiffToAvg = 100 - (rTopFinal.at<uchar>(j,i) * 100 / currentDiffToAvg);
}
if(currentDiffToAvg > thresholdDifferenceToAvg){
possibleKeyPoint.differenceToAvg = currentDiffToAvg;
possibleKeyPoint.keyPoint.x = i;
possibleKeyPoint.keyPoint.y = j;
possibleKeyPoints.append(possibleKeyPoint);
}
}
}
Mat contourImg(rTopFinal.rows, rTopFinal.cols, CV_8UC1, Scalar(0,0,0));
Point p;
for (int i = 0; i < possibleKeyPoints.size(); ++i) {
if(i % 3 == 0)
circle(rTopFinal, p, 1, Scalar(255,255,255));
p = possibleKeyPoints.at(i).keyPoint;
contourImg.at<uchar>(p.y, p.x) = 255;
}
Mat _img;
double otsu_thresh_val = cv::threshold(
contourImg, _img, 0, 255, CV_THRESH_BINARY | CV_THRESH_OTSU
);
Canny(contourImg, contourImg, otsu_thresh_val*0.5, otsu_thresh_val);
vector<vector<Point> > contours;
vector<Vec4i> hierarchy;
findContours( contourImg, contours, hierarchy, CV_RETR_TREE, CV_CHAIN_APPROX_SIMPLE, Point(0, 0) );
for( uint i = 0; i< contours.size(); i++ ){
drawContours( contourImg, contours, i, Scalar(255,255,255), 1, 8, hierarchy, 1, Point() );
}
keyPoint2.y = 1000;
for (int i = 0; i < rTopFinal.rows; ++i) {
for (int j = rTopFinal.cols/2; j > 0; --j) {
if(contourImg.at<uchar>(i,j) == 255){
if(keyPoint2.y >= i){
keyPoint2.y = i;
keyPoint2.x = j;
}
}
}
}
keyPoint4.y = 1000;
for (int i = 0; i < rTopFinal.rows; ++i) {
for (int j = rTopFinal.cols/2; j < rTopFinal.cols; ++j) {
if(contourImg.at<uchar>(i,j) == 255){
if(keyPoint4.y >= i){
keyPoint4.y = i;
keyPoint4.x = j;
}
}
}
}
keyPoint3.y = 0;
int kp2kp3Width = keyPoint4.x - keyPoint2.x;
kp2kp3Width = kp2kp3Width/2;
for (int i = keyPoint2.x; i < keyPoint4.x; ++i) {
for (int j = 0; j < keyPoint1.y-14; ++j) {
if(contourImg.at<uchar>(j,i) == 255){
if(keyPoint3.y <= j && i <= (keyPoint2.x + kp2kp3Width + (kp2kp3Width*0.2)) ){
keyPoint3.y = j;
keyPoint3.x = i;
}
}
}
}
}
void KeyPointsDeliverer::extractLowerLipKeyPoint(int thresholdDifferenceToAvg, int totalLineCheck)
{
QList<PossibleKeyPoint> possibleKeyPoints;
PossibleKeyPoint possibleKeyPoint;
QList<QList<int> > val;
int aPixelsumme;
int dPixeldurchschnitt;
int diffPixeldurchschnitt;
int rlow;
for (int i = 0; i < rLowFinal.cols; ++i) {
for (int j = rLowFinal.rows/2; j < rLowFinal.rows-totalLineCheck/2; ++j) {
QList<int> brabs;
int currentDiffToAvg = 0;
for (int k = 1; k < totalLineCheck/2 + 1; ++k) {
currentDiffToAvg += rLowFinal.at<uchar>(j-k,i) + rLowFinal.at<uchar>(j+k,i);
}
aPixelsumme = currentDiffToAvg;
currentDiffToAvg = currentDiffToAvg / totalLineCheck;
dPixeldurchschnitt = currentDiffToAvg;
if(currentDiffToAvg > 0){
currentDiffToAvg = 100 - (rLowFinal.at<uchar>(j,i) * 100 / currentDiffToAvg);
}
diffPixeldurchschnitt = currentDiffToAvg;
rlow = rLowFinal.at<uchar>(j,i);
if(currentDiffToAvg > thresholdDifferenceToAvg){
brabs << aPixelsumme << dPixeldurchschnitt << diffPixeldurchschnitt << rlow;
val << brabs;
possibleKeyPoint.differenceToAvg = currentDiffToAvg;
possibleKeyPoint.keyPoint.x = i;
possibleKeyPoint.keyPoint.y = j;
possibleKeyPoints.append(possibleKeyPoint);
}
}
}
// foreach (QList<int> v, val) {
// ROS_INFO("#################1##################");
// ROS_INFO("aPixelsumme %d", v.at(0));
// ROS_INFO("dPixeldurchschnitt %d", v.at(1));
// ROS_INFO("diffPixeldurchschnitt %d Rlow %d", v.at(2), v.at(3));
// ROS_INFO("#################2##################");
// }
Mat contourImg(rLowFinal.rows, rLowFinal.cols, CV_8UC1, Scalar(0,0,0));
Point p;
for (int i = 0; i < possibleKeyPoints.size(); ++i) {
if(i % 3 == 0)
circle(rLowFinal, p, 1, Scalar(255,255,255));
p = possibleKeyPoints.at(i).keyPoint;
contourImg.at<uchar>(p.y, p.x) = 255;
}
Mat _img;
double otsu_thresh_val = cv::threshold(
contourImg, _img, 0, 255, CV_THRESH_BINARY | CV_THRESH_OTSU
);
Canny(contourImg, contourImg, otsu_thresh_val*0.5, otsu_thresh_val);
vector<vector<Point> > contours;
vector<Vec4i> hierarchy;
findContours( contourImg, contours, hierarchy, CV_RETR_TREE, CV_CHAIN_APPROX_SIMPLE, Point(0, 0) );
for( uint i = 0; i< contours.size(); i++ ){
drawContours( contourImg, contours, i, Scalar(255,255,255), 1, 8, hierarchy, 1, Point() );
}
Point kpTmp;
kpTmp.y = 0;
int kp2kp3Width = keyPoint4.x - keyPoint2.x;
kp2kp3Width = kp2kp3Width/2;
for (int i = keyPoint2.x; i < keyPoint4.x; ++i) {
for (int j = rLowFinal.rows-(rLowFinal.rows*0.2); j > keyPoint1.y; --j) {
if(contourImg.at<uchar>(j, i) == 255){
if(kpTmp.y <= j && i <= (keyPoint2.x + kp2kp3Width)){
kpTmp.y = j;
keyPoint6.y = j;
keyPoint6.x = i;
break;
}
}
}
}
}
// Detect and Identify the hand gesture present in the image
std::vector<Hand> GestureDetector::detect(cv::Mat &image, cv::Mat &filtered)
{
std::vector<Hand> hands;
// find contours in the filtered image
std::vector< std::vector<cv::Point> > contours;
std::vector<cv::Vec4i> hierarchy;
cv::findContours(filtered,
contours, // a vector of contours
hierarchy, // a hierarchy of contours if there are parent
//child relations in the image
CV_RETR_EXTERNAL, // retrieve the external contours
CV_CHAIN_APPROX_TC89_L1); // an approximation algorithm
//------------------Find Face----------------
//preprocess for face recognition
std::vector< cv::Rect > faces;
cv::Mat gray;
//shrink the image for speed
cv::resize(image, gray, cv::Size2i(image.cols/4, image.rows/4));
cv::cvtColor(gray, gray, CV_BGR2GRAY);
cv::equalizeHist(gray, gray);
cascadeFace.detectMultiScale(gray, faces);
// draw bounds for faces
for (unsigned int i = 0; i < faces.size(); i++ )
{
//resize the rectangle to match the image
faces[i] += cv::Point(faces[i].x * 3,faces[i].y * 3);
faces[i] += faces[i].size() +faces[i].size() +faces[i].size();
rectangle(image, faces[i], cv::Scalar(0,0,204), 3);
}
//----------------End Face--------------------
//------------Find all skin regions-----------
QString join("");
int massMin = 5000, massMax = 24000; // minimum contour mass
cv::Scalar color( 100, 150, 255 ); //random color
int idx = 0;
// iterate through all the top-level contours
for( ; idx >= 0; idx = hierarchy[idx][0] )
{
// compute all moments and mass
cv::Moments mom = cv::moments(cv::Mat(contours[idx]));
cv::Point center = cv::Point(mom.m10/mom.m00,mom.m01/mom.m00);
int cMass = mom.m00;
//calculate the rotated and bounding rectangles of the blobs
cv::RotatedRect rotRect = cv::minAreaRect( cv::Mat(contours[idx]) );
cv::Rect bRect = rotRect.boundingRect();
// find intersections of detected faces and blobs
bool faceOverlap = false;
for(unsigned int i = 0; i < faces.size(); i++)
{
cv::Rect intersect = faces[i] & bRect;
if(intersect.width > 30 || intersect.height > 30)
faceOverlap = true;
}
// Eliminate contours that are too small,
// and ones that coincide with one of the found faces
if(cMass > massMin && cMass < massMax && !faceOverlap)
{
Hand curHand(rotRect, mom);
hands.push_back(curHand); //push the detected hand onto the list
// draw each connected component
cv::Mat contourImg(image.size(), image.type(), cv::Scalar(0));
cv::drawContours( contourImg, contours, idx, color, CV_FILLED, 8, hierarchy );
cv::GaussianBlur(contourImg, contourImg, cv::Size(5,5), 0);
image += contourImg;
// draw bounding rotated rectangles
for( int j = 0; j < 4; j++ )
line( image, curHand.rotPoints[j], curHand.rotPoints[(j+1)%4], cv::Scalar(0,102,204), 3, 8 );
rectangle(image, curHand.boxRect, cv::Scalar(0,204,102), 3);
}
}
if(hands.size() == 0)
{ //if no hands were found create a 'NONE' hand
hands.push_back(Hand());
}
return hands;
}
