void FingerTracker::Setup() { VideoCapture capture(0); if (!capture.isOpened()) { throw std::runtime_error("Could not start camera capture"); } int windowSize = 25; int Xpos = 200; int Ypos = 50; int update = 0; int buttonClicked = 0; namedWindow("RGB", CV_WINDOW_AUTOSIZE); createTrackbar("X", "RGB", &Xpos, 320, TrackBarCallback, (void*)&update); createTrackbar("Y", "RGB", &Ypos, 240, TrackBarCallback, (void*)&update); createTrackbar("Size", "RGB", &windowSize, 100, TrackBarCallback, (void*)&update); setMouseCallback("RGB", MouseCallback, (void*)&buttonClicked); Mat fingerWindowBackground, fingerWindowBackgroundGray; m_calibrationData.reset(new CalibrationData()); bool ticking = false; std::chrono::system_clock::time_point start = std::chrono::system_clock::now(); while (true) { Mat frame, frameHSV; if (capture.read(frame)) { flip(frame, frame, 1); pyrDown(frame, frame, Size(frame.cols / 2, frame.rows / 2)); Rect fingerWindow(Point(Xpos, Ypos), Size(windowSize, windowSize*3)); if (Xpos + windowSize >= frame.cols || Ypos + windowSize*3 >= frame.rows) { windowSize = 20; Xpos = 200; Ypos = 50; update = 0; } else if (buttonClicked == 1) { frame(fingerWindow).copyTo(fingerWindowBackground); cvtColor(fingerWindowBackground, fingerWindowBackgroundGray, CV_BGR2GRAY); buttonClicked = 0; update = 0; cvDestroyAllWindows(); } if (fingerWindowBackgroundGray.rows && !m_calibrationData->m_ready) { Mat diff, thd; absdiff(frame(fingerWindow), fingerWindowBackground, diff); std::vector<Mat> ch; split(diff, ch); threshold(ch[0], ch[0], m_calibrationDiffThreshold, 255, 0); threshold(ch[1], ch[1], m_calibrationDiffThreshold, 255, 0); threshold(ch[2], ch[2], m_calibrationDiffThreshold, 255, 0); thd = ch[0]; add(thd, ch[1], thd); add(thd, ch[2], thd); medianBlur(thd, thd, 5); Mat top, middle, bottom; Rect r1 = Rect(0, 0, thd.cols, thd.rows/3); Rect r2 = Rect(0, thd.rows / 3 + 1, thd.cols, thd.rows/3); Rect r3 = Rect(0, thd.rows * 2 / 3 + 1, thd.cols, thd.rows - thd.rows * 2 / 3 - 1); top = thd(r1); middle = thd(r2); bottom = thd(r3); auto percentageTop = countNonZero(top) * 100.0 / top.size().area(); auto percentageMiddle = countNonZero(middle) * 100.0 / middle.size().area(); auto percentageBottom = countNonZero(bottom) * 100.0 / bottom.size().area(); bool topReady = false; bool middleReady = false; bool bottomReady = false; Scalar c1, c2, c3; if (percentageTop > m_calibrationTopLowerThd && percentageTop < m_calibrationTopUpperThd) { topReady = true; c1 = Scalar(0, 255, 255); } else { c1 = Scalar(0, 0, 255); } if (percentageMiddle > m_calibrationMiddleLowerThd && percentageMiddle < m_calibrationMiddleUppperThd) { middleReady = true; c2 = Scalar(0, 255, 255); } else { c2 = Scalar(0, 0, 255); } if (percentageBottom > m_calibrationBottomLowerThd && percentageBottom < m_calibrationBottomUpperThd) { bottomReady = true; c3 = Scalar(0, 255, 255); } else { c3 = Scalar(0, 0, 255); } bool readyToGo = false; if (middleReady && topReady && bottomReady) { c1 = Scalar(0, 255, 0); c2 = Scalar(0, 255, 0); c3 = Scalar(0, 255, 0); if (!ticking) { start = std::chrono::system_clock::now(); ticking = true; } if (std::chrono::system_clock::now() - start > std::chrono::seconds(1)) { readyToGo = true; } } else { ticking = false; } #if ENABLE_DEBUG_WINDOWS std::stringstream ss; ss << percentageTop << ", " << percentageMiddle << ", " << percentageBottom; putText(frame, ss.str(), Point(0, getTextSize(ss.str(), 0, 0.5, 1, nullptr).height), 0, 0.5, Scalar::all(255), 1); cv::imshow("Thresholded", thd); #endif if (percentageTop >= m_calibrationTopUpperThd && percentageBottom >= m_calibrationBottomUpperThd && percentageMiddle >= m_calibrationMiddleUppperThd) { putText(frame, "Move finger away from camera", Point(0, getTextSize("Move finger away from camera", 0, 0.5, 1, nullptr).height), 0, 0.5, Scalar::all(255), 1); } else if (percentageTop <= m_calibrationTopLowerThd && percentageBottom <= m_calibrationBottomLowerThd && percentageMiddle <= m_calibrationMiddleLowerThd) { putText(frame, "Move finger closer to camera", Point(0, getTextSize("Move finger closer to camera", 0, 0.5, 1, nullptr).height), 0, 0.5, Scalar::all(255), 1); } if (readyToGo) { Mat framePatchHSV; cvtColor(frame(fingerWindow), framePatchHSV, CV_BGR2HSV); cvtColor(frame, frameHSV, CV_BGR2HSV); MatND hist; calcHist(&framePatchHSV, 1, m_calibrationData->m_channels, thd, hist, 2, m_calibrationData->m_histSize, (const float**)m_calibrationData->m_ranges, true, false); m_calibrationData->m_hist = hist; normalize(m_calibrationData->m_hist, m_calibrationData->m_hist, 0, 255, NORM_MINMAX, -1, Mat()); #if ENABLE_DEBUG_WINDOWS double maxVal=0; minMaxLoc(m_calibrationData->m_hist, 0, &maxVal, 0, 0); int scale = 10; Mat histImg = Mat::zeros(m_calibrationData->m_sbins*scale, m_calibrationData->m_hbins*10, CV_8UC3); for( int h = 0; h < m_calibrationData->m_hbins; h++) { for( int s = 0; s < m_calibrationData->m_sbins; s++ ) { float binVal = m_calibrationData->m_hist.at<float>(h, s); int intensity = cvRound(binVal*255/maxVal); rectangle( histImg, Point(h*scale, s*scale), Point( (h+1)*scale - 1, (s+1)*scale - 1), Scalar::all(intensity), CV_FILLED ); } } imshow("H-S Histogram", histImg); #endif m_calibrationData->m_ready = true; frame(fingerWindow).copyTo(m_calibrationData->m_fingerPatch); m_calibrationData->m_fingerRect = fingerWindow; m_currentCandidate.m_windowRect = fingerWindow; m_currentCandidate.m_fingerPosition = fingerWindow.tl(); return; } rectangle(frame, r1.tl() + fingerWindow.tl(), r1.br() + fingerWindow.tl(), c1); rectangle(frame, r2.tl() + fingerWindow.tl(), r2.br() + fingerWindow.tl(), c2); rectangle(frame, r3.tl() + fingerWindow.tl(), r3.br() + fingerWindow.tl(), c3); imshow("Calibration", frame); } else { int baseline = 0; putText(frame, "Adjust calibration window, click when ready", Point(0, getTextSize("Adjust calibration window", 0, 0.4, 2, &baseline).height), 0, 0.4, Scalar::all(255), 1); rectangle(frame, fingerWindow.tl(), fingerWindow.br(), Scalar(0, 0, 255)); imshow("RGB", frame); } auto key = cvWaitKey(10); if (char(key) == 27) { break; } } } capture.release(); }