void main() {
using namespace std;
using namespace cv;
using namespace TUIO;
cv::Mat depthFrameRaw(480, 640, CV_16UC1);
cv::Mat x(480, 640, CV_32FC1);
cv::Mat y(480, 640, CV_32FC1);
cv::Mat z(480, 640, CV_32FC1);
Mat1f debugFrame(480, 640);
CLNUICamera cam = CreateNUICamera();
const float rMin = 25;
stringstream buffer;
unsigned short* depthFrameRawData = (unsigned short*) depthFrameRaw.data;
int key;
bool tuio3dMode = false;
TuioServer* tuio = new TuioServer(tuio3dMode);
TuioTime time;
bool debug = true;
float zMin = 0.0f;
float zMax = 0.75f;
if ( StartNUICamera(cam) ) {
cout << "cam started" << endl;
while ( (key = cvWaitKey(1)) != 27) {
if ( GetNUICameraDepthFrameRAW(cam, (PUSHORT)depthFrameRaw.data) ) {
unproject(depthFrameRawData, (float*)x.data, (float*)y.data, (float*)z.data);
debugFrame = z * 0.1;
if (key == 32) {
debug = !debug;
}
std::vector<cv::Point2i> fingerTips;
if (debug) {
fingerTips = detectFingertips(z, 0, 0.75, debugFrame);
} else {
// find fingertips
fingerTips = detectFingertips(z);
// draw fingetips
for(vector<Point2i>::iterator it = fingerTips.begin(); it != fingerTips.end(); it++) {
circle(debugFrame, (*it), 10, Scalar(1.0f), -1);
}
}
// send fingertip positions via TUIO
time = TuioTime::getSessionTime();
tuio->initFrame(time);
for(vector<Point2i>::iterator it = fingerTips.begin(); it != fingerTips.end(); it++) {
// pixel coordinates
int u = (int)(*it).x;
int v = (int)(*it).y;
float pX = 1.0f - u / 640.0f;
float pY = v / 480.0f;
float pZ = (z.at<float>(v,u) - zMin) / (zMax - zMin);
TUIO::TuioCursor* cursor = tuio->getClosestTuioCursor(pX,pY,pZ);
if (cursor == NULL || cursor->getDistance(pX,pY,pZ) > 0.05) {
tuio->addTuioCursor(pX,pY,pZ);
} else if(cursor->getTuioTime() != time) {
tuio->updateTuioCursor(cursor, pX, pY, pZ);
}
}
tuio->stopUntouchedMovingCursors();
tuio->removeUntouchedStoppedCursors();
tuio->commitFrame();
cout << tuio->getTuioCursors().size() << endl;
// draw our debugframe
imshow("debugFrame", debugFrame);
}
}
} else {
cout << "could not start cam" << endl;
}
StopNUICamera(cam);
}
int main() {
const unsigned int nBackgroundTrain = 30;
const unsigned short touchDepthMin = 10;
const unsigned short touchDepthMax = 20;
const unsigned int touchMinArea = 50;
const bool localClientMode = true; // connect to a local client
const double debugFrameMaxDepth = 4000; // maximal distance (in millimeters) for 8 bit debug depth frame quantization
const char* windowName = "Debug";
const char* colorWindowName = "image";
const Scalar debugColor0(0, 0, 128);
const Scalar debugColor1(255, 0, 0);
const Scalar debugColor2(255, 255, 255);
const Scalar debugColor3(0, 255, 255);
const Scalar debugColor4(255, 0, 255);
int xMin = 50;
int xMax = 550;
int yMin = 50;
int yMax = 300;
Mat1s depth(480, 640); // 16 bit depth (in millimeters)
Mat1b depth8(480, 640); // 8 bit depth
Mat3b rgb(480, 640); // 8 bit depth
Mat3b debug(480, 640); // debug visualization
Mat1s foreground(640, 480);
Mat1b foreground8(640, 480);
Mat1b touch(640, 480); // touch mask
Mat1s background(480, 640);
vector<Mat1s> buffer(nBackgroundTrain);
IplImage * image = cvCreateImage(cvSize(640, 480), 8, 3);
IplImage * convertedImage = cvCreateImage(cvSize(640, 480), 8, 3);
initOpenNI("niConfig.xml");
// TUIO server object
TuioServer* tuio;
if (localClientMode) {
tuio = new TuioServer();
} else {
tuio = new TuioServer("192.168.0.2", 3333, false);
}
TuioTime time;
namedWindow(colorWindowName);
createTrackbar("xMin", colorWindowName, &xMin, 640);
createTrackbar("xMax", colorWindowName, &xMax, 640);
createTrackbar("yMin", colorWindowName, &yMin, 480);
createTrackbar("yMax", colorWindowName, &yMax, 480);
// create some sliders
namedWindow(windowName);
createTrackbar("xMin", windowName, &xMin, 640);
createTrackbar("xMax", windowName, &xMax, 640);
createTrackbar("yMin", windowName, &yMin, 480);
createTrackbar("yMax", windowName, &yMax, 480);
Keyboard * piano = new Keyboard();
(*piano).initKeyMap();
system("qjackctl &");
sleep(4);
JackByTheNotes * notesJack = new JackByTheNotes();
notesJack->connect();
sleep(2);
system("sudo jack_connect Piano:Rubinstein system:playback_1 &");
map<double, timeval> keys;
// create background model (average depth)
for (unsigned int i = 0; i < nBackgroundTrain; i++) {
xnContext.WaitAndUpdateAll();
depth.data = (uchar*) xnDepthGenerator.GetDepthMap();
buffer[i] = depth;
}
average(buffer, background);
while (waitKey(1) != 27) {
// read available data
xnContext.WaitAndUpdateAll();
// update 16 bit depth matrix
depth.data = (uchar*) xnDepthGenerator.GetDepthMap();
//xnImgeGenertor.GetGrayscale8ImageMap()
XnRGB24Pixel* xnRgb =
const_cast<XnRGB24Pixel*>(xnImgeGenertor.GetRGB24ImageMap());
// IplImage * image = cvCreateImage(cvSize(640, 480), 8, 3);
// IplImage * convertedImage = cvCreateImage(cvSize(640, 480), 8, 3);
cvSetData (image, xnRgb, 640 * 3);
cvConvertImage(image, convertedImage, CV_CVTIMG_SWAP_RB);
bool color = true;
rgb = convertedImage;
// cvtColor(rgb,rgb,CV_RGB2BGR);
// update rgb image
// rgb.data = (uchar*) xnImgeGenertor.GetRGB24ImageMap(); // segmentation fault here
// cvCvtColor(rgb, rgb, CV_BGR2RGB);
// extract foreground by simple subtraction of very basic background model
foreground = background - depth;
// find touch mask by thresholding (points that are close to background = touch points)
touch = (foreground > touchDepthMin) & (foreground < touchDepthMax);
// extract ROI
Rect roi(xMin, yMin, xMax - xMin, yMax - yMin);
Mat touchRoi = touch(roi);
// find touch points
vector<vector<Point2i> > contours;
vector<Point2f> touchPoints;
findContours(touchRoi, contours, CV_RETR_LIST, CV_CHAIN_APPROX_SIMPLE,
Point2i(xMin, yMin));
for (unsigned int i = 0; i < contours.size(); i++) {
Mat contourMat(contours[i]);
// find touch points by area thresholding
if (contourArea(contourMat) > touchMinArea) {
Scalar center = mean(contourMat);
Point2i touchPoint(center[0], center[1]);
touchPoints.push_back(touchPoint);
}
}
// send TUIO cursors
time = TuioTime::getSessionTime();
tuio->initFrame(time);
for (unsigned int i = 0; i < touchPoints.size(); i++) { // touch points
float cursorX = (touchPoints[i].x - xMin) / (xMax - xMin);
float cursorY = 1 - (touchPoints[i].y - yMin) / (yMax - yMin);
TuioCursor* cursor = tuio->getClosestTuioCursor(cursorX, cursorY);
// TODO improve tracking (don't move cursors away, that might be closer to another touch point)
if (cursor == NULL || cursor->getTuioTime() == time) {
tuio->addTuioCursor(cursorX, cursorY);
} else {
tuio->updateTuioCursor(cursor, cursorX, cursorY);
}
}
tuio->stopUntouchedMovingCursors();
tuio->removeUntouchedStoppedCursors();
tuio->commitFrame();
// draw debug frame
depth.convertTo(depth8, CV_8U, 255 / debugFrameMaxDepth); // render depth to debug frame
cvtColor(depth8, debug, CV_GRAY2BGR);
debug.setTo(debugColor0, touch); // touch mask
rectangle(debug, roi, debugColor1, 2); // surface boundaries
if (color)
rectangle(rgb, roi, debugColor1, 2); // surface boundaries
// draw 10 white keys within the roi
int stride = (xMax - xMin) / 10;
for (int keys = 1; keys < 10; keys++) {
Point lower(xMin + keys * stride, yMax);
if (keys == 3 || keys == 7) {
Point upper(xMin + keys * stride, yMin);
line(debug, upper, lower, debugColor3, 2, 0);
if (color)
line(rgb, upper, lower, debugColor3, 2, 0);
continue;
} else {
Point upper(xMin + keys * stride, (yMin + yMax) / 2);
line(debug, upper, lower, debugColor3, 2, 0);
if (color)
line(rgb, upper, lower, debugColor3, 2, 0);
}
Point blkUpper(xMin + keys * stride - stride / 3, yMin);
Point blkLower(xMin + keys * stride + stride / 3,
(yMin + yMax) / 2);
rectangle(debug, blkUpper, blkLower, debugColor4, 2);
if (color)
rectangle(rgb, blkUpper, blkLower, debugColor4, 2);
}
for (unsigned int i = 0; i < touchPoints.size(); i++) { // touch points
circle(debug, touchPoints[i], 5, debugColor2, CV_FILLED);
if (color)
circle(rgb, touchPoints[i], 5, debugColor2, CV_FILLED);
double frequency = piano->keyFrequency(touchPoints[i].y - 50,
touchPoints[i].x - 50);
cout << frequency << " " << touchPoints[i].y - 50 << " "
<< touchPoints[i].x - 50 << endl;
if (keys.find(frequency) == keys.end()) {
Note * note = new Note(frequency, 2, 4000);
notesJack->playNote(*note);
timeval now;
gettimeofday(&now, NULL);
keys[frequency] = now;
} else {
timeval now;
gettimeofday(&now, NULL);
if ((now.tv_sec - keys[frequency].tv_sec) * 1000
+ (now.tv_usec - keys[frequency].tv_usec) / 1000
> 1000) {
Note * note = new Note(frequency, 2, 4000);
notesJack->playNote(*note);
keys[frequency] = now;
}
}
}
// render debug frame (with sliders)
// IplImage grayScale = debug;
// cvFlip(&grayScale, NULL, 1);
// Mat gray(&grayScale);
// imshow(windowName, gray);
//
// IplImage colorful = rgb;
// cvFlip(&colorful, NULL, 1);
// Mat real(&colorful);
// imshow("image", real);
imshow(windowName, debug);
imshow("image", rgb);
// cvShowImage("image", image);
}
return 0;
}
int main() {
const unsigned int nBackgroundTrain = 30;
const unsigned short touchDepthMin = 10;
const unsigned short touchDepthMax = 20;
const unsigned int touchMinArea = 50;
const bool localClientMode = true; // connect to a local client
const double debugFrameMaxDepth = 4000; // maximal distance (in millimeters) for 8 bit debug depth frame quantization
const char* windowName = "Debug";
const Scalar debugColor0(0,0,128);
const Scalar debugColor1(255,0,0);
const Scalar debugColor2(255,255,255);
int xMin = 110;
int xMax = 560;
int yMin = 120;
int yMax = 320;
Mat1s depth(480, 640); // 16 bit depth (in millimeters)
Mat1b depth8(480, 640); // 8 bit depth
Mat3b rgb(480, 640); // 8 bit depth
Mat3b debug(480, 640); // debug visualization
Mat1s foreground(640, 480);
Mat1b foreground8(640, 480);
Mat1b touch(640, 480); // touch mask
Mat1s background(480, 640);
vector<Mat1s> buffer(nBackgroundTrain);
CHECK_RC(initOpenNI("niConfig.xml"), "initOpenNI");
// TUIO server object
TuioServer* tuio;
if (localClientMode) {
tuio = new TuioServer();
} else {
tuio = new TuioServer("192.168.0.1",3333,false);
}
TuioTime time;
// create some sliders
namedWindow(windowName);
createTrackbar("xMin", windowName, &xMin, 640);
createTrackbar("xMax", windowName, &xMax, 640);
createTrackbar("yMin", windowName, &yMin, 480);
createTrackbar("yMax", windowName, &yMax, 480);
// create background model (average depth)
for (unsigned int i=0; i<nBackgroundTrain; i++) {
CHECK_RC(xnContext.WaitAndUpdateAll(), "xnContext.WaitAndUpdateAll()");
depth.data = (uchar*) xnDepthGenerator.GetDepthMap();
buffer[i] = depth;
}
average(buffer, background);
while ( waitKey(1) != 27 ) {
// read available data
xnContext.WaitAndUpdateAll();
// update 16 bit depth matrix
depth.data = (uchar*) xnDepthGenerator.GetDepthMap();
//xnImgeGenertor.GetGrayscale8ImageMap()
// update rgb image
//rgb.data = (uchar*) xnImgeGenertor.GetRGB24ImageMap(); // segmentation fault here
//cvtColor(rgb, rgb, CV_RGB2BGR);
// extract foreground by simple subtraction of very basic background model
foreground = background - depth;
// find touch mask by thresholding (points that are close to background = touch points)
touch = (foreground > touchDepthMin) & (foreground < touchDepthMax);
// extract ROI
Rect roi(xMin, yMin, xMax - xMin, yMax - yMin);
Mat touchRoi = touch(roi);
// find touch points
vector< vector<Point2i> > contours;
vector<Point2f> touchPoints;
findContours(touchRoi, contours, CV_RETR_LIST, CV_CHAIN_APPROX_SIMPLE, Point2i(xMin, yMin));
for (unsigned int i=0; i<contours.size(); i++) {
Mat contourMat(contours[i]);
// find touch points by area thresholding
if ( contourArea(contourMat) > touchMinArea ) {
Scalar center = mean(contourMat);
Point2i touchPoint(center[0], center[1]);
touchPoints.push_back(touchPoint);
}
}
// send TUIO cursors
time = TuioTime::getSessionTime();
tuio->initFrame(time);
for (unsigned int i=0; i<touchPoints.size(); i++) { // touch points
float cursorX = (touchPoints[i].x - xMin) / (xMax - xMin);
float cursorY = 1 - (touchPoints[i].y - yMin)/(yMax - yMin);
TuioCursor* cursor = tuio->getClosestTuioCursor(cursorX,cursorY);
// TODO improve tracking (don't move cursors away, that might be closer to another touch point)
if (cursor == NULL || cursor->getTuioTime() == time) {
tuio->addTuioCursor(cursorX,cursorY);
} else {
tuio->updateTuioCursor(cursor, cursorX, cursorY);
}
}
tuio->stopUntouchedMovingCursors();
tuio->removeUntouchedStoppedCursors();
tuio->commitFrame();
// draw debug frame
depth.convertTo(depth8, CV_8U, 255 / debugFrameMaxDepth); // render depth to debug frame
cvtColor(depth8, debug, CV_GRAY2BGR);
debug.setTo(debugColor0, touch); // touch mask
rectangle(debug, roi, debugColor1, 2); // surface boundaries
for (unsigned int i=0; i<touchPoints.size(); i++) { // touch points
circle(debug, touchPoints[i], 5, debugColor2, CV_FILLED);
}
// render debug frame (with sliders)
imshow(windowName, debug);
//imshow("image", rgb);
}
return 0;
}
int main() {
const unsigned int nBackgroundTrain = 30; // サンプリング回数
int touchDepthMin = 10; // タッチ判定の最小値(defautl:10)
int touchDepthMax = 20; // タッチ判定の最大値(default:20)
int touchMinArea = 50; // このエリアよりも輪郭が大きいなら、タッチ箇所とみなす
const bool localClientMode = false; // connect to a local client
const double debugFrameMaxDepth = 4000; // maximal distance (in millimeters) for 8 bit debug depth frame quantization. 4000mm === 4m
const char* windowName = "TouchReader"; // ウィンドウ名
const Scalar debugColor0(0, 0, 128); // タッチ近似領域の色：Scalr(Blue, Green, Red) === (0x800000) === red
const Scalar debugColor1(255, 0, 0); // ROIを囲む枠線の色
const Scalar debugColor2(255, 255, 255); // タッチの色
const int xMin = 0;
const int xMax = 640;
const int yMin = 00;
const int yMax = 480;
Mat1s depth(480, 640); // 16 bit depth (in millimeters) <Mat1s === short>
Mat1b depth8(480, 640); // 8 bit depth <Mat1b === uchar>
//Mat3b rgb(480, 640); // 8 bit depth <Mat3b === Vec3b<r,g,b>>
Mat3b debug(480, 640); // debug visualization
Mat1s foreground(640, 480);
Mat1b foreground8(640, 480);
Mat1b touch(640, 480); // touch mask
Mat1s background(480, 640);
vector<Mat1s> buffer(nBackgroundTrain);
if (initKinnect() != 0) {
printf("initKinnect Error\n");
return -1;
}
// TUIO server object
TuioServer* tuio;
if (localClientMode) {
printf("connect \"LOCAL\" TuioServer\n");
tuio = new TuioServer();
} else {
printf("connect TuioServer 150.43.77.24:3333\n");
tuio = new TuioServer("150.43.74.5", 3333, false); // 安東さん
//tuio = new TuioServer("150.43.77.24", 3333, false); // 龍さん
}
TuioTime time;
// create some sliders
namedWindow(windowName);
/*
createTrackbar("xMin", windowName, &xMin, 640);
createTrackbar("xMax", windowName, &xMax, 640);
createTrackbar("yMin", windowName, &yMin, 480);
createTrackbar("yMax", windowName, &yMax, 480);
//*/
createTrackbar("touchDepthMin", windowName, &touchDepthMin, 100);
createTrackbar("touchDepthMax", windowName, &touchDepthMax, 100);
createTrackbar("touchMinArea", windowName, &touchMinArea, 100);
// create background model (average depth)
for (unsigned int i=0; i<nBackgroundTrain; i++) {
updateKinnect();
depth.data = getKinnectDepthMap();
buffer[i] = depth;
}
average(buffer, background);
while ( waitKey(1) != 27 ) {
// データ読み取り
updateKinnect();
// update 16 bit depth matrix
depth.data = getKinnectDepthMap();
//xnImgeGenertor.GetGrayscale8ImageMap()
// update rgb image
//rgb.data = (uchar*) xnImgeGenertor.GetRGB24ImageMap(); // segmentation fault here
//cvtColor(rgb, rgb, CV_RGB2BGR);
// extract foreground by simple subtraction of very basic background model
foreground = background - depth;
// タッチマスク
// find touch mask by thresholding (points that are close to background = touch points)
touch = (foreground > touchDepthMin) & (foreground < touchDepthMax);
// extract ROI
Rect roi(xMin, yMin, xMax - xMin, yMax - yMin);
Mat touchRoi = touch(roi);
// タッチ位置を探す
vector< vector<Point2i> > contours;
vector<Point2f> touchPoints;//タッチ位置
findContours(touchRoi, contours, CV_RETR_LIST, CV_CHAIN_APPROX_SIMPLE, Point2i(xMin, yMin));//輪郭を探しだす by OpenCV
for (unsigned int i=0; i<contours.size(); i++) {
Mat contourMat(contours[i]);
// find touch points by area thresholding
if ( contourArea(contourMat) > touchMinArea ) { // 小さすぎる点はタッチと見なさない
Scalar center = mean(contourMat);
Point2i touchPoint(center[0], center[1]);
touchPoints.push_back(touchPoint);
}
}
// send TUIO cursors
time = TuioTime::getSessionTime();
tuio->initFrame(time);
for (unsigned int i = 0; i < touchPoints.size(); i++) { // touch points
//float cursorX2 = (touchPoints[i].x - xMin) / (xMax - xMin);
float cursorX = 1- (touchPoints[i].x - xMin) / (xMax - xMin);
//float cursorY2 = 1 - (touchPoints[i].y - yMin)/(yMax - yMin);
float cursorY = (touchPoints[i].y - yMin)/(yMax - yMin);
cursorY = 1 - (touchPoints[i].y - yMin)/(yMax - yMin);//安東さん用
TuioCursor* cursor = tuio->getClosestTuioCursor(cursorX,cursorY);
// TODO improve tracking (don't move cursors away, that might be closer to another touch point)
if (cursor == NULL || cursor->getTuioTime() == time) {
tuio->addTuioCursor(cursorX, cursorY);
//printf("addTuioCursor TuioServer(%f, %f)\n", cursorX, cursorY);
//printf("addTuioCursor TuioServer(%f, %f)2\n", cursorX, cursorY2);
} else {
tuio->updateTuioCursor(cursor, cursorX, cursorY);
//printf("updateTuioCursor TuioServer(%f, %f)\n", cursorX, cursorY);
//printf("updateTuioCursor TuioServer(%f, %f)2\n", cursorX, cursorY2);
}
}
tuio->stopUntouchedMovingCursors();
tuio->removeUntouchedStoppedCursors();
tuio->commitFrame();
//--------------------
// draw debug frame
//--------------------
// render depth to debug frame
depth.convertTo(depth8, CV_8U, 255 / debugFrameMaxDepth);
cvtColor(/* in */depth8, /* out*/debug, /* 変換方法 */CV_GRAY2BGR);
// ヒートマップの描画
debug.setTo(debugColor0, touch); // touch mask
//rectangle(debug, roi, debugColor1, 2); // surface boundaries
// タッチ位置の描画
for (unsigned int i = 0; i < touchPoints.size(); i++) { // touch points
circle(debug, touchPoints[i], 5, debugColor2, CV_FILLED);
}
// render debug frame (with sliders)
imshow(windowName, debug);
//imshow("image", rgb);
}
die++;
sleep(1);
printf("main thread finished.\n");
printf("\ttouchDepthMin = %d\n", touchDepthMin);
printf("\ttouchDepthMax = %d\n", touchDepthMax);
printf("\ttouchMinArea = %d\n", touchMinArea);
return 0;
}
