void SimpleSimulator::mouseDragged(float x, float y) {
//printf("dragged %f %f\n",x,y);
TuioCursor *cursor = NULL;
float distance = width;
for (std::list<TuioCursor*>::iterator iter = activeCursorList.begin(); iter!=activeCursorList.end(); iter++) {
TuioCursor *tcur = (*iter);
float test = tcur->getDistance(x,y);
if (test<distance) {
distance = test;
cursor = tcur;
}
}
if (cursor==NULL) return;
if (cursor->getTuioTime()==frameTime) return;
std::list<TuioCursor*>::iterator joint = std::find( jointCursorList.begin(), jointCursorList.end(), cursor );
if( joint != jointCursorList.end() ) {
float dx = x-cursor->getX();
float dy = y-cursor->getY();
for (std::list<TuioCursor*>::iterator iter = jointCursorList.begin(); iter!=jointCursorList.end(); iter++) {
TuioCursor *jointCursor = (*iter);
tuioServer->updateTuioCursor(jointCursor,jointCursor->getX()+dx,jointCursor->getY()+dy);
}
} else tuioServer->updateTuioCursor(cursor,x,y);
}
std::list<TuioCursor*> TuioManager::getUntouchedCursors() {
std::list<TuioCursor*> untouched;
for (std::list<TuioCursor*>::iterator tuioCursor = cursorList.begin(); tuioCursor!=cursorList.end(); tuioCursor++) {
TuioCursor *tcur = (*tuioCursor);
if (tcur->getTuioTime()!=currentFrameTime) untouched.push_back(tcur);
}
return untouched;
}
void TuioServer::commitFrame()
{
if (updateCursor)
{
startCursorBundle();
for (std::list<TuioCursor *>::iterator tuioCursor = cursorList.begin(); tuioCursor != cursorList.end(); tuioCursor++)
{
// start a new packet if we exceed the packet capacity
if ((oscPacket->Capacity() - oscPacket->Size()) < CUR_MESSAGE_SIZE)
{
sendCursorBundle(currentFrame);
startCursorBundle();
}
TuioCursor *tcur = (*tuioCursor);
if ((full_update) || (tcur->getTuioTime() == currentFrameTime))
addCursorMessage(tcur);
}
sendCursorBundle(currentFrame);
}
else if ((!periodic_update) && (lastCursorUpdate < currentFrameTime.getSeconds()))
{
lastCursorUpdate = currentFrameTime.getSeconds();
startCursorBundle();
sendCursorBundle(currentFrame);
}
updateCursor = false;
if (updateObject)
{
startObjectBundle();
for (std::list<TuioObject *>::iterator tuioObject = objectList.begin(); tuioObject != objectList.end(); tuioObject++)
{
// start a new packet if we exceed the packet capacity
if ((oscPacket->Capacity() - oscPacket->Size()) < OBJ_MESSAGE_SIZE)
{
sendObjectBundle(currentFrame);
startObjectBundle();
}
TuioObject *tobj = (*tuioObject);
if ((full_update) || (tobj->getTuioTime() == currentFrameTime))
addObjectMessage(tobj);
}
sendObjectBundle(currentFrame);
}
else if ((!periodic_update) && (lastObjectUpdate < currentFrameTime.getSeconds()))
{
lastObjectUpdate = currentFrameTime.getSeconds();
startObjectBundle();
sendObjectBundle(currentFrame);
}
updateObject = false;
}
void TuioServer::removeUntouchedStoppedCursors() {
for (std::list<TuioCursor*>::iterator tuioCursor = cursorList.begin(); tuioCursor!=cursorList.end(); tuioCursor++) {
TuioCursor *tcur = (*tuioCursor);
if ((tcur->getTuioTime()!=currentFrameTime) && (!tcur->getTuioState()==TUIO_ADDED) && (!tcur->isMoving())) {
removeTuioCursor(tcur);
removeUntouchedStoppedCursors();
break;
}
}
}
void TuioManager::removeUntouchedStoppedCursors() {
if (cursorList.size()==0) return;
std::list<TuioCursor*>::iterator tuioCursor = cursorList.begin();
while (tuioCursor!=cursorList.end()) {
TuioCursor *tcur = (*tuioCursor);
if ((tcur->getTuioTime()!=currentFrameTime) && (!tcur->isMoving())) {
removeTuioCursor(tcur);
tuioCursor = cursorList.begin();
} else tuioCursor++;
}
}
void TuioManager::stopUntouchedMovingCursors() {
std::list<TuioCursor*> untouched;
for (std::list<TuioCursor*>::iterator tuioCursor = cursorList.begin(); tuioCursor!=cursorList.end(); tuioCursor++) {
TuioCursor *tcur = (*tuioCursor);
if ((tcur->getTuioTime()!=currentFrameTime) && (tcur->isMoving())) {
tcur->stop(currentFrameTime);
updateCursor = true;
if (verbose)
std::cout << "set cur " << tcur->getCursorID() << " (" << tcur->getSessionID() << ") " << tcur->getX() << " " << tcur->getY()
<< " " << tcur->getXSpeed() << " " << tcur->getYSpeed()<< " " << tcur->getMotionAccel() << " " << std::endl;
}
}
}
int main() {
const unsigned int nBackgroundTrain = 30;
const unsigned short touchDepthMin = 10;
const unsigned short touchDepthMax = 20;
const unsigned int touchMinArea = 50;
const bool localClientMode = true; // connect to a local client
const double debugFrameMaxDepth = 4000; // maximal distance (in millimeters) for 8 bit debug depth frame quantization
const char* windowName = "Debug";
const Scalar debugColor0(0,0,128);
const Scalar debugColor1(255,0,0);
const Scalar debugColor2(255,255,255);
int xMin = 110;
int xMax = 560;
int yMin = 120;
int yMax = 320;
Mat1s depth(480, 640); // 16 bit depth (in millimeters)
Mat1b depth8(480, 640); // 8 bit depth
Mat3b rgb(480, 640); // 8 bit depth
Mat3b debug(480, 640); // debug visualization
Mat1s foreground(640, 480);
Mat1b foreground8(640, 480);
Mat1b touch(640, 480); // touch mask
Mat1s background(480, 640);
vector<Mat1s> buffer(nBackgroundTrain);
CHECK_RC(initOpenNI("niConfig.xml"), "initOpenNI");
// TUIO server object
TuioServer* tuio;
if (localClientMode) {
tuio = new TuioServer();
} else {
tuio = new TuioServer("192.168.0.1",3333,false);
}
TuioTime time;
// create some sliders
namedWindow(windowName);
createTrackbar("xMin", windowName, &xMin, 640);
createTrackbar("xMax", windowName, &xMax, 640);
createTrackbar("yMin", windowName, &yMin, 480);
createTrackbar("yMax", windowName, &yMax, 480);
// create background model (average depth)
for (unsigned int i=0; i<nBackgroundTrain; i++) {
CHECK_RC(xnContext.WaitAndUpdateAll(), "xnContext.WaitAndUpdateAll()");
depth.data = (uchar*) xnDepthGenerator.GetDepthMap();
buffer[i] = depth;
}
average(buffer, background);
while ( waitKey(1) != 27 ) {
// read available data
xnContext.WaitAndUpdateAll();
// update 16 bit depth matrix
depth.data = (uchar*) xnDepthGenerator.GetDepthMap();
//xnImgeGenertor.GetGrayscale8ImageMap()
// update rgb image
//rgb.data = (uchar*) xnImgeGenertor.GetRGB24ImageMap(); // segmentation fault here
//cvtColor(rgb, rgb, CV_RGB2BGR);
// extract foreground by simple subtraction of very basic background model
foreground = background - depth;
// find touch mask by thresholding (points that are close to background = touch points)
touch = (foreground > touchDepthMin) & (foreground < touchDepthMax);
// extract ROI
Rect roi(xMin, yMin, xMax - xMin, yMax - yMin);
Mat touchRoi = touch(roi);
// find touch points
vector< vector<Point2i> > contours;
vector<Point2f> touchPoints;
findContours(touchRoi, contours, CV_RETR_LIST, CV_CHAIN_APPROX_SIMPLE, Point2i(xMin, yMin));
for (unsigned int i=0; i<contours.size(); i++) {
Mat contourMat(contours[i]);
// find touch points by area thresholding
if ( contourArea(contourMat) > touchMinArea ) {
Scalar center = mean(contourMat);
Point2i touchPoint(center[0], center[1]);
touchPoints.push_back(touchPoint);
}
}
// send TUIO cursors
time = TuioTime::getSessionTime();
tuio->initFrame(time);
for (unsigned int i=0; i<touchPoints.size(); i++) { // touch points
float cursorX = (touchPoints[i].x - xMin) / (xMax - xMin);
float cursorY = 1 - (touchPoints[i].y - yMin)/(yMax - yMin);
TuioCursor* cursor = tuio->getClosestTuioCursor(cursorX,cursorY);
// TODO improve tracking (don't move cursors away, that might be closer to another touch point)
if (cursor == NULL || cursor->getTuioTime() == time) {
tuio->addTuioCursor(cursorX,cursorY);
} else {
tuio->updateTuioCursor(cursor, cursorX, cursorY);
}
}
tuio->stopUntouchedMovingCursors();
tuio->removeUntouchedStoppedCursors();
tuio->commitFrame();
// draw debug frame
depth.convertTo(depth8, CV_8U, 255 / debugFrameMaxDepth); // render depth to debug frame
cvtColor(depth8, debug, CV_GRAY2BGR);
debug.setTo(debugColor0, touch); // touch mask
rectangle(debug, roi, debugColor1, 2); // surface boundaries
for (unsigned int i=0; i<touchPoints.size(); i++) { // touch points
circle(debug, touchPoints[i], 5, debugColor2, CV_FILLED);
}
// render debug frame (with sliders)
imshow(windowName, debug);
//imshow("image", rgb);
}
return 0;
}
int main() {
const unsigned int nBackgroundTrain = 30;
const unsigned short touchDepthMin = 10;
const unsigned short touchDepthMax = 20;
const unsigned int touchMinArea = 50;
const bool localClientMode = true; // connect to a local client
const double debugFrameMaxDepth = 4000; // maximal distance (in millimeters) for 8 bit debug depth frame quantization
const char* windowName = "Debug";
const char* colorWindowName = "image";
const Scalar debugColor0(0, 0, 128);
const Scalar debugColor1(255, 0, 0);
const Scalar debugColor2(255, 255, 255);
const Scalar debugColor3(0, 255, 255);
const Scalar debugColor4(255, 0, 255);
int xMin = 50;
int xMax = 550;
int yMin = 50;
int yMax = 300;
Mat1s depth(480, 640); // 16 bit depth (in millimeters)
Mat1b depth8(480, 640); // 8 bit depth
Mat3b rgb(480, 640); // 8 bit depth
Mat3b debug(480, 640); // debug visualization
Mat1s foreground(640, 480);
Mat1b foreground8(640, 480);
Mat1b touch(640, 480); // touch mask
Mat1s background(480, 640);
vector<Mat1s> buffer(nBackgroundTrain);
IplImage * image = cvCreateImage(cvSize(640, 480), 8, 3);
IplImage * convertedImage = cvCreateImage(cvSize(640, 480), 8, 3);
initOpenNI("niConfig.xml");
// TUIO server object
TuioServer* tuio;
if (localClientMode) {
tuio = new TuioServer();
} else {
tuio = new TuioServer("192.168.0.2", 3333, false);
}
TuioTime time;
namedWindow(colorWindowName);
createTrackbar("xMin", colorWindowName, &xMin, 640);
createTrackbar("xMax", colorWindowName, &xMax, 640);
createTrackbar("yMin", colorWindowName, &yMin, 480);
createTrackbar("yMax", colorWindowName, &yMax, 480);
// create some sliders
namedWindow(windowName);
createTrackbar("xMin", windowName, &xMin, 640);
createTrackbar("xMax", windowName, &xMax, 640);
createTrackbar("yMin", windowName, &yMin, 480);
createTrackbar("yMax", windowName, &yMax, 480);
Keyboard * piano = new Keyboard();
(*piano).initKeyMap();
system("qjackctl &");
sleep(4);
JackByTheNotes * notesJack = new JackByTheNotes();
notesJack->connect();
sleep(2);
system("sudo jack_connect Piano:Rubinstein system:playback_1 &");
map<double, timeval> keys;
// create background model (average depth)
for (unsigned int i = 0; i < nBackgroundTrain; i++) {
xnContext.WaitAndUpdateAll();
depth.data = (uchar*) xnDepthGenerator.GetDepthMap();
buffer[i] = depth;
}
average(buffer, background);
while (waitKey(1) != 27) {
// read available data
xnContext.WaitAndUpdateAll();
// update 16 bit depth matrix
depth.data = (uchar*) xnDepthGenerator.GetDepthMap();
//xnImgeGenertor.GetGrayscale8ImageMap()
XnRGB24Pixel* xnRgb =
const_cast<XnRGB24Pixel*>(xnImgeGenertor.GetRGB24ImageMap());
// IplImage * image = cvCreateImage(cvSize(640, 480), 8, 3);
// IplImage * convertedImage = cvCreateImage(cvSize(640, 480), 8, 3);
cvSetData (image, xnRgb, 640 * 3);
cvConvertImage(image, convertedImage, CV_CVTIMG_SWAP_RB);
bool color = true;
rgb = convertedImage;
// cvtColor(rgb,rgb,CV_RGB2BGR);
// update rgb image
// rgb.data = (uchar*) xnImgeGenertor.GetRGB24ImageMap(); // segmentation fault here
// cvCvtColor(rgb, rgb, CV_BGR2RGB);
// extract foreground by simple subtraction of very basic background model
foreground = background - depth;
// find touch mask by thresholding (points that are close to background = touch points)
touch = (foreground > touchDepthMin) & (foreground < touchDepthMax);
// extract ROI
Rect roi(xMin, yMin, xMax - xMin, yMax - yMin);
Mat touchRoi = touch(roi);
// find touch points
vector<vector<Point2i> > contours;
vector<Point2f> touchPoints;
findContours(touchRoi, contours, CV_RETR_LIST, CV_CHAIN_APPROX_SIMPLE,
Point2i(xMin, yMin));
for (unsigned int i = 0; i < contours.size(); i++) {
Mat contourMat(contours[i]);
// find touch points by area thresholding
if (contourArea(contourMat) > touchMinArea) {
Scalar center = mean(contourMat);
Point2i touchPoint(center[0], center[1]);
touchPoints.push_back(touchPoint);
}
}
// send TUIO cursors
time = TuioTime::getSessionTime();
tuio->initFrame(time);
for (unsigned int i = 0; i < touchPoints.size(); i++) { // touch points
float cursorX = (touchPoints[i].x - xMin) / (xMax - xMin);
float cursorY = 1 - (touchPoints[i].y - yMin) / (yMax - yMin);
TuioCursor* cursor = tuio->getClosestTuioCursor(cursorX, cursorY);
// TODO improve tracking (don't move cursors away, that might be closer to another touch point)
if (cursor == NULL || cursor->getTuioTime() == time) {
tuio->addTuioCursor(cursorX, cursorY);
} else {
tuio->updateTuioCursor(cursor, cursorX, cursorY);
}
}
tuio->stopUntouchedMovingCursors();
tuio->removeUntouchedStoppedCursors();
tuio->commitFrame();
// draw debug frame
depth.convertTo(depth8, CV_8U, 255 / debugFrameMaxDepth); // render depth to debug frame
cvtColor(depth8, debug, CV_GRAY2BGR);
debug.setTo(debugColor0, touch); // touch mask
rectangle(debug, roi, debugColor1, 2); // surface boundaries
if (color)
rectangle(rgb, roi, debugColor1, 2); // surface boundaries
// draw 10 white keys within the roi
int stride = (xMax - xMin) / 10;
for (int keys = 1; keys < 10; keys++) {
Point lower(xMin + keys * stride, yMax);
if (keys == 3 || keys == 7) {
Point upper(xMin + keys * stride, yMin);
line(debug, upper, lower, debugColor3, 2, 0);
if (color)
line(rgb, upper, lower, debugColor3, 2, 0);
continue;
} else {
Point upper(xMin + keys * stride, (yMin + yMax) / 2);
line(debug, upper, lower, debugColor3, 2, 0);
if (color)
line(rgb, upper, lower, debugColor3, 2, 0);
}
Point blkUpper(xMin + keys * stride - stride / 3, yMin);
Point blkLower(xMin + keys * stride + stride / 3,
(yMin + yMax) / 2);
rectangle(debug, blkUpper, blkLower, debugColor4, 2);
if (color)
rectangle(rgb, blkUpper, blkLower, debugColor4, 2);
}
for (unsigned int i = 0; i < touchPoints.size(); i++) { // touch points
circle(debug, touchPoints[i], 5, debugColor2, CV_FILLED);
if (color)
circle(rgb, touchPoints[i], 5, debugColor2, CV_FILLED);
double frequency = piano->keyFrequency(touchPoints[i].y - 50,
touchPoints[i].x - 50);
cout << frequency << " " << touchPoints[i].y - 50 << " "
<< touchPoints[i].x - 50 << endl;
if (keys.find(frequency) == keys.end()) {
Note * note = new Note(frequency, 2, 4000);
notesJack->playNote(*note);
timeval now;
gettimeofday(&now, NULL);
keys[frequency] = now;
} else {
timeval now;
gettimeofday(&now, NULL);
if ((now.tv_sec - keys[frequency].tv_sec) * 1000
+ (now.tv_usec - keys[frequency].tv_usec) / 1000
> 1000) {
Note * note = new Note(frequency, 2, 4000);
notesJack->playNote(*note);
keys[frequency] = now;
}
}
}
// render debug frame (with sliders)
// IplImage grayScale = debug;
// cvFlip(&grayScale, NULL, 1);
// Mat gray(&grayScale);
// imshow(windowName, gray);
//
// IplImage colorful = rgb;
// cvFlip(&colorful, NULL, 1);
// Mat real(&colorful);
// imshow("image", real);
imshow(windowName, debug);
imshow("image", rgb);
// cvShowImage("image", image);
}
return 0;
}
int main() {
const unsigned int nBackgroundTrain = 30; // サンプリング回数
int touchDepthMin = 10; // タッチ判定の最小値(defautl:10)
int touchDepthMax = 20; // タッチ判定の最大値(default:20)
int touchMinArea = 50; // このエリアよりも輪郭が大きいなら、タッチ箇所とみなす
const bool localClientMode = false; // connect to a local client
const double debugFrameMaxDepth = 4000; // maximal distance (in millimeters) for 8 bit debug depth frame quantization. 4000mm === 4m
const char* windowName = "TouchReader"; // ウィンドウ名
const Scalar debugColor0(0, 0, 128); // タッチ近似領域の色：Scalr(Blue, Green, Red) === (0x800000) === red
const Scalar debugColor1(255, 0, 0); // ROIを囲む枠線の色
const Scalar debugColor2(255, 255, 255); // タッチの色
const int xMin = 0;
const int xMax = 640;
const int yMin = 00;
const int yMax = 480;
Mat1s depth(480, 640); // 16 bit depth (in millimeters) <Mat1s === short>
Mat1b depth8(480, 640); // 8 bit depth <Mat1b === uchar>
//Mat3b rgb(480, 640); // 8 bit depth <Mat3b === Vec3b<r,g,b>>
Mat3b debug(480, 640); // debug visualization
Mat1s foreground(640, 480);
Mat1b foreground8(640, 480);
Mat1b touch(640, 480); // touch mask
Mat1s background(480, 640);
vector<Mat1s> buffer(nBackgroundTrain);
if (initKinnect() != 0) {
printf("initKinnect Error\n");
return -1;
}
// TUIO server object
TuioServer* tuio;
if (localClientMode) {
printf("connect \"LOCAL\" TuioServer\n");
tuio = new TuioServer();
} else {
printf("connect TuioServer 150.43.77.24:3333\n");
tuio = new TuioServer("150.43.74.5", 3333, false); // 安東さん
//tuio = new TuioServer("150.43.77.24", 3333, false); // 龍さん
}
TuioTime time;
// create some sliders
namedWindow(windowName);
/*
createTrackbar("xMin", windowName, &xMin, 640);
createTrackbar("xMax", windowName, &xMax, 640);
createTrackbar("yMin", windowName, &yMin, 480);
createTrackbar("yMax", windowName, &yMax, 480);
//*/
createTrackbar("touchDepthMin", windowName, &touchDepthMin, 100);
createTrackbar("touchDepthMax", windowName, &touchDepthMax, 100);
createTrackbar("touchMinArea", windowName, &touchMinArea, 100);
// create background model (average depth)
for (unsigned int i=0; i<nBackgroundTrain; i++) {
updateKinnect();
depth.data = getKinnectDepthMap();
buffer[i] = depth;
}
average(buffer, background);
while ( waitKey(1) != 27 ) {
// データ読み取り
updateKinnect();
// update 16 bit depth matrix
depth.data = getKinnectDepthMap();
//xnImgeGenertor.GetGrayscale8ImageMap()
// update rgb image
//rgb.data = (uchar*) xnImgeGenertor.GetRGB24ImageMap(); // segmentation fault here
//cvtColor(rgb, rgb, CV_RGB2BGR);
// extract foreground by simple subtraction of very basic background model
foreground = background - depth;
// タッチマスク
// find touch mask by thresholding (points that are close to background = touch points)
touch = (foreground > touchDepthMin) & (foreground < touchDepthMax);
// extract ROI
Rect roi(xMin, yMin, xMax - xMin, yMax - yMin);
Mat touchRoi = touch(roi);
// タッチ位置を探す
vector< vector<Point2i> > contours;
vector<Point2f> touchPoints;//タッチ位置
findContours(touchRoi, contours, CV_RETR_LIST, CV_CHAIN_APPROX_SIMPLE, Point2i(xMin, yMin));//輪郭を探しだす by OpenCV
for (unsigned int i=0; i<contours.size(); i++) {
Mat contourMat(contours[i]);
// find touch points by area thresholding
if ( contourArea(contourMat) > touchMinArea ) { // 小さすぎる点はタッチと見なさない
Scalar center = mean(contourMat);
Point2i touchPoint(center[0], center[1]);
touchPoints.push_back(touchPoint);
}
}
// send TUIO cursors
time = TuioTime::getSessionTime();
tuio->initFrame(time);
for (unsigned int i = 0; i < touchPoints.size(); i++) { // touch points
//float cursorX2 = (touchPoints[i].x - xMin) / (xMax - xMin);
float cursorX = 1- (touchPoints[i].x - xMin) / (xMax - xMin);
//float cursorY2 = 1 - (touchPoints[i].y - yMin)/(yMax - yMin);
float cursorY = (touchPoints[i].y - yMin)/(yMax - yMin);
cursorY = 1 - (touchPoints[i].y - yMin)/(yMax - yMin);//安東さん用
TuioCursor* cursor = tuio->getClosestTuioCursor(cursorX,cursorY);
// TODO improve tracking (don't move cursors away, that might be closer to another touch point)
if (cursor == NULL || cursor->getTuioTime() == time) {
tuio->addTuioCursor(cursorX, cursorY);
//printf("addTuioCursor TuioServer(%f, %f)\n", cursorX, cursorY);
//printf("addTuioCursor TuioServer(%f, %f)2\n", cursorX, cursorY2);
} else {
tuio->updateTuioCursor(cursor, cursorX, cursorY);
//printf("updateTuioCursor TuioServer(%f, %f)\n", cursorX, cursorY);
//printf("updateTuioCursor TuioServer(%f, %f)2\n", cursorX, cursorY2);
}
}
tuio->stopUntouchedMovingCursors();
tuio->removeUntouchedStoppedCursors();
tuio->commitFrame();
//--------------------
// draw debug frame
//--------------------
// render depth to debug frame
depth.convertTo(depth8, CV_8U, 255 / debugFrameMaxDepth);
cvtColor(/* in */depth8, /* out*/debug, /* 変換方法 */CV_GRAY2BGR);
// ヒートマップの描画
debug.setTo(debugColor0, touch); // touch mask
//rectangle(debug, roi, debugColor1, 2); // surface boundaries
// タッチ位置の描画
for (unsigned int i = 0; i < touchPoints.size(); i++) { // touch points
circle(debug, touchPoints[i], 5, debugColor2, CV_FILLED);
}
// render debug frame (with sliders)
imshow(windowName, debug);
//imshow("image", rgb);
}
die++;
sleep(1);
printf("main thread finished.\n");
printf("\ttouchDepthMin = %d\n", touchDepthMin);
printf("\ttouchDepthMax = %d\n", touchDepthMax);
printf("\ttouchMinArea = %d\n", touchMinArea);
return 0;
}