~KinectSample()
{
// 終了処理
if ( kinect != 0 ) {
kinect->NuiShutdown();
kinect->Release();
}
}
int _tmain(int argc, _TCHAR* argv[])
{
cv::setUseOptimized( true );
// Kinectのインスタンス生成、初期化
INuiSensor* pSensor;
HRESULT hResult = S_OK;
hResult = NuiCreateSensorByIndex( 0, &pSensor );
if( FAILED( hResult ) ){
std::cerr << "Error : NuiCreateSensorByIndex" << std::endl;
return -1;
}
hResult = pSensor->NuiInitialize( NUI_INITIALIZE_FLAG_USES_COLOR );
if( FAILED( hResult ) ){
std::cerr << "Error : NuiInitialize" << std::endl;
return -1;
}
// Colorストリームの開始
HANDLE hColorEvent = INVALID_HANDLE_VALUE;
HANDLE hColorHandle = INVALID_HANDLE_VALUE;
hColorEvent = CreateEvent( nullptr, true, false, nullptr );
hResult = pSensor->NuiImageStreamOpen( NUI_IMAGE_TYPE_COLOR, NUI_IMAGE_RESOLUTION_640x480, 0, 2, hColorEvent, &hColorHandle );
if( FAILED( hResult ) ){
std::cerr << "Error : NuiImageStreamOpen( COLOR )" << std::endl;
return -1;
}
HANDLE hEvents[1] = { hColorEvent };
cv::namedWindow( "Color" );
while( 1 ){
// フレームの更新待ち
ResetEvent( hColorEvent );
WaitForMultipleObjects( ARRAYSIZE( hEvents ), hEvents, true, INFINITE );
// Colorカメラからフレームを取得
NUI_IMAGE_FRAME pColorImageFrame = { 0 };
hResult = pSensor->NuiImageStreamGetNextFrame( hColorHandle, 0, &pColorImageFrame );
if( FAILED( hResult ) ){
std::cerr << "Error : NuiImageStreamGetNextFrame( COLOR )" << std::endl;
return -1;
}
// Color画像データの取得
INuiFrameTexture* pColorFrameTexture = pColorImageFrame.pFrameTexture;
NUI_LOCKED_RECT sColorLockedRect;
pColorFrameTexture->LockRect( 0, &sColorLockedRect, nullptr, 0 );
// 表示
cv::Mat colorMat( 480, 640, CV_8UC4, reinterpret_cast<uchar*>( sColorLockedRect.pBits ) );
cv::imshow( "Color", colorMat );
// フレームの解放
pColorFrameTexture->UnlockRect( 0 );
pSensor->NuiImageStreamReleaseFrame( hColorHandle, &pColorImageFrame );
// ループの終了判定(Escキー)
if( cv::waitKey( 30 ) == VK_ESCAPE ){
break;
}
}
// Kinectの終了処理
pSensor->NuiShutdown();
CloseHandle( hColorEvent );
CloseHandle( hColorHandle );
cv::destroyAllWindows();
return 0;
}
// connect to a Kinect sensor and stream its data into LSL
void stream_from_sensor(int sensornum, bool smoothdata) {
HRESULT hr=0;
try {
// instantiate sensor
cout << "initializing sensor " << sensornum << "..." << endl;
INuiSensor *sensor;
if (FAILED(hr=NuiCreateSensorByIndex(sensornum,&sensor)))
throw runtime_error("Could not instantiate sensor.");
// initialize and enable skeletal tracking
if (FAILED(hr = sensor->NuiInitialize(NUI_INITIALIZE_FLAG_USES_SKELETON)))
throw runtime_error("Could not initialize NUI functionality.");
if (FAILED(hr = sensor->NuiSkeletonTrackingEnable(NULL,NUI_SKELETON_TRACKING_FLAG_SUPPRESS_NO_FRAME_DATA)))
throw runtime_error("Could not enable skeletal tracking.");
cout << "opening outlet..." << endl;
char tmp[1024]; ;
stream_info info(string("KinectMocap")+=boost::lexical_cast<string>(sensornum),"Mocap",NUM_CHANNELS_PER_STREAM,30,cf_float32,string(tmp,tmp+wcstombs(tmp,sensor->NuiUniqueId(),sizeof(tmp))));
// physical setup
xml_element setup = info.desc().append_child("setup");
xml_element cam = setup.append_child("cameras").append_child("camera");
cam.append_child_value("label","Kinect");
cam.append_child("position")
.append_child_value("X","0.0")
.append_child_value("Y","0.0")
.append_child_value("Z","0.0");
cam.append_child_value("diagonal_fov",boost::lexical_cast<string>(NUI_CAMERA_DEPTH_NOMINAL_DIAGONAL_FOV).c_str());
cam.append_child_value("horizontal_fov",boost::lexical_cast<string>(NUI_CAMERA_DEPTH_NOMINAL_HORIZONTAL_FOV).c_str());
cam.append_child_value("vertical_fov",boost::lexical_cast<string>(NUI_CAMERA_DEPTH_NOMINAL_VERTICAL_FOV).c_str());
// markers
xml_element mrks = setup.append_child("markers");
for (int k=0;k<NUI_SKELETON_POSITION_COUNT;k++) {
mrks.append_child("marker")
.append_child_value("label",joint_names[k])
.append_child_value("id",boost::lexical_cast<std::string>(k).c_str());
}
// channel layout
xml_element channels = info.desc().append_child("channels");
for (int s=0;s<NUI_SKELETON_MAX_TRACKED_COUNT;s++) {
for (int k=0;k<NUI_SKELETON_POSITION_COUNT;k++) {
channels.append_child("channel")
.append_child_value("label",(string(joint_names[k])+="_X").c_str())
.append_child_value("marker",joint_names[k])
.append_child_value("type","PositionX")
.append_child_value("unit","meters");
channels.append_child("channel")
.append_child_value("label",(string(joint_names[k])+="_Y").c_str())
.append_child_value("marker",joint_names[k])
.append_child_value("type","PositionY")
.append_child_value("unit","meters");
channels.append_child("channel")
.append_child_value("label",(string(joint_names[k])+="_Z").c_str())
.append_child_value("marker",joint_names[k])
.append_child_value("type","PositionZ")
.append_child_value("unit","meters");
channels.append_child("channel")
.append_child_value("label",(string(joint_names[k])+="_Conf").c_str())
.append_child_value("marker",joint_names[k])
.append_child_value("type","Confidence")
.append_child_value("unit","normalized");
}
channels.append_child("channel")
.append_child_value("label",(string("SkeletonTrackingId")+=boost::lexical_cast<string>(s)).c_str())
.append_child_value("type","TrackingId");
channels.append_child("channel")
.append_child_value("label",(string("SkeletonQualityFlags")+=boost::lexical_cast<string>(s)).c_str());
}
// misc meta-data
info.desc().append_child("acquisition")
.append_child_value("manufacturer","Microsoft")
.append_child_value("model","Kinect 1.0");
if (smoothdata)
info.desc().append_child("filtering")
.append_child("holt_double_exponential")
.append_child_value("smoothing","0.5")
.append_child_value("correction","0.5")
.append_child_value("jitter_radius","0.05")
.append_child_value("max_deviation_radius","0.04");
stream_outlet outlet(info);
// acquisition loop
cout << "starting to track on sensor " << sensornum << endl;
while (true) {
// get next frame
NUI_SKELETON_FRAME frame = {0};
if (FAILED(sensor->NuiSkeletonGetNextFrame(100,&frame)))
continue;
// for each skeleton index s
vector<float> sample(NUM_CHANNELS_PER_STREAM);
for (int s=0,i=0; s<NUI_SKELETON_COUNT; s++) {
if (frame.SkeletonData[s].eTrackingState == NUI_SKELETON_TRACKED) {
// optionally smooth the data
if (smoothdata)
sensor->NuiTransformSmooth(&frame,NULL);
// assign the sample data (at slot i)
for (int k=0;k<NUI_SKELETON_POSITION_COUNT;k++) {
sample[i*NUM_CHANNELS_PER_SKELETON + k*NUM_CHANNELS_PER_JOINT + 0] = frame.SkeletonData[s].SkeletonPositions[k].x;
sample[i*NUM_CHANNELS_PER_SKELETON + k*NUM_CHANNELS_PER_JOINT + 1] = frame.SkeletonData[s].SkeletonPositions[k].y;
sample[i*NUM_CHANNELS_PER_SKELETON + k*NUM_CHANNELS_PER_JOINT + 2] = frame.SkeletonData[s].SkeletonPositions[k].z;
sample[i*NUM_CHANNELS_PER_SKELETON + k*NUM_CHANNELS_PER_JOINT + 3] = frame.SkeletonData[s].eSkeletonPositionTrackingState[k] / 2.0f; // 0.0, 0.5, or 1.0
}
sample[i*NUM_CHANNELS_PER_SKELETON + NUM_CHANNELS_PER_JOINT*NUI_SKELETON_POSITION_COUNT] = frame.SkeletonData[s].dwTrackingID;
sample[i*NUM_CHANNELS_PER_SKELETON + NUM_CHANNELS_PER_JOINT*NUI_SKELETON_POSITION_COUNT] = frame.SkeletonData[s].dwQualityFlags;
i++;
}
}
// push the sample into LSL
outlet.push_sample(sample);
}
// shut down
sensor->NuiShutdown();
sensor->Release();
}
catch(exception &e) {
cerr << "Error trying to stream from sensor " << sensornum << ": " << e.what() << endl;
}
}
int _tmain(int argc, _TCHAR* argv[])
{
cv::setUseOptimized( true );
INuiSensor* pSensor;
HRESULT hResult = S_OK;
hResult = NuiCreateSensorByIndex( 0, &pSensor );
if( FAILED( hResult ) ){
std::cerr << "Error : NuiCreateSensorByIndex" << std::endl;
return -1;
}
hResult = pSensor->NuiInitialize( NUI_INITIALIZE_FLAG_USES_COLOR | NUI_INITIALIZE_FLAG_USES_DEPTH_AND_PLAYER_INDEX | NUI_INITIALIZE_FLAG_USES_SKELETON );
if( FAILED( hResult ) ){
std::cerr << "Error : NuiInitialize" << std::endl;
return -1;
}
HANDLE hColorEvent = INVALID_HANDLE_VALUE;
HANDLE hColorHandle = INVALID_HANDLE_VALUE;
hColorEvent = CreateEvent( nullptr, true, false, nullptr );
hResult = pSensor->NuiImageStreamOpen( NUI_IMAGE_TYPE_COLOR, NUI_IMAGE_RESOLUTION_640x480, 0, 2, hColorEvent, &hColorHandle );
if( FAILED( hResult ) ){
std::cerr << "Error : NuiImageStreamOpen( COLOR )" << std::endl;
return -1;
}
HANDLE hDepthPlayerEvent = INVALID_HANDLE_VALUE;
HANDLE hDepthPlayerHandle = INVALID_HANDLE_VALUE;
hDepthPlayerEvent = CreateEvent( nullptr, true, false, nullptr );
hResult = pSensor->NuiImageStreamOpen( NUI_IMAGE_TYPE_DEPTH_AND_PLAYER_INDEX, NUI_IMAGE_RESOLUTION_640x480, 0, 2, hDepthPlayerEvent, &hDepthPlayerHandle );
if( FAILED( hResult ) ){
std::cerr << "Error : NuiImageStreamOpen( DEPTH&PLAYER )" << std::endl;
return -1;
}
HANDLE hSkeletonEvent = INVALID_HANDLE_VALUE;
hSkeletonEvent = CreateEvent( nullptr, true, false, nullptr );
hResult = pSensor->NuiSkeletonTrackingEnable( hSkeletonEvent, 0 );
if( FAILED( hResult ) ){
std::cerr << "Error : NuiSkeletonTrackingEnable" << std::endl;
return -1;
}
unsigned long refWidth = 0;
unsigned long refHeight = 0;
NuiImageResolutionToSize( NUI_IMAGE_RESOLUTION_640x480, refWidth, refHeight );
int width = static_cast<int>( refWidth );
int height = static_cast<int>( refHeight );
INuiCoordinateMapper* pCordinateMapper;
hResult = pSensor->NuiGetCoordinateMapper( &pCordinateMapper );
if( FAILED( hResult ) ){
std::cerr << "Error : NuiGetCoordinateMapper" << std::endl;
return -1;
}
std::vector<NUI_COLOR_IMAGE_POINT> pColorPoint( width * height );
HANDLE hEvents[3] = { hColorEvent, hDepthPlayerEvent, hSkeletonEvent };
cv::Vec3b color[7];
color[0] = cv::Vec3b( 0, 0, 0 );
color[1] = cv::Vec3b( 255, 0, 0 );
color[2] = cv::Vec3b( 0, 255, 0 );
color[3] = cv::Vec3b( 0, 0, 255 );
color[4] = cv::Vec3b( 255, 255, 0 );
color[5] = cv::Vec3b( 255, 0, 255 );
color[6] = cv::Vec3b( 0, 255, 255 );
cv::namedWindow( "Color" );
cv::namedWindow( "Depth" );
cv::namedWindow( "Player" );
cv::namedWindow( "Skeleton" );
while( 1 ){
ResetEvent( hColorEvent );
ResetEvent( hDepthPlayerEvent );
ResetEvent( hSkeletonEvent );
WaitForMultipleObjects( ARRAYSIZE( hEvents ), hEvents, true, INFINITE );
NUI_IMAGE_FRAME colorImageFrame = { 0 };
hResult = pSensor->NuiImageStreamGetNextFrame( hColorHandle, 0, &colorImageFrame );
if( FAILED( hResult ) ){
std::cerr << "Error : NuiImageStreamGetNextFrame( COLOR )" << std::endl;
return -1;
}
INuiFrameTexture* pColorFrameTexture = colorImageFrame.pFrameTexture;
NUI_LOCKED_RECT colorLockedRect;
pColorFrameTexture->LockRect( 0, &colorLockedRect, nullptr, 0 );
NUI_IMAGE_FRAME depthPlayerImageFrame = { 0 };
hResult = pSensor->NuiImageStreamGetNextFrame( hDepthPlayerHandle, 0, &depthPlayerImageFrame );
if( FAILED( hResult ) ){
std::cerr << "Error : NuiImageStreamGetNextFrame( DEPTH&PLAYER )" << std::endl;
return -1;
}
BOOL nearMode = false;
INuiFrameTexture* pDepthPlayerFrameTexture = nullptr;
pSensor->NuiImageFrameGetDepthImagePixelFrameTexture( hDepthPlayerHandle, &depthPlayerImageFrame, &nearMode, &pDepthPlayerFrameTexture );
NUI_LOCKED_RECT depthPlayerLockedRect;
pDepthPlayerFrameTexture->LockRect( 0, &depthPlayerLockedRect, nullptr, 0 );
NUI_SKELETON_FRAME skeletonFrame = { 0 };
hResult = pSensor->NuiSkeletonGetNextFrame( 0, &skeletonFrame );
if( FAILED( hResult ) ){
std::cout << "Error : NuiSkeletonGetNextFrame" << std::endl;
return -1;
}
/*
NUI_TRANSFORM_SMOOTH_PARAMETERS smoothParameter;
smoothParameter.fSmoothing = 0.5;
smoothParameter.fCorrection = 0.5;
smoothParameter.fPrediction = 0.0f;
smoothParameter.fJitterRadius = 0.05f;
smoothParameter.fMaxDeviationRadius = 0.04f;
hResult = NuiTransformSmooth( &skeletonFrame, &smoothParameter );
*/
cv::Mat colorMat( height, width, CV_8UC4, reinterpret_cast<unsigned char*>( colorLockedRect.pBits ) );
cv::Mat bufferMat = cv::Mat::zeros( height, width, CV_16UC1 );
cv::Mat playerMat = cv::Mat::zeros( height, width, CV_8UC3 );
NUI_DEPTH_IMAGE_PIXEL* pDepthPlayerPixel = reinterpret_cast<NUI_DEPTH_IMAGE_PIXEL*>( depthPlayerLockedRect.pBits );
pCordinateMapper->MapDepthFrameToColorFrame( NUI_IMAGE_RESOLUTION_640x480, width * height, pDepthPlayerPixel, NUI_IMAGE_TYPE_COLOR, NUI_IMAGE_RESOLUTION_640x480, width * height, &pColorPoint[0] );
for( int y = 0; y < height; y++ ){
for( int x = 0; x < width; x++ ){
unsigned int index = y * width + x;
bufferMat.at<unsigned short>( pColorPoint[index].y, pColorPoint[index].x ) = pDepthPlayerPixel[index].depth;
playerMat.at<cv::Vec3b>( pColorPoint[index].y, pColorPoint[index].x ) = color[pDepthPlayerPixel[index].playerIndex];
}
}
cv::Mat depthMat( height, width, CV_8UC1 );
bufferMat.convertTo( depthMat, CV_8U, -255.0f / 10000.0f, 255.0f );
cv::Mat skeletonMat = cv::Mat::zeros( height, width, CV_8UC3 );
NUI_COLOR_IMAGE_POINT colorPoint;
for( int count = 0; count < NUI_SKELETON_COUNT; count++ ){
NUI_SKELETON_DATA skeletonData = skeletonFrame.SkeletonData[count];
if( skeletonData.eTrackingState == NUI_SKELETON_TRACKED ){
for( int position = 0; position < NUI_SKELETON_POSITION_COUNT; position++ ){
pCordinateMapper->MapSkeletonPointToColorPoint( &skeletonData.SkeletonPositions[position], NUI_IMAGE_TYPE_COLOR, NUI_IMAGE_RESOLUTION_640x480, &colorPoint );
if( ( colorPoint.x >= 0 ) && ( colorPoint.x < width ) && ( colorPoint.y >= 0 ) && ( colorPoint.y < height ) ){
cv::circle( skeletonMat, cv::Point( colorPoint.x, colorPoint.y ), 10, static_cast<cv::Scalar>( color[count + 1] ), -1, CV_AA );
}
}
std::stringstream ss;
ss << skeletonData.SkeletonPositions[NUI_SKELETON_POSITION_HIP_CENTER].z;
pCordinateMapper->MapSkeletonPointToColorPoint( &skeletonData.SkeletonPositions[NUI_SKELETON_POSITION_HEAD], NUI_IMAGE_TYPE_COLOR, NUI_IMAGE_RESOLUTION_640x480, &colorPoint );
if( ( colorPoint.x >= 0 ) && ( colorPoint.x < width ) && ( colorPoint.y >= 0 ) && ( colorPoint.y < height ) ){
cv::putText( skeletonMat, ss.str(), cv::Point( colorPoint.x - 50, colorPoint.y - 20 ), cv::FONT_HERSHEY_SIMPLEX, 1.5f, static_cast<cv::Scalar>( color[count + 1] ) );
}
}
else if( skeletonData.eTrackingState == NUI_SKELETON_POSITION_ONLY ){
pCordinateMapper->MapSkeletonPointToColorPoint( &skeletonData.SkeletonPositions[NUI_SKELETON_POSITION_HIP_CENTER], NUI_IMAGE_TYPE_COLOR, NUI_IMAGE_RESOLUTION_640x480, &colorPoint );
if( ( colorPoint.x >= 0 ) && ( colorPoint.x < width ) && ( colorPoint.y >= 0 ) && ( colorPoint.y < height ) ){
cv::circle( skeletonMat, cv::Point( colorPoint.x, colorPoint.y ), 10, static_cast<cv::Scalar>( color[count + 1] ), -1, CV_AA );
}
}
}
cv::imshow( "Color", colorMat );
cv::imshow( "Depth", depthMat );
cv::imshow( "Player", playerMat );
cv::imshow( "Skeleton", skeletonMat );
pColorFrameTexture->UnlockRect( 0 );
pDepthPlayerFrameTexture->UnlockRect( 0 );
pSensor->NuiImageStreamReleaseFrame( hColorHandle, &colorImageFrame );
pSensor->NuiImageStreamReleaseFrame( hDepthPlayerHandle, &depthPlayerImageFrame );
if( cv::waitKey( 30 ) == VK_ESCAPE ){
break;
}
}
pSensor->NuiShutdown();
pSensor->NuiSkeletonTrackingDisable();
pCordinateMapper->Release();
CloseHandle( hColorEvent );
CloseHandle( hDepthPlayerEvent );
CloseHandle( hSkeletonEvent );
cv::destroyAllWindows();
return 0;
}
int _tmain( int argc, _TCHAR* argv[] )
{
cv::setUseOptimized( true );
// Kinectのインスタンス生成、初期化
INuiSensor* pSensor;
HRESULT hResult = S_OK;
hResult = NuiCreateSensorByIndex( 0, &pSensor );
if( FAILED( hResult ) ){
std::cerr << "Error : NuiCreateSensorByIndex" << std::endl;
return -1;
}
hResult = pSensor->NuiInitialize( NUI_INITIALIZE_FLAG_USES_AUDIO );
if( FAILED( hResult ) ){
std::cerr << "Error : NuiInitialize" << std::endl;
return -1;
}
// Audioストリームの初期化(InitializeAudioStream)
std::cout << "InitializeAudioStream" << std::endl;
INuiAudioBeam* pNuiAudioSource;
hResult = pSensor->NuiGetAudioSource( &pNuiAudioSource );
if( FAILED( hResult ) ){
std::cerr << "Error : NuiGetAudioSource" << std::endl;
return -1;
}
IMediaObject* pMediaObject = nullptr;
IPropertyStore* pPropertyStore = nullptr;
pNuiAudioSource->QueryInterface( IID_IMediaObject, reinterpret_cast<void**>( &pMediaObject ) );
pNuiAudioSource->QueryInterface( IID_IPropertyStore, reinterpret_cast<void**>( &pPropertyStore ) );
PROPVARIANT propvariant;
PropVariantInit( &propvariant );
propvariant.vt = VT_I4;
propvariant.lVal = static_cast<LONG>( 4 );
pPropertyStore->SetValue( MFPKEY_WMAAECMA_SYSTEM_MODE, propvariant );
PropVariantClear( &propvariant );
WAVEFORMATEX waveFormat = { AudioFormat, AudioChannels, AudioSamplesPerSecond, AudioAverageBytesPerSecond, AudioBlockAlign, AudioBitsPerSample, 0 };
DMO_MEDIA_TYPE mediaType = { 0 };
MoInitMediaType( &mediaType, sizeof( WAVEFORMATEX ) );
mediaType.majortype = MEDIATYPE_Audio;
mediaType.subtype = MEDIASUBTYPE_PCM;
mediaType.lSampleSize = 0;
mediaType.bFixedSizeSamples = true;
mediaType.bTemporalCompression = false;
mediaType.formattype = FORMAT_WaveFormatEx;
memcpy( mediaType.pbFormat, &waveFormat, sizeof( WAVEFORMATEX ) );
pMediaObject->SetOutputType( 0, &mediaType, 0 );
KinectAudioStream* audioStream = new KinectAudioStream( pMediaObject );
IStream* pStream = nullptr;
audioStream->QueryInterface( IID_IStream, reinterpret_cast<void**>( &pStream ) );
CoInitialize( nullptr );
ISpStream* pSpeechStream = nullptr;
CoCreateInstance( CLSID_SpStream, NULL, CLSCTX_INPROC_SERVER, __uuidof(ISpStream), reinterpret_cast<void**>( &pSpeechStream ) );
pSpeechStream->SetBaseStream( pStream, SPDFID_WaveFormatEx, &waveFormat );
MoFreeMediaType( &mediaType );
pStream->Release();
pPropertyStore->Release();
pMediaObject->Release();
pNuiAudioSource->Release();
// 音声認識器を作成(CreateSpeechRecognizer)
std::cout << "CreateSpeechRecognizer" << std::endl;
ISpRecognizer* pSpeechRecognizer;
CoCreateInstance( CLSID_SpInprocRecognizer, nullptr, CLSCTX_INPROC_SERVER, __uuidof(ISpRecognizer), reinterpret_cast<void**>( &pSpeechRecognizer ) );
pSpeechRecognizer->SetInput( pSpeechStream, false );
/*
// If can use ATL, easier to using SpFindBestToken(sphelper.h). When using Professional or more.
ISpObjectToken* pEngineToken = nullptr;
SpFindBestToken( SPCAT_RECOGNIZERS, L"Language=411;Kinect=True", NULL, &pEngineToken ); // Japanese "Language=411;Kinect=True" English "Language=409;Kinect=True"
*/
///*
// If can't use ATL, alternative to using SpFIndBestToken(sphelper.h). When using Express.
const wchar_t* pVendorPreferred = L"VendorPreferred";
const unsigned long lengthVendorPreferred = static_cast<unsigned long>( wcslen( pVendorPreferred ) );
unsigned long length;
ULongAdd( lengthVendorPreferred, 1, &length );
wchar_t* pAttribsVendorPreferred = new wchar_t[ length ];
StringCchCopyW( pAttribsVendorPreferred, length, pVendorPreferred );
ISpObjectTokenCategory* pTokenCategory = nullptr;
CoCreateInstance( CLSID_SpObjectTokenCategory, nullptr, CLSCTX_ALL, __uuidof(ISpObjectTokenCategory), reinterpret_cast<void**>( &pTokenCategory ) );
pTokenCategory->SetId( SPCAT_RECOGNIZERS, false );
IEnumSpObjectTokens* pEnumTokens = nullptr;
CoCreateInstance( CLSID_SpMMAudioEnum, nullptr, CLSCTX_ALL, __uuidof(IEnumSpObjectTokens), reinterpret_cast<void**>( &pEnumTokens ) );
pTokenCategory->EnumTokens( L"Language=411;Kinect=True", pAttribsVendorPreferred, &pEnumTokens ); // Japanese "Language=411;Kinect=True" English "Language=409;Kinect=True"
delete[] pAttribsVendorPreferred;
ISpObjectToken* pEngineToken = nullptr;
pEnumTokens->Next( 1, &pEngineToken, nullptr );
//*/
pSpeechRecognizer->SetRecognizer( pEngineToken );
ISpRecoContext* pSpeechContext;
pSpeechRecognizer->CreateRecoContext( &pSpeechContext );
pEngineToken->Release();
///*
pTokenCategory->Release();
pEnumTokens->Release();
//*/
// 音声認識辞書の作成(LoadSpeechGrammar)
std::cout << "LoadSpeechGrammar" << std::endl;
ISpRecoGrammar* pSpeechGrammar;
pSpeechContext->CreateGrammar( 1, &pSpeechGrammar );
pSpeechGrammar->LoadCmdFromFile( L"SpeechRecognition_Ja.grxml", /*SPLO_STATIC*/SPLO_DYNAMIC ); // http://www.w3.org/TR/speech-grammar/ (UTF-8/CRLF)
audioStream->StartCapture();
pSpeechGrammar->SetRuleState( nullptr, nullptr, SPRS_ACTIVE );
pSpeechRecognizer->SetRecoState( SPRST_ACTIVE_ALWAYS );
pSpeechContext->SetInterest( SPFEI( SPEI_RECOGNITION ), SPFEI( SPEI_RECOGNITION ) );
pSpeechContext->Resume( 0 );
HANDLE hSpeechEvent = INVALID_HANDLE_VALUE;
hSpeechEvent = pSpeechContext->GetNotifyEventHandle();
HANDLE hEvents[1] = { hSpeechEvent };
int width = 640;
int height = 480;
cv::Mat audioMat = cv::Mat::zeros( height, width, CV_8UC3 );
cv::namedWindow( "Audio" );
bool exit = false;
std::cout << std::endl << "Speech Recognition Start..." << std::endl << std::endl;
while( 1 ){
// イベントの更新待ち
ResetEvent( hSpeechEvent );
unsigned long waitObject = MsgWaitForMultipleObjectsEx( ARRAYSIZE( hEvents ), hEvents, INFINITE, QS_ALLINPUT, MWMO_INPUTAVAILABLE );
if( waitObject == WAIT_OBJECT_0 ){
// イベントの取得
const float confidenceThreshold = 0.3f;
SPEVENT eventStatus;
unsigned long eventFetch = 0;
pSpeechContext->GetEvents( 1, &eventStatus, &eventFetch );
while( eventFetch > 0 ){
switch( eventStatus.eEventId ){
// 音声認識イベント(SPEI_HYPOTHESIS:推定またはSPEI_RECOGNITION:認識)
case SPEI_HYPOTHESIS:
case SPEI_RECOGNITION:
if( eventStatus.elParamType == SPET_LPARAM_IS_OBJECT ){
// フレーズの取得
ISpRecoResult* pRecoResult = reinterpret_cast<ISpRecoResult*>( eventStatus.lParam );
SPPHRASE* pPhrase = nullptr;
hResult = pRecoResult->GetPhrase( &pPhrase );
if( SUCCEEDED( hResult ) ){
if( ( pPhrase->pProperties != nullptr ) && ( pPhrase->pProperties->pFirstChild != nullptr ) ){
// 辞書のフレーズタグと比較
const SPPHRASEPROPERTY* pSemantic = pPhrase->pProperties->pFirstChild;
if( pSemantic->SREngineConfidence > confidenceThreshold ){
if( wcscmp( L"あか", pSemantic->pszValue ) == 0 ){
std::cout << "あか" << std::endl;
audioMat = cv::Scalar( 0, 0, 255 );
}
else if( wcscmp( L"みどり", pSemantic->pszValue ) == 0 ){
std::cout << "みどり" << std::endl;
audioMat = cv::Scalar( 0, 255, 0 );
}
else if( wcscmp( L"あお", pSemantic->pszValue ) == 0 ){
std::cout << "あお" << std::endl;
audioMat = cv::Scalar( 255, 0, 0 );
}
else if( wcscmp( L"おわり", pSemantic->pszValue ) == 0 ){
exit = true;
}
}
}
CoTaskMemFree( pPhrase );
}
}
break;
default:
break;
}
pSpeechContext->GetEvents( 1, &eventStatus, &eventFetch );
}
}
// 表示
cv::imshow( "Audio", audioMat );
// ループの終了判定(Escキー)
if( cv::waitKey( 30 ) == VK_ESCAPE || exit ){
break;
}
}
// 終了処理
audioStream->StopCapture();
pSpeechRecognizer->SetRecoState( SPRST_INACTIVE );
CoUninitialize();
pSensor->NuiShutdown();
CloseHandle( hSpeechEvent );
cv::destroyAllWindows();
return 0;
}
int _tmain(int argc, _TCHAR* argv[])
{
cv::setUseOptimized( true );
// Kinectのインスタンス生成、初期化
INuiSensor* pSensor;
HRESULT hResult = S_OK;
hResult = NuiCreateSensorByIndex( 0, &pSensor );
if( FAILED( hResult ) ){
std::cerr << "Error : NuiCreateSensorByIndex" << std::endl;
return -1;
}
hResult = pSensor->NuiInitialize( NUI_INITIALIZE_FLAG_USES_COLOR | NUI_INITIALIZE_FLAG_USES_DEPTH_AND_PLAYER_INDEX | NUI_INITIALIZE_FLAG_USES_SKELETON );
if( FAILED( hResult ) ){
std::cerr << "Error : NuiInitialize" << std::endl;
return -1;
}
// Colorストリーム
HANDLE hColorEvent = INVALID_HANDLE_VALUE;
HANDLE hColorHandle = INVALID_HANDLE_VALUE;
hColorEvent = CreateEvent( nullptr, true, false, nullptr );
hResult = pSensor->NuiImageStreamOpen( NUI_IMAGE_TYPE_COLOR, NUI_IMAGE_RESOLUTION_640x480, 0, 2, hColorEvent, &hColorHandle );
if( FAILED( hResult ) ){
std::cerr << "Error : NuiImageStreamOpen( COLOR )" << std::endl;
return -1;
}
// Depth&Playerストリーム
HANDLE hDepthPlayerEvent = INVALID_HANDLE_VALUE;
HANDLE hDepthPlayerHandle = INVALID_HANDLE_VALUE;
hDepthPlayerEvent = CreateEvent( nullptr, true, false, nullptr );
hResult = pSensor->NuiImageStreamOpen( NUI_IMAGE_TYPE_DEPTH_AND_PLAYER_INDEX, NUI_IMAGE_RESOLUTION_640x480, 0, 2, hDepthPlayerEvent, &hDepthPlayerHandle );
if( FAILED( hResult ) ){
std::cerr << "Error : NuiImageStreamOpen( DEPTH&PLAYER )" << std::endl;
return -1;
}
// Skeletonストリーム
HANDLE hSkeletonEvent = INVALID_HANDLE_VALUE;
hSkeletonEvent = CreateEvent( nullptr, true, false, nullptr );
hResult = pSensor->NuiSkeletonTrackingEnable( hSkeletonEvent, 0 );
if( FAILED( hResult ) ){
std::cerr << "Error : NuiSkeletonTrackingEnable" << std::endl;
return -1;
}
HANDLE hEvents[3] = { hColorEvent, hDepthPlayerEvent, hSkeletonEvent };
// カラーテーブル
cv::Vec3b color[7];
color[0] = cv::Vec3b( 0, 0, 0 );
color[1] = cv::Vec3b( 255, 0, 0 );
color[2] = cv::Vec3b( 0, 255, 0 );
color[3] = cv::Vec3b( 0, 0, 255 );
color[4] = cv::Vec3b( 255, 255, 0 );
color[5] = cv::Vec3b( 255, 0, 255 );
color[6] = cv::Vec3b( 0, 255, 255 );
cv::namedWindow( "Color" );
cv::namedWindow( "Depth" );
cv::namedWindow( "Player" );
cv::namedWindow( "Skeleton" );
while( 1 ){
// フレームの更新待ち
ResetEvent( hColorEvent );
ResetEvent( hDepthPlayerEvent );
ResetEvent( hSkeletonEvent );
WaitForMultipleObjects( ARRAYSIZE( hEvents ), hEvents, true, INFINITE );
// Colorカメラからフレームを取得
NUI_IMAGE_FRAME pColorImageFrame = { 0 };
hResult = pSensor->NuiImageStreamGetNextFrame( hColorHandle, 0, &pColorImageFrame );
if( FAILED( hResult ) ){
std::cerr << "Error : NuiImageStreamGetNextFrame( COLOR )" << std::endl;
return -1;
}
// Depthセンサーからフレームを取得
NUI_IMAGE_FRAME pDepthPlayerImageFrame = { 0 };
hResult = pSensor->NuiImageStreamGetNextFrame( hDepthPlayerHandle, 0, &pDepthPlayerImageFrame );
if( FAILED( hResult ) ){
std::cerr << "Error : NuiImageStreamGetNextFrame( DEPTH&PLAYER )" << std::endl;
return -1;
}
// Skeletonフレームを取得
NUI_SKELETON_FRAME pSkeletonFrame = { 0 };
hResult = pSensor->NuiSkeletonGetNextFrame( 0, &pSkeletonFrame );
if( FAILED( hResult ) ){
std::cout << "Error : NuiSkeletonGetNextFrame" << std::endl;
return -1;
}
// Color画像データの取得
INuiFrameTexture* pColorFrameTexture = pColorImageFrame.pFrameTexture;
NUI_LOCKED_RECT sColorLockedRect;
pColorFrameTexture->LockRect( 0, &sColorLockedRect, nullptr, 0 );
// Depthデータの取得
INuiFrameTexture* pDepthPlayerFrameTexture = pDepthPlayerImageFrame.pFrameTexture;
NUI_LOCKED_RECT sDepthPlayerLockedRect;
pDepthPlayerFrameTexture->LockRect( 0, &sDepthPlayerLockedRect, nullptr, 0 );
// 表示
cv::Mat colorMat( 480, 640, CV_8UC4, reinterpret_cast<uchar*>( sColorLockedRect.pBits ) );
LONG registX = 0;
LONG registY = 0;
ushort* pBuffer = reinterpret_cast<ushort*>( sDepthPlayerLockedRect.pBits );
cv::Mat bufferMat = cv::Mat::zeros( 480, 640, CV_16UC1 );
cv::Mat playerMat = cv::Mat::zeros( 480, 640, CV_8UC3 );
for( int y = 0; y < 480; y++ ){
for( int x = 0; x < 640; x++ ){
pSensor->NuiImageGetColorPixelCoordinatesFromDepthPixelAtResolution( NUI_IMAGE_RESOLUTION_640x480, NUI_IMAGE_RESOLUTION_640x480, nullptr, x, y, *pBuffer, ®istX, ®istY );
if( ( registX >= 0 ) && ( registX < 640 ) && ( registY >= 0 ) && ( registY < 480 ) ){
bufferMat.at<ushort>( registY, registX ) = *pBuffer & 0xFFF8;
playerMat.at<cv::Vec3b>( registY, registX ) = color[*pBuffer & 0x7];
}
pBuffer++;
}
}
cv::Mat depthMat( 480, 640, CV_8UC1 );
bufferMat.convertTo( depthMat, CV_8UC3, -255.0f / NUI_IMAGE_DEPTH_MAXIMUM, 255.0f );
cv::Mat skeletonMat = cv::Mat::zeros( 480, 640, CV_8UC3 );
cv::Point2f point;
for( int count = 0; count < NUI_SKELETON_COUNT; count++ ){
NUI_SKELETON_DATA skeleton = pSkeletonFrame.SkeletonData[count];
if( skeleton.eTrackingState == NUI_SKELETON_TRACKED ){
for( int position = 0; position < NUI_SKELETON_POSITION_COUNT; position++ ){
NuiTransformSkeletonToDepthImage( skeleton.SkeletonPositions[position], &point.x, &point.y, NUI_IMAGE_RESOLUTION_640x480 );
cv::circle( skeletonMat, point, 10, static_cast<cv::Scalar>( color[count + 1] ), -1, CV_AA );
}
}
}
cv::imshow( "Color", colorMat );
cv::imshow( "Depth", depthMat );
cv::imshow( "Player", playerMat );
cv::imshow( "Skeleton", skeletonMat );
// フレームの解放
pColorFrameTexture->UnlockRect( 0 );
pDepthPlayerFrameTexture->UnlockRect( 0 );
pSensor->NuiImageStreamReleaseFrame( hColorHandle, &pColorImageFrame );
pSensor->NuiImageStreamReleaseFrame( hDepthPlayerHandle, &pDepthPlayerImageFrame );
// ループの終了判定(Escキー)
if( cv::waitKey( 30 ) == VK_ESCAPE ){
break;
}
}
// Kinectの終了処理
pSensor->NuiShutdown();
pSensor->NuiSkeletonTrackingDisable();
CloseHandle( hColorEvent );
CloseHandle( hDepthPlayerEvent );
CloseHandle( hSkeletonEvent );
CloseHandle( hColorHandle );
CloseHandle( hDepthPlayerHandle );
cv::destroyAllWindows();
return 0;
}
int _tmain(int argc, _TCHAR* argv[])
{
cv::setUseOptimized( true );
INuiSensor* pSensor;
HRESULT hResult = S_OK;
hResult = NuiCreateSensorByIndex( 0, &pSensor );
if( FAILED( hResult ) ){
std::cerr << "Error : NuiCreateSensorByIndex" << std::endl;
return -1;
}
hResult = pSensor->NuiInitialize( NUI_INITIALIZE_FLAG_USES_COLOR );
if( FAILED( hResult ) ){
std::cerr << "Error : NuiInitialize" << std::endl;
return -1;
}
HANDLE hColorEvent = INVALID_HANDLE_VALUE;
HANDLE hColorHandle = INVALID_HANDLE_VALUE;
hColorEvent = CreateEvent( nullptr, true, false, nullptr );
hResult = pSensor->NuiImageStreamOpen( NUI_IMAGE_TYPE_COLOR, NUI_IMAGE_RESOLUTION_640x480, 0, 2, hColorEvent, &hColorHandle );
if( FAILED( hResult ) ){
std::cerr << "Error : NuiImageStreamOpen( COLOR )" << std::endl;
return -1;
}
INuiColorCameraSettings* pCameraSettings;
hResult = pSensor->NuiGetColorCameraSettings( &pCameraSettings );
if( FAILED( hResult ) ){
std::cerr << "Error : NuiGetColorCameraSettings" << std::endl;
return -1;
}
unsigned long refWidth = 0;
unsigned long refHeight = 0;
NuiImageResolutionToSize( NUI_IMAGE_RESOLUTION_640x480, refWidth, refHeight );
int width = static_cast<int>( refWidth );
int height = static_cast<int>( refHeight );
HANDLE hEvents[1] = { hColorEvent };
cv::namedWindow( "Color" );
while( 1 ){
ResetEvent( hColorEvent );
WaitForMultipleObjects( ARRAYSIZE( hEvents ), hEvents, true, INFINITE );
NUI_IMAGE_FRAME colorImageFrame = { 0 };
hResult = pSensor->NuiImageStreamGetNextFrame( hColorHandle, 0, &colorImageFrame );
if( FAILED( hResult ) ){
std::cerr << "Error : NuiImageStreamGetNextFrame( COLOR )" << std::endl;
return -1;
}
INuiFrameTexture* pColorFrameTexture = colorImageFrame.pFrameTexture;
NUI_LOCKED_RECT colorLockedRect;
pColorFrameTexture->LockRect( 0, &colorLockedRect, nullptr, 0 );
cv::Mat colorMat( height, width, CV_8UC4, reinterpret_cast<unsigned char*>( colorLockedRect.pBits ) );
cv::imshow( "Color", colorMat );
pColorFrameTexture->UnlockRect( 0 );
pSensor->NuiImageStreamReleaseFrame( hColorHandle, &colorImageFrame );
int key = cv::waitKey( 30 );
if( key == VK_ESCAPE ){
break;
}
}
pSensor->NuiShutdown();
pCameraSettings->Release();
CloseHandle( hColorEvent );
cv::destroyAllWindows();
return 0;
}