/// <summary>
/// Create speech recognizer that will read Kinect audio stream data.
/// </summary>
/// <returns>
/// <para>S_OK on success, otherwise failure code.</para>
/// </returns>
HRESULT KinectReader::CreateSpeechRecognizer()
{
ISpObjectToken *pEngineToken = NULL;
HRESULT hr = CoCreateInstance(CLSID_SpInprocRecognizer, NULL, CLSCTX_INPROC_SERVER, __uuidof(ISpRecognizer), (void**)&m_pSpeechRecognizer);
if (SUCCEEDED(hr))
{
m_pSpeechRecognizer->SetInput(m_pSpeechStream, FALSE);
hr = SpFindBestToken(SPCAT_RECOGNIZERS,L"Language=409;Kinect=True",NULL,&pEngineToken);
if (SUCCEEDED(hr))
{
m_pSpeechRecognizer->SetRecognizer(pEngineToken);
hr = m_pSpeechRecognizer->CreateRecoContext(&m_pSpeechContext);
// For long recognition sessions (a few hours or more), it may be beneficial to turn off adaptation of the acoustic model.
// This will prevent recognition accuracy from degrading over time.
//if (SUCCEEDED(hr))
//{
// hr = m_pSpeechRecognizer->SetPropertyNum(L"AdaptationOn", 0);
//}
}
}
SafeRelease(pEngineToken);
return hr;
}
HRESULT KinectSpeechRecognizer::CreateRecognizer()
{
ISpObjectToken *engineToken = NULL;
//Create singleton instance of Speech Recognizer
HRESULT hr = CoCreateInstance(CLSID_SpInprocRecognizer, NULL, CLSCTX_INPROC_SERVER, __uuidof(ISpRecognizer), (void**)&speechRecognizer);
if (SUCCEEDED(hr))
{
speechRecognizer->SetInput(speechStream, TRUE);
// If this fails here, you have not installed the acoustic models for Kinect
hr = SpFindBestToken(SPCAT_RECOGNIZERS, L"Language=409;Kinect=True", NULL, &engineToken);
if (SUCCEEDED(hr))
{
speechRecognizer->SetRecognizer(engineToken);
hr = speechRecognizer->CreateRecoContext(&speechContext);
// For long recognition sessions (a few hours or more), it may be beneficial to turn off adaptation of the acoustic model.
// This will prevent recognition accuracy from degrading over time.
if (SUCCEEDED(hr))
{
hr = speechRecognizer->SetPropertyNum(L"AdaptationOn", 0);
}
}
}
//Release the engine token
if(engineToken)
{
engineToken->Release();
engineToken = NULL;
}
return hr;
}
mssapi_captions::mssapi_captions(
captions_cb callback,
const std::string &lang) try
: captions_handler(callback, AUDIO_FORMAT_16BIT, 16000)
{
HRESULT hr;
std::wstring wlang;
wlang.resize(lang.size());
for (size_t i = 0; i < lang.size(); i++)
wlang[i] = (wchar_t)lang[i];
LCID lang_id = LocaleNameToLCID(wlang.c_str(), 0);
wchar_t lang_str[32];
_snwprintf(lang_str, 31, L"language=%x", (int)lang_id);
stop = CreateEvent(nullptr, false, false, nullptr);
if (!stop.Valid())
throw "Failed to create event";
hr = SpFindBestToken(SPCAT_RECOGNIZERS, lang_str, nullptr, &token);
if (FAILED(hr))
throw HRError("SpFindBestToken failed", hr);
hr = CoCreateInstance(CLSID_SpInprocRecognizer, nullptr, CLSCTX_ALL,
__uuidof(ISpRecognizer), (void**)&recognizer);
if (FAILED(hr))
throw HRError("CoCreateInstance for recognizer failed", hr);
hr = recognizer->SetRecognizer(token);
if (FAILED(hr))
throw HRError("SetRecognizer failed", hr);
hr = recognizer->SetRecoState(SPRST_INACTIVE);
if (FAILED(hr))
throw HRError("SetRecoState(SPRST_INACTIVE) failed", hr);
hr = recognizer->CreateRecoContext(&context);
if (FAILED(hr))
throw HRError("CreateRecoContext failed", hr);
ULONGLONG interest = SPFEI(SPEI_RECOGNITION) |
SPFEI(SPEI_END_SR_STREAM);
hr = context->SetInterest(interest, interest);
if (FAILED(hr))
throw HRError("SetInterest failed", hr);
hr = context->SetNotifyWin32Event();
if (FAILED(hr))
throw HRError("SetNotifyWin32Event", hr);
notify = context->GetNotifyEventHandle();
if (notify == INVALID_HANDLE_VALUE)
throw HRError("GetNotifyEventHandle failed", E_NOINTERFACE);
size_t sample_rate = audio_output_get_sample_rate(obs_get_audio());
audio = new CaptionStream((DWORD)sample_rate, this);
audio->Release();
hr = recognizer->SetInput(audio, false);
if (FAILED(hr))
throw HRError("SetInput failed", hr);
hr = context->CreateGrammar(1, &grammar);
if (FAILED(hr))
throw HRError("CreateGrammar failed", hr);
hr = grammar->LoadDictation(nullptr, SPLO_STATIC);
if (FAILED(hr))
throw HRError("LoadDictation failed", hr);
try {
t = std::thread([this] () {main_thread();});
} catch (...) {
throw "Failed to create thread";
}
} catch (const char *err) {
blog(LOG_WARNING, "%s: %s", __FUNCTION__, err);
throw CAPTIONS_ERROR_GENERIC_FAIL;
} catch (HRError err) {
blog(LOG_WARNING, "%s: %s (%lX)", __FUNCTION__, err.str, err.hr);
throw CAPTIONS_ERROR_GENERIC_FAIL;
}
// 初始化语音识别
HRESULT ThisApp::init_speech_recognizer(){
HRESULT hr = S_OK;
// 创建语音输入流
if (SUCCEEDED(hr)){
hr = CoCreateInstance(CLSID_SpStream, nullptr, CLSCTX_INPROC_SERVER, __uuidof(ISpStream), (void**)&m_pSpeechStream);;
}
// 与我们的Kinect语音输入相连接
if (SUCCEEDED(hr)){
WAVEFORMATEX wft = {
WAVE_FORMAT_PCM, // PCM编码
1, // 单声道
16000, // 采样率为16KHz
32000, // 每分钟数据流 = 采样率 * 对齐
2, // 对齐 : 单声道 * 样本深度 = 2byte
16, // 样本深度 16BIT
0 // 额外数据
};
// 设置状态
hr = m_pSpeechStream->SetBaseStream(m_p16BitPCMAudioStream, SPDFID_WaveFormatEx, &wft);
}
// 创建语音识别对象
if (SUCCEEDED(hr)){
ISpObjectToken *pEngineToken = nullptr;
// 创建语言识别器
hr = CoCreateInstance(CLSID_SpInprocRecognizer, nullptr, CLSCTX_INPROC_SERVER, __uuidof(ISpRecognizer), (void**)&m_pSpeechRecognizer);
if (SUCCEEDED(hr)) {
// 连接我们创建的语音输入流对象
m_pSpeechRecognizer->SetInput(m_pSpeechStream, TRUE);
// 创建待识别语言 这里选择大陆汉语(zh-cn)
// 目前没有Kinect的汉语语音识别包 有的话可以设置"language=804;Kinect=Ture"
hr = SpFindBestToken(SPCAT_RECOGNIZERS, L"Language=804", nullptr, &pEngineToken);
if (SUCCEEDED(hr)) {
// 设置待识别语言
m_pSpeechRecognizer->SetRecognizer(pEngineToken);
// 创建语音识别上下文
hr = m_pSpeechRecognizer->CreateRecoContext(&m_pSpeechContext);
// 适应性 ON! 防止因长时间的处理而导致识别能力的退化
if (SUCCEEDED(hr)) {
hr = m_pSpeechRecognizer->SetPropertyNum(L"AdaptationOn", 0);
}
}
}
SafeRelease(pEngineToken);
}
// 创建语法
if (SUCCEEDED(hr)){
hr = m_pSpeechContext->CreateGrammar(1, &m_pSpeechGrammar);
}
// 加载静态SRGS语法文件
if (SUCCEEDED(hr)){
hr = m_pSpeechGrammar->LoadCmdFromFile(s_GrammarFileName, SPLO_STATIC);
}
// 激活语法规则
if (SUCCEEDED(hr)){
hr = m_pSpeechGrammar->SetRuleState(nullptr, nullptr, SPRS_ACTIVE);
}
// 设置识别器一直读取数据
if (SUCCEEDED(hr)){
hr = m_pSpeechRecognizer->SetRecoState(SPRST_ACTIVE_ALWAYS);
}
// 设置对识别事件感兴趣
if (SUCCEEDED(hr)){
hr = m_pSpeechContext->SetInterest(SPFEI(SPEI_RECOGNITION), SPFEI(SPEI_RECOGNITION));
}
// 保证语音识别处于激活状态
if (SUCCEEDED(hr)){
hr = m_pSpeechContext->Resume(0);
}
// 获取识别事件
if (SUCCEEDED(hr)){
m_p16BitPCMAudioStream->SetSpeechState(TRUE);
m_hSpeechEvent = m_pSpeechContext->GetNotifyEventHandle();
printf_s("init_speech_recognizer succeeded\n");
}
#ifdef _DEBUG
else
printf_s("init_speech_recognizer failed\n");
#endif
return hr;
}
void Sound::test() {
ISpVoice * pVoice = NULL;
ISpObjectToken* pVoiceToken=nullptr;
IEnumSpObjectTokens* pEnum;
ULONG ulCount = 0;
if (FAILED(::CoInitialize(NULL)))
{
return;
}
HRESULT hr = S_OK;
// Find the best matching installed en-us recognizer.
CComPtr<ISpObjectToken> cpRecognizerToken;
if (SUCCEEDED(hr))
{
hr = SpFindBestToken(SPCAT_RECOGNIZERS, L"language=409", NULL, &cpRecognizerToken);
}
// Create the in-process recognizer and immediately set its state to inactive.
CComPtr<ISpRecognizer> cpRecognizer;
if (SUCCEEDED(hr))
{
hr = cpRecognizer.CoCreateInstance(CLSID_SpInprocRecognizer);
}
if (SUCCEEDED(hr))
{
hr = cpRecognizer->SetRecognizer(cpRecognizerToken);
}
if (SUCCEEDED(hr))
{
hr = cpRecognizer->SetRecoState(SPRST_INACTIVE);
}
// Create a new recognition context from the recognizer.
CComPtr<ISpRecoContext> cpContext;
if (SUCCEEDED(hr))
{
hr = cpRecognizer->CreateRecoContext(&cpContext);
}
// Subscribe to the speech recognition event and end stream event.
if (SUCCEEDED(hr))
{
ULONGLONG ullEventInterest = SPFEI(SPEI_RECOGNITION);
hr = cpContext->SetInterest(ullEventInterest, ullEventInterest);
}
// Establish a Win32 event to signal when speech events are available.
HANDLE hSpeechNotifyEvent = INVALID_HANDLE_VALUE;
if (SUCCEEDED(hr))
{
hr = cpContext->SetNotifyWin32Event();
}
if (SUCCEEDED(hr))
{
hSpeechNotifyEvent = cpContext->GetNotifyEventHandle();
if (INVALID_HANDLE_VALUE == hSpeechNotifyEvent)
{
// Notification handle unsupported.
hr = E_NOINTERFACE;
}
}
// Initialize an audio object to use the default audio input of the system and set the recognizer to use it.
CComPtr<ISpAudio> cpAudioIn;
if (SUCCEEDED(hr))
{
hr = cpAudioIn.CoCreateInstance(CLSID_SpMMAudioIn);
}
if (SUCCEEDED(hr))
{
hr = cpRecognizer->SetInput(cpAudioIn, TRUE);
}
// Populate a WAVEFORMATEX struct with our desired output audio format. information.
WAVEFORMATEX* pWfexCoMemRetainedAudioFormat = NULL;
GUID guidRetainedAudioFormat = GUID_NULL;
if (SUCCEEDED(hr))
{
hr = SpConvertStreamFormatEnum(SPSF_16kHz16BitMono, &guidRetainedAudioFormat, &pWfexCoMemRetainedAudioFormat);
}
// Instruct the recognizer to retain the audio from its recognition results.
if (SUCCEEDED(hr))
{
hr = cpContext->SetAudioOptions(SPAO_RETAIN_AUDIO, &guidRetainedAudioFormat, pWfexCoMemRetainedAudioFormat);
}
if (NULL != pWfexCoMemRetainedAudioFormat)
{
CoTaskMemFree(pWfexCoMemRetainedAudioFormat);
}
// Create a new grammar and load an SRGS grammar from file.
CComPtr<ISpRecoGrammar> cpGrammar;
if (SUCCEEDED(hr))
{
hr = cpContext->CreateGrammar(0, &cpGrammar);
}
if (SUCCEEDED(hr))
{
hr = cpGrammar->LoadCmdFromFile(L"grammar.grxml", SPLO_STATIC);
}
// Set all top-level rules in the new grammar to the active state.
if (SUCCEEDED(hr))
{
hr = cpGrammar->SetRuleState(NULL, NULL, SPRS_ACTIVE);
}
// Set the recognizer state to active to begin recognition.
if (SUCCEEDED(hr))
{
hr = cpRecognizer->SetRecoState(SPRST_ACTIVE_ALWAYS);
}
// Establish a separate Win32 event to signal the event loop exit.
HANDLE hExitEvent = CreateEventW(NULL, FALSE, FALSE, NULL);
// Collect the events listened for to pump the speech event loop.
HANDLE rghEvents[] = { hSpeechNotifyEvent, hExitEvent };
// Speech recognition event loop.
BOOL fContinue = TRUE;
while (fContinue && SUCCEEDED(hr))
{
// Wait for either a speech event or an exit event, with a 15 second timeout.
DWORD dwMessage = WaitForMultipleObjects(sp_countof(rghEvents), rghEvents, FALSE, 15000);
switch (dwMessage)
{
// With the WaitForMultipleObjects call above, WAIT_OBJECT_0 is a speech event from hSpeechNotifyEvent.
case WAIT_OBJECT_0:
{
// Sequentially grab the available speech events from the speech event queue.
CSpEvent spevent;
while (S_OK == spevent.GetFrom(cpContext))
{
switch (spevent.eEventId)
{
case SPEI_RECOGNITION:
{
// Retrieve the recognition result and output the text of that result.
ISpRecoResult* pResult = spevent.RecoResult();
LPWSTR pszCoMemResultText = NULL;
hr = pResult->GetText(SP_GETWHOLEPHRASE, SP_GETWHOLEPHRASE, TRUE, &pszCoMemResultText, NULL);
if (SUCCEEDED(hr))
{
wprintf(L"Recognition event received, text=\"%s\"\r\n", pszCoMemResultText);
}
// Also retrieve the retained audio we requested.
CComPtr<ISpStreamFormat> cpRetainedAudio;
if (SUCCEEDED(hr))
{
hr = pResult->GetAudio(0, 0, &cpRetainedAudio);
}
// To demonstrate, we'll speak the retained audio back using ISpVoice.
CComPtr<ISpVoice> cpVoice;
if (SUCCEEDED(hr))
{
hr = cpVoice.CoCreateInstance(CLSID_SpVoice);
}
if (SUCCEEDED(hr))
{
hr = cpVoice->SpeakStream(cpRetainedAudio, SPF_DEFAULT, 0);
}
if (NULL != pszCoMemResultText)
{
CoTaskMemFree(pszCoMemResultText);
}
break;
}
}
}
break;
}
case WAIT_OBJECT_0 + 1:
case WAIT_TIMEOUT:
{
// Exit event or timeout; discontinue the speech loop.
fContinue = FALSE;
//break;
}
}
}
CoUninitialize();
CComPtr <ISpVoice> cpVoice;
CComPtr <ISpStream> cpStream;
CSpStreamFormat cAudioFmt;
//Create a SAPI Voice
hr = cpVoice.CoCreateInstance(CLSID_SpVoice);
//Set the audio format
if (SUCCEEDED(hr))
{
hr = cAudioFmt.AssignFormat(SPSF_22kHz16BitMono);
}
//Call SPBindToFile, a SAPI helper method, to bind the audio stream to the file
if (SUCCEEDED(hr))
{
hr = SPBindToFile(L"c:\\ttstemp.wav", SPFM_CREATE_ALWAYS,
&cpStream, &cAudioFmt.FormatId(), cAudioFmt.WaveFormatExPtr());
}
//set the output to cpStream so that the output audio data will be stored in cpStream
if (SUCCEEDED(hr))
{
hr = cpVoice->SetOutput(cpStream, TRUE);
}
//Speak the text "hello world" synchronously
if (SUCCEEDED(hr))
{
hr = cpVoice->Speak(L"Hello World", SPF_DEFAULT, NULL);
}
//close the stream
if (SUCCEEDED(hr))
{
hr = cpStream->Close();
}
//Release the stream and voice object
cpStream.Release();
cpVoice.Release();
CComPtr<ISpGrammarBuilder> cpGrammarBuilder;
SPSTATEHANDLE hStateTravel;
// Create (if rule does not already exist)
// top-level Rule, defaulting to Active.
hr = cpGrammarBuilder->GetRule(L"Travel", 0, SPRAF_TopLevel | SPRAF_Active, TRUE, &hStateTravel);
// Approach 1: List all possible phrases.
// This is the most intuitive approach, and it does not sacrifice efficiency
// because the grammar builder will merge shared sub-phrases when possible.
// There is only one root state, hStateTravel, and the terminal NULL state,
// and there are six unique transitions between root state and NULL state.
/* XML Approximation:
<rule id="Travel">
<item> fly to Seattle </item>
<item> fly to New York </item>
<item> fly to Washington DC </item>
<item> drive to Seattle </item>
<item> drive to New York </item>
<item> drive to Washington DC </item>
</rule>
*/
// Create set of peer phrases, each containing complete phrase.
// Note: the word delimiter is set as " ", so that the text we
// attach to the transition can be multiple words (for example,
// "fly to Seattle" is implicitly "fly" + "to" + "Seattle"):
if (SUCCEEDED(hr))
{
hr = cpGrammarBuilder->AddWordTransition(hStateTravel, NULL, L"fly to Seattle", L" ", SPWT_LEXICAL, 1, NULL);
}
if (SUCCEEDED(hr))
{
hr = cpGrammarBuilder->AddWordTransition(hStateTravel, NULL, L"fly to New York", L" ", SPWT_LEXICAL, 1, NULL);
}
if (SUCCEEDED(hr))
{
hr = cpGrammarBuilder->AddWordTransition(hStateTravel, NULL, L"fly to Washington DC", L" ", SPWT_LEXICAL, 1, NULL);
}
if (SUCCEEDED(hr))
{
hr = cpGrammarBuilder->AddWordTransition(hStateTravel, NULL, L"drive to Seattle", L" ", SPWT_LEXICAL, 1, NULL);
}
if (SUCCEEDED(hr))
{
hr = cpGrammarBuilder->AddWordTransition(hStateTravel, NULL, L"drive to New York", L" ", SPWT_LEXICAL, 1, NULL);
}
if (SUCCEEDED(hr))
{
hr = cpGrammarBuilder->AddWordTransition(hStateTravel, NULL, L"drive to Washington DC", L" ", SPWT_LEXICAL, 1, NULL);
}
// Find the best matching installed en-US recognizer.
//CComPtr<ISpObjectToken> cpRecognizerToken;
if (SUCCEEDED(hr))
{
hr = SpFindBestToken(SPCAT_RECOGNIZERS, L"language=409", NULL, &cpRecognizerToken);
}
// Create the in-process recognizer and immediately set its state to inactive.
//CComPtr<ISpRecognizer> cpRecognizer;
if (SUCCEEDED(hr))
{
hr = cpRecognizer.CoCreateInstance(CLSID_SpInprocRecognizer);
}
if (SUCCEEDED(hr))
{
hr = cpRecognizer->SetRecognizer(cpRecognizerToken);
}
if (SUCCEEDED(hr))
{
hr = cpRecognizer->SetRecoState(SPRST_INACTIVE);
}
// Create a new recognition context from the recognizer.
//CComPtr<ISpRecoContext> cpContext;
if (SUCCEEDED(hr))
{
hr = cpRecognizer->CreateRecoContext(&cpContext);
}
// Subscribe to the speech recognition event and end stream event.
if (SUCCEEDED(hr))
{
ULONGLONG ullEventInterest = SPFEI(SPEI_RECOGNITION) | SPFEI(SPEI_END_SR_STREAM);
hr = cpContext->SetInterest(ullEventInterest, ullEventInterest);
}
// Establish a Win32 event to signal when speech events are available.
//HANDLE hSpeechNotifyEvent = INVALID_HANDLE_VALUE;
if (SUCCEEDED(hr))
{
hr = cpContext->SetNotifyWin32Event();
}
if (SUCCEEDED(hr))
{
hr = cpContext->SetNotifyWin32Event();
}
if (SUCCEEDED(hr))
{
hSpeechNotifyEvent = cpContext->GetNotifyEventHandle();
if (INVALID_HANDLE_VALUE == hSpeechNotifyEvent)
{
// Notification handle unsupported
//hr = SPERR_UNITIALIZED;
}
}
// Set up an audio input stream using a .wav file and set the recognizer's input.
CComPtr<ISpStream> cpInputStream;
if (SUCCEEDED(hr))
{
hr = SPBindToFile(L"Test.wav", SPFM_OPEN_READONLY, &cpInputStream);
}
if (SUCCEEDED(hr))
{
hr = cpRecognizer->SetInput(cpInputStream, TRUE);
}
// Create a new grammar and load an SRGS grammar from file.
//CComPtr<ISpRecoGrammar> cpGrammar;
if (SUCCEEDED(hr))
{
hr = cpContext->CreateGrammar(0, &cpGrammar);
}
if (SUCCEEDED(hr))
{
hr = cpGrammar->LoadCmdFromFile(L"grammar.grxml", SPLO_STATIC);
}
// Set all top-level rules in the new grammar to the active state.
if (SUCCEEDED(hr))
{
hr = cpGrammar->SetRuleState(NULL, NULL, SPRS_ACTIVE);
}
// Finally, set the recognizer state to active to begin recognition.
if (SUCCEEDED(hr))
{
hr = cpRecognizer->SetRecoState(SPRST_ACTIVE_ALWAYS);
}
hr = CoCreateInstance(CLSID_SpVoice, NULL, CLSCTX_ALL, IID_ISpVoice, (void **)&pVoice);
if (SUCCEEDED(hr)) {
hr = SpEnumTokens(SPCAT_VOICES, L"Gender=Female", NULL, &pEnum);
if (SUCCEEDED(hr))
{
// Get the number of voices.
hr = pEnum->GetCount(&ulCount);
}
// Obtain a list of available voice tokens, set
// the voice to the token, and call Speak.
while (SUCCEEDED(hr) && ulCount--) {
if (pVoiceToken != nullptr) {
pVoiceToken->Release();
}
if (SUCCEEDED(hr))
{
hr = pEnum->Next(1, &pVoiceToken, NULL);
}
if (SUCCEEDED(hr))
{
hr = pVoice->SetVoice(pVoiceToken);
}
if (SUCCEEDED(hr))
{
wchar_t* start = L"<?xml version=\"1.0\" encoding=\"ISO - 8859 - 1\"?><speak version = \"1.0\" xmlns = \"http://www.w3.org/2001/10/synthesis\" xml:lang = \"en-US\">";
wchar_t* end = L"</speak>";
const wchar_t *xml = L"<voice required = \"Gender=Male\"> hi! <prosody pitch=\"fast\"> This is low pitch. </prosody><prosody volume=\"x - loud\"> This is extra loud volume. </prosody>";
wstring s = start;
s += xml;
s += end;
hr = pVoice->Speak(xml, SPF_IS_XML| SPF_ASYNC, 0);
//hr = pVoice->Speak(L"How are you?", SPF_DEFAULT, NULL);
}
}
/*
if (SUCCEEDED(hr)) {
hr = pEnum->Next(1, &pVoiceToken, NULL);
if (SUCCEEDED(hr)) {
hr = pVoice->SetVoice(pVoiceToken);
// Set the output to the default audio device.
if (SUCCEEDED(hr)) {
hr = pVoice->SetOutput(NULL, TRUE);
if (SUCCEEDED(hr)) {
hr = pVoice->Speak(L"Hello, world!", SPF_DEFAULT, 0);
}
}
}
}
*/
pVoice->Release();
}
::CoUninitialize();
}
