/**
* <EN>
* @brief Top function to start input processing
*
* If threading mode is enabled, this function simply enters to
* adin_thread_process() to process triggered samples detected by
* another running A/D-in thread.
*
* If threading mode is not available or disabled by either device requirement
* or OS capability, this function simply calls adin_cut() to detect speech
* segment from input device and process them concurrently by one process.
* </EN>
* <JA>
* @brief 入力処理を行うトップ関数
*
* スレッドモードでは，この関数は adin_thead_process() を呼び出し，
* 非スレッドモードでは adin_cut() を直接呼び出す. 引数や返り値は
* adin_cut() と同一である.
* </JA>
*
* @param ad_process [in] function to process triggerted input.
* @param ad_check [in] function to be called periodically.
* @param recog [in] engine instance
*
* @return 2 when input termination requested by ad_process(), 1 when
* if detect end of an input segment (down trigger detected after up
* trigger), 0 when reached end of input device, -1 on error, -2 when
* input termination requested by ad_check().
*
* @callergraph
* @callgraph
*
*/
int
adin_go(int (*ad_process)(SP16 *, int, Recog *), int (*ad_check)(Recog *), Recog *recog)
{
/* output listening start message */
callback_exec(CALLBACK_EVENT_SPEECH_READY, recog);
#ifdef HAVE_PTHREAD
if (recog->adin->enable_thread) {
return(adin_thread_process(ad_process, ad_check, recog));
}
#endif
return(adin_cut(ad_process, ad_check, recog));
}
/**
* <JA>
* @brief GMMの計算を終了し，結果を出力する.
*
* gmm_proceed() によって累積された各フレームごとのスコアから，
* 最大スコアのGMMを決定する. その事後確率に基づく信頼度を計算し
* 最終的な結果を result_gmm() によって出力する.
*
* @param recog [i/o] エンジンインスタンス
* </JA>
* <EN>
* @brief Finish the GMM computation for an input, and output the result.
*
* The GMM of the maximum score is finally determined from the accumulated
* scores computed by gmm_proceed(), and compute the confidence score of the
* maximum GMM using posterior probability. Then the result will be output
* using result_gmm().
*
* @param recog [i/o] engine instance
* </EN>
*
* @callgraph
* @callergraph
*/
void
gmm_end(Recog *recog)
{
HTK_HMM_INFO *gmm;
LOGPROB *score;
HTK_HMM_Data *d;
LOGPROB maxprob;
HTK_HMM_Data *dmax;
#ifdef CONFIDENCE_MEASURE
LOGPROB sum;
#endif
int i;
int maxid;
if (recog->gc->framecount == 0) return;
gmm = recog->gmm;
score = recog->gc->gmm_score;
/* get max score */
i = 0;
maxprob = LOG_ZERO;
dmax = NULL;
maxid = 0;
for(d=gmm->start;d;d=d->next) {
if (maxprob < score[i]) {
dmax = d;
maxprob = score[i];
maxid = i;
}
i++;
}
recog->gc->max_d = dmax;
recog->gc->max_i = maxid;
#ifdef CONFIDENCE_MEASURE
/* compute CM */
sum = 0.0;
i = 0;
for(d=gmm->start;d;d=d->next) {
//sum += pow(10, recog->jconf->annotate.cm_alpha * (score[i] - maxprob));
sum += pow(10, 0.05 * (score[i] - maxprob));
i++;
}
recog->gc->gmm_max_cm = 1.0 / sum;
#endif
/* output result */
callback_exec(CALLBACK_RESULT_GMM, recog);
}
/**
* <JA>
* @brief セグメントの認識再開処理
*
* この関数はデコーダベースVADやショートポーズセグメンテーションによって
* 入力がセグメントに切られた場合に，その後の認識の再開に関する処理を行う.
* 具体的には，入力の認識を開始する前に，前回の入力セグメントにおける
* 巻戻し分のMFCC列から認識を開始する. さらに，前回のセグメンテーション時に
* 未処理だった残りの音声サンプルがあればそれも処理する.
*
* @param recog [i/o] エンジンインスタンス
*
* @return エラー時 -1，正常時 0 を返す. また，この入力断片の処理中に
* 文章の区切りが見つかったときは第1パスをここで中断するために 1 を返す.
* </JA>
* </JA>
* <EN>
* @brief Resuming recognition for short pause segmentation.
*
* This function process overlapped data and remaining speech prior
* to the next input when input was segmented at last processing.
*
* @param recog [i/o] engine instance
*
* @return -1 on error (tell caller to terminate), 0 on success (allow caller
* to call me for the next segment), or 1 when an end-of-sentence detected
* at this point (in that case caller will stop input and go to 2nd pass)
* </EN>
*
* @callgraph
* @callergraph
*
*/
int
RealTimeResume(Recog *recog)
{
MFCCCalc *mfcc;
RealBeam *r;
boolean ok_p;
#ifdef SPSEGMENT_NAIST
RecogProcess *p;
#endif
PROCESS_AM *am;
r = &(recog->real);
/* 計算用のワークエリアを準備 */
/* prepare work area for calculation */
if (recog->jconf->input.type == INPUT_WAVEFORM) {
reset_mfcc(recog);
}
/* 音響尤度計算用キャッシュを準備 */
/* prepare cache area for acoustic computation of HMM states and mixtures */
for(am=recog->amlist;am;am=am->next) {
outprob_prepare(&(am->hmmwrk), r->maxframelen);
}
/* param にある全パラメータを処理する準備 */
/* prepare to process all data in param */
for(mfcc = recog->mfcclist; mfcc; mfcc = mfcc->next) {
if (mfcc->param->samplenum == 0) mfcc->valid = FALSE;
else mfcc->valid = TRUE;
#ifdef RDEBUG
printf("Resume: %02d: f=%d\n", mfcc->id, mfcc->mfcc->param->samplenum-1);
#endif
/* フレーム数をリセット */
/* reset frame count */
mfcc->f = 0;
/* MAP-CMN の初期化 */
/* Prepare for MAP-CMN */
if (mfcc->para->cmn || mfcc->para->cvn) CMN_realtime_prepare(mfcc->cmn.wrk);
}
#ifdef BACKEND_VAD
if (recog->jconf->decodeopt.segment) {
spsegment_init(recog);
}
/* not exec pass1 begin callback here */
#else
recog->triggered = FALSE;
for (mfcc = recog->mfcclist; mfcc; mfcc = mfcc->next) {
if (!mfcc->valid) continue;
callback_exec(CALLBACK_EVENT_SEGMENT_BEGIN, recog);
callback_exec(CALLBACK_EVENT_PASS1_BEGIN, recog);
recog->triggered = TRUE;
break;
}
#endif
/* param 内の全フレームについて認識処理を進める */
/* proceed recognition for all frames in param */
while(1) {
ok_p = TRUE;
for (mfcc = recog->mfcclist; mfcc; mfcc = mfcc->next) {
if (! mfcc->valid) continue;
if (mfcc->f < mfcc->param->samplenum) {
mfcc->valid = TRUE;
ok_p = FALSE;
} else {
mfcc->valid = FALSE;
}
}
if (ok_p) {
/* すべての MFCC が終わりに達したのでループ終了 */
/* all MFCC has been processed, end of loop */
break;
}
/* 各インスタンスについて mfcc->f の認識処理を1フレーム進める */
switch (decode_proceed(recog)) {
case -1: /* error */
return -1;
break;
case 0: /* success */
break;
case 1: /* segmented */
/* segmented, end procs ([0..f])*/
r->last_is_segmented = TRUE;
return 1; /* segmented by this function */
}
#ifdef BACKEND_VAD
/* check up trigger in case of VAD segmentation */
if (recog->jconf->decodeopt.segment) {
if (recog->triggered == FALSE) {
if (spsegment_trigger_sync(recog)) {
callback_exec(CALLBACK_EVENT_SEGMENT_BEGIN, recog);
callback_exec(CALLBACK_EVENT_PASS1_BEGIN, recog);
recog->triggered = TRUE;
}
}
}
#endif
/* call frame-wise callback */
callback_exec(CALLBACK_EVENT_PASS1_FRAME, recog);
/* 1フレーム処理が進んだのでポインタを進める */
/* proceed frame pointer */
for (mfcc = recog->mfcclist; mfcc; mfcc = mfcc->next) {
if (!mfcc->valid) continue;
mfcc->f++;
}
}
/* 前回のセグメント時に入力をシフトしていない分をシフトする */
/* do the last shift here */
if (recog->jconf->input.type == INPUT_WAVEFORM) {
memmove(r->window, &(r->window[recog->jconf->input.frameshift]), sizeof(SP16) * (r->windowlen - recog->jconf->input.frameshift));
r->windownum -= recog->jconf->input.frameshift;
/* これで再開の準備が整ったので,まずは前回の処理で残っていた音声データから
処理する */
/* now that the search status has been prepared for the next input, we
first process the rest unprocessed samples at the last session */
if (r->rest_len > 0) {
return(RealTimePipeLine(r->rest_Speech, r->rest_len, recog));
}
}
/* 新規の入力に対して認識処理は続く… */
/* the recognition process will continue for the newly incoming samples... */
return 0;
}
static int
proceed_one_frame(Recog *recog)
{
MFCCCalc *mfcc;
RealBeam *r;
int maxf;
PROCESS_AM *am;
int rewind_frame;
boolean reprocess;
boolean ok_p;
r = &(recog->real);
/* call recognition start callback */
ok_p = FALSE;
maxf = 0;
for (mfcc = recog->mfcclist; mfcc; mfcc = mfcc->next) {
if (!mfcc->valid) continue;
if (maxf < mfcc->f) maxf = mfcc->f;
if (mfcc->f == 0) {
ok_p = TRUE;
}
}
if (ok_p && maxf == 0) {
/* call callback when at least one of MFCC has initial frame */
if (recog->jconf->decodeopt.segment) {
#ifdef BACKEND_VAD
/* not exec pass1 begin callback here */
#else
if (!recog->process_segment) {
callback_exec(CALLBACK_EVENT_RECOGNITION_BEGIN, recog);
}
callback_exec(CALLBACK_EVENT_SEGMENT_BEGIN, recog);
callback_exec(CALLBACK_EVENT_PASS1_BEGIN, recog);
recog->triggered = TRUE;
#endif
} else {
callback_exec(CALLBACK_EVENT_RECOGNITION_BEGIN, recog);
callback_exec(CALLBACK_EVENT_PASS1_BEGIN, recog);
recog->triggered = TRUE;
}
}
/* 各インスタンスについて mfcc->f の認識処理を1フレーム進める */
switch (decode_proceed(recog)) {
case -1: /* error */
return -1;
break;
case 0: /* success */
break;
case 1: /* segmented */
/* 認識処理のセグメント要求で終わったことをフラグにセット */
/* set flag which indicates that the input has ended with segmentation request */
r->last_is_segmented = TRUE;
/* tell the caller to be segmented by this function */
/* 呼び出し元に，ここで入力を切るよう伝える */
return 1;
}
#ifdef BACKEND_VAD
/* check up trigger in case of VAD segmentation */
if (recog->jconf->decodeopt.segment) {
if (recog->triggered == FALSE) {
if (spsegment_trigger_sync(recog)) {
if (!recog->process_segment) {
callback_exec(CALLBACK_EVENT_RECOGNITION_BEGIN, recog);
}
callback_exec(CALLBACK_EVENT_SEGMENT_BEGIN, recog);
callback_exec(CALLBACK_EVENT_PASS1_BEGIN, recog);
recog->triggered = TRUE;
}
}
}
#endif
if (spsegment_need_restart(recog, &rewind_frame, &reprocess) == TRUE) {
/* set total length to the current frame */
for (mfcc = recog->mfcclist; mfcc; mfcc = mfcc->next) {
if (!mfcc->valid) continue;
mfcc->param->header.samplenum = mfcc->f + 1;
mfcc->param->samplenum = mfcc->f + 1;
}
/* do rewind for all mfcc here */
spsegment_restart_mfccs(recog, rewind_frame, reprocess);
/* also tell adin module to rehash the concurrent audio input */
recog->adin->rehash = TRUE;
/* reset outprob cache for all AM */
for(am=recog->amlist;am;am=am->next) {
outprob_prepare(&(am->hmmwrk), am->mfcc->param->samplenum);
}
if (reprocess) {
/* process the backstep MFCCs here */
while(1) {
ok_p = TRUE;
for (mfcc = recog->mfcclist; mfcc; mfcc = mfcc->next) {
if (! mfcc->valid) continue;
mfcc->f++;
if (mfcc->f < mfcc->param->samplenum) {
mfcc->valid = TRUE;
ok_p = FALSE;
} else {
mfcc->valid = FALSE;
}
}
if (ok_p) {
/* すべての MFCC が終わりに達したのでループ終了 */
/* all MFCC has been processed, end of loop */
for (mfcc = recog->mfcclist; mfcc; mfcc = mfcc->next) {
if (! mfcc->valid) continue;
mfcc->f--;
}
break;
}
/* 各インスタンスについて mfcc->f の認識処理を1フレーム進める */
switch (decode_proceed(recog)) {
case -1: /* error */
return -1;
break;
case 0: /* success */
break;
case 1: /* segmented */
/* ignore segmentation while in the backstep segment */
break;
}
/* call frame-wise callback */
callback_exec(CALLBACK_EVENT_PASS1_FRAME, recog);
}
}
}
/* call frame-wise callback if at least one of MFCC is valid at this frame */
for (mfcc = recog->mfcclist; mfcc; mfcc = mfcc->next) {
if (mfcc->valid) {
callback_exec(CALLBACK_EVENT_PASS1_FRAME, recog);
break;
}
}
return 0;
}
/**
* <JA>
* @brief 第1パス平行認識処理の終了処理を行う.
*
* この関数は第1パス終了時に呼ばれ，入力長を確定したあと，
* decode_end() （セグメントで終了したときは decode_end_segmented()）を
* 呼び出して第1パス終了処理を行う.
*
* もし音声入力ストリームの終了によって認識が終わった場合（ファイル入力で
* 終端に達した場合など）は，デルタバッファに未処理の入力が残っているので，
* それをここで処理する.
*
* @param recog [i/o] エンジンインスタンス
*
* @return 処理成功時 TRUE, エラー時 FALSE を返す.
* </JA>
* <EN>
* @brief Finalize the 1st pass on-the-fly decoding.
*
* This function will be called after the 1st pass processing ends.
* It fix the input length of parameter vector sequence, call
* decode_end() (or decode_end_segmented() when last input was ended
* by segmentation) to finalize the 1st pass.
*
* If the last input was ended by end-of-stream (in case input reached
* EOF in file input etc.), process the rest samples remaining in the
* delta buffers.
*
* @param recog [i/o] engine instance
*
* @return TRUE on success, or FALSE on error.
* </EN>
*/
boolean
RealTimeParam(Recog *recog)
{
boolean ret1, ret2;
RealBeam *r;
int ret;
int maxf;
boolean ok_p;
MFCCCalc *mfcc;
Value *para;
#ifdef RDEBUG
int i;
#endif
r = &(recog->real);
if (r->last_is_segmented) {
/* RealTimePipeLine で認識処理側の理由により認識が中断した場合,
現状態のMFCC計算データをそのまま次回へ保持する必要があるので,
MFCC計算終了処理を行わずに第１パスの結果のみ出力して終わる. */
/* When input segmented by recognition process in RealTimePipeLine(),
we have to keep the whole current status of MFCC computation to the
next call. So here we only output the 1st pass result. */
for (mfcc = recog->mfcclist; mfcc; mfcc = mfcc->next) {
mfcc->param->header.samplenum = mfcc->f + 1;/* len = lastid + 1 */
mfcc->param->samplenum = mfcc->f + 1;
}
decode_end_segmented(recog);
/* この区間の param データを第２パスのために返す */
/* return obtained parameter for 2nd pass */
return(TRUE);
}
if (recog->jconf->input.type == INPUT_VECTOR) {
/* finalize real-time 1st pass */
for (mfcc = recog->mfcclist; mfcc; mfcc = mfcc->next) {
mfcc->param->header.samplenum = mfcc->f;
mfcc->param->samplenum = mfcc->f;
}
/* 最終フレーム処理を行い，認識の結果出力と終了処理を行う */
decode_end(recog);
return TRUE;
}
/* MFCC計算の終了処理を行う: 最後の遅延フレーム分を処理 */
/* finish MFCC computation for the last delayed frames */
for (mfcc = recog->mfcclist; mfcc; mfcc = mfcc->next) {
if (mfcc->para->delta || mfcc->para->acc) {
mfcc->valid = TRUE;
} else {
mfcc->valid = FALSE;
}
}
/* loop until all data has been flushed */
while (1) {
/* if all mfcc became invalid, exit loop here */
ok_p = FALSE;
for (mfcc = recog->mfcclist; mfcc; mfcc = mfcc->next) {
if (mfcc->valid) {
ok_p = TRUE;
break;
}
}
if (!ok_p) break;
/* try to get 1 frame for all mfcc instances */
for (mfcc = recog->mfcclist; mfcc; mfcc = mfcc->next) {
para = mfcc->para;
if (! mfcc->valid) continue;
/* check if there is data in cycle buffer of delta */
ret1 = WMP_deltabuf_flush(mfcc->db);
#ifdef RDEBUG
printf("DeltaBufLast: ret=%d, status=", ret1);
for(i=0;i<mfcc->db->len;i++) {
printf("%d", mfcc->db->is_on[i]);
}
printf(", nextstore=%d\n", mfcc->db->store);
#endif
if (ret1) {
/* uncomputed delta has flushed, compute it with tmpmfcc */
if (para->energy && para->absesup) {
memcpy(mfcc->tmpmfcc, mfcc->db->vec, sizeof(VECT) * (para->baselen - 1));
memcpy(&(mfcc->tmpmfcc[para->baselen-1]), &(mfcc->db->vec[para->baselen]), sizeof(VECT) * para->baselen);
} else {
memcpy(mfcc->tmpmfcc, mfcc->db->vec, sizeof(VECT) * para->baselen * 2);
}
if (para->acc) {
/* this new delta should be given to the accel cycle buffer */
ret2 = WMP_deltabuf_proceed(mfcc->ab, mfcc->tmpmfcc);
#ifdef RDEBUG
printf("AccelBuf: ret=%d, status=", ret2);
for(i=0;i<mfcc->ab->len;i++) {
printf("%d", mfcc->ab->is_on[i]);
}
printf(", nextstore=%d\n", mfcc->ab->store);
#endif
if (ret2) {
/* uncomputed accel was given, compute it with tmpmfcc */
memcpy(mfcc->tmpmfcc, mfcc->ab->vec, sizeof(VECT) * (para->veclen - para->baselen));
memcpy(&(mfcc->tmpmfcc[para->veclen - para->baselen]), &(mfcc->ab->vec[para->veclen - para->baselen]), sizeof(VECT) * para->baselen);
} else {
/* still no input is given: */
/* in case of very short input: go on to the next input */
continue;
}
}
} else {
/* no data left in the delta buffer */
if (para->acc) {
/* no new data, just flush the accel buffer */
ret2 = WMP_deltabuf_flush(mfcc->ab);
#ifdef RDEBUG
printf("AccelBuf: ret=%d, status=", ret2);
for(i=0;i<mfcc->ab->len;i++) {
printf("%d", mfcc->ab->is_on[i]);
}
printf(", nextstore=%d\n", mfcc->ab->store);
#endif
if (ret2) {
/* uncomputed data has flushed, compute it with tmpmfcc */
memcpy(mfcc->tmpmfcc, mfcc->ab->vec, sizeof(VECT) * (para->veclen - para->baselen));
memcpy(&(mfcc->tmpmfcc[para->veclen - para->baselen]), &(mfcc->ab->vec[para->veclen - para->baselen]), sizeof(VECT) * para->baselen);
} else {
/* actually no data exists in both delta and accel */
mfcc->valid = FALSE; /* disactivate this instance */
continue; /* end this loop */
}
} else {
/* only delta: input fully flushed */
mfcc->valid = FALSE; /* disactivate this instance */
continue; /* end this loop */
}
}
/* a new frame has been obtained from delta buffer to tmpmfcc */
if(para->cmn || para->cvn) CMN_realtime(mfcc->cmn.wrk, mfcc->tmpmfcc);
if (param_alloc(mfcc->param, mfcc->f + 1, mfcc->param->veclen) == FALSE) {
jlog("ERROR: failed to allocate memory for incoming MFCC vectors\n");
return FALSE;
}
/* store to mfcc->f */
memcpy(mfcc->param->parvec[mfcc->f], mfcc->tmpmfcc, sizeof(VECT) * mfcc->param->veclen);
#ifdef ENABLE_PLUGIN
/* call postprocess plugin if any */
plugin_exec_vector_postprocess(mfcc->param->parvec[mfcc->f], mfcc->param->veclen, mfcc->f);
#endif
}
/* call recognition start callback */
ok_p = FALSE;
maxf = 0;
for (mfcc = recog->mfcclist; mfcc; mfcc = mfcc->next) {
if (!mfcc->valid) continue;
if (maxf < mfcc->f) maxf = mfcc->f;
if (mfcc->f == 0) {
ok_p = TRUE;
}
}
if (ok_p && maxf == 0) {
/* call callback when at least one of MFCC has initial frame */
if (recog->jconf->decodeopt.segment) {
#ifdef BACKEND_VAD
/* not exec pass1 begin callback here */
#else
if (!recog->process_segment) {
callback_exec(CALLBACK_EVENT_RECOGNITION_BEGIN, recog);
}
callback_exec(CALLBACK_EVENT_SEGMENT_BEGIN, recog);
callback_exec(CALLBACK_EVENT_PASS1_BEGIN, recog);
recog->triggered = TRUE;
#endif
} else {
callback_exec(CALLBACK_EVENT_RECOGNITION_BEGIN, recog);
callback_exec(CALLBACK_EVENT_PASS1_BEGIN, recog);
recog->triggered = TRUE;
}
}
/* proceed for the curent frame */
ret = decode_proceed(recog);
if (ret == -1) { /* error */
return -1;
} else if (ret == 1) { /* segmented */
/* loop out */
break;
} /* else no event occured */
#ifdef BACKEND_VAD
/* check up trigger in case of VAD segmentation */
if (recog->jconf->decodeopt.segment) {
if (recog->triggered == FALSE) {
if (spsegment_trigger_sync(recog)) {
if (!recog->process_segment) {
callback_exec(CALLBACK_EVENT_RECOGNITION_BEGIN, recog);
}
callback_exec(CALLBACK_EVENT_SEGMENT_BEGIN, recog);
callback_exec(CALLBACK_EVENT_PASS1_BEGIN, recog);
recog->triggered = TRUE;
}
}
}
#endif
/* call frame-wise callback */
callback_exec(CALLBACK_EVENT_PASS1_FRAME, recog);
/* move to next */
for (mfcc = recog->mfcclist; mfcc; mfcc = mfcc->next) {
if (! mfcc->valid) continue;
mfcc->f++;
if (mfcc->f > r->maxframelen) mfcc->valid = FALSE;
}
}
/* finalize real-time 1st pass */
for (mfcc = recog->mfcclist; mfcc; mfcc = mfcc->next) {
mfcc->param->header.samplenum = mfcc->f;
mfcc->param->samplenum = mfcc->f;
}
/* 最終フレーム処理を行い，認識の結果出力と終了処理を行う */
decode_end(recog);
return(TRUE);
}
/**
* <EN>
* @brief Main A/D-in and sound detection function
*
* This function read inputs from device and do sound detection
* (both up trigger and down trigger) until end of device.
*
* In threaded mode, this function will detach and loop forever as
* ad-in thread, (adin_thread_create()) storing triggered samples in
* speech[], and telling the status to another process thread via @a
* transfer_online in work area. The process thread, called from
* adin_go(), polls the length of speech[] and transfer_online in work area
* and process them if new samples has been stored.
*
* In non-threaded mode, this function will be called directly from
* adin_go(), and triggered samples are immediately processed within here.
*
* Threaded mode should be used for "live" input such as microphone input
* where input is infinite and capture delay is severe. For file input,
* adinnet input and other "buffered" input, non-threaded mode will be used.
*
* Argument "ad_process()" should be a function to process the triggered
* input samples. On real-time recognition, a frame-synchronous search
* function for the first pass will be specified by the caller. The current
* input will be segmented if it returns 1, and will be terminated as error
* if it returns -1.
*
* When the argument "ad_check()" specified, it will be called periodically.
* When it returns less than 0, this function will be terminated.
*
* </EN>
* <JA>
* @brief 音声入力と音検出を行うメイン関数
*
* ここでは音声入力の取り込み，音区間の開始・終了の検出を行います.
*
* スレッドモード時，この関数は独立したAD-inスレッドとしてデタッチされます.
* (adin_thread_create()), 音入力を検知するとこの関数はワークエリア内の
* speech[] にトリガしたサンプルを記録し，かつ transfer_online を TRUE に
* セットします. Julius のメイン処理スレッド (adin_go()) は
* adin_thread_process() に移行し，そこで transfer_online 時に speech[] を
* 参照しながら認識処理を行います.
*
* 非スレッドモード時は，メイン処理関数 adin_go() は直接この関数を呼び，
* 認識処理はこの内部で直接行われます.
*
* スレッドモードはマイク入力など，入力が無限で処理の遅延がデータの
* 取りこぼしを招くような live input で用いられます. 一方，ファイル入力
* やadinnet 入力のような buffered input では非スレッドモードが用いられます.
*
* 引数の ad_process は，取り込んだサンプルに対して処理を行う関数を
* 指定します. リアルタイム認識を行う場合は，ここに第1パスの認識処理を
* 行う関数が指定されます. 返り値が 1 であれば，入力をここで区切ります.
* -1 であればエラー終了します.
*
* 引数の ad_check は一定処理ごとに繰り返し呼ばれる関数を指定します. この
* 関数の返り値が 0 以下だった場合，入力を即時中断して関数を終了します.
* </JA>
*
* @param ad_process [in] function to process triggerted input.
* @param ad_check [in] function to be called periodically.
* @param recog [in] engine instance
*
* @return 2 when input termination requested by ad_process(), 1 when
* if detect end of an input segment (down trigger detected after up
* trigger), 0 when reached end of input device, -1 on error, -2 when
* input termination requested by ad_check().
*
* @callergraph
* @callgraph
*
*/
static int
adin_cut(int (*ad_process)(SP16 *, int, Recog *), int (*ad_check)(Recog *), Recog *recog)
{
ADIn *a;
int i;
int ad_process_ret;
int imax, len, cnt;
int wstep;
int end_status = 0; /* return value */
boolean transfer_online_local; /* local repository of transfer_online */
int zc; /* count of zero cross */
a = recog->adin;
/*
* there are 3 buffers:
* temporary storage queue: buffer[]
* cycle buffer for zero-cross counting: (in zc_e)
* swap buffer for re-starting after short tail silence
*
* Each samples are first read to buffer[], then passed to count_zc_e()
* to find trigger. Samples between trigger and end of speech are
* passed to (*ad_process) with pointer to the first sample and its length.
*
*/
if (a->need_init) {
a->bpmax = MAXSPEECHLEN;
a->bp = 0;
a->is_valid_data = FALSE;
/* reset zero-cross status */
if (a->adin_cut_on) {
reset_count_zc_e(&(a->zc), a->thres, a->c_length, a->c_offset);
}
a->end_of_stream = FALSE;
a->nc = 0;
a->sblen = 0;
a->need_init = FALSE; /* for next call */
}
/****************/
/* resume input */
/****************/
// if (!a->adin_cut_on && a->is_valid_data == TRUE) {
// callback_exec(CALLBACK_EVENT_SPEECH_START, recog);
// }
/*************/
/* main loop */
/*************/
for (;;) {
/* check end of input by end of stream */
if (a->end_of_stream && a->bp == 0) break;
/****************************/
/* read in new speech input */
/****************************/
if (a->end_of_stream) {
/* already reaches end of stream, just process the rest */
a->current_len = a->bp;
} else {
/*****************************************************/
/* get samples from input device to temporary buffer */
/*****************************************************/
/* buffer[0..bp] is the current remaining samples */
/*
mic input - samples exist in a device buffer
tcpip input - samples exist in a socket
file input - samples in a file
Return value is the number of read samples.
If no data exists in the device (in case of mic input), ad_read()
will return 0. If reached end of stream (in case end of file or
receive end ack from tcpip client), it will return -1.
If error, returns -2. If the device requests segmentation, returns -3.
*/
if (a->down_sample) {
/* get 48kHz samples to temporal buffer */
cnt = (*(a->ad_read))(a->buffer48, (a->bpmax - a->bp) * a->io_rate);
} else {
cnt = (*(a->ad_read))(&(a->buffer[a->bp]), a->bpmax - a->bp);
}
if (cnt < 0) { /* end of stream / segment or error */
/* set the end status */
switch(cnt) {
case -1: /* end of stream */
a->input_side_segment = FALSE;
end_status = 0;
break;
case -2:
a->input_side_segment = FALSE;
end_status = -1;
break;
case -3:
a->input_side_segment = TRUE;
end_status = 0;
}
/* now the input has been ended,
we should not get further speech input in the next loop,
instead just process the samples in the temporary buffer until
the entire data is processed. */
a->end_of_stream = TRUE;
cnt = 0; /* no new input */
/* in case the first trial of ad_read() fails, exit this loop */
if (a->bp == 0) break;
}
if (a->down_sample && cnt != 0) {
/* convert to 16kHz */
cnt = ds48to16(&(a->buffer[a->bp]), a->buffer48, cnt, a->bpmax - a->bp, a->ds);
if (cnt < 0) { /* conversion error */
jlog("ERROR: adin_cut: error in down sampling\n");
end_status = -1;
a->end_of_stream = TRUE;
cnt = 0;
if (a->bp == 0) break;
}
}
if (cnt > 0 && a->level_coef != 1.0) {
/* scale the level of incoming input */
for (i = a->bp; i < a->bp + cnt; i++) {
a->buffer[i] = (SP16) ((float)a->buffer[i] * a->level_coef);
}
}
/*************************************************/
/* execute callback here for incoming raw data stream.*/
/* the content of buffer[bp...bp+cnt-1] or the */
/* length can be modified in the functions. */
/*************************************************/
if (cnt > 0) {
#ifdef ENABLE_PLUGIN
plugin_exec_adin_captured(&(a->buffer[a->bp]), cnt);
#endif
callback_exec_adin(CALLBACK_ADIN_CAPTURED, recog, &(a->buffer[a->bp]), cnt);
/* record total number of captured samples */
a->total_captured_len += cnt;
}
/*************************************************/
/* some speech processing for the incoming input */
/*************************************************/
if (cnt > 0) {
if (a->strip_flag) {
/* strip off successive zero samples */
len = strip_zero(&(a->buffer[a->bp]), cnt);
if (len != cnt) cnt = len;
}
if (a->need_zmean) {
/* remove DC offset */
sub_zmean(&(a->buffer[a->bp]), cnt);
}
}
/* current len = current samples in buffer */
a->current_len = a->bp + cnt;
}
#ifdef THREAD_DEBUG
if (a->end_of_stream) {
jlog("DEBUG: adin_cut: stream already ended\n");
}
if (cnt > 0) {
jlog("DEBUG: adin_cut: get %d samples [%d-%d]\n", a->current_len - a->bp, a->bp, a->current_len);
}
#endif
/**************************************************/
/* call the periodic callback (non threaded mode) */
/*************************************************/
/* this function is mainly for periodic checking of incoming command
in module mode */
/* in threaded mode, this will be done in process thread, not here in adin thread */
if (ad_check != NULL
#ifdef HAVE_PTHREAD
&& !a->enable_thread
#endif
) {
/* if ad_check() returns value < 0, termination of speech input is required */
if ((i = (*ad_check)(recog)) < 0) { /* -1: soft termination -2: hard termination */
// if ((i == -1 && current_len == 0) || i == -2) {
if (i == -2 ||
(i == -1 && a->is_valid_data == FALSE)) {
end_status = -2; /* recognition terminated by outer function */
/* execute callback */
if (a->current_len > 0) {
callback_exec(CALLBACK_EVENT_SPEECH_STOP, recog);
}
a->need_init = TRUE; /* bufer status shoule be reset at next call */
goto break_input;
}
}
}
/***********************************************************************/
/* if no data has got but not end of stream, repeat next input samples */
/***********************************************************************/
if (a->current_len == 0) continue;
/* When not adin_cut mode, all incoming data is valid.
So is_valid_data should be set to TRUE when some input first comes
till this input ends. So, if some data comes, set is_valid_data to
TRUE here. */
if (!a->adin_cut_on && a->is_valid_data == FALSE && a->current_len > 0) {
a->is_valid_data = TRUE;
callback_exec(CALLBACK_EVENT_SPEECH_START, recog);
}
/******************************************************/
/* prepare for processing samples in temporary buffer */
/******************************************************/
wstep = a->chunk_size; /* process unit (should be smaller than cycle buffer) */
/* imax: total length that should be processed at one ad_read() call */
/* if in real-time mode and not threaded, recognition process
will be called and executed as the ad_process() callback within
this function. If the recognition speed is over the real time,
processing all the input samples at the loop below may result in the
significant delay of getting next input, that may result in the buffer
overflow of the device (namely a microphone device will suffer from
this). So, in non-threaded mode, in order to avoid buffer overflow and
input frame dropping, we will leave here by processing
only one segment [0..wstep], and leave the rest in the temporary buffer.
*/
#ifdef HAVE_PTHREAD
if (a->enable_thread) imax = a->current_len; /* process whole */
else imax = (a->current_len < wstep) ? a->current_len : wstep; /* one step */
#else
imax = (a->current_len < wstep) ? a->current_len : wstep; /* one step */
#endif
/* wstep: unit length for the loop below */
if (wstep > a->current_len) wstep = a->current_len;
#ifdef THREAD_DEBUG
jlog("DEBUG: process %d samples by %d step\n", imax, wstep);
#endif
#ifdef HAVE_PTHREAD
if (a->enable_thread) {
/* get transfer status to local */
pthread_mutex_lock(&(a->mutex));
transfer_online_local = a->transfer_online;
pthread_mutex_unlock(&(a->mutex));
}
#endif
/*********************************************************/
/* start processing buffer[0..current_len] by wstep step */
/*********************************************************/
i = 0;
while (i + wstep <= imax) {
if (a->adin_cut_on) {
/********************/
/* check triggering */
/********************/
/* the cycle buffer in count_zc_e() holds the last
samples of (head_margin) miliseconds, and the zerocross
over the threshold level are counted within the cycle buffer */
/* store the new data to cycle buffer and update the count */
/* return zero-cross num in the cycle buffer */
zc = count_zc_e(&(a->zc), &(a->buffer[i]), wstep);
if (zc > a->noise_zerocross) { /* now triggering */
if (a->is_valid_data == FALSE) {
/*****************************************************/
/* process off, trigger on: detect speech triggering */
/*****************************************************/
a->is_valid_data = TRUE; /* start processing */
a->nc = 0;
#ifdef THREAD_DEBUG
jlog("DEBUG: detect on\n");
#endif
/* record time */
a->last_trigger_sample = a->total_captured_len - a->current_len + i + wstep - a->zc.valid_len;
callback_exec(CALLBACK_EVENT_SPEECH_START, recog);
a->last_trigger_len = 0;
if (a->zc.valid_len > wstep) {
a->last_trigger_len += a->zc.valid_len - wstep;
}
/****************************************/
/* flush samples stored in cycle buffer */
/****************************************/
/* (last (head_margin) msec samples */
/* if threaded mode, processing means storing them to speech[].
if ignore_speech_while_recog is on (default), ignore the data
if transfer is offline (=while processing second pass).
Else, datas are stored even if transfer is offline */
if ( ad_process != NULL
#ifdef HAVE_PTHREAD
&& (!a->enable_thread || !a->ignore_speech_while_recog || transfer_online_local)
#endif
) {
/* copy content of cycle buffer to cbuf */
zc_copy_buffer(&(a->zc), a->cbuf, &len);
/* Note that the last 'wstep' samples are the same as
the current samples 'buffer[i..i+wstep]', and
they will be processed later. So, here only the samples
cbuf[0...len-wstep] will be processed
*/
if (len - wstep > 0) {
#ifdef THREAD_DEBUG
jlog("DEBUG: callback for buffered samples (%d bytes)\n", len - wstep);
#endif
#ifdef ENABLE_PLUGIN
plugin_exec_adin_triggered(a->cbuf, len - wstep);
#endif
callback_exec_adin(CALLBACK_ADIN_TRIGGERED, recog, a->cbuf, len - wstep);
ad_process_ret = (*ad_process)(a->cbuf, len - wstep, recog);
switch(ad_process_ret) {
case 1: /* segmentation notification from process callback */
#ifdef HAVE_PTHREAD
if (a->enable_thread) {
/* in threaded mode, just stop transfer */
pthread_mutex_lock(&(a->mutex));
a->transfer_online = transfer_online_local = FALSE;
pthread_mutex_unlock(&(a->mutex));
} else {
/* in non-threaded mode, set end status and exit loop */
end_status = 2;
adin_purge(a, i);
goto break_input;
}
break;
#else
/* in non-threaded mode, set end status and exit loop */
end_status = 2;
adin_purge(a, i);
goto break_input;
#endif
case -1: /* error occured in callback */
/* set end status and exit loop */
end_status = -1;
goto break_input;
}
}
}
} else { /* is_valid_data == TRUE */
/******************************************************/
/* process on, trigger on: we are in a speech segment */
/******************************************************/
if (a->nc > 0) {
/*************************************/
/* re-triggering in trailing silence */
/*************************************/
#ifdef THREAD_DEBUG
jlog("DEBUG: re-triggered\n");
#endif
/* reset noise counter */
a->nc = 0;
if (a->sblen > 0) {
a->last_trigger_len += a->sblen;
}
#ifdef TMP_FIX_200602
if (ad_process != NULL
#ifdef HAVE_PTHREAD
&& (!a->enable_thread || !a->ignore_speech_while_recog || transfer_online_local)
#endif
) {
#endif
/*************************************************/
/* process swap buffer stored while tail silence */
/*************************************************/
/* In trailing silence, the samples within the tail margin length
will be processed immediately, but samples after the tail
margin will not be processed, instead stored in swapbuf[].
If re-triggering occurs while in the trailing silence,
the swapped samples should be processed now to catch up
with current input
*/
if (a->sblen > 0) {
#ifdef THREAD_DEBUG
jlog("DEBUG: callback for swapped %d samples\n", a->sblen);
#endif
#ifdef ENABLE_PLUGIN
plugin_exec_adin_triggered(a->swapbuf, a->sblen);
#endif
callback_exec_adin(CALLBACK_ADIN_TRIGGERED, recog, a->swapbuf, a->sblen);
ad_process_ret = (*ad_process)(a->swapbuf, a->sblen, recog);
a->sblen = 0;
switch(ad_process_ret) {
case 1: /* segmentation notification from process callback */
#ifdef HAVE_PTHREAD
if (a->enable_thread) {
/* in threaded mode, just stop transfer */
pthread_mutex_lock(&(a->mutex));
a->transfer_online = transfer_online_local = FALSE;
pthread_mutex_unlock(&(a->mutex));
} else {
/* in non-threaded mode, set end status and exit loop */
end_status = 2;
adin_purge(a, i);
goto break_input;
}
break;
#else
/* in non-threaded mode, set end status and exit loop */
end_status = 2;
adin_purge(a, i);
goto break_input;
#endif
case -1: /* error occured in callback */
/* set end status and exit loop */
end_status = -1;
goto break_input;
}
}
#ifdef TMP_FIX_200602
}
#endif
}
}
} else if (a->is_valid_data == TRUE) {
/*******************************************************/
/* process on, trigger off: processing tailing silence */
/*******************************************************/
#ifdef THREAD_DEBUG
jlog("DEBUG: TRAILING SILENCE\n");
#endif
if (a->nc == 0) {
/* start of tail silence: prepare valiables for start swapbuf[] */
a->rest_tail = a->sbsize - a->c_length;
a->sblen = 0;
#ifdef THREAD_DEBUG
jlog("DEBUG: start tail silence, rest_tail = %d\n", a->rest_tail);
#endif
}
/* increment noise counter */
a->nc++;
}
} /* end of triggering handlers */
/********************************************************************/
/* process the current segment buffer[i...i+wstep] if process == on */
/********************************************************************/
if (a->adin_cut_on && a->is_valid_data && a->nc > 0 && a->rest_tail == 0) {
/* The current trailing silence is now longer than the user-
specified tail margin length, so the current samples
should not be processed now. But if 're-triggering'
occurs in the trailing silence later, they should be processed
then. So we just store the overed samples in swapbuf[] and
not process them now */
#ifdef THREAD_DEBUG
jlog("DEBUG: tail silence over, store to swap buffer (nc=%d, rest_tail=%d, sblen=%d-%d)\n", a->nc, a->rest_tail, a->sblen, a->sblen+wstep);
#endif
if (a->sblen + wstep > a->sbsize) {
jlog("ERROR: adin_cut: swap buffer for re-triggering overflow\n");
}
memcpy(&(a->swapbuf[a->sblen]), &(a->buffer[i]), wstep * sizeof(SP16));
a->sblen += wstep;
} else {
/* we are in a normal speech segment (nc == 0), or
trailing silence (shorter than tail margin length) (nc>0,rest_tail>0)
The current trailing silence is shorter than the user-
specified tail margin length, so the current samples
should be processed now as same as the normal speech segment */
#ifdef TMP_FIX_200602
if (!a->adin_cut_on || a->is_valid_data == TRUE) {
#else
if(
(!a->adin_cut_on || a->is_valid_data == TRUE)
#ifdef HAVE_PTHREAD
&& (!a->enable_thread || !a->ignore_speech_while_recog || transfer_online_local)
#endif
) {
#endif
if (a->nc > 0) {
/* if we are in a trailing silence, decrease the counter to detect
start of swapbuf[] above */
if (a->rest_tail < wstep) a->rest_tail = 0;
else a->rest_tail -= wstep;
#ifdef THREAD_DEBUG
jlog("DEBUG: %d processed, rest_tail=%d\n", wstep, a->rest_tail);
#endif
}
a->last_trigger_len += wstep;
#ifdef TMP_FIX_200602
if (ad_process != NULL
#ifdef HAVE_PTHREAD
&& (!a->enable_thread || !a->ignore_speech_while_recog || transfer_online_local)
#endif
) {
#else
if ( ad_process != NULL ) {
#endif
#ifdef THREAD_DEBUG
jlog("DEBUG: callback for input sample [%d-%d]\n", i, i+wstep);
#endif
/* call external function */
#ifdef ENABLE_PLUGIN
plugin_exec_adin_triggered(&(a->buffer[i]), wstep);
#endif
callback_exec_adin(CALLBACK_ADIN_TRIGGERED, recog, &(a->buffer[i]), wstep);
ad_process_ret = (*ad_process)(&(a->buffer[i]), wstep, recog);
switch(ad_process_ret) {
case 1: /* segmentation notification from process callback */
#ifdef HAVE_PTHREAD
if (a->enable_thread) {
/* in threaded mode, just stop transfer */
pthread_mutex_lock(&(a->mutex));
a->transfer_online = transfer_online_local = FALSE;
pthread_mutex_unlock(&(a->mutex));
} else {
/* in non-threaded mode, set end status and exit loop */
adin_purge(a, i+wstep);
end_status = 2;
goto break_input;
}
break;
#else
/* in non-threaded mode, set end status and exit loop */
adin_purge(a, i+wstep);
end_status = 2;
goto break_input;
#endif
case -1: /* error occured in callback */
/* set end status and exit loop */
end_status = -1;
goto break_input;
}
}
}
} /* end of current segment processing */
if (a->adin_cut_on && a->is_valid_data && a->nc >= a->nc_max) {
/*************************************/
/* process on, trailing silence over */
/* = end of input segment */
/*************************************/
#ifdef THREAD_DEBUG
jlog("DEBUG: detect off\n");
#endif
/* end input by silence */
a->is_valid_data = FALSE; /* turn off processing */
a->sblen = 0;
callback_exec(CALLBACK_EVENT_SPEECH_STOP, recog);
#ifdef HAVE_PTHREAD
if (a->enable_thread) { /* just stop transfer */
pthread_mutex_lock(&(a->mutex));
a->transfer_online = transfer_online_local = FALSE;
pthread_mutex_unlock(&(a->mutex));
} else {
adin_purge(a, i+wstep);
end_status = 1;
goto break_input;
}
#else
adin_purge(a, i+wstep);
end_status = 1;
goto break_input;
#endif
}
/*********************************************************/
/* end of processing buffer[0..current_len] by wstep step */
/*********************************************************/
i += wstep; /* increment to next wstep samples */
}
/* purge processed samples and update queue */
adin_purge(a, i);
}
break_input:
/****************/
/* pause input */
/****************/
if (a->end_of_stream) { /* input already ends */
/* execute callback */
callback_exec(CALLBACK_EVENT_SPEECH_STOP, recog);
if (a->bp == 0) { /* rest buffer successfully flushed */
/* reset status */
a->need_init = TRUE; /* bufer status shoule be reset at next call */
}
if (end_status >= 0) {
end_status = (a->bp) ? 1 : 0;
}
}
return(end_status);
}
#ifdef HAVE_PTHREAD
/***********************/
/* threading functions */
/***********************/
/*************************/
/* adin thread functions */
/*************************/
/**
* <EN>
* Callback to store triggered samples within A/D-in thread.
* </EN>
* <JA>
* A/D-in スレッドにてトリガした入力サンプルを保存するコールバック.
* </JA>
*
* @param now [in] triggered fragment
* @param len [in] length of above
* @param recog [in] engine instance
*
* @return always 0, to tell caller to just continue the input
*/
static int
adin_store_buffer(SP16 *now, int len, Recog *recog)
{
ADIn *a;
a = recog->adin;
if (a->speechlen + len > MAXSPEECHLEN) {
/* just mark as overflowed, and continue this thread */
pthread_mutex_lock(&(a->mutex));
a->adinthread_buffer_overflowed = TRUE;
pthread_mutex_unlock(&(a->mutex));
return(0);
}
pthread_mutex_lock(&(a->mutex));
memcpy(&(a->speech[a->speechlen]), now, len * sizeof(SP16));
a->speechlen += len;
pthread_mutex_unlock(&(a->mutex));
#ifdef THREAD_DEBUG
jlog("DEBUG: input: stored %d samples, total=%d\n", len, a->speechlen);
#endif
return(0); /* continue */
}
/**
* <EN>
* A/D-in thread main function.
* </EN>
* <JA>
* A/D-in スレッドのメイン関数.
* </JA>
*
* @param dummy [in] a dummy data, not used.
*/
static void
adin_thread_input_main(void *dummy)
{
Recog *recog;
int ret;
recog = dummy;
ret = adin_cut(adin_store_buffer, NULL, recog);
if (ret == -2) { /* termination request by ad_check? */
jlog("Error: adin thread exit with termination request by checker\n");
} else if (ret == -1) { /* error */
jlog("Error: adin thread exit with error\n");
} else if (ret == 0) { /* EOF */
jlog("Stat: adin thread end with EOF\n");
}
recog->adin->adinthread_ended = TRUE;
/* return to end this thread */
}
/**
* <JA>
* @brief フレーム同期ビーム探索メイン関数（バッチ処理用）
*
* 与えられた入力ベクトル列に対して第１パス(フレーム同期ビーム探索)を
* 行い，その結果を出力する. また全フレームに渡る単語終端を，第２パス
* のために単語トレリス構造体に格納する.
*
* この関数は入力ベクトル列があらかじめ得られている場合に用いられる.
* 第１パスが入力と並列して実行されるオンライン認識の場合，
* この関数は用いられず，代わりにこのファイルで定義されている各サブ関数が
* 直接 realtime-1stpass.c 内から呼ばれる.
*
* @param recog [in] エンジンインスタンス
* </JA>
* <EN>
* @brief Frame synchronous beam search: the main (for batch mode)
*
* This function perform the 1st recognition pass of frame-synchronous beam
* search and output the result. It also stores all the word ends in every
* input frame to word trellis structure.
*
* This function will be called if the whole input vector is already given
* to the end. When online recognition, where the 1st pass will be
* processed in parallel with input, this function will not be used.
* In that case, functions defined in this file will be directly called
* from functions in realtime-1stpass.c.
*
* @param recog [in] engine instance
* </EN>
* @callgraph
* @callergraph
*/
boolean
get_back_trellis(Recog *recog)
{
boolean ok_p;
MFCCCalc *mfcc;
int rewind_frame;
PROCESS_AM *am;
boolean reprocess;
/* initialize mfcc instances */
for(mfcc=recog->mfcclist;mfcc;mfcc=mfcc->next) {
/* mark all as valid, since all frames are fully prepared beforehand */
if (mfcc->param->samplenum == 0) mfcc->valid = FALSE;
else mfcc->valid = TRUE;
/* set frame pointers to 0 */
mfcc->f = 0;
}
/* callback of process start */
#ifdef BACKEND_VAD
if (recog->jconf->decodeopt.segment) {
/* at first time, recognition does not start yet */
/* reset segmentation flags */
spsegment_init(recog);
} else {
/* execute callback for pass1 begin here */
callback_exec(CALLBACK_EVENT_RECOGNITION_BEGIN, recog);
callback_exec(CALLBACK_EVENT_PASS1_BEGIN, recog);
recog->triggered = TRUE;
}
#else
if (recog->jconf->decodeopt.segment) {
if (!recog->process_segment) {
callback_exec(CALLBACK_EVENT_RECOGNITION_BEGIN, recog);
}
callback_exec(CALLBACK_EVENT_SEGMENT_BEGIN, recog);
} else {
callback_exec(CALLBACK_EVENT_RECOGNITION_BEGIN, recog);
}
callback_exec(CALLBACK_EVENT_PASS1_BEGIN, recog);
recog->triggered = TRUE;
#endif
while(1) {
ok_p = TRUE;
for (mfcc = recog->mfcclist; mfcc; mfcc = mfcc->next) {
if (! mfcc->valid) continue;
if (mfcc->f < mfcc->param->samplenum) {
mfcc->valid = TRUE;
ok_p = FALSE;
} else {
mfcc->valid = FALSE;
}
}
if (ok_p) {
/* すべての MFCC が終わりに達したのでループ終了 */
/* all MFCC has been processed, end of loop */
break;
}
switch (decode_proceed(recog)) {
case -1: /* error */
return FALSE;
break;
case 0: /* success */
break;
case 1: /* segmented */
/* 探索中断: 処理された入力は 0 から t-2 まで */
/* search terminated: processed input = [0..t-2] */
/* この時点で第1パスを終了する */
/* end the 1st pass at this point */
decode_end_segmented(recog);
/* terminate 1st pass here */
return TRUE;
}
#ifdef BACKEND_VAD
/* check up trigger in case of VAD segmentation */
if (recog->jconf->decodeopt.segment) {
if (recog->triggered == FALSE) {
if (spsegment_trigger_sync(recog)) {
if (!recog->process_segment) {
callback_exec(CALLBACK_EVENT_RECOGNITION_BEGIN, recog);
}
callback_exec(CALLBACK_EVENT_SEGMENT_BEGIN, recog);
callback_exec(CALLBACK_EVENT_PASS1_BEGIN, recog);
recog->triggered = TRUE;
}
}
}
#endif
if (spsegment_need_restart(recog, &rewind_frame, &reprocess) == TRUE) {
/* do rewind for all mfcc here */
spsegment_restart_mfccs(recog, rewind_frame, reprocess);
/* reset outprob cache for all AM */
for(am=recog->amlist;am;am=am->next) {
outprob_prepare(&(am->hmmwrk), am->mfcc->param->samplenum);
}
}
/* call frame-wise callback */
callback_exec(CALLBACK_EVENT_PASS1_FRAME, recog);
/* 1フレーム処理が進んだのでポインタを進める */
/* proceed frame pointer */
for (mfcc = recog->mfcclist; mfcc; mfcc = mfcc->next) {
if (!mfcc->valid) continue;
mfcc->f++;
}
if (recog->process_want_terminate) {
/* termination requested */
decode_end_segmented(recog);
return TRUE;
}
}
/* 最終フレーム処理を行い，認識の結果出力と終了処理を行う */
decode_end(recog);
return TRUE;
}
/**
* <EN>
* @brief Process one input frame for all recognition process instance.
*
* This function proceeds the recognition for one frame. All
* recognition process instance will be processed synchronously.
* The input frame for each instance is stored in mfcc->f, where mfcc
* is the MFCC calculation instance assigned to each process instance.
*
* If an instance's mfcc->invalid is set to TRUE, its processing will
* be skipped.
*
* When using GMM, GMM computation will also be executed here.
* If GMM_VAD is defined, GMM-based voice detection will be performed
* inside this function, by using a scheme of short-pause segmentation.
*
* This function also handles segmentation of recognition process. A
* segmentation will occur when end of speech is detected by level-based
* sound detection or GMM-based / decoder-based VAD, or by request from
* application. When segmented, it stores current frame and return with
* that status.
*
* The frame-wise callbacks will be executed inside this function,
* when at least one valid recognition process instances exists.
*
* </EN>
* <JA>
* @brief 全ての認識処理インスタンス処理を1フレーム分進める.
*
* 全ての認識処理インスタンスについて，割り付けられているMFCC計算インスタンス
* の mfcc->f をカレントフレームとして処理を1フレーム進める.
*
* なお，mfcc->invalid が TRUE となっている処理インスタンスの処理はスキップ
* される.
*
* GMMの計算もここで呼び出される. GMM_VAD 定義時は，GMM による
* 発話区間開始・終了の検出がここで行われる. また，GMMの計算結果，
* あるいは認識処理内のショートポーズセグメンテーション判定やデバイス・外部
* からの要求によりセグメンテーションが要求されたかどうかの判定も行う.
*
* フレーム単位で呼び出されるコールバックが登録されている場合は，それらの
* 呼出しも行う.
* </JA>
*
* @param recog [in] engine instance
*
* @return 0 on success, -1 on error, or 1 when an input segmentation
* occured/requested inside this function.
*
* @callgraph
* @callergraph
*
*/
int
decode_proceed(Recog *recog)
{
MFCCCalc *mfcc;
boolean break_flag;
boolean break_decode;
RecogProcess *p;
boolean ok_p;
#ifdef GMM_VAD
GMMCalc *gmm;
boolean break_gmm;
#endif
break_decode = FALSE;
for(p = recog->process_list; p; p = p->next) {
#ifdef DETERMINE
p->have_determine = FALSE;
#endif
p->have_interim = FALSE;
}
for (mfcc = recog->mfcclist; mfcc; mfcc = mfcc->next) {
mfcc->segmented = FALSE;
}
#ifdef POWER_REJECT
for (mfcc = recog->mfcclist; mfcc; mfcc = mfcc->next) {
if (!mfcc->valid) continue;
if (mfcc->f == 0) {
mfcc->avg_power = 0.0;
if (debug2_flag) jlog("STAT: power_reject: reset\n");
}
}
#endif
#ifdef GMM_VAD
if (recog->gmm != NULL) {
/* reset flags */
break_gmm = FALSE;
recog->gc->want_rewind = FALSE;
}
#endif
if (recog->gmm != NULL && recog->gmmmfcc->valid) {
/* GMM 計算を行う */
if (recog->gmmmfcc->f == 0) {
/* GMM 計算の初期化 */
gmm_prepare(recog);
}
/* このフレームに対するGMMの尤度を計算 */
gmm_proceed(recog);
#ifdef GMM_VAD
/* Check for GMM-based VAD */
gmm = recog->gc;
gmm_check_trigger(recog);
if (gmm->after_trigger) {
/* after trigger, in speech area */
if (gmm->down_trigger) {
/* down trigger, end segment */
#ifdef GMM_VAD_DEBUG
printf("GMM_VAD: %d: down trigger\n", recog->gmmmfcc->f);
#endif
recog->gmmmfcc->sparea_start = recog->gmmmfcc->f + 1 - recog->jconf->detect.gmm_margin;
if (recog->gmmmfcc->sparea_start < 0) recog->gmmmfcc->sparea_start = 0;
gmm->after_trigger = FALSE;
recog->gmmmfcc->segmented = TRUE;
break_gmm = TRUE;
} else {
/* keep recognition */
}
} else {
/* before trigger, in noise area */
if (gmm->up_trigger) {
/* start recognition */
/* request caller to rewind to the backstep point and
re-start with normal search */
if (recog->gmmmfcc->f + 1 < recog->jconf->detect.gmm_margin) {
gmm->rewind_frame = 0;
} else {
gmm->rewind_frame = recog->gmmmfcc->f + 1 - recog->jconf->detect.gmm_margin;
}
#ifdef GMM_VAD_DEBUG
printf("GMM_VAD: %d: up trigger, start recognition with %d frame rewind\n", recog->gmmmfcc->f, recog->gmmmfcc->f - gmm->rewind_frame);
#endif
gmm->want_rewind = TRUE;
gmm->want_rewind_reprocess = TRUE;
gmm->after_trigger = TRUE;
return 0;
} else {
/* before trigger, noise continues */
/* if noise goes more than a certain frame, shrink the noise area
to avoid unlimited memory usage */
if (recog->gmmmfcc->f + 1 > GMM_VAD_AUTOSHRINK_LIMIT) {
gmm->want_rewind = TRUE;
gmm->want_rewind_reprocess = FALSE;
gmm->rewind_frame = recog->gmmmfcc->f + 1 - recog->jconf->detect.gmm_margin;
if (debug2_flag) {
jlog("DEBUG: GMM_VAD: pause exceeded %d, rewind\n", GMM_VAD_AUTOSHRINK_LIMIT);
}
}
/* skip recognition processing */
return 0;
}
}
#endif /* GMM_VAD */
}
for(p = recog->process_list; p; p = p->next) {
if (!p->live) continue;
mfcc = p->am->mfcc;
if (!mfcc->valid) {
/* このフレームの処理をスキップ */
/* skip processing the frame */
continue;
}
/* mfcc-f のフレームについて認識処理(フレーム同期ビーム探索)を進める */
/* proceed beam search for mfcc->f */
if (mfcc->f == 0) {
/* 最初のフレーム: 探索処理を初期化 */
/* initial frame: initialize search process */
if (get_back_trellis_init(mfcc->param, p) == FALSE) {
jlog("ERROR: %02d %s: failed to initialize the 1st pass\n", p->config->id, p->config->name);
return -1;
}
}
if (mfcc->f > 0 || p->am->hmminfo->multipath) {
/* 1フレーム探索を進める */
/* proceed search for 1 frame */
if (get_back_trellis_proceed(mfcc->f, mfcc->param, p, FALSE) == FALSE) {
mfcc->segmented = TRUE;
break_decode = TRUE;
}
if (p->config->successive.enabled) {
if (detect_end_of_segment(p, mfcc->f - 1)) {
/* セグメント終了検知: 第１パスここで中断 */
mfcc->segmented = TRUE;
break_decode = TRUE;
}
}
}
}
/* セグメントすべきかどうか最終的な判定を行う．
デコーダベースVADあるいは spsegment の場合，複数インスタンス間で OR
を取る．また，GMMなど複数基準がある場合は基準間で AND を取る．*/
/* determine whether to segment at here
If multiple segmenter exists, take their AND */
break_flag = FALSE;
if (break_decode
#ifdef GMM_VAD
|| (recog->gmm != NULL && break_gmm)
#endif
) {
break_flag = TRUE;
}
if (break_flag) {
/* 探索処理の終了が発生したのでここで認識を終える.
最初のフレームから [f-1] 番目までが認識されたことになる
*/
/* the recognition process tells us to stop recognition, so
recognition should be terminated here.
the recognized data are [0..f-1] */
/* 最終フレームを last_time にセット */
/* set the last frame to last_time */
for (mfcc = recog->mfcclist; mfcc; mfcc = mfcc->next) {
mfcc->last_time = mfcc->f - 1;
}
if (! recog->jconf->decodeopt.segment) {
/* ショートポーズ以外で切れた場合，残りのサンプルは認識せずに捨てる */
/* drop rest inputs if segmented by error */
for (mfcc = recog->mfcclist; mfcc; mfcc = mfcc->next) {
mfcc->param->header.samplenum = mfcc->f;
mfcc->param->samplenum = mfcc->f;
}
}
return 1;
}
/* call frame-wise callback for the processing results if any */
#ifdef DETERMINE
ok_p = FALSE;
for(p=recog->process_list;p;p=p->next) {
if (!p->live) continue;
if (p->have_determine) {
ok_p = TRUE;
}
}
if (ok_p) callback_exec(CALLBACK_RESULT_PASS1_DETERMINED, recog);
#endif
ok_p = FALSE;
for(p=recog->process_list;p;p=p->next) {
if (!p->live) continue;
if (p->have_interim) {
ok_p = TRUE;
}
}
if (ok_p) callback_exec(CALLBACK_RESULT_PASS1_INTERIM, recog);
return 0;
}
