long CSphinxConf::HypConf (search_hyp_t *hyp) { search_hyp_t *h; int32 w1, w2, finishwid, startwid, type[4096]; int32 i, k, t; finishwid = kb_get_word_id ("</s>"); startwid = kb_get_word_id ("<s>"); if ((finishwid < 0) || (startwid < 0)) return -1; w1 = finishwid; w2 = startwid; type[0] = 3; // Dummy trigram entry before utterance k = 1; for (h = hyp; h; h = h->next) { if (k > 4095) return -1; lm_tg_score (w1, w2, h->wid); type[k++] = lm3g_access_type(); w1 = w2; w2 = h->wid; } lm_tg_score (w1, w2, finishwid); type[k++] = lm3g_access_type(); type[k++] = 3; // Dummy trigram entries after utterance for (i = 1, h = hyp; i < k-2; i++, h = h->next) { t = type[i-1] + type[i] + ((type[i+1] + type[i+2])<<1); h->conf = (float)((double)(t-6)/12.0); } return 0; }
void vithist_rescore (vithist_t *vh, kbcore_t *kbc, s3wid_t wid, int32 ef, int32 score, int32 pred, int32 type) { vithist_entry_t *pve, tve; s3lmwid_t lwid; int32 se, fe; int32 i; assert (vh->n_frm == ef); pve = vh->entry[VITHIST_ID2BLK(pred)] + VITHIST_ID2BLKOFFSET(pred); /* Create a temporary entry with all the info currently available */ tve.wid = wid; tve.sf = pve->ef + 1; tve.ef = ef; tve.type = type; tve.valid = 1; tve.ascr = score - pve->score; if (pred == 0) { /* Special case for the initial <s> entry */ se = 0; fe = 1; } else { se = vh->frame_start[pve->ef]; fe = vh->frame_start[pve->ef + 1]; } if (dict_filler_word (kbcore_dict(kbc), wid)) { tve.lscr = fillpen (kbcore_fillpen(kbc), wid); tve.score = score + tve.lscr; tve.pred = pred; tve.lmstate.lm3g = pve->lmstate.lm3g; vithist_enter (vh, kbc, &tve); } else { lwid = kbcore_dict2lmwid (kbc, wid); tve.lmstate.lm3g.lwid[0] = lwid; for (i = se; i < fe; i++) { pve = vh->entry[VITHIST_ID2BLK(i)] + VITHIST_ID2BLKOFFSET(i); if (pve->valid) { tve.lscr = lm_tg_score (kbcore_lm(kbc), pve->lmstate.lm3g.lwid[1], pve->lmstate.lm3g.lwid[0], lwid); tve.score = pve->score + tve.ascr + tve.lscr; if ((tve.score - vh->wbeam) >= vh->bestscore[vh->n_frm]) { tve.pred = i; tve.lmstate.lm3g.lwid[1] = pve->lmstate.lm3g.lwid[0]; vithist_enter (vh, kbc, &tve); } } } } }
/** * Scores the given N-gram using the given language model. * * args: * wid - the IDs of the sequence of words in the n-gram * nwd - the number of words in the n-gram * lm - the language model to use * * return: the language model score of the given sequence of words */ int score_ngram(s3lmwid32_t * wid, int nwd, lm_t * lm) { int32 score; score = 0; if (nwd == 3) { /* The last argument is a hack: the information there - the dict * ID - is never used if LM classes are not used, and classes * are not used in this code. Therefore, the last argument here * is a nop. */ score = lm_tg_score(lm, wid[0], wid[1], wid[2], 0); } else if (nwd == 2) { /* Ditto. */ score = lm_bg_score(lm, wid[0], wid[1], 0); } else if (nwd == 1) { /* Ditto. */ score = lm_ug_score(lm, wid[0], 0); } else { printf("%d grams not supported\n", nwd); } return score; }
int32 vithist_partialutt_end (vithist_t *vh, kbcore_t *kbc) { int32 f, i, b, l; int32 sv, nsv, scr, bestscore, bestvh; vithist_entry_t *ve, *bestve; s3lmwid_t endwid; lm_t *lm; /* Find last frame with entries in vithist table */ for (f = vh->n_frm-1; f >= 0; --f) { sv = vh->frame_start[f]; /* First vithist entry in frame f */ nsv = vh->frame_start[f+1]; /* First vithist entry in next frame (f+1) */ if (sv < nsv) break; } if (f < 0) return -1; if (f != vh->n_frm-1){ E_ERROR("No word exits from last frame in block %d\n",vh->n_frm-1); return -1; } /* Terminate in a final </s> node (make this optional?) */ lm = kbcore_lm (kbc); endwid = lm_finishwid (lm); bestscore = MAX_NEG_INT32; bestvh = -1; for (i = sv; i < nsv; i++) { b = VITHIST_ID2BLK (i); l = VITHIST_ID2BLKOFFSET (i); ve = vh->entry[b] + l; scr = ve->score; scr += lm_tg_score (lm, ve->lmstate.lm3g.lwid[1], ve->lmstate.lm3g.lwid[0], endwid); if (bestscore < scr) { bestscore = scr; bestvh = i; bestve = ve; } } return bestvh; }
/** * Scores the given N-gram using the given language model. * * args: * wid - the IDs of the sequence of words in the n-gram * nwd - the number of words in the n-gram * lm - the language model to use * * return: the language model score of the given sequence of words */ int score_ngram(s3lmwid_t *wid, int nwd, lm_t *lm) { int32 score, tgscr; score = 0; if (nwd == 3) { score = lm_tg_score(lm, wid[0], wid[1], wid[2]); } else if (nwd == 2) { score = lm_bg_score(lm, wid[0], wid[1]); } else if (nwd == 1) { score = lm_ug_score(lm, wid[0]); } else { printf("%d grams not supported\n", nwd); } return score; }
int32 vithist_utt_end (vithist_t *vh, kbcore_t *kbc) { int32 f, i, b, l; int32 sv, nsv, scr, bestscore, bestvh, vhid; vithist_entry_t *ve, *bestve=0; s3lmwid_t endwid; lm_t *lm; dict_t *dict; /* Find last frame with entries in vithist table */ for (f = vh->n_frm-1; f >= 0; --f) { sv = vh->frame_start[f]; /* First vithist entry in frame f */ nsv = vh->frame_start[f+1]; /* First vithist entry in next frame (f+1) */ if (sv < nsv) break; } if (f < 0) return -1; if (f != vh->n_frm-1) E_ERROR("No word exit in frame %d, using exits from frame %d\n", vh->n_frm-1, f); /* Terminate in a final </s> node (make this optional?) */ lm = kbcore_lm (kbc); endwid = lm_finishwid (lm); bestscore = MAX_NEG_INT32; bestvh = -1; for (i = sv; i < nsv; i++) { b = VITHIST_ID2BLK (i); l = VITHIST_ID2BLKOFFSET (i); ve = vh->entry[b] + l; scr = ve->score; scr += lm_tg_score (lm, ve->lmstate.lm3g.lwid[1], ve->lmstate.lm3g.lwid[0], endwid); if (bestscore < scr) { bestscore = scr; bestvh = i; bestve = ve; } } assert (bestvh >= 0); dict = kbcore_dict (kbc); if (f != vh->n_frm-1) { E_ERROR("No word exit in frame %d, using exits from frame %d\n", vh->n_frm-1, f); /* Add a dummy silwid covering the remainder of the utterance */ assert (vh->frame_start[vh->n_frm-1] == vh->frame_start[vh->n_frm]); vh->n_frm -= 1; vithist_rescore (vh, kbc, dict_silwid (dict), vh->n_frm, bestve->score, bestvh, -1); vh->n_frm += 1; vh->frame_start[vh->n_frm] = vh->n_entry; return vithist_utt_end (vh, kbc); } /* Create an </s> entry */ vhid = vh->n_entry; ve = vithist_entry_alloc (vh); ve->wid = dict_finishwid (dict); ve->sf = (bestve->ef == BAD_S3FRMID) ? 0 : bestve->ef + 1; ve->ef = vh->n_frm; ve->ascr = 0; ve->lscr = bestscore - bestve->score; ve->score = bestscore; ve->pred = bestvh; ve->type = 0; ve->valid = 1; ve->lmstate.lm3g.lwid[0] = endwid; ve->lmstate.lm3g.lwid[1] = ve->lmstate.lm3g.lwid[0]; return vhid; }