static int32 lm3g_tg_score(NGRAM_MODEL_TYPE *model, int32 lw1, int32 lw2, int32 lw3, int32 *n_used) { ngram_model_t *base = &model->base; int32 i, n, score; trigram_t *tg; tginfo_t *tginfo, *prev_tginfo; if ((base->n < 3) || (lw1 < 0) || (lw2 < 0)) return (lm3g_bg_score(model, lw2, lw3, n_used)); sbmtx_lock(model->lm3g.tginfo_mtx); prev_tginfo = NULL; for (tginfo = model->lm3g.tginfo[lw2]; tginfo; tginfo = tginfo->next) { if (tginfo->w1 == lw1) break; prev_tginfo = tginfo; } if (!tginfo) { load_tginfo(model, lw1, lw2); tginfo = model->lm3g.tginfo[lw2]; } else if (prev_tginfo) { prev_tginfo->next = tginfo->next; tginfo->next = model->lm3g.tginfo[lw2]; model->lm3g.tginfo[lw2] = tginfo; } tginfo->used = 1; /* Trigrams for w1,w2 now pointed to by tginfo */ n = tginfo->n_tg; tg = tginfo->tg; if ((i = find_tg(tg, n, lw3)) >= 0) { /* Access mode = trigram */ if (n_used) *n_used = 3; score = model->lm3g.prob3[tg[i].prob3].l; } else { score = tginfo->bowt + lm3g_bg_score(model, lw2, lw3, n_used); } sbmtx_unlock(model->lm3g.tginfo_mtx); return (score); }
int32 lm_bg_score (int32 w1, int32 w2) { int32 cscr, tscr, remwt; lm_t *lm3g; if (! clm) return (lm3g_bg_score (w1, w2)); lm3g = lm_get_current (); /* Get cache LM score and apply language weight */ cscr = cache_lm_score (clm, w1, w2, &remwt); cscr *= lm3g->lw; /* Get static trigram LM score, apply remaining weight */ tscr = lm3g_bg_score (w1, w2); tscr += remwt * lm3g->lw; /* Return MAX of static trigram LM and dynamic cache LM scores (approx to sum) */ return (cscr > tscr) ? cscr : tscr; }
static int32 lm3g_template_score(ngram_model_t *base, int32 wid, int32 *history, int32 n_hist, int32 *n_used) { NGRAM_MODEL_TYPE *model = (NGRAM_MODEL_TYPE *)base; switch (n_hist) { case 0: /* Access mode: unigram */ *n_used = 1; return model->lm3g.unigrams[wid].prob1.l; case 1: return lm3g_bg_score(model, history[0], wid, n_used); case 2: default: /* Anything greater than 2 is the same as a trigram for now. */ return lm3g_tg_score(model, history[1], history[0], wid, n_used); } }
static int32 lm3g_template_raw_score(ngram_model_t *base, int32 wid, int32 *history, int32 n_hist, int32 *n_used) { NGRAM_MODEL_TYPE *model = (NGRAM_MODEL_TYPE *)base; int32 score; switch (n_hist) { case 0: /* Access mode: unigram */ *n_used = 1; /* Undo insertion penalty. */ score = model->lm3g.unigrams[wid].prob1.l - base->log_wip; /* Undo language weight. */ score = (int32)(score / base->lw); /* Undo unigram interpolation */ if (strcmp(base->word_str[wid], "<s>") != 0) { /* FIXME: configurable start_sym */ /* This operation is numerically unstable, so try to avoid it * as possible */ if (base->log_uniform + base->log_uniform_weight > logmath_get_zero(base->lmath)) { score = logmath_log(base->lmath, logmath_exp(base->lmath, score) - logmath_exp(base->lmath, base->log_uniform + base->log_uniform_weight)); } } return score; case 1: score = lm3g_bg_score(model, history[0], wid, n_used); break; case 2: default: /* Anything greater than 2 is the same as a trigram for now. */ score = lm3g_tg_score(model, history[1], history[0], wid, n_used); break; } /* FIXME (maybe): This doesn't undo unigram weighting in backoff cases. */ return (int32)((score - base->log_wip) / base->lw); }