AltWordendHyp *FakeSEpath (DecoderInst *dec, RelToken *tok, Boolean useLM)
{
AltWordendHyp *alt = NULL;
PronId endPronId;
LMState dest;
LMTokScore lmScore;
endPronId = dec->net->end->data.pron;
if (useLM)
lmScore = LMCacheTransProb (dec, dec->lm, tok->lmState, endPronId, &dest);
else
lmScore = 0.0;
if (lmScore > LSMALL) { /* transition for END state possible? */
assert (!useLM || dest == (Ptr) 0xfffffffe);
lmScore += dec->insPen;
alt = (AltWordendHyp *) New (&dec->altweHypHeap, sizeof (AltWordendHyp));
alt->next = NULL;
if (!dec->fastlmla) {
assert (lmScore <= tok->lmscore + 0.1); /* might not be true for more aggressive LMLA? */
}
/* temporarily store full score in altWEhyp */
alt->score = tok->delta + (lmScore - tok->lmscore);
alt->lm = lmScore;
alt->prev = tok->path;
}
return alt;
}
/* HandleWordend
update traceback, add LM, update LM state, recombine tokens
ln->type must be LN_WORDEND
*/
void Decoder::HandleWordend (LexNode *ln) {
WordendHyp *prev;
LMState dest;
PronId pronid;
RelTokScore newDelta, bestDelta, deltaLimit;
ModendHyp *modpath;
LexNodeInst *inst = ln->inst;
TokenSet *ts = inst->ts;
/* main beam pruning for reltoks */
/* main and relative beam pruning */
deltaLimit = std::max(_dec->beamLimit - ts->score, _dec->relBeamWidth);
/* for each token i in set, take transition in LM
recombine tokens in same LMState
newN is (current) number of new tokens (newN <= ts->n) */
pronid = (PronId) ln->data.pron;
int newN = 0;
bestDelta = LZERO;
for (int i = 0; i < ts->n; ++i) {
auto tok = &ts->relTok[i];
// Pruning
if (tok->delta < deltaLimit) continue;
auto lmScore = LMCacheTransProb (_dec->lm, tok->lmState, pronid, &dest);
// word insertion penalty
lmScore += _dec->insPen;
/* remember prev path now, as we might overwrite it below */
prev = tok->path;
modpath = tok->modpath;
/* subtract lookahead which has already been applied */
newDelta = tok->delta + (lmScore - tok->lmscore);
// Prune again
if (newDelta < deltaLimit) continue;
bestDelta = std::max(bestDelta, newDelta);
int j;
/* insert in list */
for (j = 0; j < newN; ++j) { /* is there already a token in state dest? */
auto tokJ = &ts->relTok[j];
if (tokJ->lmState != dest)
continue;
if (!_dec->latgen) {
// replace tokJ
if (newDelta > tokJ->delta) {
tokJ->delta = newDelta;
tokJ->lmscore = 0.0; // reset lookahead
tokJ->modpath = modpath;
// Update path -- weHyp exists, pron is the same anyway, update rest
tokJ->path->prev = prev;
tokJ->path->score = ts->score + newDelta;
tokJ->path->lm = lmScore;
tokJ->path->modpath = modpath;
}
/* else just toss token */
}
else { /* latgen */
auto alt = (AltWordendHyp *) New (&_dec->altweHypHeap, sizeof (AltWordendHyp));
if (newDelta > tokJ->delta) {
/* move tokJ->path to alt */
alt->prev = tokJ->path->prev;
alt->score = tokJ->path->score;
alt->lm = tokJ->path->lm;
alt->modpath = tokJ->path->modpath;
/* replace tokJ */
tokJ->delta = newDelta;
tokJ->lmscore = 0.0; /* reset lookahead */
tokJ->modpath = modpath;
tokJ->path->modpath = modpath;
/* store new tok info in path
weHyp exists, pron is the same anyway, update rest */
tokJ->path->prev = prev;
tokJ->path->score = ts->score + newDelta;
tokJ->path->lm = lmScore;
}
else {
/* store new tok info in alt */
alt->prev = prev;
alt->score = ts->score + newDelta;
alt->lm = lmScore;
alt->modpath = modpath;
}
/* attach alt to tokJ's weHyp */
alt->next = tokJ->path->alt;
tokJ->path->alt = alt;
}
break; /* leave j loop */
}
if (j == newN) { /* no token in state dest yet */
/* find spot to insert LMState dest */
for (j = 0; j < newN; ++j)
if (ts->relTok[j].lmState > dest)
break;
/* move any following reltokens up one slot */
for (int k = newN ; k > j; --k)
ts->relTok[k] = ts->relTok[k-1];
++newN;
/* new wordendHyp */
auto weHyp = (WordendHyp *) New (&_dec->weHypHeap, sizeof (WordendHyp));
weHyp->prev = prev;
weHyp->pron = ln->data.pron;
weHyp->score = ts->score + newDelta;
weHyp->lm = lmScore;
weHyp->frame = _dec->frame;
weHyp->alt = NULL;
weHyp->user = 0;
weHyp->modpath = modpath;
auto tokJ = &ts->relTok[j];
tokJ->modpath = modpath;
tokJ->path = weHyp;
tokJ->delta = newDelta;
tokJ->lmState = dest;
tokJ->we_tag = (void *) ln;
tokJ->lmscore = 0.0;
}
} /* for token i */
ts->n = newN;
if (newN > 0) {
/* renormalise to new best score */
for (int i = 0; i < ts->n; ++i) {
auto tok = &ts->relTok[i];
tok->delta -= bestDelta;
/* convert alt wordendHyp scores to deltas relativ to main weHyp */
for (auto alt = tok->path->alt; alt; alt = alt->next)
alt->score = alt->score - tok->path->score;
}
ts->score += bestDelta;
ts->id = ++_dec->tokSetIdCount;
}
else {
ts->id = 0;
ts->score = LZERO;
}
inst->best = ts->score;
}
