/**
* Constructs an empty backward language model state.
*
* This state will correspond with a translation hypothesis
* where no source words have been translated.
*
* In a forward language model, the language model state of an empty hypothesis
* would store the beginning of sentence marker <s>.
*
* Because this is a backward language model, the language model state returned by this method
* instead stores the end of sentence marker </s>.
*/
template <class Model> const FFState *BackwardLanguageModel<Model>::EmptyHypothesisState(const InputType &/*input*/) const
{
BackwardLMState *ret = new BackwardLMState();
lm::ngram::RuleScore<Model> ruleScore(*m_ngram, ret->state);
ruleScore.Terminal(m_ngram->GetVocabulary().EndSentence());
// float score =
ruleScore.Finish();
// VERBOSE(1, "BackwardLM EmptyHypothesisState has score " << score);
return ret;
}
void KENLM<Model>::EvaluateWhenApplied(const SCFG::Manager &mgr,
const SCFG::Hypothesis &hypo, int featureID, Scores &scores,
FFState &state) const
{
LanguageModelChartStateKenLM &newState = static_cast<LanguageModelChartStateKenLM&>(state);
lm::ngram::RuleScore<Model> ruleScore(*m_ngram, newState.GetChartState());
const SCFG::TargetPhraseImpl &target = hypo.GetTargetPhrase();
const AlignmentInfo::NonTermIndexMap &nonTermIndexMap =
target.GetAlignNonTerm().GetNonTermIndexMap();
const size_t size = target.GetSize();
size_t phrasePos = 0;
// Special cases for first word.
if (size) {
const SCFG::Word &word = target[0];
if (word[m_factorType] == m_bos) {
// Begin of sentence
ruleScore.BeginSentence();
phrasePos++;
} else if (word.isNonTerminal) {
// Non-terminal is first so we can copy instead of rescoring.
const SCFG::Hypothesis *prevHypo = hypo.GetPrevHypo(nonTermIndexMap[phrasePos]);
const lm::ngram::ChartState &prevState = static_cast<const LanguageModelChartStateKenLM*>(prevHypo->GetState(featureID))->GetChartState();
ruleScore.BeginNonTerminal(prevState);
phrasePos++;
}
}
for (; phrasePos < size; phrasePos++) {
const SCFG::Word &word = target[phrasePos];
if (word.isNonTerminal) {
const SCFG::Hypothesis *prevHypo = hypo.GetPrevHypo(nonTermIndexMap[phrasePos]);
const lm::ngram::ChartState &prevState = static_cast<const LanguageModelChartStateKenLM*>(prevHypo->GetState(featureID))->GetChartState();
ruleScore.NonTerminal(prevState);
} else {
ruleScore.Terminal(TranslateID(word));
}
}
float score = ruleScore.Finish();
score = TransformLMScore(score);
// take out score from loading. This needs reworking
//score -= target.GetScores().GetScores(*this)[0];
bool OOVFeatureEnabled = false;
if (OOVFeatureEnabled) {
std::vector<float> scoresVec(2);
scoresVec[0] = score;
scoresVec[1] = 0.0;
scores.PlusEquals(mgr.system, *this, scoresVec);
} else {
scores.PlusEquals(mgr.system, *this, score);
}
}
