/** * Constructs an empty backward language model state. * * This state will correspond with a translation hypothesis * where no source words have been translated. * * In a forward language model, the language model state of an empty hypothesis * would store the beginning of sentence marker <s>. * * Because this is a backward language model, the language model state returned by this method * instead stores the end of sentence marker </s>. */ template <class Model> const FFState *BackwardLanguageModel<Model>::EmptyHypothesisState(const InputType &/*input*/) const { BackwardLMState *ret = new BackwardLMState(); lm::ngram::RuleScore<Model> ruleScore(*m_ngram, ret->state); ruleScore.Terminal(m_ngram->GetVocabulary().EndSentence()); // float score = ruleScore.Finish(); // VERBOSE(1, "BackwardLM EmptyHypothesisState has score " << score); return ret; }
void KENLM<Model>::EvaluateWhenApplied(const SCFG::Manager &mgr, const SCFG::Hypothesis &hypo, int featureID, Scores &scores, FFState &state) const { LanguageModelChartStateKenLM &newState = static_cast<LanguageModelChartStateKenLM&>(state); lm::ngram::RuleScore<Model> ruleScore(*m_ngram, newState.GetChartState()); const SCFG::TargetPhraseImpl &target = hypo.GetTargetPhrase(); const AlignmentInfo::NonTermIndexMap &nonTermIndexMap = target.GetAlignNonTerm().GetNonTermIndexMap(); const size_t size = target.GetSize(); size_t phrasePos = 0; // Special cases for first word. if (size) { const SCFG::Word &word = target[0]; if (word[m_factorType] == m_bos) { // Begin of sentence ruleScore.BeginSentence(); phrasePos++; } else if (word.isNonTerminal) { // Non-terminal is first so we can copy instead of rescoring. const SCFG::Hypothesis *prevHypo = hypo.GetPrevHypo(nonTermIndexMap[phrasePos]); const lm::ngram::ChartState &prevState = static_cast<const LanguageModelChartStateKenLM*>(prevHypo->GetState(featureID))->GetChartState(); ruleScore.BeginNonTerminal(prevState); phrasePos++; } } for (; phrasePos < size; phrasePos++) { const SCFG::Word &word = target[phrasePos]; if (word.isNonTerminal) { const SCFG::Hypothesis *prevHypo = hypo.GetPrevHypo(nonTermIndexMap[phrasePos]); const lm::ngram::ChartState &prevState = static_cast<const LanguageModelChartStateKenLM*>(prevHypo->GetState(featureID))->GetChartState(); ruleScore.NonTerminal(prevState); } else { ruleScore.Terminal(TranslateID(word)); } } float score = ruleScore.Finish(); score = TransformLMScore(score); // take out score from loading. This needs reworking //score -= target.GetScores().GetScores(*this)[0]; bool OOVFeatureEnabled = false; if (OOVFeatureEnabled) { std::vector<float> scoresVec(2); scoresVec[0] = score; scoresVec[1] = 0.0; scores.PlusEquals(mgr.system, *this, scoresVec); } else { scores.PlusEquals(mgr.system, *this, score); } }