void LanguageModel::EvaluateWhenApplied(const ManagerBase &mgr,
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
FFState &state) const
{
const LMState &prevLMState = static_cast<const LMState &>(prevState);
size_t numWords = prevLMState.numWords;
// context is held backwards
vector<const Factor*> context(numWords);
for (size_t i = 0; i < numWords; ++i) {
context[i] = prevLMState.lastWords[i];
}
//DebugContext(context);
SCORE score = 0;
std::pair<SCORE, void*> fromScoring;
const TargetPhrase<Moses2::Word> &tp = hypo.GetTargetPhrase();
for (size_t i = 0; i < tp.GetSize(); ++i) {
const Word &word = tp[i];
const Factor *factor = word[m_factorType];
ShiftOrPush(context, factor);
fromScoring = Score(context);
score += fromScoring.first;
}
const Bitmap &bm = hypo.GetBitmap();
if (bm.IsComplete()) {
// everything translated
ShiftOrPush(context, m_eos);
fromScoring = Score(context);
score += fromScoring.first;
fromScoring.second = NULL;
context.clear();
} else {
assert(context.size());
if (context.size() == m_order) {
context.resize(context.size() - 1);
}
}
scores.PlusEquals(mgr.system, *this, score);
// return state
//DebugContext(context);
LMState &stateCast = static_cast<LMState&>(state);
MemPool &pool = mgr.GetPool();
stateCast.Set(pool, fromScoring.second, context);
}
void LanguageModel::EvaluateInIsolation(MemPool &pool, const System &system,
const Phrase<Moses2::Word> &source, const TargetPhraseImpl &targetPhrase, Scores &scores,
SCORE &estimatedScore) const
{
if (targetPhrase.GetSize() == 0) {
return;
}
SCORE score = 0;
SCORE nonFullScore = 0;
vector<const Factor*> context;
// context.push_back(m_bos);
context.reserve(m_order);
for (size_t i = 0; i < targetPhrase.GetSize(); ++i) {
const Factor *factor = targetPhrase[i][m_factorType];
ShiftOrPush(context, factor);
if (context.size() == m_order) {
std::pair<SCORE, void*> fromScoring = Score(context);
score += fromScoring.first;
} else {
std::pair<SCORE, void*> fromScoring = Score(context);
nonFullScore += fromScoring.first;
}
}
scores.PlusEquals(system, *this, score);
SCORE weightedScore = Scores::CalcWeightedScore(system, *this, nonFullScore);
estimatedScore += weightedScore;
}
void LM::Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, Scores &scores
, Scores &estimatedFutureScore) const
{
SCORE all = 0, ngram = 0;
PhraseVec phraseVec;
phraseVec.reserve(m_order);
for (size_t pos = 0; pos < targetPhrase.GetSize(); ++pos) {
const Word &word = targetPhrase.GetWord(pos);
ShiftOrPush(phraseVec, word);
SCORE score = GetValueCache(phraseVec);
all += score;
if (phraseVec.size() == m_order) {
ngram += score;
}
}
SCORE estimated = all - ngram;
scores.Add(*this, ngram);
estimatedFutureScore.Add(*this, estimated);
}