LMResult LanguageModelDALM::GetValue(const vector<const Word*> &contextFactor, State* finalState) const { LMResult ret; // initialize DALM array DALM::VocabId ngram[m_nGramOrder]; for(size_t i = 0; i < m_nGramOrder; i++){ ngram[i] = wid_start; } DALM::VocabId wid; for (size_t i = 0; i < contextFactor.size(); ++i) { const Word &word = *contextFactor[i]; wid = GetVocabId(word.GetFactor(m_factorType)); push(ngram, m_nGramOrder, wid); } // last word is unk? ret.unknown = (wid == m_vocab->unk()); // calc score. Doesn't handle unk yet float score = m_lm->query(ngram, m_nGramOrder); score = TransformLMScore(score); ret.score = score; (*finalState) = (void *)m_lm->get_state(ngram, m_nGramOrder); return ret; }
LMResult LanguageModelDALM::GetValue(const vector<const Word*> &contextFactor, State* finalState) const { LMResult ret; // initialize DALM array DALM::VocabId ngram[m_nGramOrder]; for(size_t i = 0; i < m_nGramOrder; i++){ ngram[i] = wid_start; } DALM::VocabId wid; for (size_t i = 0; i < contextFactor.size(); ++i) { const Word &word = *contextFactor[i]; wid = GetVocabId(word.GetFactor(m_factorType)); push(ngram, m_nGramOrder, wid); } // last word is unk? ret.unknown = (wid == DALM_UNK_WORD); // calc score. Doesn't handle unk yet float score = m_lm->query(ngram, m_nGramOrder); score = TransformLMScore(score); ret.score = score; // hash of n-1 words to use as state size_t startPos = (contextFactor.size() < m_nGramOrder) ? 0 : 1; size_t hash = 0; for (size_t i = startPos; i < contextFactor.size(); ++i) { const Word &word = *contextFactor[i]; const Factor *factor = word.GetFactor(m_factorType); boost::hash_combine(hash, factor); } (*finalState) = (State*) hash; return ret; }
LMResult LanguageModelDALM::GetValue(const vector<const Word*> &contextFactor, State* finalState) const { LMResult ret; // initialize DALM array DALM::VocabId ngram[m_nGramOrder]; for(size_t i = 0; i < m_nGramOrder; i++){ ngram[i] = wid_start; } DALM::VocabId wid; for (size_t i = 0; i < contextFactor.size(); ++i) { const Word &word = *contextFactor[i]; wid = GetVocabId(word.GetFactor(m_factorType)); push(ngram, m_nGramOrder, wid); } // last word ret.unknown = (wid == DALM_UNK_WORD); float prob = m_lm->query(ngram, m_nGramOrder); ret.score = TransformLMScore(prob); // use last word as state info const Factor *factor; size_t hash_value(const Factor &f); if (contextFactor.size()) { factor = contextFactor.back()->GetFactor(m_factorType); } else { factor = NULL; } (*finalState) = (State*) factor; return ret; }
LMResult LanguageModelDALM::GetValue(const Word &word, DALM::State* finalState) const { DALM::VocabId wid = GetVocabId(word.GetFactor(m_factorType)); return GetValue(wid, finalState); }