LMResult LanguageModelDALM::GetValue(const vector<const Word*> &contextFactor, State* finalState) const
{
LMResult ret;
// initialize DALM array
DALM::VocabId ngram[m_nGramOrder];
for(size_t i = 0; i < m_nGramOrder; i++){
ngram[i] = wid_start;
}
DALM::VocabId wid;
for (size_t i = 0; i < contextFactor.size(); ++i) {
const Word &word = *contextFactor[i];
wid = GetVocabId(word.GetFactor(m_factorType));
push(ngram, m_nGramOrder, wid);
}
// last word is unk?
ret.unknown = (wid == m_vocab->unk());
// calc score. Doesn't handle unk yet
float score = m_lm->query(ngram, m_nGramOrder);
score = TransformLMScore(score);
ret.score = score;
(*finalState) = (void *)m_lm->get_state(ngram, m_nGramOrder);
return ret;
}
LMResult LanguageModelDALM::GetValue(const vector<const Word*> &contextFactor, State* finalState) const
{
LMResult ret;
// initialize DALM array
DALM::VocabId ngram[m_nGramOrder];
for(size_t i = 0; i < m_nGramOrder; i++){
ngram[i] = wid_start;
}
DALM::VocabId wid;
for (size_t i = 0; i < contextFactor.size(); ++i) {
const Word &word = *contextFactor[i];
wid = GetVocabId(word.GetFactor(m_factorType));
push(ngram, m_nGramOrder, wid);
}
// last word is unk?
ret.unknown = (wid == DALM_UNK_WORD);
// calc score. Doesn't handle unk yet
float score = m_lm->query(ngram, m_nGramOrder);
score = TransformLMScore(score);
ret.score = score;
// hash of n-1 words to use as state
size_t startPos = (contextFactor.size() < m_nGramOrder) ? 0 : 1;
size_t hash = 0;
for (size_t i = startPos; i < contextFactor.size(); ++i) {
const Word &word = *contextFactor[i];
const Factor *factor = word.GetFactor(m_factorType);
boost::hash_combine(hash, factor);
}
(*finalState) = (State*) hash;
return ret;
}
LMResult LanguageModelDALM::GetValue(const vector<const Word*> &contextFactor, State* finalState) const
{
LMResult ret;
// initialize DALM array
DALM::VocabId ngram[m_nGramOrder];
for(size_t i = 0; i < m_nGramOrder; i++){
ngram[i] = wid_start;
}
DALM::VocabId wid;
for (size_t i = 0; i < contextFactor.size(); ++i) {
const Word &word = *contextFactor[i];
wid = GetVocabId(word.GetFactor(m_factorType));
push(ngram, m_nGramOrder, wid);
}
// last word
ret.unknown = (wid == DALM_UNK_WORD);
float prob = m_lm->query(ngram, m_nGramOrder);
ret.score = TransformLMScore(prob);
// use last word as state info
const Factor *factor;
size_t hash_value(const Factor &f);
if (contextFactor.size()) {
factor = contextFactor.back()->GetFactor(m_factorType);
} else {
factor = NULL;
}
(*finalState) = (State*) factor;
return ret;
}
LMResult LanguageModelDALM::GetValue(const Word &word, DALM::State* finalState) const
{
DALM::VocabId wid = GetVocabId(word.GetFactor(m_factorType));
return GetValue(wid, finalState);
}
