float language_model::log_prob(const token_list& tokens) const
{
float prob = 0.0f;
lm::lm_state state;
lm::lm_state state_next;
for (const auto& token : tokens.tokens())
{
prob += score(state, token, state_next);
state = state_next;
}
return prob;
}
float language_model::log_prob(const token_list& tokens) const
{
using diff_type = decltype(tokens.tokens().begin())::difference_type;
float prob = 0.0f;
// tokens < N
for (uint64_t i = 0; i < N_ - 1 && i < tokens.size(); ++i)
{
prob += prob_calc(tokens.tokens().begin(),
tokens.tokens().begin() + static_cast<diff_type>(i)
+ 1);
}
// tokens >= N
for (uint64_t i = N_ - 1; i < tokens.size(); ++i)
{
prob += prob_calc(
tokens.tokens().begin() + static_cast<diff_type>(i - N_ + 1),
tokens.tokens().begin() + static_cast<diff_type>(i) + 1);
}
return prob;
}
