float language_model::log_prob(const token_list& tokens) const { float prob = 0.0f; lm::lm_state state; lm::lm_state state_next; for (const auto& token : tokens.tokens()) { prob += score(state, token, state_next); state = state_next; } return prob; }
float language_model::log_prob(const token_list& tokens) const { using diff_type = decltype(tokens.tokens().begin())::difference_type; float prob = 0.0f; // tokens < N for (uint64_t i = 0; i < N_ - 1 && i < tokens.size(); ++i) { prob += prob_calc(tokens.tokens().begin(), tokens.tokens().begin() + static_cast<diff_type>(i) + 1); } // tokens >= N for (uint64_t i = N_ - 1; i < tokens.size(); ++i) { prob += prob_calc( tokens.tokens().begin() + static_cast<diff_type>(i - N_ + 1), tokens.tokens().begin() + static_cast<diff_type>(i) + 1); } return prob; }