/** * Constructor. Initializes on-the-fly composition with a language model. * \param fst Machine you want to apply the language to. Pass a delayed machine if you can, as it will expand it in constructor. * \param model A KenLM language model * \param epsilons List of words to work as epsilons * \param natlog Use or not natural logs * \param lmscale Language model scale */ ApplyLanguageModelOnTheFly ( const Fst<Arc>& fst, KenLMModelT& model, #ifndef USE_GOOGLE_SPARSE_HASH unordered_set<Label>& epsilons, #else google::dense_hash_set<Label>& epsilons, #endif bool natlog, float lmscale , float lmwp, const IdBridgeT& idbridge ) : composed_ ( NULL ) , natlog10_ ( natlog ? -lmscale* ::log ( 10.0 ) : -lmscale ), fst_ ( fst ), lmmodel_ ( model ), vocab_ ( model.GetVocabulary() ), wp_ ( lmwp ) , epsilons_ ( epsilons ) , history ( model.Order(), 0), idbridge_ (idbridge) { #ifdef USE_GOOGLE_SPARSE_HASH stateexistence_.set_empty_key ( numeric_limits<ull>::max() ); statemap_.set_empty_key ( numeric_limits<uint64_t>::max() ); basic_string<unsigned> aux (KENLM_MAX_ORDER, numeric_limits<unsigned>::max() ); seenlmstates_.set_empty_key ( aux ); #endif buffersize = ( model.Order() - 1 ) * sizeof ( unsigned int ); buffer = const_cast<unsigned *> ( history.c_str() ); if (!fst_.NumStates() ) { LWARN ("Empty lattice"); return; } composed_ = new VectorFst<Arc>; typename KenLMModelT::State bs = model.NullContextState(); ///Initialize with first state pair<StateId, bool> nextp = add ( bs, fst_.Start(), fst_.Final ( fst_.Start() ) ); qc_.push ( nextp.first ); composed_->SetStart ( nextp.first ); };