int main ( int argc, const char* argv[] ) { ucam::util::initLogger ( argc, argv ); FORCELINFO ( argv[0] << " starts!" ); ucam::util::RegistryPO rg ( argc, argv ); FORCELINFO ( rg.dump ( "CONFIG parameters:\n=====================", "=====================" ) ); if (rg.get<std::string> (HifstConstants::kAction) == HifstConstants::kActionProjectweight2 ) { run<fst::LexStdArc , fst::LexStdArc , fst::GenericWeightMapper<fst::LexStdArc, fst::LexStdArc, fst::MakeWeight2<fst::LexStdArc> >, fst::MakeWeight2<fst::LexStdArc> > (rg); } else if (rg.get<std::string> (HifstConstants::kAction) == HifstConstants::kActionLex2std) { run<fst::LexStdArc , fst::StdArc , fst::GenericWeightMapper<fst::LexStdArc, fst::StdArc, fst::LexToStd >, fst::LexToStd > (rg); } else if (rg.get<std::string> (HifstConstants::kAction) == HifstConstants::kActionStd2lex) { run<fst::StdArc , fst::LexStdArc , fst::GenericWeightMapper<fst::StdArc, fst::LexStdArc, fst::MakeWeight2<fst::LexStdArc> >, fst::MakeWeight2<fst::LexStdArc> > (rg); } else { LERROR ("Action not recognized! Check program option."); } FORCELINFO ( argv[0] << " finished!" ); }
int main (int argc, const char* argv[] ) { ucam::util::initLogger ( argc, argv ); FORCELINFO ( argv[0] << " starts!" ); ucam::util::RegistryPO rg ( argc, argv ); FORCELINFO ( rg.dump ( "CONFIG parameters:\n=====================", "=====================" ) ) ; ucam::util::PatternAddress<unsigned> pi (rg.get<std::string> (HifstConstants::kInput) ); ucam::util::PatternAddress<unsigned> po (rg.get<std::string> (HifstConstants::kOutput) ); for ( ucam::util::IntRangePtr ir (ucam::util::IntRangeFactory ( rg, HifstConstants::kRangeOne ) ); !ir->done(); ir->next() ) { fst::VectorFst<fst::StdArc> *mfst = fst::VectorFstRead<fst::StdArc> (pi ( ir->get() ) ); if (!mfst) return 1; TopSort (mfst); boost::multiprecision::uint128_t j = countstrings<fst::StdArc, boost::multiprecision::uint128_t> (*mfst); std::stringstream ss; ss << j; ucam::util::oszfstream o (po (ir->get() ), true); o << ss.str() << std::endl; LINFO ( pi (ir->get() ) << ":" << ss.str() ) ; o.close(); delete mfst; } FORCELINFO ( argv[0] << " ends!" ); }
MainClass(int argc, const char *argv[]) :argv_(argv) { rg_.reset(new RegistryPO(argc,argv)); util::initLogger ( argc, argv ); FORCELINFO ( argv[0] << " starts!" ); FORCELINFO ( rg_->dump ( "CONFIG parameters:\n=====================" , "=====================" ) ); }
int main ( int argc, const char *argv[] ) { ucam::util::initLogger ( argc, argv ); FORCELINFO ( argv[0] << " starts!" ); ucam::util::RegistryPO rg ( argc, argv ); FORCELINFO ( rg.dump ( "CONFIG parameters:\n=====================" , "=====================" ) ); ( ucam::util::Runner2<ucam::lmbr::SingleThreadedLmbrTask<> , ucam::lmbr::MultiThreadedLmbrTask<> > ( rg ) ) (); FORCELINFO ( argv[0] << " ends!" ); return 0; }
int main (int argc, const char* argv[] ) { ucam::util::initLogger ( argc, argv ); FORCELINFO ( argv[0] << " starts!" ); ucam::util::RegistryPO rg ( argc, argv ); FORCELINFO ( rg.dump ( "CONFIG parameters:\n=====================", "=====================" ) ) ; std::string semiring = rg.get<std::string> (HifstConstants::kHifstSemiring); if (semiring == HifstConstants::kHifstSemiringStdArc) { run<fst::StdArc> (rg); } else if (semiring == HifstConstants::kHifstSemiringLexStdArc) { run<fst::LexStdArc> (rg); } else if (semiring == HifstConstants::kHifstSemiringTupleArc) { run<TupleArc32> (rg); } else { LERROR ("Sorry, semiring option not correctly defined"); } FORCELINFO ( argv[0] << " ends!" ); }
/** * \brief General run method from TaskInterface. Dumps all stats to a file * \param &d: general Data structure containing a pointer to StatsData structure */ bool run ( Data& d ) { if ( statsoutput_() == "" ) return false; d_ = &d; FORCELINFO ( "Writing stats to " << statsoutput_ ( d.sidx ) ); ucam::util::oszfstream o ( statsoutput_ ( d.sidx ) ); if ( writeCYKStats_ ) writeCYKStats ( o ); o << "=================================================================" << std::endl; o << "Local pruning during lattice construction" << std::endl; writePruneStats ( o ); o.close(); return false; };
int run ( ucam::util::RegistryPO const& rg) { ucam::util::PatternAddress<unsigned> input (rg.get<std::string> (HifstConstants::kInput) ); ucam::util::PatternAddress<unsigned> output (rg.get<std::string> (HifstConstants::kOutput) ); WeightFunctorT mwcopy; for ( ucam::util::IntRangePtr ir (ucam::util::IntRangeFactory ( rg, HifstConstants::kRangeOne ) ); !ir->done(); ir->next() ) { FORCELINFO ("Processing file " << input ( ir->get() ) ); boost::scoped_ptr< fst::VectorFst<ArcT> > ifst (fst::VectorFstRead<ArcT> ( input ( ir->get() ) ) ); boost::scoped_ptr< fst::VectorFst<Arc2T> > ofst (new fst::VectorFst<Arc2T>); Map ( *ifst, &*ofst, MapperT ( mwcopy ) ); fst::FstWrite<Arc2T> ( *ofst, output (ir->get() ) ); } };
/** * \brief Runs the lattice building procedure. * \param d Contains the data structure with all the necessary elements (i.e. cykdata) and in which will be store a pointer to the * output lattice. */ bool run ( Data& d ) { cykfstresult_.DeleteStates(); this->d_ = &d; hieroindexexistence_.clear(); LINFO ( "Running HiFST" ); //Reset one-time warnings for inexistent language models. warned_ = false; pdtparens_.clear(); cykdata_ = d.cykdata; if ( !USER_CHECK ( cykdata_, "cyk parse has not been executed previously?" ) ) { resetExternalData (d); return true; } if ( d.cykdata->success == CYK_RETURN_FAILURE ) { ///Keep calm, return empty lattice and carry on fst::VectorFst<Arc> aux; d.fsts[outputkey_] = &cykfstresult_; d.vcat = cykdata_->vcat; resetExternalData (d); return false; } ///If not yet, initialize now functor with local conditions. initLocalConditions(); rtn_ = new RTNT; if ( localprune_ ) rtnnumstates_ = new ExpandedNumStatesRTNT; rfba_ = new ReplaceFstByArcT ( cykdata_->vcat, replacefstbyarc_, replacefstbyarcexceptions_, aligner_, replacefstbynumstates_ ); piscount_ = 0; //reset pruning-in-search count to 0 LINFO ( "Second Pass: FST-building!" ); d.stats->setTimeStart ( "lattice-construction" ); //Owned by rtn_; fst::Fst<Arc> *sfst = buildRTN ( cykdata_->categories["S"], 0, cykdata_->sentence.size() - 1 ); d.stats->setTimeEnd ( "lattice-construction" ); cykfstresult_ = (*sfst); LINFO ( "Final - RTN head optimizations !" ); optimize ( &cykfstresult_ , std::numeric_limits<unsigned>::max() , !hipdtmode_ && optimize_ ); FORCELINFO ("Stats for Sentence " << d.sidx << ": local pruning, number of times=" << piscount_); d.stats->lpcount = piscount_; //store local pruning counts in stats LINFO ("RTN expansion starts now!"); //Expand... { ///Define hieroindex Label hieroindex = APBASETAG + 1 * APCCTAG + 0 * APXTAG + ( cykdata_->sentence.size() - 1 ) * APYTAG; if ( hieroindexexistence_.find ( hieroindex ) == hieroindexexistence_.end() ) pairlabelfsts_.push_back ( pair< Label, const fst::Fst<Arc> * > ( hieroindex, &cykfstresult_ ) ); ///Optimizations over the rtn -- they are generally worth doing... fst::ReplaceUtil<Arc> replace_util (pairlabelfsts_, hieroindex, !aligner_); //has ownership of modified rtn fsts if (rtnopt_) { LINFO ("rtn optimizations..."); d_->stats->setTimeStart ("replace-opts"); replace_util.ReplaceTrivial(); replace_util.ReplaceUnique(); replace_util.Connect(); pairlabelfsts_.clear(); replace_util.GetFstPairs (&pairlabelfsts_); d_->stats->setTimeEnd ("replace-opts"); } //After optimizations, we can write RTN if required by user writeRTN(); boost::scoped_ptr< fst::VectorFst<Arc> > efst (new fst::VectorFst<Arc>); if (!hipdtmode_ ) { LINFO ("Final Replace (RTN->FSA), main index=" << hieroindex); d_->stats->setTimeStart ("replace-rtn-final"); Replace (pairlabelfsts_, &*efst, hieroindex, !aligner_); d_->stats->setTimeEnd ("replace-rtn-final"); } else { LINFO ("Final Replace (RTN->PDA)"); d_->stats->setTimeStart ("replace-pdt-final"); Replace (pairlabelfsts_, &*efst, &pdtparens_, hieroindex); d_->stats->setTimeEnd ("replace-pdt-final"); LINFO ("Number of pdtparens=" << pdtparens_.size() ); } LINFO ("Removing Epsilons..."); fst::RmEpsilon<Arc> ( &*efst ); LINFO ("Done! NS=" << efst->NumStates() ); //Apply filters applyFilters ( &*efst ); //Compose with full reference lattice to ensure that final lattice is correct. if ( d.fsts.find ( fullreferencelatticekey_ ) != d.fsts.end() ) { if ( static_cast< fst::VectorFst<Arc> * > (d.fsts[fullreferencelatticekey_])->NumStates() > 0 ) { LINFO ( "Composing with full reference lattice, NS=" << static_cast< fst::VectorFst<Arc> * > (d.fsts[fullreferencelatticekey_])->NumStates() ); fst::Compose<Arc> ( *efst, * ( static_cast<fst::VectorFst<Arc> * > (d.fsts[fullreferencelatticekey_]) ), &*efst ); LINFO ( "After composition: NS=" << efst->NumStates() ); } else { LINFO ( "No composition with full ref lattice" ); }; } else { LINFO ( "No composition with full ref lattice" ); }; //Apply language model fst::VectorFst<Arc> *res = NULL; if (efst->NumStates() ) res = applyLanguageModel ( *efst ); else { LWARN ("Empty lattice -- skipping LM application"); } if ( res != NULL ) { boost::shared_ptr<fst::VectorFst<Arc> >latlm ( res ); if ( latlm.get() == efst.get() ) { LWARN ( "Yikes! Unexpected situation! Will it crash? (muhahaha) " ); } //Todo: union with shortest path... if ( pruneweight_ < std::numeric_limits<float>::max() ) { if (!hipdtmode_ || pdtparens_.empty() ) { LINFO ("Pruning, weight=" << pruneweight_); fst::Prune<Arc> (*latlm, &cykfstresult_, mw_ ( pruneweight_ ) ); } else { LINFO ("Expanding, weight=" << pruneweight_); fst::ExpandOptions<Arc> eopts (true, false, mw_ ( pruneweight_ ) ); Expand ( *latlm, pdtparens_, &cykfstresult_, eopts); pdtparens_.clear(); } } else { LINFO ("Copying through full lattice with lm scores"); cykfstresult_ = *latlm; } } else { LINFO ("Copying through full lattice (no lm)"); cykfstresult_ = *efst; } if ( hieroindexexistence_.find ( hieroindex ) == hieroindexexistence_.end() ) pairlabelfsts_.pop_back(); } pairlabelfsts_.clear(); LINFO ( "Reps" ); fst::RmEpsilon ( &cykfstresult_ ); LINFO ( "NS=" << cykfstresult_.NumStates() ); //This should delete all pertinent fsas... LINFO ( "deleting data stuff..." ); delete rtn_; if ( localprune_ ) delete rtnnumstates_; delete rfba_; d.vcat = cykdata_->vcat; resetExternalData (d); d.fsts[outputkey_] = &cykfstresult_; if (hipdtmode_ && pdtparens_.size() ) d.fsts[outputkey_ + ".parens" ] = &pdtparens_; LINFO ( "done..." ); FORCELINFO ( "End Sentence ******************************************************" ); d.stats->setTimeEnd ( "sent-dec" ); d.stats->message += "[" + ucam::util::getTimestamp() + "] End Sentence\n"; return false; };
~MainClass() { FORCELINFO ( argv_[0] << " ends!" ); }