bool PhraseDictionaryInterpolated::Load( const std::vector<FactorType> &input , const std::vector<FactorType> &output , const std::vector<std::string>& config , const std::vector<float> &weightT , size_t tableLimit , const LMList &languageModels , float weightWP) { m_languageModels = &languageModels; m_weightT = weightT; m_tableLimit = tableLimit; m_weightWP = weightWP; //The config should be as follows: //0-3: type factor factor num-components (as usual) //4: combination mode (e.g. naive) //5-(length-2): List of phrase-table files //length-1: Weight string, in the same format as used for tmcombine UTIL_THROW_IF(config.size() < 7, util::Exception, "Missing fields from phrase table configuration: expected at least 7"); UTIL_THROW_IF(config[4] != "naive", util::Exception, "Unsupported combination mode: '" << config[4] << "'"); // Create the dictionaries for (size_t i = 5; i < config.size()-1; ++i) { m_dictionaries.push_back(DictionaryHandle(new PhraseDictionaryTreeAdaptor( GetFeature()->GetNumScoreComponents(), GetFeature()->GetNumInputScores(), GetFeature()))); bool ret = m_dictionaries.back()->Load( input, output, config[i], weightT, 0, languageModels, weightWP); if (!ret) return ret; } //Parse the weight strings for (util::TokenIter<util::SingleCharacter, false> featureWeights(config.back(), util::SingleCharacter(';')); featureWeights; ++featureWeights) { m_weights.push_back(vector<float>()); float sum = 0; for (util::TokenIter<util::SingleCharacter, false> tableWeights(*featureWeights, util::SingleCharacter(',')); tableWeights; ++tableWeights) { const float weight = boost::lexical_cast<float>(*tableWeights); m_weights.back().push_back(weight); sum += weight; } UTIL_THROW_IF(m_weights.back().size() != m_dictionaries.size(), util::Exception, "Number of weights (" << m_weights.back().size() << ") does not match number of dictionaries to combine (" << m_dictionaries.size() << ")"); UTIL_THROW_IF(abs(sum - 1) > 0.01, util::Exception, "Weights not normalised"); } //check number of weight sets. Make sure there is a weight for every score component //except for the last - which is assumed to be the phrase penalty. UTIL_THROW_IF(m_weights.size() != 1 && m_weights.size() != GetFeature()->GetNumScoreComponents()-1, util::Exception, "Unexpected number of weight sets"); //if 1 weight set, then repeat if (m_weights.size() == 1) { while(m_weights.size() < GetFeature()->GetNumScoreComponents()-1) { m_weights.push_back(m_weights[0]); } } return true; }
void ODE_SearchReplace :: Deinit ( ) { dictionary = DictionaryHandle(); }