bool PhraseDictionaryInterpolated::Load(
const std::vector<FactorType> &input
, const std::vector<FactorType> &output
, const std::vector<std::string>& config
, const std::vector<float> &weightT
, size_t tableLimit
, const LMList &languageModels
, float weightWP) {
m_languageModels = &languageModels;
m_weightT = weightT;
m_tableLimit = tableLimit;
m_weightWP = weightWP;
//The config should be as follows:
//0-3: type factor factor num-components (as usual)
//4: combination mode (e.g. naive)
//5-(length-2): List of phrase-table files
//length-1: Weight string, in the same format as used for tmcombine
UTIL_THROW_IF(config.size() < 7, util::Exception, "Missing fields from phrase table configuration: expected at least 7");
UTIL_THROW_IF(config[4] != "naive", util::Exception, "Unsupported combination mode: '" << config[4] << "'");
// Create the dictionaries
for (size_t i = 5; i < config.size()-1; ++i) {
m_dictionaries.push_back(DictionaryHandle(new PhraseDictionaryTreeAdaptor(
GetFeature()->GetNumScoreComponents(),
GetFeature()->GetNumInputScores(),
GetFeature())));
bool ret = m_dictionaries.back()->Load(
input,
output,
config[i],
weightT,
0,
languageModels,
weightWP);
if (!ret) return ret;
}
//Parse the weight strings
for (util::TokenIter<util::SingleCharacter, false> featureWeights(config.back(), util::SingleCharacter(';')); featureWeights; ++featureWeights) {
m_weights.push_back(vector<float>());
float sum = 0;
for (util::TokenIter<util::SingleCharacter, false> tableWeights(*featureWeights, util::SingleCharacter(',')); tableWeights; ++tableWeights) {
const float weight = boost::lexical_cast<float>(*tableWeights);
m_weights.back().push_back(weight);
sum += weight;
}
UTIL_THROW_IF(m_weights.back().size() != m_dictionaries.size(), util::Exception,
"Number of weights (" << m_weights.back().size() <<
") does not match number of dictionaries to combine (" << m_dictionaries.size() << ")");
UTIL_THROW_IF(abs(sum - 1) > 0.01, util::Exception, "Weights not normalised");
}
//check number of weight sets. Make sure there is a weight for every score component
//except for the last - which is assumed to be the phrase penalty.
UTIL_THROW_IF(m_weights.size() != 1 && m_weights.size() != GetFeature()->GetNumScoreComponents()-1, util::Exception, "Unexpected number of weight sets");
//if 1 weight set, then repeat
if (m_weights.size() == 1) {
while(m_weights.size() < GetFeature()->GetNumScoreComponents()-1) {
m_weights.push_back(m_weights[0]);
}
}
return true;
}
void ODE_SearchReplace :: Deinit ( )
{
dictionary = DictionaryHandle();
}
