void setMosesWeights(const FVector& currentWeights) {
PhraseFeature::updateWeights(currentWeights);
StaticData& staticData =
const_cast<StaticData&>(StaticData::Instance());
TranslationSystem& system =
const_cast<TranslationSystem&>(staticData.GetTranslationSystem(
TranslationSystem::DEFAULT));
ScoreComponentCollection mosesWeights = staticData.GetAllWeights();
for (LMList::const_iterator i = system.GetLanguageModels().begin();
i != system.GetLanguageModels().end(); ++i) {
LanguageModel* lm = const_cast<LanguageModel*>(*i);
float lmWeight = currentWeights[lm->GetScoreProducerDescription()];
//lm->SetWeight(lmWeight);
mosesWeights.Assign(lm,lmWeight);
}
const ScoreProducer* wp = system.GetWordPenaltyProducer();
const string wpName = wp->GetScoreProducerDescription();
//staticData.SetWeightWordPenalty(currentWeights[wpName]);
mosesWeights.Assign(wp,currentWeights[wpName]);
const ScoreProducer* dp = system.GetDistortionProducer();
string distName = dp->GetScoreProducerDescription();
//staticData.SetWeightDistortion(currentWeights[distName]);
mosesWeights.Assign(dp, currentWeights[distName]);
staticData.SetAllWeights(mosesWeights);
}
void PhraseLengthFeature::EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{
// get length of source and target phrase
size_t targetLength = targetPhrase.GetSize();
size_t sourceLength = source.GetSize();
// create feature names
stringstream nameSource;
nameSource << "s" << sourceLength;
stringstream nameTarget;
nameTarget << "t" << targetLength;
stringstream nameBoth;
nameBoth << sourceLength << "," << targetLength;
// increase feature counts
scoreBreakdown.PlusEquals(this,nameSource.str(),1);
scoreBreakdown.PlusEquals(this,nameTarget.str(),1);
scoreBreakdown.PlusEquals(this,nameBoth.str(),1);
//cerr << nameSource.str() << " " << nameTarget.str() << " " << nameBoth.str() << endl;
}
void LanguageModel::EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{
// contains factors used by this LM
float fullScore, nGramScore;
size_t oovCount;
CalcScore(targetPhrase, fullScore, nGramScore, oovCount);
float estimateScore = fullScore - nGramScore;
if (StaticData::Instance().GetLMEnableOOVFeature()) {
vector<float> scores(2), estimateScores(2);
scores[0] = nGramScore;
scores[1] = oovCount;
scoreBreakdown.Assign(this, scores);
estimateScores[0] = estimateScore;
estimateScores[1] = 0;
estimatedFutureScore.Assign(this, estimateScores);
} else {
scoreBreakdown.Assign(this, nGramScore);
estimatedFutureScore.Assign(this, estimateScore);
}
}
void OutputFeatureScores( std::ostream& out
, const ScoreComponentCollection &features
, const FeatureFunction *ff
, std::string &lastName )
{
const StaticData &staticData = StaticData::Instance();
bool labeledOutput = staticData.IsLabeledNBestList();
// regular features (not sparse)
if (ff->GetNumScoreComponents() != 0) {
if( labeledOutput && lastName != ff->GetScoreProducerDescription() ) {
lastName = ff->GetScoreProducerDescription();
out << " " << lastName << "=";
}
vector<float> scores = features.GetScoresForProducer( ff );
for (size_t j = 0; j<scores.size(); ++j) {
out << " " << scores[j];
}
}
// sparse features
const FVector scores = features.GetVectorForProducer( ff );
for(FVector::FNVmap::const_iterator i = scores.cbegin(); i != scores.cend(); i++) {
out << " " << i->first << "= " << i->second;
}
}
void
ChartSearchGraphWriterHypergraph::
WriteHypos(const ChartHypothesisCollection& hypos,
const map<unsigned, bool> &reachable) const
{
ChartHypothesisCollection::const_iterator iter;
for (iter = hypos.begin() ; iter != hypos.end() ; ++iter) {
const ChartHypothesis* mainHypo = *iter;
if (!StaticData::Instance().GetUnprunedSearchGraph() &&
reachable.find(mainHypo->GetId()) == reachable.end()) {
//Ignore non reachable nodes
continue;
}
(*m_out) << "# node " << m_nodeId << endl;
m_hypoIdToNodeId[mainHypo->GetId()] = m_nodeId;
++m_nodeId;
vector<const ChartHypothesis*> edges;
edges.push_back(mainHypo);
const ChartArcList *arcList = (*iter)->GetArcList();
if (arcList) {
ChartArcList::const_iterator iterArc;
for (iterArc = arcList->begin(); iterArc != arcList->end(); ++iterArc) {
const ChartHypothesis* arc = *iterArc;
if (reachable.find(arc->GetId()) != reachable.end()) {
edges.push_back(arc);
}
}
}
(*m_out) << edges.size() << endl;
for (vector<const ChartHypothesis*>::const_iterator ei = edges.begin();
ei != edges.end(); ++ei) {
const ChartHypothesis* hypo = *ei;
const TargetPhrase& target = hypo->GetCurrTargetPhrase();
size_t ntIndex = 0;
for (size_t i = 0; i < target.GetSize(); ++i) {
const Word& word = target.GetWord(i);
if (word.IsNonTerminal()) {
size_t hypoId = hypo->GetPrevHypos()[ntIndex++]->GetId();
(*m_out) << "[" << m_hypoIdToNodeId[hypoId] << "]";
} else {
(*m_out) << word.GetFactor(0)->GetString();
}
(*m_out) << " ";
}
(*m_out) << " ||| ";
ScoreComponentCollection scores = hypo->GetScoreBreakdown();
HypoList::const_iterator hi;
for (hi = hypo->GetPrevHypos().begin(); hi != hypo->GetPrevHypos().end(); ++hi) {
scores.MinusEquals((*hi)->GetScoreBreakdown());
}
scores.Save(*m_out, false);
(*m_out) << " ||| ";
(*m_out) << hypo->GetCurrSourceRange().GetNumWordsCovered();
(*m_out) << endl;
}
}
}
BOOST_FIXTURE_TEST_CASE(ctor, MockProducers)
{
ScoreComponentCollection scc;
BOOST_CHECK_EQUAL(scc.GetScoreForProducer(&single),0);
float expected[] = {0,0,0,0,0};
std::vector<float> actual= scc.GetScoresForProducer(&multi);
BOOST_CHECK_EQUAL_COLLECTIONS(expected, expected+5, actual.begin(), actual.begin()+5);
}
ChartTrellisPath::ChartTrellisPath(const ChartTrellisDetour &detour)
: m_finalNode(new ChartTrellisNode(detour, m_deviationPoint))
, m_scoreBreakdown(detour.GetBasePath().m_scoreBreakdown)
, m_totalScore(0)
{
UTIL_THROW_IF(m_deviationPoint == NULL, util::Exception, "No deviation point");
ScoreComponentCollection scoreChange;
scoreChange = detour.GetReplacementHypo().GetScoreBreakdown();
scoreChange.MinusEquals(detour.GetSubstitutedNode().GetHypothesis().GetScoreBreakdown());
m_scoreBreakdown.PlusEquals(scoreChange);
m_totalScore = m_scoreBreakdown.GetWeightedScore();
}
BOOST_FIXTURE_TEST_CASE(sparse_feature, MockProducers)
{
ScoreComponentCollection scc;
scc.Assign(&sparse, "first", 1.3f);
scc.Assign(&sparse, "second", 2.1f);
BOOST_CHECK_EQUAL( scc.GetScoreForProducer(&sparse,"first"), 1.3f);
BOOST_CHECK_EQUAL( scc.GetScoreForProducer(&sparse,"second"), 2.1f);
BOOST_CHECK_EQUAL( scc.GetScoreForProducer(&sparse,"third"), 0.0f);
scc.Assign(&sparse, "first", -1.9f);
BOOST_CHECK_EQUAL( scc.GetScoreForProducer(&sparse,"first"), -1.9f);
scc.PlusEquals(&sparse, StringPiece("first"), -1.9f);
BOOST_CHECK_EQUAL( scc.GetScoreForProducer(&sparse,"first"), -3.8f);
}
void TargetPhrase::EvaluateWithSourceContext(const InputType &input, const InputPath &inputPath)
{
const std::vector<FeatureFunction*> &ffs = FeatureFunction::GetFeatureFunctions();
const StaticData &staticData = StaticData::Instance();
ScoreComponentCollection futureScoreBreakdown;
for (size_t i = 0; i < ffs.size(); ++i) {
const FeatureFunction &ff = *ffs[i];
if (! staticData.IsFeatureFunctionIgnored( ff )) {
ff.EvaluateWithSourceContext(input, inputPath, *this, NULL, m_scoreBreakdown, &futureScoreBreakdown);
}
}
float weightedScore = m_scoreBreakdown.GetWeightedScore();
m_futureScore += futureScoreBreakdown.GetWeightedScore();
m_fullScore = weightedScore + m_futureScore;
}
void SkeletonChangeInput::EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{
// dense scores
vector<float> newScores(m_numScoreComponents);
newScores[0] = 1.5;
newScores[1] = 0.3;
scoreBreakdown.PlusEquals(this, newScores);
// sparse scores
scoreBreakdown.PlusEquals(this, "sparse-name", 2.4);
}
void RuleScope::EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{
// adjacent non-term count as 1 ammbiguity, rather than 2 as in rule scope
// source can't be empty, right?
float score = 0;
int count = 0;
for (size_t i = 0; i < source.GetSize() - 0; ++i) {
const Word &word = source.GetWord(i);
bool ambiguous = IsAmbiguous(word, m_sourceSyntax);
if (ambiguous) {
++count;
}
else {
if (count > 0) {
score += count;
}
count = -1;
}
}
// 1st & last always adjacent to ambiguity
++count;
if (count > 0) {
score += count;
}
scoreBreakdown.PlusEquals(this, score);
}
// assumes that source-side syntax labels are stored in the target non-terminal field of the rules
void SourceGHKMTreeInputMatchFeature::EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection *estimatedScores) const
{
const Range& range = inputPath.GetWordsRange();
size_t startPos = range.GetStartPos();
size_t endPos = range.GetEndPos();
const TreeInput& treeInput = static_cast<const TreeInput&>(input);
const NonTerminalSet& treeInputLabels = treeInput.GetLabelSet(startPos,endPos);
const Word& lhsLabel = targetPhrase.GetTargetLHS();
const StaticData& staticData = StaticData::Instance();
const Word& outputDefaultNonTerminal = staticData.GetOutputDefaultNonTerminal();
std::vector<float> newScores(m_numScoreComponents,0.0); // m_numScoreComponents == 2 // first fires for matches, second for mismatches
if ( (treeInputLabels.find(lhsLabel) != treeInputLabels.end()) && (lhsLabel != outputDefaultNonTerminal) ) {
// match
newScores[0] = 1.0;
} else {
// mismatch
newScores[1] = 1.0;
}
scoreBreakdown.PlusEquals(this, newScores);
}
void PhraseFeature::updateWeights(const FVector& weights) {
for (set<PhraseFeature*>::iterator i = s_phraseFeatures.begin();
i != s_phraseFeatures.end(); ++i) {
PhraseFeature* pf = *i;
vector<float> newWeights(pf->m_featureNames.size());
for (size_t j = 0; j < pf->m_featureNames.size(); ++j) {
FValue weight = weights[pf->m_featureNames[j]];
newWeights[j] = weight;
}
ScoreComponentCollection mosesWeights = StaticData::Instance().GetAllWeights();
mosesWeights.Assign(pf->m_phraseDictionary,newWeights);
(const_cast<StaticData&>(StaticData::Instance()))
.SetAllWeights(mosesWeights);
//pf->m_phraseDictionary->GetFeature()->SetWeightTransModel(newWeights);
}
}
void PhrasePenalty::Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{
scoreBreakdown.Assign(this, 1.0f);
}
void RulePairUnlexicalizedSource::EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{
const Factor* targetPhraseLHS = targetPhrase.GetTargetLHS()[0];
if ( !m_glueRules && (targetPhraseLHS == m_glueTargetLHS) ) {
return;
}
if ( !m_nonGlueRules && (targetPhraseLHS != m_glueTargetLHS) ) {
return;
}
for (size_t posS=0; posS<source.GetSize(); ++posS) {
const Word &wordS = source.GetWord(posS);
if ( !wordS.IsNonTerminal() ) {
return;
}
}
ostringstream namestr;
for (size_t posT=0; posT<targetPhrase.GetSize(); ++posT) {
const Word &wordT = targetPhrase.GetWord(posT);
const Factor* factorT = wordT[0];
if ( wordT.IsNonTerminal() ) {
namestr << "[";
}
namestr << factorT->GetString();
if ( wordT.IsNonTerminal() ) {
namestr << "]";
}
namestr << "|";
}
namestr << targetPhraseLHS->GetString() << "|";
for (AlignmentInfo::const_iterator it=targetPhrase.GetAlignNonTerm().begin();
it!=targetPhrase.GetAlignNonTerm().end(); ++it) {
namestr << "|" << it->first << "-" << it->second;
}
scoreBreakdown.PlusEquals(this, namestr.str(), 1);
if ( targetPhraseLHS != m_glueTargetLHS ) {
scoreBreakdown.PlusEquals(this, 1);
}
}
void WordPenaltyProducer::Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{
float score = - (float) targetPhrase.GetNumTerminals();
scoreBreakdown.Assign(this, score);
}
void TargetPhrase::EvaluateInIsolation(const Phrase &source, const std::vector<FeatureFunction*> &ffs)
{
if (ffs.size()) {
const StaticData &staticData = StaticData::Instance();
ScoreComponentCollection futureScoreBreakdown;
for (size_t i = 0; i < ffs.size(); ++i) {
const FeatureFunction &ff = *ffs[i];
if (! staticData.IsFeatureFunctionIgnored( ff )) {
ff.EvaluateInIsolation(source, *this, m_scoreBreakdown, futureScoreBreakdown);
}
}
float weightedScore = m_scoreBreakdown.GetWeightedScore();
m_futureScore += futureScoreBreakdown.GetWeightedScore();
m_fullScore = weightedScore + m_futureScore;
}
}
/** Score due to one segment */
void SingleStateFeatureFunction::doSingleUpdate(
const TranslationOption* option,
const TargetGap& gap, FVector& scores) {
ScoreComponentCollection accumulator;
//the previous state of the (new) hypo
StateHandle prevState = m_prevStates[gap.segment.GetStartPos()];
//Evaluate the score of inserting this hypo, and get the prev state
//for the next hypo.
prevState.reset(m_mosesFeature->Evaluate(
*option,prevState.get(),&accumulator));
//if there's a hypo on the right, then evaluate it
if (gap.rightHypo) {
prevState.reset(m_mosesFeature->Evaluate(
gap.rightHypo->GetTranslationOption(),prevState.get(),&accumulator));
}
scores += accumulator.GetScoresVector();
}
void Model1Feature::EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection *estimatedFutureScore) const
{
const Sentence& sentence = static_cast<const Sentence&>(input);
float score = 0.0;
float norm = TransformScore(1+sentence.GetSize());
for (size_t posT=0; posT<targetPhrase.GetSize(); ++posT) {
const Word &wordT = targetPhrase.GetWord(posT);
if ( !wordT.IsNonTerminal() ) {
float thisWordProb = m_model1.GetProbability(m_emptyWord,wordT[0]); // probability conditioned on empty word
// cache lookup
bool foundInCache = false;
{
#ifdef WITH_THREADS
boost::shared_lock<boost::shared_mutex> read_lock(m_accessLock);
#endif
boost::unordered_map<const InputType*, boost::unordered_map<const Factor*, float> >::const_iterator sentenceCache = m_cache.find(&input);
if (sentenceCache != m_cache.end()) {
boost::unordered_map<const Factor*, float>::const_iterator cacheHit = sentenceCache->second.find(wordT[0]);
if (cacheHit != sentenceCache->second.end()) {
foundInCache = true;
score += cacheHit->second;
FEATUREVERBOSE(3, "Cached score( " << wordT << " ) = " << cacheHit->second << std::endl);
}
}
}
if (!foundInCache) {
for (size_t posS=1; posS<sentence.GetSize()-1; ++posS) { // ignore <s> and </s>
const Word &wordS = sentence.GetWord(posS);
float modelProb = m_model1.GetProbability(wordS[0],wordT[0]);
FEATUREVERBOSE(4, "p( " << wordT << " | " << wordS << " ) = " << modelProb << std::endl);
thisWordProb += modelProb;
}
float thisWordScore = TransformScore(thisWordProb) - norm;
FEATUREVERBOSE(3, "score( " << wordT << " ) = " << thisWordScore << std::endl);
{
#ifdef WITH_THREADS
// need to update cache; write lock
boost::unique_lock<boost::shared_mutex> lock(m_accessLock);
#endif
m_cache[&input][wordT[0]] = thisWordScore;
}
score += thisWordScore;
}
}
}
scoreBreakdown.PlusEquals(this, score);
}
/** Score due to two segments.
The left and right refer to the target positions.**/
void SingleStateFeatureFunction::doContiguousPairedUpdate(
const TranslationOption* leftOption,
const TranslationOption* rightOption,
const TargetGap& gap, FVector& scores) {
ScoreComponentCollection accumulator;
//The previous state of the (new) current hypo.
StateHandle prevState(m_prevStates[gap.segment.GetStartPos()]);
//Evaluate the hypos in the gap
prevState.reset(m_mosesFeature->Evaluate(
*leftOption,prevState.get(),&accumulator));
prevState.reset(m_mosesFeature->Evaluate(
*rightOption,prevState.get(),&accumulator));
//if there's a hypo on the right, evaluate it
if (gap.rightHypo) {
prevState.reset(m_mosesFeature->Evaluate(
gap.rightHypo->GetTranslationOption(),prevState.get(),&accumulator));
}
scores += accumulator.GetScoresVector();
}
void SkeletonStatelessFF::EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection *estimatedScores) const
{
if (targetPhrase.GetNumNonTerminals()) {
vector<float> newScores(m_numScoreComponents);
newScores[0] = - std::numeric_limits<float>::infinity();
scoreBreakdown.PlusEquals(this, newScores);
}
}
void PhrasePenalty::EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{
if (m_perPhraseTable) {
const PhraseDictionary *pt = targetPhrase.GetContainer();
if (pt) {
size_t ptId = pt->GetId();
UTIL_THROW_IF2(ptId >= m_numScoreComponents, "Wrong number of scores");
vector<float> scores(m_numScoreComponents, 0);
scores[ptId] = 1.0f;
scoreBreakdown.Assign(this, scores);
}
}
else {
scoreBreakdown.Assign(this, 1.0f);
}
}
//Find top n translations of source, and send them to output
static void outputTopN(const StringPiece& sourcePhraseString, PhraseDictionary* phraseTable, const std::vector<FactorType> &input, ostream& out) {
//get list of target phrases
Phrase sourcePhrase;
sourcePhrase.CreateFromString(Input,input,sourcePhraseString,NULL);
InputPath inputPath(sourcePhrase, NonTerminalSet(), WordsRange(0,sourcePhrase.GetSize()-1),NULL,NULL);
InputPathList inputPaths;
inputPaths.push_back(&inputPath);
phraseTable->GetTargetPhraseCollectionBatch(inputPaths);
const TargetPhraseCollection* targetPhrases = inputPath.GetTargetPhrases(*phraseTable);
//print phrases
const std::vector<FactorType>& output = StaticData::Instance().GetOutputFactorOrder();
if (targetPhrases) {
//if (targetPhrases->GetSize() > 10) cerr << "src " << sourcePhrase << " tgt count " << targetPhrases->GetSize() << endl;
for (TargetPhraseCollection::const_iterator i = targetPhrases->begin(); i != targetPhrases->end(); ++i) {
const TargetPhrase* targetPhrase = *i;
out << sourcePhrase.GetStringRep(input);
out << " ||| ";
out << targetPhrase->GetStringRep(output);
out << " ||| ";
const ScoreComponentCollection scores = targetPhrase->GetScoreBreakdown();
vector<float> phraseScores = scores.GetScoresForProducer(phraseTable);
for (size_t j = 0; j < phraseScores.size(); ++j) {
out << exp(phraseScores[j]) << " ";
}
out << "||| ";
const AlignmentInfo& align = targetPhrase->GetAlignTerm();
for (AlignmentInfo::const_iterator j = align.begin(); j != align.end(); ++j) {
out << j->first << "-" << j->second << " ";
}
out << endl;
}
}
}
size_t Perceptron::updateWeightsHopeFear(
ScoreComponentCollection& weightUpdate,
const vector< vector<ScoreComponentCollection> >& featureValuesHope,
const vector< vector<ScoreComponentCollection> >& featureValuesFear,
const vector< vector<float> >& dummy1,
const vector< vector<float> >& dummy2,
const vector< vector<float> >& dummy3,
const vector< vector<float> >& dummy4,
float perceptron_learning_rate,
size_t rank,
size_t epoch,
int updatePosition)
{
cerr << "Rank " << rank << ", epoch " << epoch << ", hope: " << featureValuesHope[0][0] << endl;
cerr << "Rank " << rank << ", epoch " << epoch << ", fear: " << featureValuesFear[0][0] << endl;
ScoreComponentCollection featureValueDiff = featureValuesHope[0][0];
featureValueDiff.MinusEquals(featureValuesFear[0][0]);
cerr << "Rank " << rank << ", epoch " << epoch << ", hope - fear: " << featureValueDiff << endl;
featureValueDiff.MultiplyEquals(perceptron_learning_rate);
weightUpdate.PlusEquals(featureValueDiff);
cerr << "Rank " << rank << ", epoch " << epoch << ", update: " << featureValueDiff << endl;
return 0;
}
void DeleteRules::EvaluateInIsolation(const Phrase &source
, const TargetPhrase &target
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedScores) const
{
// dense scores
size_t hash = 0;
boost::hash_combine(hash, source);
boost::hash_combine(hash, target);
boost::unordered_set<size_t>::const_iterator iter;
iter = m_ruleHashes.find(hash);
if (iter != m_ruleHashes.end()) {
scoreBreakdown.PlusEquals(this, -std::numeric_limits<float>::infinity());
}
}
void InputFeature::EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection *estimatedScores) const
{
if (m_legacy) {
//binary phrase-table does input feature itself
return;
} else if (input.GetType() == WordLatticeInput) {
const ScorePair *scores = inputPath.GetInputScore();
if (scores) {
scoreBreakdown.PlusEquals(this, *scores);
}
}
}
void OpSequenceModel:: Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{
osmHypothesis obj;
obj.setState(OSM->NullContextState());
WordsBitmap myBitmap(source.GetSize());
vector <string> mySourcePhrase;
vector <string> myTargetPhrase;
vector<float> scores(5);
vector <int> alignments;
int startIndex = 0;
int endIndex = source.GetSize();
const AlignmentInfo &align = targetPhrase.GetAlignTerm();
AlignmentInfo::const_iterator iter;
for (iter = align.begin(); iter != align.end(); ++iter) {
alignments.push_back(iter->first);
alignments.push_back(iter->second);
}
for (int i = 0; i < targetPhrase.GetSize(); i++) {
if (targetPhrase.GetWord(i).IsOOV())
myTargetPhrase.push_back("_TRANS_SLF_");
else
myTargetPhrase.push_back(targetPhrase.GetWord(i).GetFactor(0)->GetString().as_string());
}
for (int i = 0; i < source.GetSize(); i++) {
mySourcePhrase.push_back(source.GetWord(i).GetFactor(0)->GetString().as_string());
}
obj.setPhrases(mySourcePhrase , myTargetPhrase);
obj.constructCepts(alignments,startIndex,endIndex-1,targetPhrase.GetSize());
obj.computeOSMFeature(startIndex,myBitmap);
obj.calculateOSMProb(*OSM);
obj.populateScores(scores);
estimatedFutureScore.PlusEquals(this, scores);
}
BOOST_FIXTURE_TEST_CASE(plusequals, MockProducers)
{
float arr1[] = {1,2,3,4,5};
float arr2[] = {2,4,6,8,10};
std::vector<float> vec1(arr1,arr1+5);
std::vector<float> vec2(arr2,arr2+5);
ScoreComponentCollection scc;
scc.PlusEquals(&single, 3.4f);
BOOST_CHECK_EQUAL(scc.GetScoreForProducer(&single), 3.4f);
scc.PlusEquals(&multi,vec1);
std::vector<float> actual = scc.GetScoresForProducer(&multi);
BOOST_CHECK_EQUAL_COLLECTIONS(vec1.begin(),vec1.end()
,actual.begin(), actual.end());
scc.PlusEquals(&multi,vec1);
actual = scc.GetScoresForProducer(&multi);
BOOST_CHECK_EQUAL_COLLECTIONS(vec2.begin(),vec2.end(),
actual.begin(), actual.end());
BOOST_CHECK_EQUAL(scc.GetScoreForProducer(&single), 3.4f);
}
void CountNonTerms::Evaluate(const Phrase &sourcePhrase
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{
const StaticData &staticData = StaticData::Instance();
vector<float> scores(m_numScoreComponents, 0);
size_t indScore = 0;
if (m_all) {
for (size_t i = 0; i < targetPhrase.GetSize(); ++i) {
const Word &word = targetPhrase.GetWord(i);
if (word.IsNonTerminal()) {
++scores[indScore];
}
}
++indScore;
}
if (m_targetSyntax) {
for (size_t i = 0; i < targetPhrase.GetSize(); ++i) {
const Word &word = targetPhrase.GetWord(i);
if (word.IsNonTerminal() && word != staticData.GetOutputDefaultNonTerminal()) {
++scores[indScore];
}
}
++indScore;
}
if (m_sourceSyntax) {
for (size_t i = 0; i < sourcePhrase.GetSize(); ++i) {
const Word &word = sourcePhrase.GetWord(i);
if (word.IsNonTerminal() && word != staticData.GetInputDefaultNonTerminal()) {
++scores[indScore];
}
}
++indScore;
}
scoreBreakdown.PlusEquals(this, scores);
}
void CoveredReferenceFeature::EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection *estimatedFutureScore) const
{
long id = input.GetTranslationId();
boost::unordered_map<long, std::multiset<string> >::const_iterator refIt = m_refs.find(id);
multiset<string> wordsInPhrase = GetWordsInPhrase(targetPhrase);
multiset<string> covered;
set_intersection(wordsInPhrase.begin(), wordsInPhrase.end(),
refIt->second.begin(), refIt->second.end(),
inserter(covered, covered.begin()));
vector<float> scores;
scores.push_back(covered.size());
scoreBreakdown.Assign(this, scores);
estimatedFutureScore->Assign(this, scores);
}
