void LexicalReordering:: SetCache(TranslationOption& to) const { if (to.GetLexReorderingScores(this)) return; // Scores were were set already (e.g., by sampling phrase table) if (m_table) { Phrase const& sphrase = to.GetInputPath().GetPhrase(); Phrase const& tphrase = to.GetTargetPhrase(); to.CacheLexReorderingScores(*this, this->GetProb(sphrase,tphrase)); } else { // e.g. OOV with Mmsapt // Scores vals(GetNumScoreComponents(), 0); // to.CacheLexReorderingScores(*this, vals); } }
void DecodeStepGeneration::Process(const TranslationOption &inputPartialTranslOpt , const DecodeStep &decodeStep , PartialTranslOptColl &outputPartialTranslOptColl , TranslationOptionCollection * /* toc */ , bool /*adhereTableLimit*/) const { if (inputPartialTranslOpt.GetTargetPhrase().GetSize() == 0) { // word deletion TranslationOption *newTransOpt = new TranslationOption(inputPartialTranslOpt); outputPartialTranslOptColl.Add(newTransOpt); return; } // normal generation step const GenerationDictionary* generationDictionary = decodeStep.GetGenerationDictionaryFeature(); const Phrase &targetPhrase = inputPartialTranslOpt.GetTargetPhrase(); const InputPath &inputPath = inputPartialTranslOpt.GetInputPath(); size_t targetLength = targetPhrase.GetSize(); // generation list for each word in phrase vector< WordList > wordListVector(targetLength); // create generation list int wordListVectorPos = 0; for (size_t currPos = 0 ; currPos < targetLength ; currPos++) { // going thorugh all words // generatable factors for this word to be put in wordList WordList &wordList = wordListVector[wordListVectorPos]; const Word &word = targetPhrase.GetWord(currPos); // consult dictionary for possible generations for this word const OutputWordCollection *wordColl = generationDictionary->FindWord(word); if (wordColl == NULL) { // word not found in generation dictionary //toc->ProcessUnknownWord(sourceWordsRange.GetStartPos(), factorCollection); return; // can't be part of a phrase, special handling } else { // sort(*wordColl, CompareWordCollScore); OutputWordCollection::const_iterator iterWordColl; for (iterWordColl = wordColl->begin() ; iterWordColl != wordColl->end(); ++iterWordColl) { const Word &outputWord = (*iterWordColl).first; const ScoreComponentCollection& score = (*iterWordColl).second; // enter into word list generated factor(s) and its(their) score(s) wordList.push_back(WordPair(outputWord, score)); } wordListVectorPos++; // done, next word } } // use generation list (wordList) // set up iterators (total number of expansions) size_t numIteration = 1; vector< WordListIterator > wordListIterVector(targetLength); vector< const Word* > mergeWords(targetLength); for (size_t currPos = 0 ; currPos < targetLength ; currPos++) { wordListIterVector[currPos] = wordListVector[currPos].begin(); numIteration *= wordListVector[currPos].size(); } // go thru each possible factor for each word & create hypothesis for (size_t currIter = 0 ; currIter < numIteration ; currIter++) { ScoreComponentCollection generationScore; // total score for this string of words // create vector of words with new factors for last phrase for (size_t currPos = 0 ; currPos < targetLength ; currPos++) { const WordPair &wordPair = *wordListIterVector[currPos]; mergeWords[currPos] = &(wordPair.first); generationScore.PlusEquals(wordPair.second); } // merge with existing trans opt Phrase genPhrase( mergeWords); if (IsFilteringStep()) { if (!inputPartialTranslOpt.IsCompatible(genPhrase, m_conflictFactors)) continue; } const TargetPhrase &inPhrase = inputPartialTranslOpt.GetTargetPhrase(); TargetPhrase outPhrase(inPhrase); outPhrase.GetScoreBreakdown().PlusEquals(generationScore); outPhrase.MergeFactors(genPhrase, m_newOutputFactors); outPhrase.Evaluate(inputPath.GetPhrase(), m_featuresToApply); const WordsRange &sourceWordsRange = inputPartialTranslOpt.GetSourceWordsRange(); TranslationOption *newTransOpt = new TranslationOption(sourceWordsRange, outPhrase); assert(newTransOpt); newTransOpt->SetInputPath(inputPath); outputPartialTranslOptColl.Add(newTransOpt); // increment iterators IncrementIterators(wordListIterVector, wordListVector); } }
void SparseReordering::CopyScores( const TranslationOption& currentOpt, const TranslationOption* previousOpt, const InputType& input, LexicalReorderingState::ReorderingType reoType, LexicalReorderingConfiguration::Direction direction, ScoreComponentCollection* scores) const { if (m_useBetween && direction == LexicalReorderingConfiguration::Backward && (reoType == LexicalReorderingState::D || reoType == LexicalReorderingState::DL || reoType == LexicalReorderingState::DR)) { size_t gapStart, gapEnd; //NB: Using a static cast for speed, but could be nasty if //using non-sentence input const Sentence& sentence = static_cast<const Sentence&>(input); const WordsRange& currentRange = currentOpt.GetSourceWordsRange(); if (previousOpt) { const WordsRange& previousRange = previousOpt->GetSourceWordsRange(); if (previousRange < currentRange) { gapStart = previousRange.GetEndPos() + 1; gapEnd = currentRange.GetStartPos(); } else { gapStart = currentRange.GetEndPos() + 1; gapEnd = previousRange.GetStartPos(); } } else { //start of sentence gapStart = 0; gapEnd = currentRange.GetStartPos(); } assert(gapStart < gapEnd); for (size_t i = gapStart; i < gapEnd; ++i) { AddFeatures(SparseReorderingFeatureKey::Between, SparseReorderingFeatureKey::Source, sentence.GetWord(i), SparseReorderingFeatureKey::First, reoType, scores); } } //std::cerr << "SR " << topt << " " << reoType << " " << direction << std::endl; //phrase (backward) //stack (forward) SparseReorderingFeatureKey::Type type; if (direction == LexicalReorderingConfiguration::Forward) { if (!m_useStack) return; type = SparseReorderingFeatureKey::Stack; } else if (direction == LexicalReorderingConfiguration::Backward) { if (!m_usePhrase) return; type = SparseReorderingFeatureKey::Phrase; } else { //Shouldn't be called for bidirectional //keep compiler happy type = SparseReorderingFeatureKey::Phrase; assert(!"Shouldn't call CopyScores() with bidirectional direction"); } const Phrase& sourcePhrase = currentOpt.GetInputPath().GetPhrase(); AddFeatures(type, SparseReorderingFeatureKey::Source, sourcePhrase.GetWord(0), SparseReorderingFeatureKey::First, reoType, scores); AddFeatures(type, SparseReorderingFeatureKey::Source, sourcePhrase.GetWord(sourcePhrase.GetSize()-1), SparseReorderingFeatureKey::Last, reoType, scores); const Phrase& targetPhrase = currentOpt.GetTargetPhrase(); AddFeatures(type, SparseReorderingFeatureKey::Target, targetPhrase.GetWord(0), SparseReorderingFeatureKey::First, reoType, scores); AddFeatures(type, SparseReorderingFeatureKey::Target, targetPhrase.GetWord(targetPhrase.GetSize()-1), SparseReorderingFeatureKey::Last, reoType, scores); }