/***
* print surface factor only for the given phrase
*/
void OutputSurface(std::ostream &out, const Hypothesis &edge, const std::vector<FactorType> &outputFactorOrder,
bool reportSegmentation, bool reportAllFactors)
{
CHECK(outputFactorOrder.size() > 0);
const Phrase& phrase = edge.GetCurrTargetPhrase();
if (reportAllFactors == true) {
out << phrase;
} else {
size_t size = phrase.GetSize();
for (size_t pos = 0 ; pos < size ; pos++) {
const Factor *factor = phrase.GetFactor(pos, outputFactorOrder[0]);
out << *factor;
CHECK(factor);
for (size_t i = 1 ; i < outputFactorOrder.size() ; i++) {
const Factor *factor = phrase.GetFactor(pos, outputFactorOrder[i]);
CHECK(factor);
out << "|" << *factor;
}
out << " ";
}
}
// trace option "-t"
if (reportSegmentation == true && phrase.GetSize() > 0) {
out << "|" << edge.GetCurrSourceWordsRange().GetStartPos()
<< "-" << edge.GetCurrSourceWordsRange().GetEndPos() << "| ";
}
}
/***
* print surface factor only for the given phrase
*/
void OutputSurface(std::ostream &out, const Hypothesis &edge, const std::vector<FactorType> &outputFactorOrder,
char reportSegmentation, bool reportAllFactors)
{
CHECK(outputFactorOrder.size() > 0);
const Phrase& phrase = edge.GetCurrTargetPhrase();
bool markUnknown = StaticData::Instance().GetMarkUnknown();
if (reportAllFactors == true) {
out << phrase;
} else {
FactorType placeholderFactor = StaticData::Instance().GetPlaceholderFactor().second;
size_t size = phrase.GetSize();
for (size_t pos = 0 ; pos < size ; pos++) {
const Factor *factor = phrase.GetFactor(pos, outputFactorOrder[0]);
if (placeholderFactor != NOT_FOUND) {
const Factor *origFactor = phrase.GetFactor(pos, placeholderFactor);
if (origFactor) {
factor = origFactor;
}
}
CHECK(factor);
//preface surface form with UNK if marking unknowns
const Word &word = phrase.GetWord(pos);
if(markUnknown && word.IsOOV()) {
out << "UNK" << *factor;
}
else {
out << *factor;
}
for (size_t i = 1 ; i < outputFactorOrder.size() ; i++) {
const Factor *factor = phrase.GetFactor(pos, outputFactorOrder[i]);
CHECK(factor);
out << "|" << *factor;
}
out << " ";
}
}
// trace option "-t" / "-tt"
if (reportSegmentation > 0 && phrase.GetSize() > 0) {
const WordsRange &sourceRange = edge.GetCurrSourceWordsRange();
const int sourceStart = sourceRange.GetStartPos();
const int sourceEnd = sourceRange.GetEndPos();
out << "|" << sourceStart << "-" << sourceEnd;
// enriched "-tt"
if (reportSegmentation == 2) {
out << ",0, ";
const AlignmentInfo &ai = edge.GetCurrTargetPhrase().GetAlignTerm();
OutputAlignment(out, ai, 0, 0);
}
out << "| ";
}
}
void LexicalReordering::EmptyHypothesisState(FFState &state,
const ManagerBase &mgr, const InputType &input,
const Hypothesis &hypo) const
{
BidirectionalReorderingState &stateCast =
static_cast<BidirectionalReorderingState&>(state);
stateCast.Init(NULL, hypo.GetTargetPhrase(), hypo.GetInputPath(), true,
&hypo.GetBitmap());
}
/// add phrase alignment information from a Hypothesis
void
TranslationRequest::
add_phrase_aln_info(Hypothesis const& h, vector<xmlrpc_c::value>& aInfo) const
{
if (!m_withAlignInfo) return;
WordsRange const& trg = h.GetCurrTargetWordsRange();
WordsRange const& src = h.GetCurrSourceWordsRange();
std::map<std::string, xmlrpc_c::value> pAlnInfo;
pAlnInfo["tgt-start"] = xmlrpc_c::value_int(trg.GetStartPos());
pAlnInfo["src-start"] = xmlrpc_c::value_int(src.GetStartPos());
pAlnInfo["src-end"] = xmlrpc_c::value_int(src.GetEndPos());
aInfo.push_back(xmlrpc_c::value_struct(pAlnInfo));
}
void LanguageModel::EvaluateWhenApplied(const ManagerBase &mgr,
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
FFState &state) const
{
const LMState &prevLMState = static_cast<const LMState &>(prevState);
size_t numWords = prevLMState.numWords;
// context is held backwards
vector<const Factor*> context(numWords);
for (size_t i = 0; i < numWords; ++i) {
context[i] = prevLMState.lastWords[i];
}
//DebugContext(context);
SCORE score = 0;
std::pair<SCORE, void*> fromScoring;
const TargetPhrase<Moses2::Word> &tp = hypo.GetTargetPhrase();
for (size_t i = 0; i < tp.GetSize(); ++i) {
const Word &word = tp[i];
const Factor *factor = word[m_factorType];
ShiftOrPush(context, factor);
fromScoring = Score(context);
score += fromScoring.first;
}
const Bitmap &bm = hypo.GetBitmap();
if (bm.IsComplete()) {
// everything translated
ShiftOrPush(context, m_eos);
fromScoring = Score(context);
score += fromScoring.first;
fromScoring.second = NULL;
context.clear();
} else {
assert(context.size());
if (context.size() == m_order) {
context.resize(context.size() - 1);
}
}
scores.PlusEquals(mgr.system, *this, score);
// return state
//DebugContext(context);
LMState &stateCast = static_cast<LMState&>(state);
MemPool &pool = mgr.GetPool();
stateCast.Set(pool, fromScoring.second, context);
}
void GlobalLexicalModel::Evaluate
(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{
accumulator->PlusEquals( this,
GetFromCacheOrScorePhrase(hypo.GetCurrTargetPhrase()) );
}
void generate_hypotheses(const int order, const Hypothesis & h,
const vector<ME_Model> & vme,
list<Hypothesis> & vh)
{
int n = h.vt.size();
int pred_position = -1;
double min_ent = 999999;
string pred = "";
double pred_prob = 0;
for (int j = 0; j < n; j++) {
if (h.vt[j].cprd != "") continue;
double ent = h.vent[j];
if (ent < min_ent) {
// pred = h.vvp[j].begin()->first;
// pred_prob = h.vvp[j].begin()->second;
min_ent = ent;
pred_position = j;
}
}
assert(pred_position >= 0 && pred_position < n);
for (vector<pair<string, double> >::const_iterator k = h.vvp[pred_position].begin();
k != h.vvp[pred_position].end(); k++) {
Hypothesis newh = h;
newh.vt[pred_position].cprd = k->first;
newh.order[pred_position] = order + 1;
newh.prob = h.prob * k->second;
// if (newh.IsErroneous()) {
// cout << "*errorneous" << endl;
// newh.Print();
// continue;
// }
// update the neighboring predictions
for (int j = pred_position - TAG_WINDOW_SIZE; j <= pred_position + TAG_WINDOW_SIZE; j++) {
if (j < 0 || j > n-1) continue;
if (newh.vt[j].cprd == "") newh.Update(j, vme);
}
vh.push_back(newh);
}
}
float
DistortionScoreProducer::
CalculateDistortionScore(const Hypothesis& hypo,
const Range &prev, const Range &curr, const int FirstGap)
{
// if(!StaticData::Instance().UseEarlyDistortionCost()) {
if(!hypo.GetManager().options()->reordering.use_early_distortion_cost) {
return - (float) hypo.GetInput().ComputeDistortionDistance(prev, curr);
} // else {
/* Pay distortion score as soon as possible, from Moore and Quirk MT Summit 2007
Definitions:
S : current source range
S' : last translated source phrase range
S'' : longest fully-translated initial segment
*/
int prefixEndPos = (int)FirstGap-1;
if((int)FirstGap==-1)
prefixEndPos = -1;
// case1: S is adjacent to S'' => return 0
if ((int) curr.GetStartPos() == prefixEndPos+1) {
IFVERBOSE(4) std::cerr<< "MQ07disto:case1" << std::endl;
return 0;
}
// case2: S is to the left of S' => return 2(length(S))
if ((int) curr.GetEndPos() < (int) prev.GetEndPos()) {
IFVERBOSE(4) std::cerr<< "MQ07disto:case2" << std::endl;
return (float) -2*(int)curr.GetNumWordsCovered();
}
// case3: S' is a subsequence of S'' => return 2(nbWordBetween(S,S'')+length(S))
if ((int) prev.GetEndPos() <= prefixEndPos) {
IFVERBOSE(4) std::cerr<< "MQ07disto:case3" << std::endl;
int z = (int)curr.GetStartPos()-prefixEndPos - 1;
return (float) -2*(z + (int)curr.GetNumWordsCovered());
}
// case4: otherwise => return 2(nbWordBetween(S,S')+length(S))
IFVERBOSE(4) std::cerr<< "MQ07disto:case4" << std::endl;
return (float) -2*((int)curr.GetNumWordsBetween(prev) + (int)curr.GetNumWordsCovered());
}
double combine(const Hypothesis & a, const Hypothesis & b, Hypothesis & ret) const {
ret.hook = a.hook;
ret.right_side = b.right_side;
for (int i=0;i<a.prev_hyp.size();i++) {
ret.prev_hyp.push_back(a.prev_hyp[i]);
}
ret.prev_hyp.push_back(b.id());
return 0.0;
}
std::map<size_t, const Factor*> GetPlaceholders(const Hypothesis &hypo, FactorType placeholderFactor)
{
const InputPath &inputPath = hypo.GetTranslationOption().GetInputPath();
const Phrase &inputPhrase = inputPath.GetPhrase();
std::map<size_t, const Factor*> ret;
for (size_t sourcePos = 0; sourcePos < inputPhrase.GetSize(); ++sourcePos) {
const Factor *factor = inputPhrase.GetFactor(sourcePos, placeholderFactor);
if (factor) {
std::set<size_t> targetPos = hypo.GetTranslationOption().GetTargetPhrase().GetAlignTerm().GetAlignmentsForSource(sourcePos);
CHECK(targetPos.size() == 1);
ret[*targetPos.begin()] = factor;
}
}
return ret;
}
ControlRecombinationState::ControlRecombinationState(const Hypothesis &hypo, const ControlRecombination &ff)
:m_ff(ff)
{
if (ff.GetType() == SameOutput) {
//UTIL_THROW(util::Exception, "Implemented not yet completed for phrase-based model. Need to take into account the coverage");
hypo.GetOutputPhrase(m_outputPhrase);
} else {
m_hypo = &hypo;
}
}
void PhraseBasedReorderingState::Expand(const ManagerBase &mgr,
const LexicalReordering &ff, const Hypothesis &hypo, size_t phraseTableInd,
Scores &scores, FFState &state) const
{
if ((m_direction != LRModel::Forward) || !m_first) {
LRModel const& lrmodel = m_configuration;
Range const &cur = hypo.GetInputPath().range;
LRModel::ReorderingType reoType = (
m_first ?
lrmodel.GetOrientation(cur) :
lrmodel.GetOrientation(prevPath->range, cur));
CopyScores(mgr.system, scores, hypo.GetTargetPhrase(), reoType);
}
PhraseBasedReorderingState &stateCast =
static_cast<PhraseBasedReorderingState&>(state);
stateCast.Init(this, hypo.GetTargetPhrase(), hypo.GetInputPath(), false,
NULL);
}
std::map<size_t, const Factor*>
Hypothesis::
GetPlaceholders(const Hypothesis &hypo, FactorType placeholderFactor) const
{
const InputPath &inputPath = hypo.GetTranslationOption().GetInputPath();
const Phrase &inputPhrase = inputPath.GetPhrase();
std::map<size_t, const Factor*> ret;
for (size_t sourcePos = 0; sourcePos < inputPhrase.GetSize(); ++sourcePos) {
const Factor *factor = inputPhrase.GetFactor(sourcePos, placeholderFactor);
if (factor) {
std::set<size_t> targetPos = hypo.GetTranslationOption().GetTargetPhrase().GetAlignTerm().GetAlignmentsForSource(sourcePos);
UTIL_THROW_IF2(targetPos.size() != 1,
"Placeholder should be aligned to 1, and only 1, word");
ret[*targetPos.begin()] = factor;
}
}
return ret;
}
bool WorldModelROS::hypothesisToMsg(const Hypothesis& hyp, wire_msgs::WorldState& msg) const {
ros::Time time = ros::Time::now();
msg.header.frame_id = world_model_frame_id_;
msg.header.stamp = time;
for(list<SemanticObject*>::const_iterator it = hyp.getObjects().begin(); it != hyp.getObjects().end(); ++it) {
SemanticObject* obj_clone = (*it)->clone();
obj_clone->propagate(time.toSec());
wire_msgs::ObjectState obj_msg;
if (objectToMsg(*obj_clone, obj_msg)) {
msg.objects.push_back(obj_msg);
}
delete obj_clone;
}
return true;
}
size_t LM::Evaluate(
const Hypothesis& hypo,
size_t prevState,
Scores &scores) const
{
if (m_order <= 1) {
return 0; // not sure if returning NULL is correct
}
if (hypo.targetPhrase.GetSize() == 0) {
return 0; // not sure if returning NULL is correct
}
PhraseVec m_phraseVec(m_order);
const size_t currEndPos = hypo.targetRange.endPos;
const size_t startPos = hypo.targetRange.startPos;
size_t index = 0;
for (int currPos = (int) startPos - (int) m_order + 1 ; currPos <= (int) startPos ; currPos++) {
if (currPos >= 0)
m_phraseVec[index++] = &hypo.GetWord(currPos);
else {
m_phraseVec[index++] = &m_bos;
}
}
SCORE lmScore = GetValueCache(m_phraseVec);
// main loop
size_t endPos = std::min(startPos + m_order - 2
, currEndPos);
for (size_t currPos = startPos + 1 ; currPos <= endPos ; currPos++) {
// shift all args down 1 place
for (size_t i = 0 ; i < m_order - 1 ; i++)
m_phraseVec[i] = m_phraseVec[i + 1];
// add last factor
m_phraseVec.back() = &hypo.GetWord(currPos);
lmScore += GetValueCache(m_phraseVec);
}
// end of sentence
if (hypo.GetCoverage().IsComplete()) {
const size_t size = hypo.GetSize();
m_phraseVec.back() = &m_eos;
for (size_t i = 0 ; i < m_order - 1 ; i ++) {
int currPos = (int)(size - m_order + i + 1);
if (currPos < 0)
m_phraseVec[i] = &m_bos;
else
m_phraseVec[i] = &hypo.GetWord((size_t)currPos);
}
lmScore += GetValueCache(m_phraseVec);
} else {
if (endPos < currEndPos) {
//need to get the LM state (otherwise the last LM state is fine)
for (size_t currPos = endPos+1; currPos <= currEndPos; currPos++) {
for (size_t i = 0 ; i < m_order - 1 ; i++)
m_phraseVec[i] = m_phraseVec[i + 1];
m_phraseVec.back() = &hypo.GetWord(currPos);
}
}
}
size_t state = GetLastState();
return state;
}
void GlobalLexicalModelUnlimited::Evaluate(const Hypothesis& cur_hypo, ScoreComponentCollection* accumulator) const
{
const Sentence& input = *(m_local->input);
const TargetPhrase& targetPhrase = cur_hypo.GetCurrTargetPhrase();
for(size_t targetIndex = 0; targetIndex < targetPhrase.GetSize(); targetIndex++ ) {
StringPiece targetString = targetPhrase.GetWord(targetIndex).GetString(0); // TODO: change for other factors
if (m_ignorePunctuation) {
// check if first char is punctuation
char firstChar = targetString[0];
CharHash::const_iterator charIterator = m_punctuationHash.find( firstChar );
if(charIterator != m_punctuationHash.end())
continue;
}
if (m_biasFeature) {
stringstream feature;
feature << "glm_";
feature << targetString;
feature << "~";
feature << "**BIAS**";
accumulator->SparsePlusEquals(feature.str(), 1);
}
boost::unordered_set<uint64_t> alreadyScored;
for(size_t sourceIndex = 0; sourceIndex < input.GetSize(); sourceIndex++ ) {
const StringPiece sourceString = input.GetWord(sourceIndex).GetString(0);
// TODO: change for other factors
if (m_ignorePunctuation) {
// check if first char is punctuation
char firstChar = sourceString[0];
CharHash::const_iterator charIterator = m_punctuationHash.find( firstChar );
if(charIterator != m_punctuationHash.end())
continue;
}
const uint64_t sourceHash = util::MurmurHashNative(sourceString.data(), sourceString.size());
if ( alreadyScored.find(sourceHash) == alreadyScored.end()) {
bool sourceExists, targetExists;
if (!m_unrestricted) {
sourceExists = FindStringPiece(m_vocabSource, sourceString ) != m_vocabSource.end();
targetExists = FindStringPiece(m_vocabTarget, targetString) != m_vocabTarget.end();
}
// no feature if vocab is in use and both words are not in restricted vocabularies
if (m_unrestricted || (sourceExists && targetExists)) {
if (m_sourceContext) {
if (sourceIndex == 0) {
// add <s> trigger feature for source
stringstream feature;
feature << "glm_";
feature << targetString;
feature << "~";
feature << "<s>,";
feature << sourceString;
accumulator->SparsePlusEquals(feature.str(), 1);
alreadyScored.insert(sourceHash);
}
// add source words to the right of current source word as context
for(int contextIndex = sourceIndex+1; contextIndex < input.GetSize(); contextIndex++ ) {
StringPiece contextString = input.GetWord(contextIndex).GetString(0); // TODO: change for other factors
bool contextExists;
if (!m_unrestricted)
contextExists = FindStringPiece(m_vocabSource, contextString ) != m_vocabSource.end();
if (m_unrestricted || contextExists) {
stringstream feature;
feature << "glm_";
feature << targetString;
feature << "~";
feature << sourceString;
feature << ",";
feature << contextString;
accumulator->SparsePlusEquals(feature.str(), 1);
alreadyScored.insert(sourceHash);
}
}
} else if (m_biphrase) {
// --> look backwards for constructing context
int globalTargetIndex = cur_hypo.GetSize() - targetPhrase.GetSize() + targetIndex;
// 1) source-target pair, trigger source word (can be discont.) and adjacent target word (bigram)
StringPiece targetContext;
if (globalTargetIndex > 0)
targetContext = cur_hypo.GetWord(globalTargetIndex-1).GetString(0); // TODO: change for other factors
else
targetContext = "<s>";
if (sourceIndex == 0) {
StringPiece sourceTrigger = "<s>";
AddFeature(accumulator, sourceTrigger, sourceString,
targetContext, targetString);
} else
for(int contextIndex = sourceIndex-1; contextIndex >= 0; contextIndex-- ) {
StringPiece sourceTrigger = input.GetWord(contextIndex).GetString(0); // TODO: change for other factors
bool sourceTriggerExists = false;
if (!m_unrestricted)
sourceTriggerExists = FindStringPiece(m_vocabSource, sourceTrigger ) != m_vocabSource.end();
if (m_unrestricted || sourceTriggerExists)
AddFeature(accumulator, sourceTrigger, sourceString,
targetContext, targetString);
}
// 2) source-target pair, adjacent source word (bigram) and trigger target word (can be discont.)
StringPiece sourceContext;
if (sourceIndex-1 >= 0)
sourceContext = input.GetWord(sourceIndex-1).GetString(0); // TODO: change for other factors
else
sourceContext = "<s>";
if (globalTargetIndex == 0) {
string targetTrigger = "<s>";
AddFeature(accumulator, sourceContext, sourceString,
targetTrigger, targetString);
} else
for(int globalContextIndex = globalTargetIndex-1; globalContextIndex >= 0; globalContextIndex-- ) {
StringPiece targetTrigger = cur_hypo.GetWord(globalContextIndex).GetString(0); // TODO: change for other factors
bool targetTriggerExists = false;
if (!m_unrestricted)
targetTriggerExists = FindStringPiece(m_vocabTarget, targetTrigger ) != m_vocabTarget.end();
if (m_unrestricted || targetTriggerExists)
AddFeature(accumulator, sourceContext, sourceString,
targetTrigger, targetString);
}
} else if (m_bitrigger) {
// allow additional discont. triggers on both sides
int globalTargetIndex = cur_hypo.GetSize() - targetPhrase.GetSize() + targetIndex;
if (sourceIndex == 0) {
StringPiece sourceTrigger = "<s>";
bool sourceTriggerExists = true;
if (globalTargetIndex == 0) {
string targetTrigger = "<s>";
bool targetTriggerExists = true;
if (m_unrestricted || (sourceTriggerExists && targetTriggerExists))
AddFeature(accumulator, sourceTrigger, sourceString,
targetTrigger, targetString);
} else {
// iterate backwards over target
for(int globalContextIndex = globalTargetIndex-1; globalContextIndex >= 0; globalContextIndex-- ) {
StringPiece targetTrigger = cur_hypo.GetWord(globalContextIndex).GetString(0); // TODO: change for other factors
bool targetTriggerExists = false;
if (!m_unrestricted)
targetTriggerExists = FindStringPiece(m_vocabTarget, targetTrigger ) != m_vocabTarget.end();
if (m_unrestricted || (sourceTriggerExists && targetTriggerExists))
AddFeature(accumulator, sourceTrigger, sourceString,
targetTrigger, targetString);
}
}
}
// iterate over both source and target
else {
// iterate backwards over source
for(int contextIndex = sourceIndex-1; contextIndex >= 0; contextIndex-- ) {
StringPiece sourceTrigger = input.GetWord(contextIndex).GetString(0); // TODO: change for other factors
bool sourceTriggerExists = false;
if (!m_unrestricted)
sourceTriggerExists = FindStringPiece(m_vocabSource, sourceTrigger ) != m_vocabSource.end();
if (globalTargetIndex == 0) {
string targetTrigger = "<s>";
bool targetTriggerExists = true;
if (m_unrestricted || (sourceTriggerExists && targetTriggerExists))
AddFeature(accumulator, sourceTrigger, sourceString,
targetTrigger, targetString);
} else {
// iterate backwards over target
for(int globalContextIndex = globalTargetIndex-1; globalContextIndex >= 0; globalContextIndex-- ) {
StringPiece targetTrigger = cur_hypo.GetWord(globalContextIndex).GetString(0); // TODO: change for other factors
bool targetTriggerExists = false;
if (!m_unrestricted)
targetTriggerExists = FindStringPiece(m_vocabTarget, targetTrigger ) != m_vocabTarget.end();
if (m_unrestricted || (sourceTriggerExists && targetTriggerExists))
AddFeature(accumulator, sourceTrigger, sourceString,
targetTrigger, targetString);
}
}
}
}
} else {
stringstream feature;
feature << "glm_";
feature << targetString;
feature << "~";
feature << sourceString;
accumulator->SparsePlusEquals(feature.str(), 1);
alreadyScored.insert(sourceHash);
}
}
}
}
}
}
void WordTranslationFeature::EvaluateWhenApplied
(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{
const Sentence& input = static_cast<const Sentence&>(hypo.GetInput());
const TranslationOption& transOpt = hypo.GetTranslationOption();
const TargetPhrase& targetPhrase = hypo.GetCurrTargetPhrase();
const AlignmentInfo &alignment = targetPhrase.GetAlignTerm();
// process aligned words
for (AlignmentInfo::const_iterator alignmentPoint = alignment.begin(); alignmentPoint != alignment.end(); alignmentPoint++) {
const Phrase& sourcePhrase = transOpt.GetInputPath().GetPhrase();
int sourceIndex = alignmentPoint->first;
int targetIndex = alignmentPoint->second;
Word ws = sourcePhrase.GetWord(sourceIndex);
if (m_factorTypeSource == 0 && ws.IsNonTerminal()) continue;
Word wt = targetPhrase.GetWord(targetIndex);
if (m_factorTypeSource == 0 && wt.IsNonTerminal()) continue;
StringPiece sourceWord = ws.GetFactor(m_factorTypeSource)->GetString();
StringPiece targetWord = wt.GetFactor(m_factorTypeTarget)->GetString();
if (m_ignorePunctuation) {
// check if source or target are punctuation
char firstChar = sourceWord[0];
CharHash::const_iterator charIterator = m_punctuationHash.find( firstChar );
if(charIterator != m_punctuationHash.end())
continue;
firstChar = targetWord[0];
charIterator = m_punctuationHash.find( firstChar );
if(charIterator != m_punctuationHash.end())
continue;
}
if (!m_unrestricted) {
if (FindStringPiece(m_vocabSource, sourceWord) == m_vocabSource.end())
sourceWord = "OTHER";
if (FindStringPiece(m_vocabTarget, targetWord) == m_vocabTarget.end())
targetWord = "OTHER";
}
if (m_simple) {
// construct feature name
stringstream featureName;
featureName << m_description << "_";
featureName << sourceWord;
featureName << "~";
featureName << targetWord;
accumulator->SparsePlusEquals(featureName.str(), 1);
}
if (m_domainTrigger && !m_sourceContext) {
const bool use_topicid = input.GetUseTopicId();
const bool use_topicid_prob = input.GetUseTopicIdAndProb();
if (use_topicid || use_topicid_prob) {
if(use_topicid) {
// use topicid as trigger
const long topicid = input.GetTopicId();
stringstream feature;
feature << m_description << "_";
if (topicid == -1)
feature << "unk";
else
feature << topicid;
feature << "_";
feature << sourceWord;
feature << "~";
feature << targetWord;
accumulator->SparsePlusEquals(feature.str(), 1);
} else {
// use topic probabilities
const vector<string> &topicid_prob = *(input.GetTopicIdAndProb());
if (atol(topicid_prob[0].c_str()) == -1) {
stringstream feature;
feature << m_description << "_unk_";
feature << sourceWord;
feature << "~";
feature << targetWord;
accumulator->SparsePlusEquals(feature.str(), 1);
} else {
for (size_t i=0; i+1 < topicid_prob.size(); i+=2) {
stringstream feature;
feature << m_description << "_";
feature << topicid_prob[i];
feature << "_";
feature << sourceWord;
feature << "~";
feature << targetWord;
accumulator->SparsePlusEquals(feature.str(), atof((topicid_prob[i+1]).c_str()));
}
}
}
} else {
// range over domain trigger words (keywords)
const long docid = input.GetDocumentId();
for (boost::unordered_set<std::string>::const_iterator p = m_vocabDomain[docid].begin(); p != m_vocabDomain[docid].end(); ++p) {
string sourceTrigger = *p;
stringstream feature;
feature << m_description << "_";
feature << sourceTrigger;
feature << "_";
feature << sourceWord;
feature << "~";
feature << targetWord;
accumulator->SparsePlusEquals(feature.str(), 1);
}
}
}
if (m_sourceContext) {
size_t globalSourceIndex = hypo.GetTranslationOption().GetStartPos() + sourceIndex;
if (!m_domainTrigger && globalSourceIndex == 0) {
// add <s> trigger feature for source
stringstream feature;
feature << m_description << "_";
feature << "<s>,";
feature << sourceWord;
feature << "~";
feature << targetWord;
accumulator->SparsePlusEquals(feature.str(), 1);
}
// range over source words to get context
for(size_t contextIndex = 0; contextIndex < input.GetSize(); contextIndex++ ) {
if (contextIndex == globalSourceIndex) continue;
StringPiece sourceTrigger = input.GetWord(contextIndex).GetFactor(m_factorTypeSource)->GetString();
if (m_ignorePunctuation) {
// check if trigger is punctuation
char firstChar = sourceTrigger[0];
CharHash::const_iterator charIterator = m_punctuationHash.find( firstChar );
if(charIterator != m_punctuationHash.end())
continue;
}
const long docid = input.GetDocumentId();
bool sourceTriggerExists = false;
if (m_domainTrigger)
sourceTriggerExists = FindStringPiece(m_vocabDomain[docid], sourceTrigger ) != m_vocabDomain[docid].end();
else if (!m_unrestricted)
sourceTriggerExists = FindStringPiece(m_vocabSource, sourceTrigger ) != m_vocabSource.end();
if (m_domainTrigger) {
if (sourceTriggerExists) {
stringstream feature;
feature << m_description << "_";
feature << sourceTrigger;
feature << "_";
feature << sourceWord;
feature << "~";
feature << targetWord;
accumulator->SparsePlusEquals(feature.str(), 1);
}
} else if (m_unrestricted || sourceTriggerExists) {
stringstream feature;
feature << m_description << "_";
if (contextIndex < globalSourceIndex) {
feature << sourceTrigger;
feature << ",";
feature << sourceWord;
} else {
feature << sourceWord;
feature << ",";
feature << sourceTrigger;
}
feature << "~";
feature << targetWord;
accumulator->SparsePlusEquals(feature.str(), 1);
}
}
}
if (m_targetContext) {
throw runtime_error("Can't use target words outside current translation option in a stateless feature");
/*
size_t globalTargetIndex = cur_hypo.GetCurrTargetWordsRange().GetStartPos() + targetIndex;
if (globalTargetIndex == 0) {
// add <s> trigger feature for source
stringstream feature;
feature << "wt_";
feature << sourceWord;
feature << "~";
feature << "<s>,";
feature << targetWord;
accumulator->SparsePlusEquals(feature.str(), 1);
}
// range over target words (up to current position) to get context
for(size_t contextIndex = 0; contextIndex < globalTargetIndex; contextIndex++ ) {
string targetTrigger = cur_hypo.GetWord(contextIndex).GetFactor(m_factorTypeTarget)->GetString();
if (m_ignorePunctuation) {
// check if trigger is punctuation
char firstChar = targetTrigger.at(0);
CharHash::const_iterator charIterator = m_punctuationHash.find( firstChar );
if(charIterator != m_punctuationHash.end())
continue;
}
bool targetTriggerExists = false;
if (!m_unrestricted)
targetTriggerExists = m_vocabTarget.find( targetTrigger ) != m_vocabTarget.end();
if (m_unrestricted || targetTriggerExists) {
stringstream feature;
feature << "wt_";
feature << sourceWord;
feature << "~";
feature << targetTrigger;
feature << ",";
feature << targetWord;
accumulator->SparsePlusEquals(feature.str(), 1);
}
}*/
}
}
}
void PhrasePairFeature::EvaluateWhenApplied(
const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{
const TargetPhrase& target = hypo.GetCurrTargetPhrase();
const Phrase& source = hypo.GetTranslationOption().GetInputPath().GetPhrase();
if (m_simple) {
ostringstream namestr;
namestr << "pp_";
namestr << source.GetWord(0).GetFactor(m_sourceFactorId)->GetString();
for (size_t i = 1; i < source.GetSize(); ++i) {
const Factor* sourceFactor = source.GetWord(i).GetFactor(m_sourceFactorId);
namestr << ",";
namestr << sourceFactor->GetString();
}
namestr << "~";
namestr << target.GetWord(0).GetFactor(m_targetFactorId)->GetString();
for (size_t i = 1; i < target.GetSize(); ++i) {
const Factor* targetFactor = target.GetWord(i).GetFactor(m_targetFactorId);
namestr << ",";
namestr << targetFactor->GetString();
}
accumulator->SparsePlusEquals(namestr.str(),1);
}
if (m_domainTrigger) {
const Sentence& input = static_cast<const Sentence&>(hypo.GetInput());
const bool use_topicid = input.GetUseTopicId();
const bool use_topicid_prob = input.GetUseTopicIdAndProb();
// compute pair
ostringstream pair;
pair << source.GetWord(0).GetFactor(m_sourceFactorId)->GetString();
for (size_t i = 1; i < source.GetSize(); ++i) {
const Factor* sourceFactor = source.GetWord(i).GetFactor(m_sourceFactorId);
pair << ",";
pair << sourceFactor->GetString();
}
pair << "~";
pair << target.GetWord(0).GetFactor(m_targetFactorId)->GetString();
for (size_t i = 1; i < target.GetSize(); ++i) {
const Factor* targetFactor = target.GetWord(i).GetFactor(m_targetFactorId);
pair << ",";
pair << targetFactor->GetString();
}
if (use_topicid || use_topicid_prob) {
if(use_topicid) {
// use topicid as trigger
const long topicid = input.GetTopicId();
stringstream feature;
feature << "pp_";
if (topicid == -1)
feature << "unk";
else
feature << topicid;
feature << "_";
feature << pair.str();
accumulator->SparsePlusEquals(feature.str(), 1);
} else {
// use topic probabilities
const vector<string> &topicid_prob = *(input.GetTopicIdAndProb());
if (atol(topicid_prob[0].c_str()) == -1) {
stringstream feature;
feature << "pp_unk_";
feature << pair.str();
accumulator->SparsePlusEquals(feature.str(), 1);
} else {
for (size_t i=0; i+1 < topicid_prob.size(); i+=2) {
stringstream feature;
feature << "pp_";
feature << topicid_prob[i];
feature << "_";
feature << pair.str();
accumulator->SparsePlusEquals(feature.str(), atof((topicid_prob[i+1]).c_str()));
}
}
}
} else {
// range over domain trigger words
const long docid = input.GetDocumentId();
for (set<string>::const_iterator p = m_vocabDomain[docid].begin(); p != m_vocabDomain[docid].end(); ++p) {
string sourceTrigger = *p;
ostringstream namestr;
namestr << "pp_";
namestr << sourceTrigger;
namestr << "_";
namestr << pair.str();
accumulator->SparsePlusEquals(namestr.str(),1);
}
}
}
if (m_sourceContext) {
const Sentence& input = static_cast<const Sentence&>(hypo.GetInput());
// range over source words to get context
for(size_t contextIndex = 0; contextIndex < input.GetSize(); contextIndex++ ) {
StringPiece sourceTrigger = input.GetWord(contextIndex).GetFactor(m_sourceFactorId)->GetString();
if (m_ignorePunctuation) {
// check if trigger is punctuation
char firstChar = sourceTrigger[0];
CharHash::const_iterator charIterator = m_punctuationHash.find( firstChar );
if(charIterator != m_punctuationHash.end())
continue;
}
bool sourceTriggerExists = false;
if (!m_unrestricted)
sourceTriggerExists = FindStringPiece(m_vocabSource, sourceTrigger ) != m_vocabSource.end();
if (m_unrestricted || sourceTriggerExists) {
ostringstream namestr;
namestr << "pp_";
namestr << sourceTrigger;
namestr << "~";
namestr << source.GetWord(0).GetFactor(m_sourceFactorId)->GetString();
for (size_t i = 1; i < source.GetSize(); ++i) {
const Factor* sourceFactor = source.GetWord(i).GetFactor(m_sourceFactorId);
namestr << ",";
namestr << sourceFactor->GetString();
}
namestr << "~";
namestr << target.GetWord(0).GetFactor(m_targetFactorId)->GetString();
for (size_t i = 1; i < target.GetSize(); ++i) {
const Factor* targetFactor = target.GetWord(i).GetFactor(m_targetFactorId);
namestr << ",";
namestr << targetFactor->GetString();
}
accumulator->SparsePlusEquals(namestr.str(),1);
}
}
}
}
ConstrainedDecodingState::ConstrainedDecodingState(const Hypothesis &hypo)
{
hypo.GetOutputPhrase(m_outputPhrase);
}
void Observations::setTarget(UnitObserver* o) {
UnitObserverType type = o->getType();
if (mPendingRequests.empty() && type != GLOBAL) {
//nothing to do
o->setIdle();
//EventLog::GetInstance().logEvent("IDLE "+o->getName());
return;
}
LOG_INFO("Setting new target for observer "+o->getName());
//if we have the same number of observers as hypotheses, then don't move them from their target group after it is set
if (mNumObservers >= mNumObservableGroups && o->hasTargetUnits()) {
//see if we're requesting this hypothesis again
std::deque<Hypothesis*>::iterator hit = std::find(mPendingRequests.begin(), mPendingRequests.end(), o->getHypothesis());
if (hit != mPendingRequests.end()) {
o->setTarget(o->getHypothesis(), o->getTargetGroup(), (float)o->getAltitude(), o->getTargetDuration());
mPendingRequests.erase(hit);
}
//EventLog::GetInstance().logEvent("FULL "+o->getName());
return;
}
switch (type) {
case GLOBAL: {
//move to the centre of the observed groups
/*std::vector<dtCore::RefPtr<UnitGroup> > groups = theApp->getTargets()->groupUnits(5000, RED);
if (groups.size() > 0) {
UnitGroup* g = groups[0].get();
osg::Vec3 centre = g->getCentre();
float radius = g->getRadius();
centre.z() += radius;
o->setTarget(centre, 1);
}*/
}
break;
case SEQUENTIAL: {
//nearest neighbour (+ priorities) from start position, and wait 5 seconds
float dMax = theApp->getTerrain()->GetHorizontalSize() * sqrt(2); //size of map
std::deque<Hypothesis*>::iterator it;
float dMin = std::numeric_limits<float>::max();
dtCore::RefPtr<UnitGroup> group;
Hypothesis* hypothesis = NULL;
for (it = mPendingRequests.begin(); it != mPendingRequests.end(); it++) {
dtCore::RefPtr<UnitGroup> g = (*it)->getObservationGroup();
float d = (g->getCentre() - o->getPosition()).length();
//normalise
d /= dMax;
//add in hypothesis priorities
float p = d + (*it)->getPriority();
//
LOG_INFO("Hypothesis group "+boost::lexical_cast<std::string>((*it)->getName())+": Priority "+boost::lexical_cast<std::string>(p)+" ("+boost::lexical_cast<std::string>(d)+" + "+boost::lexical_cast<std::string>((*it)->getPriority()))
//std::cout << (*it)->getName() << ":d" << d <<",t"<<(*it)->getPriority()<<",p"<<p<<std::endl;
if (p < dMin) {
dMin = p;
group = g;
hypothesis = *it;
}
}
//dtCore::RefPtr<UnitGroup> u = nearestNeighbour(o->getPosition(), mPendingRequests);
//o->setTarget(u.get(), 500, 2);
//TODO set duration and altitude properly
o->setTarget(hypothesis, group.get(), 500, theApp->getObservationDuration());
mPendingRequests.erase(std::find(mPendingRequests.begin(), mPendingRequests.end(), hypothesis));
}
break;
case ROUND_ROBIN: {
if (!mPendingRequests.empty()) {
//EventLog::GetInstance().logEvent("OBST "+mPendingRequests.front()->getName());
o->setTarget(mPendingRequests.front(), mPendingRequests.front()->getObservationGroup().get(), 500, theApp->getObservationDuration());
mPendingRequests.pop_front();
}
/*std::queue<Hypothesis*>::iterator it = mPendingRequests.begin();
if (it != mPendingRequests.end()) {
mPendingRequests.erase(it);
}*/
}
break;
case THREAT: {
//calculate resultant threat of observing each target
//decide what target each observee is going to...
//conf > 0.5. conf decays (or variance increases) without observations.
//target certainty... take closest if conf < 0.5 or if variance = global variance
//err, lets just take closest for now...
osg::Vec3 oPos = theApp->mapPointToTerrain(o->getPosition());
std::deque<Hypothesis*>::iterator it;
dtCore::RefPtr<UnitGroup> group;
Hypothesis* hypothesis = NULL;
UnitGroup* predictedGroup = NULL;
UnitGroup* closestGroup = NULL;
float confidence = 0;
float minLeeway = std::numeric_limits<float>::max();
for (it = mPendingRequests.begin(); it != mPendingRequests.end(); it++) {
dtCore::RefPtr<UnitGroup> g = (*it)->getObservationGroup();
osg::Vec3 gPos = g->getCentreCurrent();
//float gRadius = g->getRadius();
std::vector<dtCore::RefPtr<UnitGroup> > targetGroups = (*it)->getTargetGroups();
dtCore::RefPtr<UnitGroup> targetGroup;
float dMin = std::numeric_limits<float>::max();
//find closest target group for this observation group
for (std::vector<dtCore::RefPtr<UnitGroup> >::iterator it2 = targetGroups.begin(); it2 != targetGroups.end(); it2++ ) {
float d = (gPos - (*it2)->getCentreCurrent()).length();
//TODO fix: subtract radii from centre to centre distance
//d = d - gRadius - (*it2)->getRadius();
if (d < 0) d = 0;
if (d < dMin) {
dMin = d;
targetGroup = *it2;
}
}
//calc threat (time = dMin / speed)
//if targetgroup is the same as the highest confidence (>0.5) group then assume running away
UnitGroup* predictedTarget = (*it)->getTargetGroup();
float conf = (*it)->getConfidence();
if (predictedTarget != NULL && predictedTarget == targetGroup.get() && conf > 0.5) {
//dMin = v2*ds/(v2-v1)
dMin = (g->getMaxSpeed() * dMin) / (g->getMaxSpeed() - targetGroup->getMaxSpeed());
}
float attackTime = dMin / g->getMaxSpeed();
//find time for observer to travel to observation group
float travelTime = (gPos - oPos).length() / o->getMaxSpeed();
//calc leeway
float leeway = attackTime - travelTime;
std::ostringstream oss;
oss << "OBSL " << o->getName() << " " << (*it)->getName() << " " << attackTime << " " << travelTime << " " << leeway;
EventLog::GetInstance().logEvent(oss.str());
//take lowest (non-negative) leeway
if (leeway > 0 && leeway < minLeeway) {
minLeeway = leeway;
group = g;
hypothesis = *it;
predictedGroup = predictedTarget;
closestGroup = targetGroup.get();
confidence = conf;
}
}
//if hypothesis has high confidence, but not at min leeway target, then see if more important target to view
//i.e., make better use of our time as min leeway isn't very likely
if (hypothesis && confidence > 0.5 && closestGroup != predictedGroup) {
float minTime = minLeeway;
for (it = mPendingRequests.begin(); it != mPendingRequests.end(); it++) {
float conf = (*it)->getConfidence();
if (conf <= 0) {
dtCore::RefPtr<UnitGroup> g = (*it)->getObservationGroup();
osg::Vec3 gPos = g->getCentreCurrent();
float travelTime = (gPos - oPos).length() / o->getMaxSpeed();
//time to get there, observe, and get back
float totalTime = (travelTime * 2) + (*it)->getSimTime();
if (totalTime < minTime) {
hypothesis = *it;
group = g;
}
}
}
}
//we didn't find any group to look at (probably they were all predicted (inf))
if (hypothesis == NULL) {
//find the hypothesis that has longest time since last obs.
double maxTime = 0;
for (it = mPendingRequests.begin(); it != mPendingRequests.end(); it++) {
double t = (*it)->getTimeDiff();
if (t >= maxTime) {
maxTime = t;
hypothesis = *it;
}
}
group = hypothesis->getObservationGroup();
}
o->setTarget(hypothesis, group.get(), 500, theApp->getObservationDuration());
mPendingRequests.erase(std::find(mPendingRequests.begin(), mPendingRequests.end(), hypothesis));
LOG_DEBUG("Setting target to "+hypothesis->getName())
//not sure if one target will hog the observer...
}
break;
/*case SCANNING: {
//find nearest neighbour
dtCore::RefPtr<UnitGroup> u = nearestNeighbour(o->getPosition(), mPendingRequests);
//find point where it comes into view
osg::Vec3 obsPos = getObservationPoint(o->getGroundPosition(), o->getViewRadius(), u->getCentre(), u->getRadius());
float totalTime = 2;
float timeLeft = (totalTime / 2) - 0.5; //allowing time to turn around and return
while (timeLeft > 0) {
//find next neighbour
dtCore::RefPtr<UnitGroup> u2 = nearestNeighbour(obsPos, mPendingRequests);
osg::Vec3 obsPos2 = getObservationPoint(obsPos, o->getViewRadius(), u2->getCentre(), u2->getRadius());
float nextDist = (obsPos - obsPos2).length();
//see if we can get to next nearest neighbour and back within 2 seconds
float travelTime = getTimeToTravel(nextDist);
if (travelTime < timeLeft) {
//add
o->addTarget(u.get(), 500, 2);
mPendingRequests.erase(std::find(mPendingRequests.begin(), mPendingRequests.end(), u));
}
timeLeft -= travelTime;
}
//add the targets in reverse order to get back...
o->setTarget(u.get(), 500, 2);
mPendingRequests.erase(std::find(mPendingRequests.begin(), mPendingRequests.end(), u));
}
break;*/
/*case GROUP: {
//get group
dtCore::RefPtr<UnitGroup> u = nearestNeighbour(o->getPosition(), mPendingRequests);
UnitGroup* group = theApp->getTargets()->groupObservedUnits(RED,u.get(),250);
//set target position
o->setTarget(group, 300, 2);
//remove observed units
std::vector<Unit*> units = group->getUnits();
for (unsigned int i=0; i<units.size(); i++) {
for (std::vector<dtCore::RefPtr<UnitGroup> >::iterator it = mPendingRequests.begin(); it != mPendingRequests.end(); it++) {
if ((*it)->isInGroup(units[i])) {
mPendingRequests.erase(it);
break;
}
}
}
}
break;*/
case FOLLOWING: {
//if leader, find nearest neighbour
/*dtCore::RefPtr<UnitGroup> u = nearestNeighbour(o->getPosition(), mPendingRequests);
o->setTarget(itMax->get(), 500, 0);
mPendingRequests.erase(std::find(mPendingRequests.begin(), mPendingRequests.end(), u));
*///if follower get to leaders position 2 seconds later
}
break;
}
}
/***
* print surface factor only for the given phrase
*/
void OutputSurface(std::ostream &out, const Hypothesis &edge, const std::vector<FactorType> &outputFactorOrder,
char reportSegmentation, bool reportAllFactors)
{
CHECK(outputFactorOrder.size() > 0);
const TargetPhrase& phrase = edge.GetCurrTargetPhrase();
bool markUnknown = StaticData::Instance().GetMarkUnknown();
if (reportAllFactors == true) {
out << phrase;
} else {
FactorType placeholderFactor = StaticData::Instance().GetPlaceholderFactor();
std::map<size_t, const Factor*> placeholders;
if (placeholderFactor != NOT_FOUND) {
// creates map of target position -> factor for placeholders
placeholders = GetPlaceholders(edge, placeholderFactor);
}
size_t size = phrase.GetSize();
for (size_t pos = 0 ; pos < size ; pos++) {
const Factor *factor = phrase.GetFactor(pos, outputFactorOrder[0]);
if (placeholders.size()) {
// do placeholders
std::map<size_t, const Factor*>::const_iterator iter = placeholders.find(pos);
if (iter != placeholders.end()) {
factor = iter->second;
}
}
CHECK(factor);
//preface surface form with UNK if marking unknowns
const Word &word = phrase.GetWord(pos);
if(markUnknown && word.IsOOV()) {
out << "UNK" << *factor;
} else {
out << *factor;
}
for (size_t i = 1 ; i < outputFactorOrder.size() ; i++) {
const Factor *factor = phrase.GetFactor(pos, outputFactorOrder[i]);
CHECK(factor);
out << "|" << *factor;
}
out << " ";
}
}
// trace ("report segmentation") option "-t" / "-tt"
if (reportSegmentation > 0 && phrase.GetSize() > 0) {
const WordsRange &sourceRange = edge.GetCurrSourceWordsRange();
const int sourceStart = sourceRange.GetStartPos();
const int sourceEnd = sourceRange.GetEndPos();
out << "|" << sourceStart << "-" << sourceEnd; // enriched "-tt"
if (reportSegmentation == 2) {
out << ",wa=";
const AlignmentInfo &ai = edge.GetCurrTargetPhrase().GetAlignTerm();
OutputAlignment(out, ai, 0, 0);
out << ",total=";
out << edge.GetScore() - edge.GetPrevHypo()->GetScore();
out << ",";
ScoreComponentCollection scoreBreakdown(edge.GetScoreBreakdown());
scoreBreakdown.MinusEquals(edge.GetPrevHypo()->GetScoreBreakdown());
OutputAllFeatureScores(scoreBreakdown, out);
}
out << "| ";
}
}
void
SearchSpace::
beamSearch(PhraseTable& pt, Features& weight, int beamsize, int distLimit, int tlimit, bool debug)
{
Hypothesis tmpHypo;
_hypoHeaps[0].push_back(tmpHypo);
Hypothesis& initHypo=_hypoHeaps[0][0];
initHypo.coveredWords.init(_sentence.size(),false);
initHypo.currentScore=0;
initHypo.estimatedScore=0;
initHypo.features.init();
initHypo.lastCoveredWord=-1;
initHypo.represent="";
initHypo.translation.clear();
initHypo.trace.p_prev=NULL;
initHypo.trace.p_rule=NULL;
for(size_t heapIter=0;heapIter<_sentence.size();heapIter++)
{
HypothesisHeap& hHeap=_hypoHeaps[heapIter];
hHeap.sortAndPrune(beamsize);
for(int hypoIter=0;hypoIter<(int)hHeap.size();hypoIter++)
{
Hypothesis& curHypo=hHeap[hypoIter];
if(debug)
{
cout<<"CurHypo ::"<<endl;
curHypo.print(cout);
}
BITVECTOR bvec=curHypo.coveredWords;
int firstUnCovered=bvec.firstFalse();
for(int start=0;start<(int)bvec.size();start++)
{
if(bvec[start]==true)
continue;
for(int stop=start;
stop<(int)bvec.size()&&
bvec[stop]==false&&
(stop<distLimit+firstUnCovered||start==firstUnCovered);
stop++)
{
pair<int,int> phraseSpan=make_pair(start,stop);
string candiPhrase=_auxSpace.queryPhrase(phraseSpan);
vector<PhraseRuleEntry*>* p_rules=pt.queryRulesVec(candiPhrase);
if(p_rules==NULL)continue;
int newLength=heapIter+stop-start+1;
BITVECTOR newBVec;
combine(bvec,phraseSpan,newBVec);
//cout<<"newBVec: "<<newBVec.represent()<<endl;
double futureScore=_auxSpace.queryFutureCost(newBVec);
vector<PhraseRuleEntry*>& rules=*p_rules;
for(size_t rIter=0;rIter<rules.size()&&(int)rIter<tlimit;rIter++)
{
PhraseRuleEntry& rule=*rules[rIter];
Hypothesis newHypo;
newHypo.genFromChild(curHypo,make_pair(start,stop),rule,weight,futureScore);
if(debug)
{
cout<<"newHypo :: "<<endl;
newHypo.print(cout);
}
_hypoHeaps[newLength].addHypothesis(newHypo);
}
}
}
}
}
_hypoHeaps.back().sortAndPrune(beamsize);
if(debug)
{
cout<<"final hHeap"<<endl;
for(size_t i=0;i<_hypoHeaps.back().size();i++){
Hypothesis& hypo=_hypoHeaps.back()[i];
hypo.print(cout);
}
}
}
bool TranslationOption::Overlap(const Hypothesis &hypothesis) const
{
const WordsBitmap &bitmap = hypothesis.GetWordsBitmap();
return bitmap.Overlap(GetSourceWordsRange());
}
void KENLM<Model>::EvaluateWhenApplied(const ManagerBase &mgr,
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
FFState &state) const
{
KenLMState &stateCast = static_cast<KenLMState&>(state);
const System &system = mgr.system;
const lm::ngram::State &in_state =
static_cast<const KenLMState&>(prevState).state;
if (!hypo.GetTargetPhrase().GetSize()) {
stateCast.state = in_state;
return;
}
const std::size_t begin = hypo.GetCurrTargetWordsRange().GetStartPos();
//[begin, end) in STL-like fashion.
const std::size_t end = hypo.GetCurrTargetWordsRange().GetEndPos() + 1;
const std::size_t adjust_end = std::min(end, begin + m_ngram->Order() - 1);
std::size_t position = begin;
typename Model::State aux_state;
typename Model::State *state0 = &stateCast.state, *state1 = &aux_state;
float score = m_ngram->Score(in_state, TranslateID(hypo.GetWord(position)),
*state0);
++position;
for (; position < adjust_end; ++position) {
score += m_ngram->Score(*state0, TranslateID(hypo.GetWord(position)),
*state1);
std::swap(state0, state1);
}
if (hypo.GetBitmap().IsComplete()) {
// Score end of sentence.
std::vector<lm::WordIndex> indices(m_ngram->Order() - 1);
const lm::WordIndex *last = LastIDs(hypo, &indices.front());
score += m_ngram->FullScoreForgotState(&indices.front(), last,
m_ngram->GetVocabulary().EndSentence(), stateCast.state).prob;
} else if (adjust_end < end) {
// Get state after adding a long phrase.
std::vector<lm::WordIndex> indices(m_ngram->Order() - 1);
const lm::WordIndex *last = LastIDs(hypo, &indices.front());
m_ngram->GetState(&indices.front(), last, stateCast.state);
} else if (state0 != &stateCast.state) {
// Short enough phrase that we can just reuse the state.
stateCast.state = *state0;
}
score = TransformLMScore(score);
bool OOVFeatureEnabled = false;
if (OOVFeatureEnabled) {
std::vector<float> scoresVec(2);
scoresVec[0] = score;
scoresVec[1] = 0.0;
scores.PlusEquals(system, *this, scoresVec);
} else {
scores.PlusEquals(system, *this, score);
}
}
