//increasing efficiency: no copy of alignment (calc. everything incrementally)
LogProb transpair_model4::scoreOfSwap(const alignment&a, WordIndex j1, WordIndex j2,double thisValue)const
{
WordIndex aj1=a(j1),aj2=a(j2);
if( aj1==aj2 )
return 1.0;
LogProb change=transpair_model3::scoreOfSwap(a,j1,j2,-1.0,0);
LogProb a_prob=thisValue;
if( a_prob<0.0 )
a_prob=prob_of_target_and_alignment_given_source(a,2);
massert(a_prob==prob_of_target_and_alignment_given_source(a,2));
//alignment b(a);
const_cast<alignment&>(a).set(j1,aj2);
const_cast<alignment&>(a).set(j2,aj1);
LogProb b_prob=prob_of_target_and_alignment_given_source(a,2);
const_cast<alignment&>(a).set(j1,aj1);
const_cast<alignment&>(a).set(j2,aj2);
if( verboseTP )
cerr << "scoreOfSwap: " << change << ' ' << a_prob << ' ' << b_prob << ' ' << endl;
change*=b_prob/a_prob;
if( verboseTP )
cerr << "resulting: " << change << " should be " << _scoreOfSwap(a,j1,j2) << endl;
return change;
}
LogProb transpair_model4::_scoreOfMove(const alignment&a, WordIndex new_i, WordIndex j,double)const
{
LogProb a_prob=prob_of_target_and_alignment_given_source(a);
alignment b(a);
b.set(j, new_i);
LogProb b_prob=prob_of_target_and_alignment_given_source(b);
if( a_prob )
return b_prob/a_prob;
else if( b_prob )
return 1e20;
else
return 1.0;
}
LogProb transpair_model4::_scoreOfSwap(const alignment&a, WordIndex j1, WordIndex j2,double)const
{
LogProb a_prob=prob_of_target_and_alignment_given_source(a);
alignment b(a);
b.set(j1, a(j2));
b.set(j2, a(j1));
LogProb b_prob=prob_of_target_and_alignment_given_source(b);
if( a_prob )
return b_prob/a_prob;
else if( b_prob )
return 1e20;
else
return 1.0;
}
LogProb transpair_model5::scoreOfSwap(const alignment&a, WordIndex j1, WordIndex j2,double thisValue)const
{
if( doModel4Scoring )
return transpair_model4::scoreOfSwap(a,j1,j2,thisValue);
alignment b(a);
b.set(j1,a(j2));
b.set(j2,a(j1));
LogProb change=transpair_model3::scoreOfSwap(a,j1,j2,-1.0,0);
LogProb a_prob=thisValue;
if( a_prob<0.0 )
a_prob=prob_of_target_and_alignment_given_source(a,2);
massert(a_prob==prob_of_target_and_alignment_given_source(a,2));
LogProb b_prob=prob_of_target_and_alignment_given_source(b,2);
change*=b_prob/a_prob;
return change;
}
LogProb model3::prob_of_target_given_source(tmodel<COUNT, PROB>& tTable,
Vector<WordIndex>& fs,
Vector<WordIndex>& es)
{
WordIndex x, y ;
LogProb total = 0 ;
// WordIndex l = es.size(), m = fs.size();
WordIndex l = es.size()-1, m = fs.size()-1;
Vector<WordIndex> A(fs.size(),/*-1*/0);
Vector<WordIndex> Fert(es.size(),0);
WordIndex i,j ;
for ( x = 0 ; x < pow(l+1.0, double(m)) ; x++){ // For all possible alignmets A
y = x ;
// for (j = 1 ; j < m ; j++){
for (j = 1 ; j <= m ; j++){
A[j] = y % (l+1) ;
y /= (l+1) ;
}
// for(i = 0 ; i < l ; i++)
for(i = 0 ; i <= l ; i++)
Fert[i] = 0 ;
// for (j = 1 ; j < m ; j++)
for (j = 1 ; j <= m ; j++)
Fert[A[j]]++;
// if (2 * Fert[0] < m){
if (2 * Fert[0] <= m){ /* consider alignments that has Fert[0] less than
half the length of french sentence */
total += prob_of_target_and_alignment_given_source(A, Fert, tTable, fs, es);
}
}
return(total);
}
//increasing efficiency: no copy of alignment (calc. everything incrementally)
LogProb transpair_model4::scoreOfMove(const alignment&a, WordIndex new_i, WordIndex j,double thisValue)const
{
if( a(j)==new_i )
return 1.0;
LogProb change=transpair_model3::scoreOfMove(a,new_i,j,-1.0,0);
LogProb a_prob=thisValue;
if(a_prob<0.0 )
a_prob=prob_of_target_and_alignment_given_source(a,2);
massert(a_prob==prob_of_target_and_alignment_given_source(a,2));
WordIndex old_i=a(j);
//alignment b(a);
const_cast<alignment&>(a).set(j,new_i);
LogProb b_prob=prob_of_target_and_alignment_given_source(a,2);
const_cast<alignment&>(a).set(j,old_i);
change*=b_prob/a_prob;
return change;
}
LogProb transpair_model5::_scoreOfSwap(const alignment&a, WordIndex j1, WordIndex j2,double thisValue)const
{
if( doModel4Scoring )
return transpair_model4::_scoreOfSwap(a,j1,j2,thisValue);
alignment b(a);
b.set(j1, a(j2));
b.set(j2, a(j1));
LogProb a_prob=prob_of_target_and_alignment_given_source(a);
LogProb b_prob=prob_of_target_and_alignment_given_source(b);
assert(a_prob);
assert(b_prob);
if( a_prob )
return b_prob/a_prob;
else if( b_prob )
return 1e20;
else
return 1.0;
}
//increasing efficiency: no copy of alignment (calc. everything incrementally)
LogProb transpair_model5::scoreOfMove(const alignment&a, WordIndex new_i, WordIndex j,double thisValue)const
{
if( doModel4Scoring )
return transpair_model4::scoreOfMove(a,new_i,j,thisValue);
alignment b(a);
b.set(j,new_i);
LogProb change;
const WordIndex old_i=a(j);
WordIndex f0=a.fert(0);
if (old_i == new_i)
change=1.0;
else if (old_i == 0)
change=((double)p0*p0/p1) *
((f0*(m-f0+1.0)) / ((m-2*f0+1)*(m-2*f0+2.0))) *
((PROB)(1.0)) *
(get_fertility(new_i, a.fert(new_i)+1) / get_fertility(new_i, a.fert(new_i)))*
(t(new_i, j)/t(old_i, j))*
1.0;
else if (new_i == 0)
change=(double(p1) / (p0*p0)) *
(double((m-2*f0)*(m-2*f0-1))/((1+f0)*(m-f0))) *
(1.0) *
(get_fertility(old_i, a.fert(old_i)-1) /get_fertility(old_i, a.fert(old_i)))*
(t(new_i, j) /t(old_i, j)) *
(1.0);
else
change=(1.0) *
(get_fertility(old_i,a.fert(old_i)-1) / get_fertility(old_i,a.fert(old_i))) *
(get_fertility(new_i,a.fert(new_i)+1) /get_fertility(new_i,a.fert(new_i))) *
(t(new_i,j)/t(old_i,j)) *
(1.0);
LogProb a_prob=thisValue;
if( a_prob<0.0 )
a_prob=prob_of_target_and_alignment_given_source(a,2);
massert(a_prob==prob_of_target_and_alignment_given_source(a,2));
LogProb b_prob=prob_of_target_and_alignment_given_source(b,2);
change*=b_prob/a_prob;
return change;
}
void model3::findBestAlignment(Vector<WordIndex>& es,
Vector<WordIndex>& fs,
Vector<WordIndex>& A,
Vector<WordIndex>& Fert,
LogProb& best_score,
/*tmodel<COUNT, PROB>& tTable,
amodel<PROB>& aTable, */
int i_peg = -1 ,
int j_peg = -1 )
// This finds the best Model2 alignment (i.e. no fertilities stuff) in A
// for the given sentence pair. Its score is returned in A. Its fertility
// info in Fert.
// if j_peg == -1 && i_peg == -1 then No pegging is performed.
{
WordIndex i, j, l, m, best_i=0;
LogProb temp, score, ss;
l = es.size() - 1;
m = fs.size() - 1;
for (i=0 ; i <= l ; i++)
Fert[i] = 0 ;
ss = 1 ;
if ((j_peg != -1) && (i_peg != -1)){ // if you're doing pegging
A[j_peg] = i_peg ;
Fert[i_peg] = 1 ;
ss *= double(tTable.getProb(es[i_peg], fs[j_peg])) *
double(aTable.getValue(i_peg, j_peg, l, m));
}
for (j = 1 ; j <= m ; j++){
if (int(j) != j_peg){
score = 0 ;
for (i = 0 ; i <= l ; i++){
// first make sure that connecting target word at pos j to source word
// at pos i will not lead to a violation on Fertility restrictions
// (e.g. maximum fertility for a word, max fertility for NULL word, etc)
if ((Fert[i]+1 < MAX_FERTILITY) && ((i == 0 && (m >= 2*(Fert[0]+1)))
|| (i != 0))){
temp = double(tTable.getProb(es[i], fs[j])) *
double(aTable.getValue(i, j, l, m));
if (temp > score ){
best_i = i ;
score = temp ;
} // end of if (temp > score)
} // end of if (((i == 0 ...)
} // end of for (i= 0 ...)
if (score == 0){
cerr << "WARNING: In searching for model2 best alignment\n " ;
cerr << "Nothing was set for target token " << fs[j] <<
"at position j: " << j << "\n";
for (i = 0 ; i <= l ; i++){
cerr << "i: " << i << "ttable("<<es[i]<<", "<<fs[j]<<") = " <<
tTable.getProb(es[i], fs[j]) << " atable(" << i<<", "<<j<<", "<<
l<<", "<<m<<") = "<< aTable.getValue(i, j, l, m) << " product " <<
double(tTable.getProb(es[i], fs[j])) *
double(aTable.getValue(i, j, l, m)) << '\n';
if ((Fert[i]+1 < MAX_FERTILITY) && ((i == 0 && (m >= 2*(Fert[0]+1)))
|| (i != 0)))
cerr <<"Passed fertility condition \n";
else
cerr <<"Failed fertility condition \n";
}
} // end of if (score == 0)
else {
Fert[best_i]++ ;
A[j] = best_i ;
}
ss *= score ;
} // end of if (j != j_peg)
} // end of for (j == 1 ; ...)
if (ss <= 0){
cerr << "WARNING: Model2 viterbi alignment has zero score for sentence pair:\n" ;
printSentencePair(es, fs, cerr);
}
best_score = prob_of_target_and_alignment_given_source(A, Fert, tTable, fs, es);
if (Log)
logmsg << "finding best alignment : score : " << ss <<"p(f, a/e) = "<< best_score<<"\n";
}
void model3::hillClimb(Vector<WordIndex>& es,
Vector<WordIndex>& fs,
Vector<WordIndex>& A,
Vector<WordIndex>& Fert,
LogProb& best_score,
tmodel<COUNT, PROB>& tTable,
int = -1,
int j_peg = -1)
// Hill climbing given alignment A .
// Alignment A will be updated and also best_score
// if no pegging is needed i_peg == -1, and j_peg == -1
{
WordIndex i, j, l, m, j1, old_i;
LogProb change ;
bool local_minima;
int level = 0 ;
LogProb best_change_so_far, best_change ;
Vector<WordIndex> A_so_far;
Vector<WordIndex> Fert_so_far;
l = es.size() - 1;
m = fs.size() - 1;
if (Log)
logmsg << "\nStarting hill climbing with original score: " << best_score <<"\n";
best_change = 1 ; // overall scaling factor (i.e. from the begining of climb
do {
best_change_so_far = 1 ; // best scaling factor of this level of hill climb
local_minima = true ;
for (j = 1 ; j <= m ; j++){
if (int(j) != j_peg){ // make sure not to change the pegged link
for (j1 = j + 1 ; j1 <= m; j1++){
// for all possible swaps
// make sure you are not swapping at same position
if ((A[j] != A[j1]) && (int(j1) != j_peg)){
// change = scoreOfSwap(es, fs, A, best_score, tTable, j, j1);
change = scoreOfSwap(es, fs, A, tTable, j, j1);
if (change > best_change_so_far){ // if better alignment found, keep it
local_minima = false ;
best_change_so_far = change ;
A_so_far = A ;
Fert_so_far = Fert ;
old_i = A_so_far[j] ;
A_so_far[j] = A_so_far[j1] ;
A_so_far[j1] = old_i ;
} // end of if (change > best_change_so_far)
} // end of if (A[j] != A[j1] ..)
} // of for (j1 = j+1 ....)
// for (i = 0 ; i < l ; i++){ // all possible moves
for (i = 0 ; i <= l ; i++){ // all possible moves
if (i != A[j]){ // make sure not to move to same position
if (i != 0 || (m >= 2 * (Fert[0]+1))){ // if moving to NULL word
// (pos 0), make sure not to violate the fertility restriction
// i.e. NULL can not take more than half the target words
// change = scoreOfMove(es, fs, A, Fert, best_score, tTable, j, i);
change = scoreOfMove(es, fs, A, Fert, tTable, j, i);
if (change > best_change_so_far){ // if better alignment found, keep it
best_change_so_far = change ;
local_minima = false ;
A_so_far = A ;
Fert_so_far = Fert ;
old_i = A_so_far[j] ;
A_so_far[j] = i ;
Fert_so_far[old_i]-- ;
Fert_so_far[i]++ ;
} // end of if (change > best_change_so_far)
} // end of if ((i!=0) ...
} // end of if (i != A[j] )
} // end of for (i = 0 ; ....)
} // end of if(j != j_peg)
} // end of for (j = 1 ; ...)
level++;
if (!local_minima){
if (best_change_so_far > 1){ // if current chage is improving
A = A_so_far ;
Fert = Fert_so_far ;
best_change *= best_change_so_far ;
}
else{
local_minima = true ;
}
} // end of if(!local_minima)
if (Log)
logmsg << "." ;
if (level> 15)
cerr << "." ;
} while (local_minima == false);
if (Log)
logmsg << "\n" << "Hill Climb Level: " << level << " score: scaling old: " <<(best_score*best_change) ;
if (level > 15)
cerr << "\nHill Climb Level: " << level << " score: scaling old: " <<(best_score*best_change) ;
best_score = prob_of_target_and_alignment_given_source(A, Fert, tTable, fs, es);
if (Log)
logmsg << " using new calc: " << best_score << '\n';
if (level>15)
cerr << " using new calc: " << best_score << '\n';
}
void model3::em(int noIterations, sentenceHandler& sHandler1) {
LogProb all_prob, aprob, temp;
WordIndex i, j, l, m;
time_t it_st, st, it_fn, fn;
string tfile, dfile, nfile, p0file, afile, number;
st = time(NULL) ;
cout << "\n" << "Starting Model3: Training";
// sentenceHandler sHandler1(efFilename.c_str());
sHandler1.rewind();
for (int it=1; it <= noIterations; it++) {
it_st = time(NULL) ;
cout << "\n" << "Model3: Iteration " << it;
// set up the names of the files where the tables will be printed
int n = it;
number = "";
do {
//mj changed next line
number.insert((size_t) 0, 1, (char)(n % 10 + '0'));
} while ((n /= 10) > 0);
tfile = Prefix + ".t3." + number;
afile = Prefix + ".a3." + number;
nfile = Prefix + ".n3." + number;
dfile = Prefix + ".d3." + number;
p0file = Prefix + ".p0_3." + number;
// tCountTable.clear();
dCountTable.clear();
nCountTable.clear();
p0_count = 0.0;
p1_count = 0.0;
all_prob = 0;
sentPair sent;
while (sHandler1.getNextSentence(sent)) {
Vector<WordIndex>& es = sent.eSent;
Vector<WordIndex>& fs = sent.fSent;
const float count = sent.getCount();
if ((sent.sentenceNo % 1000) == 0)
cout <<sent.sentenceNo << '\n';
Vector<WordIndex> A(fs.size(),/*-1*/0);
Vector<WordIndex> Fert(es.size(),0);
LogProb lcount=(LogProb)count;
l = es.size()-1;
m = fs.size()-1;
WordIndex x, y;
all_prob = prob_of_target_given_source(tTable, fs, es);
if (all_prob == 0)
cout << "\n" <<"all_prob = 0";
for (x = 0; x < pow(l+1.0, double(m)) ; x++) { // For all possible alignmets A
y = x;
for (j = 1; j <= m; j++) {
A[j] = y % (l+1);
y /= (l+1);
}
for (i = 0; i <= l; i++)
Fert[i] = 0;
for (j = 1; j <= m; j++)
Fert[A[j]]++;
if (2 * Fert[0] <= m) { /* consider alignments that has Fert[0] less than
half the number of words in French sentence */
aprob = prob_of_target_and_alignment_given_source(A, Fert,
tTable, fs, es);
temp = aprob/all_prob;
LogProb templcount = temp*lcount;
for (j = 1; j <= m; j++) {
tTable.incCount(es[A[j]], fs[j], templcount);
if (0 != A[j])
dCountTable.addValue(j, A[j], l, m, templcount);
}
for (i = 0; i <= l; i++) {
nCountTable.addValue(es[i], Fert[i], templcount);
//cout << "AFTER INC2: " << templcount << " " << nCountTable.getRef(es[i], Fert[i]) << '\n';
}
p1_count += double(temp) * (Fert[0] * count);
p0_count += double(temp) * ((m - 2 * Fert[0]) * count);
}
} /* of looping over all alignments */
} /* of sentence pair E, F */
sHandler1.rewind();
// normalize tables
if (OutputInAachenFormat==1)
tTable.printCountTable(tfile.c_str(), Elist.getVocabList(),
Flist.getVocabList(), 1);
tTable.normalizeTable(Elist, Flist);
aCountTable.normalize(aTable);
dCountTable.normalize(dTable);
nCountTable.normalize(nTable, &Elist.getVocabList());
// normalize p1 & p0
if (p1_count + p0_count != 0) {
p1 = p1_count / (p1_count + p0_count );
p0 = 1 - p1;
} else {
p1 = p0 = 0;
}
// print tables
if (OutputInAachenFormat==0)
tTable.printProbTable(tfile.c_str(), Elist.getVocabList(),
Flist.getVocabList(), OutputInAachenFormat);
dTable.printTable(dfile.c_str());
nTable.printNTable(Elist.uniqTokens(), nfile.c_str(),
Elist.getVocabList(), OutputInAachenFormat);
ofstream of(p0file.c_str());
of << p0;
of.close();
it_fn = time(NULL) ;
cout << "\n" << "Model3 Iteration "<<it<<" took: " << difftime(it_fn,
it_st) << " seconds\n";
} /* of iterations */
fn = time(NULL) ;
cout << "\n" << "Entire Model3 Training took: " << difftime(fn, st)
<< " seconds\n";
}
