//increasing efficiency: no copy of alignment (calc. everything incrementally) LogProb transpair_model4::scoreOfSwap(const alignment&a, WordIndex j1, WordIndex j2,double thisValue)const { WordIndex aj1=a(j1),aj2=a(j2); if( aj1==aj2 ) return 1.0; LogProb change=transpair_model3::scoreOfSwap(a,j1,j2,-1.0,0); LogProb a_prob=thisValue; if( a_prob<0.0 ) a_prob=prob_of_target_and_alignment_given_source(a,2); massert(a_prob==prob_of_target_and_alignment_given_source(a,2)); //alignment b(a); const_cast<alignment&>(a).set(j1,aj2); const_cast<alignment&>(a).set(j2,aj1); LogProb b_prob=prob_of_target_and_alignment_given_source(a,2); const_cast<alignment&>(a).set(j1,aj1); const_cast<alignment&>(a).set(j2,aj2); if( verboseTP ) cerr << "scoreOfSwap: " << change << ' ' << a_prob << ' ' << b_prob << ' ' << endl; change*=b_prob/a_prob; if( verboseTP ) cerr << "resulting: " << change << " should be " << _scoreOfSwap(a,j1,j2) << endl; return change; }
LogProb transpair_model4::_scoreOfMove(const alignment&a, WordIndex new_i, WordIndex j,double)const { LogProb a_prob=prob_of_target_and_alignment_given_source(a); alignment b(a); b.set(j, new_i); LogProb b_prob=prob_of_target_and_alignment_given_source(b); if( a_prob ) return b_prob/a_prob; else if( b_prob ) return 1e20; else return 1.0; }
LogProb transpair_model4::_scoreOfSwap(const alignment&a, WordIndex j1, WordIndex j2,double)const { LogProb a_prob=prob_of_target_and_alignment_given_source(a); alignment b(a); b.set(j1, a(j2)); b.set(j2, a(j1)); LogProb b_prob=prob_of_target_and_alignment_given_source(b); if( a_prob ) return b_prob/a_prob; else if( b_prob ) return 1e20; else return 1.0; }
LogProb transpair_model5::scoreOfSwap(const alignment&a, WordIndex j1, WordIndex j2,double thisValue)const { if( doModel4Scoring ) return transpair_model4::scoreOfSwap(a,j1,j2,thisValue); alignment b(a); b.set(j1,a(j2)); b.set(j2,a(j1)); LogProb change=transpair_model3::scoreOfSwap(a,j1,j2,-1.0,0); LogProb a_prob=thisValue; if( a_prob<0.0 ) a_prob=prob_of_target_and_alignment_given_source(a,2); massert(a_prob==prob_of_target_and_alignment_given_source(a,2)); LogProb b_prob=prob_of_target_and_alignment_given_source(b,2); change*=b_prob/a_prob; return change; }
LogProb model3::prob_of_target_given_source(tmodel<COUNT, PROB>& tTable, Vector<WordIndex>& fs, Vector<WordIndex>& es) { WordIndex x, y ; LogProb total = 0 ; // WordIndex l = es.size(), m = fs.size(); WordIndex l = es.size()-1, m = fs.size()-1; Vector<WordIndex> A(fs.size(),/*-1*/0); Vector<WordIndex> Fert(es.size(),0); WordIndex i,j ; for ( x = 0 ; x < pow(l+1.0, double(m)) ; x++){ // For all possible alignmets A y = x ; // for (j = 1 ; j < m ; j++){ for (j = 1 ; j <= m ; j++){ A[j] = y % (l+1) ; y /= (l+1) ; } // for(i = 0 ; i < l ; i++) for(i = 0 ; i <= l ; i++) Fert[i] = 0 ; // for (j = 1 ; j < m ; j++) for (j = 1 ; j <= m ; j++) Fert[A[j]]++; // if (2 * Fert[0] < m){ if (2 * Fert[0] <= m){ /* consider alignments that has Fert[0] less than half the length of french sentence */ total += prob_of_target_and_alignment_given_source(A, Fert, tTable, fs, es); } } return(total); }
//increasing efficiency: no copy of alignment (calc. everything incrementally) LogProb transpair_model4::scoreOfMove(const alignment&a, WordIndex new_i, WordIndex j,double thisValue)const { if( a(j)==new_i ) return 1.0; LogProb change=transpair_model3::scoreOfMove(a,new_i,j,-1.0,0); LogProb a_prob=thisValue; if(a_prob<0.0 ) a_prob=prob_of_target_and_alignment_given_source(a,2); massert(a_prob==prob_of_target_and_alignment_given_source(a,2)); WordIndex old_i=a(j); //alignment b(a); const_cast<alignment&>(a).set(j,new_i); LogProb b_prob=prob_of_target_and_alignment_given_source(a,2); const_cast<alignment&>(a).set(j,old_i); change*=b_prob/a_prob; return change; }
LogProb transpair_model5::_scoreOfSwap(const alignment&a, WordIndex j1, WordIndex j2,double thisValue)const { if( doModel4Scoring ) return transpair_model4::_scoreOfSwap(a,j1,j2,thisValue); alignment b(a); b.set(j1, a(j2)); b.set(j2, a(j1)); LogProb a_prob=prob_of_target_and_alignment_given_source(a); LogProb b_prob=prob_of_target_and_alignment_given_source(b); assert(a_prob); assert(b_prob); if( a_prob ) return b_prob/a_prob; else if( b_prob ) return 1e20; else return 1.0; }
//increasing efficiency: no copy of alignment (calc. everything incrementally) LogProb transpair_model5::scoreOfMove(const alignment&a, WordIndex new_i, WordIndex j,double thisValue)const { if( doModel4Scoring ) return transpair_model4::scoreOfMove(a,new_i,j,thisValue); alignment b(a); b.set(j,new_i); LogProb change; const WordIndex old_i=a(j); WordIndex f0=a.fert(0); if (old_i == new_i) change=1.0; else if (old_i == 0) change=((double)p0*p0/p1) * ((f0*(m-f0+1.0)) / ((m-2*f0+1)*(m-2*f0+2.0))) * ((PROB)(1.0)) * (get_fertility(new_i, a.fert(new_i)+1) / get_fertility(new_i, a.fert(new_i)))* (t(new_i, j)/t(old_i, j))* 1.0; else if (new_i == 0) change=(double(p1) / (p0*p0)) * (double((m-2*f0)*(m-2*f0-1))/((1+f0)*(m-f0))) * (1.0) * (get_fertility(old_i, a.fert(old_i)-1) /get_fertility(old_i, a.fert(old_i)))* (t(new_i, j) /t(old_i, j)) * (1.0); else change=(1.0) * (get_fertility(old_i,a.fert(old_i)-1) / get_fertility(old_i,a.fert(old_i))) * (get_fertility(new_i,a.fert(new_i)+1) /get_fertility(new_i,a.fert(new_i))) * (t(new_i,j)/t(old_i,j)) * (1.0); LogProb a_prob=thisValue; if( a_prob<0.0 ) a_prob=prob_of_target_and_alignment_given_source(a,2); massert(a_prob==prob_of_target_and_alignment_given_source(a,2)); LogProb b_prob=prob_of_target_and_alignment_given_source(b,2); change*=b_prob/a_prob; return change; }
void model3::findBestAlignment(Vector<WordIndex>& es, Vector<WordIndex>& fs, Vector<WordIndex>& A, Vector<WordIndex>& Fert, LogProb& best_score, /*tmodel<COUNT, PROB>& tTable, amodel<PROB>& aTable, */ int i_peg = -1 , int j_peg = -1 ) // This finds the best Model2 alignment (i.e. no fertilities stuff) in A // for the given sentence pair. Its score is returned in A. Its fertility // info in Fert. // if j_peg == -1 && i_peg == -1 then No pegging is performed. { WordIndex i, j, l, m, best_i=0; LogProb temp, score, ss; l = es.size() - 1; m = fs.size() - 1; for (i=0 ; i <= l ; i++) Fert[i] = 0 ; ss = 1 ; if ((j_peg != -1) && (i_peg != -1)){ // if you're doing pegging A[j_peg] = i_peg ; Fert[i_peg] = 1 ; ss *= double(tTable.getProb(es[i_peg], fs[j_peg])) * double(aTable.getValue(i_peg, j_peg, l, m)); } for (j = 1 ; j <= m ; j++){ if (int(j) != j_peg){ score = 0 ; for (i = 0 ; i <= l ; i++){ // first make sure that connecting target word at pos j to source word // at pos i will not lead to a violation on Fertility restrictions // (e.g. maximum fertility for a word, max fertility for NULL word, etc) if ((Fert[i]+1 < MAX_FERTILITY) && ((i == 0 && (m >= 2*(Fert[0]+1))) || (i != 0))){ temp = double(tTable.getProb(es[i], fs[j])) * double(aTable.getValue(i, j, l, m)); if (temp > score ){ best_i = i ; score = temp ; } // end of if (temp > score) } // end of if (((i == 0 ...) } // end of for (i= 0 ...) if (score == 0){ cerr << "WARNING: In searching for model2 best alignment\n " ; cerr << "Nothing was set for target token " << fs[j] << "at position j: " << j << "\n"; for (i = 0 ; i <= l ; i++){ cerr << "i: " << i << "ttable("<<es[i]<<", "<<fs[j]<<") = " << tTable.getProb(es[i], fs[j]) << " atable(" << i<<", "<<j<<", "<< l<<", "<<m<<") = "<< aTable.getValue(i, j, l, m) << " product " << double(tTable.getProb(es[i], fs[j])) * double(aTable.getValue(i, j, l, m)) << '\n'; if ((Fert[i]+1 < MAX_FERTILITY) && ((i == 0 && (m >= 2*(Fert[0]+1))) || (i != 0))) cerr <<"Passed fertility condition \n"; else cerr <<"Failed fertility condition \n"; } } // end of if (score == 0) else { Fert[best_i]++ ; A[j] = best_i ; } ss *= score ; } // end of if (j != j_peg) } // end of for (j == 1 ; ...) if (ss <= 0){ cerr << "WARNING: Model2 viterbi alignment has zero score for sentence pair:\n" ; printSentencePair(es, fs, cerr); } best_score = prob_of_target_and_alignment_given_source(A, Fert, tTable, fs, es); if (Log) logmsg << "finding best alignment : score : " << ss <<"p(f, a/e) = "<< best_score<<"\n"; }
void model3::hillClimb(Vector<WordIndex>& es, Vector<WordIndex>& fs, Vector<WordIndex>& A, Vector<WordIndex>& Fert, LogProb& best_score, tmodel<COUNT, PROB>& tTable, int = -1, int j_peg = -1) // Hill climbing given alignment A . // Alignment A will be updated and also best_score // if no pegging is needed i_peg == -1, and j_peg == -1 { WordIndex i, j, l, m, j1, old_i; LogProb change ; bool local_minima; int level = 0 ; LogProb best_change_so_far, best_change ; Vector<WordIndex> A_so_far; Vector<WordIndex> Fert_so_far; l = es.size() - 1; m = fs.size() - 1; if (Log) logmsg << "\nStarting hill climbing with original score: " << best_score <<"\n"; best_change = 1 ; // overall scaling factor (i.e. from the begining of climb do { best_change_so_far = 1 ; // best scaling factor of this level of hill climb local_minima = true ; for (j = 1 ; j <= m ; j++){ if (int(j) != j_peg){ // make sure not to change the pegged link for (j1 = j + 1 ; j1 <= m; j1++){ // for all possible swaps // make sure you are not swapping at same position if ((A[j] != A[j1]) && (int(j1) != j_peg)){ // change = scoreOfSwap(es, fs, A, best_score, tTable, j, j1); change = scoreOfSwap(es, fs, A, tTable, j, j1); if (change > best_change_so_far){ // if better alignment found, keep it local_minima = false ; best_change_so_far = change ; A_so_far = A ; Fert_so_far = Fert ; old_i = A_so_far[j] ; A_so_far[j] = A_so_far[j1] ; A_so_far[j1] = old_i ; } // end of if (change > best_change_so_far) } // end of if (A[j] != A[j1] ..) } // of for (j1 = j+1 ....) // for (i = 0 ; i < l ; i++){ // all possible moves for (i = 0 ; i <= l ; i++){ // all possible moves if (i != A[j]){ // make sure not to move to same position if (i != 0 || (m >= 2 * (Fert[0]+1))){ // if moving to NULL word // (pos 0), make sure not to violate the fertility restriction // i.e. NULL can not take more than half the target words // change = scoreOfMove(es, fs, A, Fert, best_score, tTable, j, i); change = scoreOfMove(es, fs, A, Fert, tTable, j, i); if (change > best_change_so_far){ // if better alignment found, keep it best_change_so_far = change ; local_minima = false ; A_so_far = A ; Fert_so_far = Fert ; old_i = A_so_far[j] ; A_so_far[j] = i ; Fert_so_far[old_i]-- ; Fert_so_far[i]++ ; } // end of if (change > best_change_so_far) } // end of if ((i!=0) ... } // end of if (i != A[j] ) } // end of for (i = 0 ; ....) } // end of if(j != j_peg) } // end of for (j = 1 ; ...) level++; if (!local_minima){ if (best_change_so_far > 1){ // if current chage is improving A = A_so_far ; Fert = Fert_so_far ; best_change *= best_change_so_far ; } else{ local_minima = true ; } } // end of if(!local_minima) if (Log) logmsg << "." ; if (level> 15) cerr << "." ; } while (local_minima == false); if (Log) logmsg << "\n" << "Hill Climb Level: " << level << " score: scaling old: " <<(best_score*best_change) ; if (level > 15) cerr << "\nHill Climb Level: " << level << " score: scaling old: " <<(best_score*best_change) ; best_score = prob_of_target_and_alignment_given_source(A, Fert, tTable, fs, es); if (Log) logmsg << " using new calc: " << best_score << '\n'; if (level>15) cerr << " using new calc: " << best_score << '\n'; }
void model3::em(int noIterations, sentenceHandler& sHandler1) { LogProb all_prob, aprob, temp; WordIndex i, j, l, m; time_t it_st, st, it_fn, fn; string tfile, dfile, nfile, p0file, afile, number; st = time(NULL) ; cout << "\n" << "Starting Model3: Training"; // sentenceHandler sHandler1(efFilename.c_str()); sHandler1.rewind(); for (int it=1; it <= noIterations; it++) { it_st = time(NULL) ; cout << "\n" << "Model3: Iteration " << it; // set up the names of the files where the tables will be printed int n = it; number = ""; do { //mj changed next line number.insert((size_t) 0, 1, (char)(n % 10 + '0')); } while ((n /= 10) > 0); tfile = Prefix + ".t3." + number; afile = Prefix + ".a3." + number; nfile = Prefix + ".n3." + number; dfile = Prefix + ".d3." + number; p0file = Prefix + ".p0_3." + number; // tCountTable.clear(); dCountTable.clear(); nCountTable.clear(); p0_count = 0.0; p1_count = 0.0; all_prob = 0; sentPair sent; while (sHandler1.getNextSentence(sent)) { Vector<WordIndex>& es = sent.eSent; Vector<WordIndex>& fs = sent.fSent; const float count = sent.getCount(); if ((sent.sentenceNo % 1000) == 0) cout <<sent.sentenceNo << '\n'; Vector<WordIndex> A(fs.size(),/*-1*/0); Vector<WordIndex> Fert(es.size(),0); LogProb lcount=(LogProb)count; l = es.size()-1; m = fs.size()-1; WordIndex x, y; all_prob = prob_of_target_given_source(tTable, fs, es); if (all_prob == 0) cout << "\n" <<"all_prob = 0"; for (x = 0; x < pow(l+1.0, double(m)) ; x++) { // For all possible alignmets A y = x; for (j = 1; j <= m; j++) { A[j] = y % (l+1); y /= (l+1); } for (i = 0; i <= l; i++) Fert[i] = 0; for (j = 1; j <= m; j++) Fert[A[j]]++; if (2 * Fert[0] <= m) { /* consider alignments that has Fert[0] less than half the number of words in French sentence */ aprob = prob_of_target_and_alignment_given_source(A, Fert, tTable, fs, es); temp = aprob/all_prob; LogProb templcount = temp*lcount; for (j = 1; j <= m; j++) { tTable.incCount(es[A[j]], fs[j], templcount); if (0 != A[j]) dCountTable.addValue(j, A[j], l, m, templcount); } for (i = 0; i <= l; i++) { nCountTable.addValue(es[i], Fert[i], templcount); //cout << "AFTER INC2: " << templcount << " " << nCountTable.getRef(es[i], Fert[i]) << '\n'; } p1_count += double(temp) * (Fert[0] * count); p0_count += double(temp) * ((m - 2 * Fert[0]) * count); } } /* of looping over all alignments */ } /* of sentence pair E, F */ sHandler1.rewind(); // normalize tables if (OutputInAachenFormat==1) tTable.printCountTable(tfile.c_str(), Elist.getVocabList(), Flist.getVocabList(), 1); tTable.normalizeTable(Elist, Flist); aCountTable.normalize(aTable); dCountTable.normalize(dTable); nCountTable.normalize(nTable, &Elist.getVocabList()); // normalize p1 & p0 if (p1_count + p0_count != 0) { p1 = p1_count / (p1_count + p0_count ); p0 = 1 - p1; } else { p1 = p0 = 0; } // print tables if (OutputInAachenFormat==0) tTable.printProbTable(tfile.c_str(), Elist.getVocabList(), Flist.getVocabList(), OutputInAachenFormat); dTable.printTable(dfile.c_str()); nTable.printNTable(Elist.uniqTokens(), nfile.c_str(), Elist.getVocabList(), OutputInAachenFormat); ofstream of(p0file.c_str()); of << p0; of.close(); it_fn = time(NULL) ; cout << "\n" << "Model3 Iteration "<<it<<" took: " << difftime(it_fn, it_st) << " seconds\n"; } /* of iterations */ fn = time(NULL) ; cout << "\n" << "Entire Model3 Training took: " << difftime(fn, st) << " seconds\n"; }