/*
*
* Calculates the expected counts
* based on the observed counts class
* member. It must be called after
* calculate observed counts for
* meaningful results.
*/
void WFST_Trainer_Local::calculate_expected_counts() {
int arc_id = 0;
for (StateIterator<VectorFst<LogArc> > siter(*fst); !siter.Done(); siter.Next()) {
int state_id = siter.Value();
int tmp_arc_id = arc_id;
vector<double> total_traversals(symbol_bound,-DBL_MAX);
for (ArcIterator<VectorFst<LogArc> > aiter(*fst,state_id); !aiter.Done(); aiter.Next()) {
const LogArc &arc = aiter.Value();
total_traversals[arc.ilabel]
= logadd(total_traversals[arc.ilabel],this->observed_counts[tmp_arc_id]);
tmp_arc_id += 1;
}
for (ArcIterator<VectorFst<LogArc> > aiter(*fst,state_id); !aiter.Done(); aiter.Next()) {
const LogArc &arc = aiter.Value();
double arc_prob = arc.weight.Value();
this->expected_counts[arc_id] = logadd(this->expected_counts[arc_id],
total_traversals[arc.ilabel] - arc_prob);
arc_id += 1;
}
}
}
/// Is node i an ancestor of node j or vice versa? If so, we return 1. Other-
/// wise we return a 0.
int
GATreeBASE::ancestral(unsigned int i, unsigned int j) const
{
GATreeIterBASE aiter(*this);
GATreeIterBASE biter(*this);
GANodeBASE * aparent, *a;
GANodeBASE * bparent, *b;
aparent = a = aiter.warp(i);
bparent = b = biter.warp(j);
while(aparent && aparent != b)
{
aparent = aparent->parent;
}
if(aparent == b)
{
return 1;
}
while(bparent && bparent != a)
{
bparent = bparent->parent;
}
if(bparent == a)
{
return 1;
}
return 0;
}
void M2MFstAligner::expectation( ){
/*
Here we compute the arc posteriors. This routine is almost
fun to implement in the FST paradigm.
*/
for( unsigned int i=0; i<fsas.size(); i++ ){
//Compute Forward and Backward probabilities
ShortestDistance( fsas.at(i), &alpha );
ShortestDistance( fsas.at(i), &beta, true );
//Compute the normalized Gamma probabilities and
// update our running tally
for( StateIterator<VectorFst<LogArc> > siter(fsas.at(i)); !siter.Done(); siter.Next() ){
LogArc::StateId q = siter.Value();
for( ArcIterator<VectorFst<LogArc> > aiter(fsas.at(i),q); !aiter.Done(); aiter.Next() ){
const LogArc& arc = aiter.Value();
const LogWeight& gamma = Divide(Times(Times(alpha[q], arc.weight), beta[arc.nextstate]), beta[0]);
//Check for any BadValue results, otherwise add to the tally.
//We call this 'prev_alignment_model' which may seem misleading, but
// this conventions leads to 'alignment_model' being the final version.
if( gamma.Value()==gamma.Value() ){
prev_alignment_model[arc.ilabel] = Plus(prev_alignment_model[arc.ilabel], gamma);
total = Plus(total, gamma);
}
}
}
alpha.clear();
beta.clear();
}
}
float
M2MFstAligner::maximization(bool lastiter)
{
//Maximization. Simple count normalization. Probably get an improvement
// by using a more sophisticated regularization approach.
map<LogArc::Label,LogWeight>::iterator it;
float change = abs(total.Value() - prevTotal.Value());
//cout << "Total: " << total << " Change: " << abs(total.Value()-prevTotal.Value()) << endl;
prevTotal = total;
//Normalize and iterate to the next model. We apply it dynamically
// during the expectation step.
for (it = prev_alignment_model.begin();
it != prev_alignment_model.end(); it++) {
alignment_model[(*it).first] = Divide((*it).second, total);
(*it).second = LogWeight::Zero();
}
for (int i = 0; i < fsas.size(); i++) {
for (StateIterator<VectorFst<LogArc> > siter(fsas[i]);
!siter.Done(); siter.Next()) {
LogArc::StateId q = siter.Value();
for (MutableArcIterator<VectorFst<LogArc> > aiter(&fsas[i], q); !aiter.Done(); aiter.Next()) {
LogArc arc = aiter.Value();
arc.weight = alignment_model[arc.ilabel];
aiter.SetValue(arc);
}
}
}
total = LogWeight::Zero();
return change;
}
M2MFstAligner::M2MFstAligner(string _model_file)
{
VectorFst<LogArc> *model = VectorFst<LogArc>::Read(_model_file);
for (StateIterator<VectorFst<LogArc> > siter(*model);
!siter.Done(); siter.Next()) {
LogArc::StateId q = siter.Value();
for (ArcIterator<VectorFst<LogArc> > aiter(*model, q);
!aiter.Done(); aiter.Next()) {
const LogArc & arc = aiter.Value();
alignment_model.insert(pair<LogArc::Label,LogWeight> (arc.ilabel, arc.weight));
}
}
isyms = (SymbolTable *) model->InputSymbols();
int i = 0;
eps = isyms->Find(i); //Can't write '0' here for some reason...
skip = isyms->Find(1);
vector<string> seps = split(isyms->Find(2), "_");
seq1_sep = seps[0];
seq2_sep = seps[1];
s1s2_sep = isyms->Find(3);
vector<string> params = split(isyms->Find(4), "_");
seq1_del = params[0].compare("true") ? false : true;
seq2_del = params[1].compare("true") ? false : true;
seq1_max = atoi(params[2].c_str());
seq2_max = atoi(params[3].c_str());
}
void
M2MFstAligner::write_lattice(string lattice)
{
//Write out the entire training set in lattice format
//Perform the union first. This output can then
// be plugged directly in to a counter to obtain expected
// alignment counts for the EM-trained corpus. Yields
// far higher-quality joint n-gram models, which are also
// more robust for smaller training corpora.
//Make sure you call this BEFORE any call to
// write_all_alignments
// as the latter function will override some of the weights
//Chaining the standard Union operation, including using a
// rational FST still performs very poorly in the log semiring.
//Presumably it's running push or something at each step. It
// should be fine to do that just once at the end.
//Rolling our own union turns out to be MUCH faster.
VectorFst<LogArc> ufst;
ufst.AddState();
ufst.SetStart(0);
int total_states = 0;
for (int i = 0; i < fsas.size(); i++) {
TopSort(&fsas[i]);
for (StateIterator<VectorFst<LogArc> > siter(fsas[i]);
!siter.Done(); siter.Next()) {
LogArc::StateId q = siter.Value();
LogArc::StateId r;
if (q == 0)
r = 0;
else
r = ufst.AddState();
for (ArcIterator <VectorFst<LogArc> > aiter(fsas[i], q);
!aiter.Done(); aiter.Next()) {
const LogArc & arc = aiter.Value();
ufst.AddArc(r,
LogArc(arc.ilabel, arc.ilabel, arc.weight,
arc.nextstate + total_states));
}
if (fsas[i].Final(q) != LogWeight::Zero())
ufst.SetFinal(r, LogWeight::One());
}
total_states += fsas[i].NumStates() - 1;
}
//Normalize weights
Push(&ufst, REWEIGHT_TO_INITIAL);
//Write the resulting lattice to disk
ufst.Write(lattice);
//Write the syms table too.
isyms->WriteText("lattice.syms");
return;
}
float M2MFstAligner::maximization( bool lastiter ){
//Maximization. Standard approach is simple count normalization.
//The 'penalize' option penalizes links by total length. Results seem to be inconclusive.
// Probably get an improvement by distinguishing between gaps and insertions, etc.
bool cond = false;
float change = abs(total.Value()-prevTotal.Value());
if( cond==false ){
map<LogArc::Label,LogWeight>::iterator it;
//cout << "Total: " << total << " Change: " << abs(total.Value()-prevTotal.Value()) << endl;
prevTotal = total;
//Normalize and iterate to the next model. We apply it dynamically
// during the expectation step.
for( it=prev_alignment_model.begin(); it != prev_alignment_model.end(); it++ ){
alignment_model[(*it).first] = Divide((*it).second,total);
(*it).second = LogWeight::Zero();
}
}else{
_conditional_max( true );
}
for( unsigned int i=0; i<fsas.size(); i++ ){
for( StateIterator<VectorFst<LogArc> > siter(fsas[i]); !siter.Done(); siter.Next() ){
LogArc::StateId q = siter.Value();
for( MutableArcIterator<VectorFst<LogArc> > aiter(&fsas[i], q); !aiter.Done(); aiter.Next() ){
LogArc arc = aiter.Value();
if( penalize_em==true ){
LabelDatum* ld = &penalties[arc.ilabel];
if( ld->lhs>1 && ld->rhs>1 ){
arc.weight = 99;
}else if( ld->lhsE==false && ld->rhsE==false ){
arc.weight = arc.weight.Value() * ld->tot;
}
/*
else{
arc.weight = arc.weight.Value() * (ld->tot+10);
}
*/
if( arc.weight == LogWeight::Zero() || arc.weight != arc.weight )
arc.weight = 99;
}else{
arc.weight = alignment_model[arc.ilabel];
}
aiter.SetValue(arc);
}
}
}
total = LogWeight::Zero();
return change;
}
StdArc::Weight ARPA2WFST::_get_final_bow( StdArc::StateId st ){
if( arpafst.Final(st) != StdArc::Weight::Zero() )
return arpafst.Final(st);
for( ArcIterator<VectorFst<StdArc> > aiter(arpafst,st);
!aiter.Done();
aiter.Next() ){
StdArc arc = aiter.Value();
//Assume there is never more than 1 <eps> transition
if( arc.ilabel == 0 ){
StdArc::Weight w = Times(arc.weight, _get_final_bow(arc.nextstate));
arpafst.SetFinal(st, w);
return w;
}
}
//Should never reach this place
return StdArc::Weight::Zero();
}
M2MFstAligner::M2MFstAligner( string _model_file, bool _penalize, bool _penalize_em, bool _restrict ){
/*
Initialize the aligner with a previously trained model.
The model requires that the first several symbols in the
symbols table contain the separator and other bookkeeping info.
*/
restrict = _restrict;
penalize = _penalize;
penalize_em = _penalize_em;
penalties.set_empty_key(0);
VectorFst<LogArc>* model = VectorFst<LogArc>::Read( _model_file );
for( StateIterator<VectorFst<LogArc> > siter(*model); !siter.Done(); siter.Next() ){
LogArc::StateId q = siter.Value();
for( ArcIterator<VectorFst<LogArc> > aiter(*model, q); !aiter.Done(); aiter.Next() ){
const LogArc& arc = aiter.Value();
alignment_model.insert( pair<LogArc::Label,LogWeight>(arc.ilabel,arc.weight) );
}
}
isyms = (SymbolTable*)model->InputSymbols();
int i = 0;
eps = isyms->Find(i);//Can't write '0' here for some reason...
skip = isyms->Find(1);
string tie = "_"; //tie to pack parameters
string sseps = isyms->Find(2);
vector<string> seps = tokenize_utf8_string( &sseps, &tie );
seq1_sep = seps[0];
seq2_sep = seps[1];
s1s2_sep = isyms->Find(3);
string sparams = isyms->Find(4);
vector<string> params = tokenize_utf8_string( &sparams, &tie );
seq1_del = params[0].compare("true") ? false : true;
seq2_del = params[1].compare("true") ? false : true;
seq1_max = atoi(params[2].c_str());
seq2_max = atoi(params[3].c_str());
}
void M2MFstAligner::Sequences2FST( VectorFst<LogArc>* fst, vector<string>* seq1, vector<string>* seq2 ){
/*
Build an FST that represents all possible alignments between seq1 and seq2, given the
parameter values input by the user. Here we encode the input and output labels, in fact
creating a WFSA. This simplifies the training process, but means that we can only
easily compute a joint maximization. In practice joint maximization seems to give the
best results anyway, so it probably doesn't matter.
Note: this also performs the initizization routine. It performs a UNIFORM initialization
meaning that every non-null alignment sequence is eventually initialized to 1/Num(unique_alignments).
It might be more appropriate to consider subsequence length here, but for now we stick
to the m2m-aligner approach.
TODO: Add an FST version and support for conditional maximization. May be useful for languages
like Japanese where there is a distinct imbalance in the seq1->seq2 length correspondences.
*/
int istate=0; int ostate=0;
for( unsigned int i=0; i<=seq1->size(); i++ ){
for( unsigned int j=0; j<=seq2->size(); j++ ){
fst->AddState();
istate = i*(seq2->size()+1)+j;
//Epsilon arcs for seq1
if( seq1_del==true )
for( unsigned int l=1; l<=seq2_max; l++ ){
if( j+l<=seq2->size() ){
vector<string> subseq2( seq2->begin()+j, seq2->begin()+j+l );
int is = isyms->AddSymbol(skip+s1s2_sep+vec2str(subseq2, seq2_sep));
ostate = i*(seq2->size()+1) + (j+l);
LogArc arc( is, is, 99, ostate );
fst->AddArc( istate, arc );
/*
if( prev_alignment_model.find(arc.ilabel)==prev_alignment_model.end() ){
prev_alignment_model.insert( pair<LogArc::Label,LogWeight>(arc.ilabel,arc.weight) );
_compute_penalties( arc.ilabel, 1, l, true, false );
}else{
prev_alignment_model[arc.ilabel] = Plus(prev_alignment_model[arc.ilabel],arc.weight);
}
total = Plus( total, arc.weight );
*/
}
}
//Epsilon arcs for seq2
if( seq2_del==true )
for( unsigned int k=1; k<=seq1_max; k++ ){
if( i+k<=seq1->size() ){
vector<string> subseq1( seq1->begin()+i, seq1->begin()+i+k );
int is = isyms->AddSymbol(vec2str(subseq1, seq1_sep)+s1s2_sep+skip);
ostate = (i+k)*(seq2->size()+1) + j;
LogArc arc( is, is, 99, ostate );
fst->AddArc( istate, arc );
/*
if( prev_alignment_model.find(arc.ilabel)==prev_alignment_model.end() ){
prev_alignment_model.insert( pair<LogArc::Label,LogWeight>(arc.ilabel,arc.weight) );
_compute_penalties( arc.ilabel, k, 1, false, true );
}else{
prev_alignment_model[arc.ilabel] = Plus(prev_alignment_model[arc.ilabel],arc.weight);
}
total = Plus(total, arc.weight);
*/
}
}
//All the other arcs
for( unsigned int k=1; k<=seq1_max; k++ ){
for( unsigned int l=1; l<=seq2_max; l++ ){
if( i+k<=seq1->size() && j+l<=seq2->size() ){
vector<string> subseq1( seq1->begin()+i, seq1->begin()+i+k );
string s1 = vec2str(subseq1, seq1_sep);
vector<string> subseq2( seq2->begin()+j, seq2->begin()+j+l );
string s2 = vec2str(subseq2, seq2_sep);
//This says only 1-M and N-1 allowed, no M-N links!
if( restrict==true && l>1 && k>1)
continue;
int is = isyms->AddSymbol(s1+s1s2_sep+s2);
ostate = (i+k)*(seq2->size()+1) + (j+l);
LogArc arc( is, is, LogWeight::One().Value()*(k+l), ostate );
fst->AddArc( istate, arc );
//During the initialization phase, just count non-eps transitions
//We currently initialize to uniform probability so there is also
// no need to tally anything here.
/*
if( prev_alignment_model.find(arc.ilabel)==prev_alignment_model.end() ){
prev_alignment_model.insert( pair<LogArc::Label,LogWeight>(arc.ilabel, arc.weight) );
_compute_penalties( arc.ilabel, k, l, false, false );
}else{
prev_alignment_model[arc.ilabel] = Plus(prev_alignment_model[arc.ilabel],arc.weight);
}
total = Plus( total, arc.weight );
*/
}
}
}
}
}
fst->SetStart(0);
fst->SetFinal( ((seq1->size()+1)*(seq2->size()+1))-1, LogWeight::One() );
//Unless seq1_del==true && seq2_del==true we will have unconnected states
// thus we need to run connect to clean out these states
if( seq1_del==false || seq2_del==false )
Connect(fst);
//Only add arcs that are in the FINAL fst to the model
for( StateIterator<VectorFst<LogArc> > siter(*fst); !siter.Done(); siter.Next() ){
LogArc::StateId q = siter.Value();
for( ArcIterator<VectorFst<LogArc> > aiter(*fst, q); !aiter.Done(); aiter.Next() ){
const LogArc& arc = aiter.Value();
if( prev_alignment_model.find(arc.ilabel)==prev_alignment_model.end() ){
prev_alignment_model.insert( pair<LogArc::Label,LogWeight>(arc.ilabel, arc.weight) );
string sym = isyms->Find(arc.ilabel);
size_t del = sym.find("}");
size_t ski = sym.find("_");
size_t chu = sym.find("|");
int k=1; int l=1;
bool xd = false; bool yd = false;
if( chu!=string::npos ){
if( chu<del )
k += 1;
else
l += 1;
}
if( ski!=string::npos ){
if( ski<del )
xd = true;
else
yd = true;
}
_compute_penalties( arc.ilabel, k, l, false, false );
}else{
prev_alignment_model[arc.ilabel] = Plus(prev_alignment_model[arc.ilabel],arc.weight);
}
total = Plus( total, arc.weight );
}
}
return;
}
void M2MFstAligner::_conditional_max( bool y_given_x ){
/*
Compute the conditional distribution, P(Y|X) using the WFST paradigm.
This is bassed on the approach from Shu and Hetherington 2002.
It is assumed that all WFSTs and operations use the Log semiring.
Given:
FST1 = P(X,Y)
Compute:
FST2 = P(X)
:= Map_inv(Det(RmEps(Proj_i(FST1))))
FST3 = P(Y|X)
:= Compose(FST2,FST1)
Proj_i: project on input labels
RmEps: epsilon removal
Det: determinize
Map_inv: invert weights
Notes: An analogous process may be used to compute P(X|Y). In this
case one would project on OUTPUT labels - Proj_o, and reverse the
composition order to Compose(FST1,FST2).
Future work:
What we are doing here in terms of *generating* the JOINT fst each
time is really just a dumb hack. We *should* encode the model in an
FST and encode the individual lattices, rather than doing the hacky
manual label encoding that we currently rely on.
*/
//Joint distribution that we start with
VectorFst<LogArc>* joint = new VectorFst<LogArc>();
SymbolTable* misyms = new SymbolTable("misyms");
SymbolTable* mosyms = new SymbolTable("mosyms");
joint->AddState();
joint->AddState();
joint->SetStart(0);
joint->SetFinal(1,LogArc::Weight::One());
map<LogArc::Label,LogWeight>::iterator it;
for( it=prev_alignment_model.begin(); it != prev_alignment_model.end(); it++ ){
string isym = isyms->Find((*it).first);
vector<string> io = tokenize_utf8_string( &isym, &s1s2_sep );
LogArc arc( misyms->AddSymbol(io[0]), mosyms->AddSymbol(io[1]), (*it).second, 1 );
joint->AddArc( 0, arc );
}
//VectorFst<LogArc>* joint = new VectorFst<LogArc>();
//Push<LogArc,REWEIGHT_TO_FINAL>(*_joint, joint, kPushWeights);
//joint->SetFinal(1,LogWeight::One());
joint->Write("m2mjoint.fst");
//BEGIN COMPUTE MARGINAL P(X)
VectorFst<LogArc>* dmarg;
if( y_given_x )
dmarg = new VectorFst<LogArc>(ProjectFst<LogArc>(*joint, PROJECT_INPUT));
else
dmarg = new VectorFst<LogArc>(ProjectFst<LogArc>(*joint, PROJECT_OUTPUT));
RmEpsilon(dmarg);
VectorFst<LogArc>* marg = new VectorFst<LogArc>();
Determinize(*dmarg, marg);
ArcMap(marg, InvertWeightMapper<LogArc>());
if( y_given_x )
ArcSort(marg, OLabelCompare<LogArc>());
else
ArcSort(marg, ILabelCompare<LogArc>());
//END COMPUTE MARGINAL P(X)
marg->Write("marg.fst");
//CONDITIONAL P(Y|X)
VectorFst<LogArc>* cond = new VectorFst<LogArc>();
if( y_given_x )
Compose(*marg, *joint, cond);
else
Compose(*joint, *marg, cond);
//cond now contains the conditional distribution P(Y|X)
cond->Write("cond.fst");
//Now update the model with the new values
for( MutableArcIterator<VectorFst<LogArc> > aiter(cond, 0); !aiter.Done(); aiter.Next() ){
LogArc arc = aiter.Value();
string lab = misyms->Find(arc.ilabel)+"}"+mosyms->Find(arc.olabel);
int labi = isyms->Find(lab);
alignment_model[labi] = arc.weight;
prev_alignment_model[labi] = LogWeight::Zero();
}
delete joint, marg, cond, dmarg;
delete misyms, mosyms;
return;
}
vector<PathData> M2MFstAligner::write_alignment(const VectorFst<LogArc> &ifst,
int nbest)
{
//Generic alignment generator
VectorFst<StdArc> fst;
Map(ifst, &fst, LogToStdMapper());
for (StateIterator<VectorFst<StdArc> > siter(fst); !siter.Done();
siter.Next()) {
StdArc::StateId q = siter.Value();
for (MutableArcIterator<VectorFst<StdArc> > aiter(&fst, q);
!aiter.Done(); aiter.Next()) {
//Prior to decoding we make several 'heuristic' modifications to the weights:
// 1. A multiplier is applied to any multi-token substrings
// 2. Any LogWeight::Zero() arc weights are reset to '99'.
// We are basically resetting 'Infinity' values to a 'smallest non-Infinity'
// so that the ShortestPath algorithm actually produces something no matter what.
// 3. Any arcs that consist of subseq1:subseq2 being the same length and subseq1>1
// are set to '99' this forces shortestpath to choose arcs where one of the
// following conditions holds true
// * len(subseq1)>1 && len(subseq2)!=len(subseq1)
// * len(subseq2)>1 && len(subseq1)!=len(subseq2)
// * len(subseq1)==len(subseq2)==1
//I suspect these heuristics can be eliminated with a better choice of the initialization
// function and maximization function, but this is the way that m2m-aligner works, so
// it makes sense for our first cut implementation.
//In any case, this guarantees that M2MFstAligner produces results identical to those
// produced by m2m-aligner - but with a bit more reliability.
//UPDATE: this now produces a better alignment than m2m-aligner.
// The maxl heuristic is still in place. The aligner will produce *better* 1-best alignments
// *without* the maxl heuristic below, BUT this comes at the cost of producing a less
// flexible corpus. That is, for a small training corpus like nettalk, if we use the
// best alignment we wind up with more 'chunks' and thus get a worse coverage for unseen
// data. Using the aignment lattices to train the joint ngram model solves this problem.
// Oh baby. Can't wait to for everyone to see the paper!
//NOTE: this is going to fail if we encounter any alignments in a new test item that never
// occurred in the original model.
StdArc
arc = aiter.Value();
int
maxl = get_max_length(isyms->Find(arc.ilabel));
if (maxl == -1) {
arc.weight = 999;
}
else {
//Optionally penalize m-to-1 / 1-to-m links. This produces
// WORSE 1-best alignments, but results in better joint n-gram
// models for small training corpora when using only the 1-best
// alignment. By further favoring 1-to-1 alignments the 1-best
// alignment corpus results in a more flexible joint n-gram model
// with regard to previously unseen data.
//if( penalize==true ){
arc.weight = alignment_model[arc.ilabel].Value() * maxl;
//}else{
//For larger corpora this is probably unnecessary.
//arc.weight = alignment_model[arc.ilabel].Value();
//}
}
if (arc.weight == LogWeight::Zero())
arc.weight = 999;
if (arc.weight != arc.weight)
arc.weight = 999;
aiter.SetValue(arc);
}
}
VectorFst<StdArc> shortest;
ShortestPath(fst, &shortest, nbest);
RmEpsilon(&shortest);
//Skip empty results. This should only happen
// in the following situations:
// 1. seq1_del=false && len(seq1)<len(seq2)
// 2. seq2_del=false && len(seq1)>len(seq2)
//In both 1.and 2. the issue is that we need to
// insert a 'skip' in order to guarantee at least
// one valid alignment path through seq1*seq2, but
// user params didn't allow us to.
//Probably better to insert these where necessary
// during initialization, regardless of user prefs.
if (shortest.NumStates() == 0) {
vector<PathData> dummy;
return dummy;
}
FstPathFinder
pathfinder(skipSeqs);
pathfinder.isyms = isyms;
pathfinder.findAllStrings(shortest);
return pathfinder.paths;
}
