int mixture::train()
{
double zf;
srand(1333);
genpmap();
if (dub()<dict->size()) {
std::stringstream ss_msg;
ss_msg << "\nERROR: DUB value is too small: the LM will possibly compute wrong probabilities if sub-LMs have different vocabularies!\n";
ss_msg << "This exception should already have been handled before!!!\n";
exit_error(IRSTLM_ERROR_MODEL, ss_msg.str());
}
cerr << "mixlm --> DUB: " << dub() << endl;
for (int i=0; i<numslm; i++) {
cerr << i << " sublm --> DUB: " << sublm[i]->dub() << endl;
cerr << "eventually generate OOV code ";
cerr << sublm[i]->dict->encode(sublm[i]->dict->OOV()) << "\n";
sublm[i]->train();
}
//initialize parameters
for (int i=0; i<=lmsize(); i++) {
l[i]=new double*[pmax];
for (int j=0; j<pmax; j++) {
l[i][j]=new double[numslm];
for (int k=0; k<numslm; k++)
l[i][j][k]=1.0/(double)numslm;
}
}
if (ipfname) {
//load parameters from file
loadpar(ipfname);
} else {
//start training of mixture model
double oldl[pmax][numslm];
char alive[pmax],used[pmax];
int totalive;
ngram ng(sublm[0]->dict);
for (int lev=1; lev<=lmsize(); lev++) {
zf=sublm[0]->zerofreq(lev);
cerr << "Starting training at lev:" << lev << "\n";
for (int i=0; i<pmax; i++) {
alive[i]=1;
used[i]=0;
}
totalive=1;
int iter=0;
while (totalive && (iter < 20) ) {
iter++;
for (int i=0; i<pmax; i++)
if (alive[i])
for (int j=0; j<numslm; j++) {
oldl[i][j]=l[lev][i][j];
l[lev][i][j]=1.0/(double)numslm;
}
sublm[0]->scan(ng,INIT,lev);
while(sublm[0]->scan(ng,CONT,lev)) {
//do not include oov for unigrams
if ((lev==1) && (*ng.wordp(1)==sublm[0]->dict->oovcode()))
continue;
int par=pmap(ng,lev);
used[par]=1;
//controllo se aggiornare il parametro
if (alive[par]) {
double backoff=(lev>1?prob(ng,lev-1):1); //backoff
double denom=0.0;
double* numer = new double[numslm];
double fstar,lambda;
//int cv=(int)floor(zf * (double)ng.freq + rand01());
//int cv=1; //old version of leaving-one-out
int cv=(int)floor(zf * (double)ng.freq)+1;
//int cv=1; //old version of leaving-one-out
//if (lev==3)q
//if (iter>10)
// cout << ng
// << " backoff " << backoff
// << " level " << lev
// << "\n";
for (int i=0; i<numslm; i++) {
//use cv if i=0
sublm[i]->discount(ng,lev,fstar,lambda,(i==0)*(cv));
numer[i]=oldl[par][i]*(fstar + lambda * backoff);
ngram ngslm(sublm[i]->dict);
ngslm.trans(ng);
if ((*ngslm.wordp(1)==sublm[i]->dict->oovcode()) &&
(dict->dub() > sublm[i]->dict->size()))
numer[i]/=(double)(dict->dub() - sublm[i]->dict->size());
denom+=numer[i];
}
for (int i=0; i<numslm; i++) {
l[lev][par][i]+=(ng.freq * (numer[i]/denom));
//if (iter>10)
//cout << ng << " l: " << l[lev][par][i] << "\n";
}
delete []numer;
}
}
//normalize all parameters
totalive=0;
for (int i=0; i<pmax; i++) {
double tot=0;
if (alive[i]) {
for (int j=0; j<numslm; j++) tot+=(l[lev][i][j]);
for (int j=0; j<numslm; j++) l[lev][i][j]/=tot;
//decide if to continue to update
if (!used[i] || (reldist(l[lev][i],oldl[i],numslm)<=0.05))
alive[i]=0;
}
totalive+=alive[i];
}
cerr << "Lev " << lev << " iter " << iter << " tot alive " << totalive << "\n";
}
}
}
if (opfname) savepar(opfname);
return 1;
}
void qMirrorModel::on_dataChanged(const QModelIndex &topLeft, const QModelIndex &bottomRight, const QVector<int> &roles)
{
qDebug() << "QMirrorModel::on_dataChanged";
dub(topLeft, bottomRight, roles);
}
SV * call_dub(SV * in) {
return newSVuv(dub(in));
}
void qMirrorModel::dub(const QModelIndex &topLeft, const QModelIndex &bottomRight, const QVector<int> &roles)
{
if(roles.contains(Qt::DisplayRole))
dub(topLeft, bottomRight);
}
void main()
{
//This file tests *all* MString functions (wip!)
char myChar[] = "Hello, this is 1234, a string...?";
char myLet = 'j';
MString fromChar(myChar);
MString fromLet(myLet);
MString fromLetRep(myLet,20);
MString fromM(fromChar);
cout << fromChar << endl
<< fromLet << endl
<< fromLetRep << endl
<< fromM << endl;
//Large file read and memory test.
/*
MString readLarge;
for (int f1 = 0; f1 < 1000; f1++) {
ifstream in;
in.open("h:\\server\\MString\\largefile.txt");
while (in) {
in >> readLarge;
}
cout << readLarge;
readLarge.Empty();
cout << "!!!! " << f1 << endl;
}
*/
//Loop test for deallocation
//also Empty()
MString test;
for (int i = 0; i < 3000; i++) {
test = "this is a test";
test.Empty();
}
//GetLength
cout << "Length of fromChar = " << fromChar.GetLength()
<< endl;
//IsEmpty
cout << "IsEmpty()\n";
if (fromChar.IsEmpty())
cout << "!!!! fromChar is EMPTY\n";
else cout << "fromChar is not EMPTY\n";
//GetAt
cout << "GetAt\n";
for (int k = 0; k < fromChar.GetLength(); k++) {
cout << fromChar.GetAt(k);
}
cout << endl;
//Rvalue operator []
cout << "Rvalue operator []\n";
for (int f = 0; f < fromLetRep.GetLength(); f++) {
fromLetRep[f] = 'w';
}
cout << fromLetRep << endl;
//SetAt
cout << "SetAt\n";
for (int g = 0; g < fromLetRep.GetLength(); g++) {
fromLetRep.SetAt(g, 'j');
}
cout << fromLetRep << endl;
// =
cout << " = \n";
test = fromLetRep;
cout << test << endl;
test = 'y';
cout << test << endl;
test = "hello and how are you?";
cout << test << endl;
test = 675444;
cout << test << endl;
test = 323.2341;
cout << test << endl;
// +
cout << " + \n";
test = fromLetRep + fromChar;
cout << test << endl;
test = fromChar + '>';
cout << test << endl;
test = '<' + fromChar;
cout << test << endl;
test = fromChar + "CHAR STRING";
cout << test << endl;
test = "CHAR STRING" + fromChar;
cout << test << endl;
// +=
cout << " += \n";
test += fromLetRep;
cout << test << endl;
test.Empty();
test += 'j';
cout << test << endl;
test.Empty();
test += "CHAR STRING????";
cout << test << endl;
test += 999999;
cout << test << endl;
test.Empty();
test += 34.2342;
cout << test << endl;
test.Empty();
test = 34.00000234;
cout << test << endl;
test.Empty();
test = 340.012;
cout << test << endl;
MString dub(456.322);
cout << dub << endl;
dub.Replace('2','6');
cout << dub << endl;
}
