void htmlFieldInfo(unsigned uTable)
{
MYSQL_RES *res;
MYSQL_ROW field;
sprintf(gcQuery,"SELECT cLabel,cOtherOptions,uField FROM tField WHERE uTable=%u AND cOtherOptions!=''",uTable);
mysql_query(&gMysql,gcQuery);
if(mysql_errno(&gMysql))
tTable(mysql_error(&gMysql));
res=mysql_store_result(&gMysql);
while((field=mysql_fetch_row(res)))
{
printf("<br><a href=?gcFunction=tField&uField=%s>%s</a> %s",field[2],field[0],field[1]);
}
mysql_free_result(res);
}//void htmlFieldInfo(unsigned uTable)
void ExttFieldButtons(void)
{
void htmlTableInfo(unsigned uTable)
{
MYSQL_RES *res;
MYSQL_ROW field;
sprintf(gcQuery,"SELECT cLabel,cSubDir,cDescription,uTable FROM tTable WHERE uTable=%u",uTable);
mysql_query(&gMysql,gcQuery);
if(mysql_errno(&gMysql))
tTable(mysql_error(&gMysql));
res=mysql_store_result(&gMysql);
if((field=mysql_fetch_row(res)))
{
printf("Table: <a href=?gcFunction=tTable&uTable=%s>%s/%s</a> %s",field[3],field[1],field[0],field[2]);
}
mysql_free_result(res);
}//void htmlTableInfo(unsigned uTable)
double StartTraining(int&result)
{
double errors=0.0;
vcbList eTrainVcbList, fTrainVcbList;
globeTrainVcbList=&eTrainVcbList;
globfTrainVcbList=&fTrainVcbList;
string repFilename = Prefix + ".gizacfg" ;
ofstream of2(repFilename.c_str());
writeParameters(of2,getGlobalParSet(),-1) ;
cout << "reading vocabulary files \n";
eTrainVcbList.setName(SourceVocabFilename.c_str());
fTrainVcbList.setName(TargetVocabFilename.c_str());
eTrainVcbList.readVocabList();
fTrainVcbList.readVocabList();
cout << "Source vocabulary list has " << eTrainVcbList.uniqTokens() << " unique tokens \n";
cout << "Target vocabulary list has " << fTrainVcbList.uniqTokens() << " unique tokens \n";
vcbList eTestVcbList(eTrainVcbList) ;
vcbList fTestVcbList(fTrainVcbList) ;
corpus = new sentenceHandler(CorpusFilename.c_str(), &eTrainVcbList, &fTrainVcbList);
if (TestCorpusFilename == "NONE")
TestCorpusFilename = "";
if (TestCorpusFilename != ""){
cout << "Test corpus will be read from: " << TestCorpusFilename << '\n';
testCorpus= new sentenceHandler(TestCorpusFilename.c_str(),
&eTestVcbList, &fTestVcbList);
cout << " Test total # sentence pairs : " <<(*testCorpus).getTotalNoPairs1()<<" weighted:"<<(*testCorpus).getTotalNoPairs2() <<'\n';
cout << "Size of the source portion of test corpus: " << eTestVcbList.totalVocab() << " tokens\n";
cout << "Size of the target portion of test corpus: " << fTestVcbList.totalVocab() << " tokens \n";
cout << "In source portion of the test corpus, only " << eTestVcbList.uniqTokensInCorpus() << " unique tokens appeared\n";
cout << "In target portion of the test corpus, only " << fTestVcbList.uniqTokensInCorpus() << " unique tokens appeared\n";
cout << "ratio (target/source) : " << double(fTestVcbList.totalVocab()) /
eTestVcbList.totalVocab() << '\n';
}
cout << " Train total # sentence pairs (weighted): " << corpus->getTotalNoPairs2() << '\n';
cout << "Size of source portion of the training corpus: " << eTrainVcbList.totalVocab()-corpus->getTotalNoPairs2() << " tokens\n";
cout << "Size of the target portion of the training corpus: " << fTrainVcbList.totalVocab() << " tokens \n";
cout << "In source portion of the training corpus, only " << eTrainVcbList.uniqTokensInCorpus() << " unique tokens appeared\n";
cout << "In target portion of the training corpus, only " << fTrainVcbList.uniqTokensInCorpus() << " unique tokens appeared\n";
cout << "lambda for PP calculation in IBM-1,IBM-2,HMM:= " << double(fTrainVcbList.totalVocab()) << "/(" << eTrainVcbList.totalVocab() << "-" << corpus->getTotalNoPairs2() << ")=";
LAMBDA = double(fTrainVcbList.totalVocab()) / (eTrainVcbList.totalVocab()-corpus->getTotalNoPairs2());
cout << "= " << LAMBDA << '\n';
// load dictionary
Dictionary *dictionary;
useDict = !dictionary_Filename.empty();
if (useDict) dictionary = new Dictionary(dictionary_Filename.c_str());
else dictionary = new Dictionary("");
int minIter=0;
#ifdef BINARY_SEARCH_FOR_TTABLE
if( CoocurrenceFile.length()==0 )
{
cerr << "ERROR: NO COOCURRENCE FILE GIVEN!\n";
abort();
}
//ifstream coocs(CoocurrenceFile.c_str());
tmodel<COUNT, PROB> tTable(CoocurrenceFile);
#else
tmodel<COUNT, PROB> tTable;
#endif
model1 m1(CorpusFilename.c_str(), eTrainVcbList, fTrainVcbList,tTable,trainPerp,
*corpus,&testPerp, testCorpus,
trainViterbiPerp, &testViterbiPerp);
amodel<PROB> aTable(false);
amodel<COUNT> aCountTable(false);
model2 m2(m1,aTable,aCountTable);
hmm h(m2);
model3 m3(m2);
if(ReadTablePrefix.length() )
{
string number = "final";
string tfile,afilennfile,dfile,d4file,p0file,afile,nfile; //d5file
tfile = ReadTablePrefix + ".t3." + number ;
afile = ReadTablePrefix + ".a3." + number ;
nfile = ReadTablePrefix + ".n3." + number ;
dfile = ReadTablePrefix + ".d3." + number ;
d4file = ReadTablePrefix + ".d4." + number ;
//d5file = ReadTablePrefix + ".d5." + number ;
p0file = ReadTablePrefix + ".p0_3." + number ;
tTable.readProbTable(tfile.c_str());
aTable.readTable(afile.c_str());
m3.dTable.readTable(dfile.c_str());
m3.nTable.readNTable(nfile.c_str());
sentPair sent ;
double p0;
ifstream p0f(p0file.c_str());
p0f >> p0;
d4model d4m(MAX_SENTENCE_LENGTH);
d4m.makeWordClasses(m1.Elist,m1.Flist,SourceVocabFilename+".classes",TargetVocabFilename+".classes");
d4m.readProbTable(d4file.c_str());
//d5model d5m(d4m);
//d5m.makeWordClasses(m1.Elist,m1.Flist,SourceVocabFilename+".classes",TargetVocabFilename+".classes");
//d5m.readProbTable(d5file.c_str());
makeSetCommand("model4smoothfactor","0.0",getGlobalParSet(),2);
//makeSetCommand("model5smoothfactor","0.0",getGlobalParSet(),2);
if( corpus||testCorpus )
{
sentenceHandler *x=corpus;
if(x==0)
x=testCorpus;
cout << "Text corpus exists.\n";
x->rewind();
while(x&&x->getNextSentence(sent)){
Vector<WordIndex>& es = sent.eSent;
Vector<WordIndex>& fs = sent.fSent;
int l=es.size()-1;
int m=fs.size()-1;
transpair_model4 tm4(es,fs,m1.tTable,m2.aTable,m3.dTable,m3.nTable,1-p0,p0,&d4m);
alignment al(l,m);
cout << "I use the alignment " << sent.sentenceNo-1 << '\n';
//convert(ReferenceAlignment[sent.sentenceNo-1],al);
transpair_model3 tm3(es,fs,m1.tTable,m2.aTable,m3.dTable,m3.nTable,1-p0,p0,0);
double p=tm3.prob_of_target_and_alignment_given_source(al,1);
cout << "Sentence " << sent.sentenceNo << " has IBM-3 prob " << p << '\n';
p=tm4.prob_of_target_and_alignment_given_source(al,3,1);
cout << "Sentence " << sent.sentenceNo << " has IBM-4 prob " << p << '\n';
//transpair_model5 tm5(es,fs,m1.tTable,m2.aTable,m3.dTable,m3.nTable,1-p0,p0,&d5m);
//p=tm5.prob_of_target_and_alignment_given_source(al,3,1);
//cout << "Sentence " << sent.sentenceNo << " has IBM-5 prob " << p << '\n';
}
}
else
{
cout << "No corpus exists.\n";
}
}
void ExttFieldCommands(pentry entries[], int x)
{
if(!strcmp(gcFunction,"tFieldTools"))
{
//ModuleFunctionProcess()
MYSQL_RES *res;
MYSQL_ROW field;
if(!strcmp(gcCommand,LANG_NB_NEW))
{
if(guPermLevel>=9)
{
ProcesstFieldVars(entries,x);
guMode=2000;
tField(LANG_NB_CONFIRMNEW);
}
else
tField("Error: Denied by permissions settings");
}
else if(!strcmp(gcCommand,LANG_NB_CONFIRMNEW))
{
if(guPermLevel>=9)
{
unsigned uContactParentCompany=0;
ProcesstFieldVars(entries,x);
GetClientOwner(guLoginClient,&uContactParentCompany);
guMode=2000;
//Check entries here
if(!cLabel[0])
tField("Error: cLabel empty");
if(strchr(cLabel,' '))
tField("Error: Blank in cLabel");
guMode=0;
uField=0;
uCreatedBy=guLoginClient;
uOwner=uContactParentCompany;
uModBy=0;//Never modified
uModDate=0;//Never modified
NewtField(0);
}
else
tField("Error: Denied by permissions settings");
}
else if(!strcmp(gcCommand,LANG_NB_DELETE))
{
ProcesstFieldVars(entries,x);
if(uAllowDel(uOwner,uCreatedBy))
{
guMode=2001;
guCookieTable=uTable;
guCookieField=uField;
SetSessionCookie();
tField(LANG_NB_CONFIRMDEL);
}
else
tField("Error: Denied by permissions settings");
}
else if(!strcmp(gcCommand,LANG_NB_CONFIRMDEL))
{
ProcesstFieldVars(entries,x);
if(uAllowDel(uOwner,uCreatedBy))
{
guMode=5;
DeletetField();
}
else
tField("Error: Denied by permissions settings");
}
else if(!strcmp(gcCommand,LANG_NB_MODIFY))
{
ProcesstFieldVars(entries,x);
if(uAllowMod(uOwner,uCreatedBy))
{
guMode=2002;
tField(LANG_NB_CONFIRMMOD);
}
else
tField("Error: Denied by permissions settings");
}
else if(!strcmp(gcCommand,LANG_NB_CONFIRMMOD))
{
ProcesstFieldVars(entries,x);
if(uAllowMod(uOwner,uCreatedBy))
{
guMode=2002;
//Check entries here
if(cFKSpec[0])
{
char cTable[100]={""};
char cField[100]={""};
char cuKey[100]={""};
if(sscanf(cFKSpec,"\"%99[a-zA-Z0-9\\.]\",\"%99[a-zA-Z0-9\\.]\",%99[a-zA-Z0-9\\.]"
,cTable,cField,cuKey)<2)
tField("cFKSpec not formatted correctly."
" E.g. <i>\"tClient\",\"cLabel\",uField</i>"
" where uClient is only required for ForeignKey like field types.");
//Now verfiy that such table and field exist
//If the field is created by a source code template on app generation
//this is hard to do.
/*
sprintf(gcQuery,"SELECT uField FROM tField,tTable"
" WHERE tField.uTable=tTable.uTable"
" AND tTable.cLabel='%s'"
" AND tField.cLabel='%s'"
" AND tField.uProject=%u"
" AND tTable.uTemplateType=%u LIMIT 1",
cTable,
cField,
uProject,
uTEMPLATETYPE_RAD4);
mysql_query(&gMysql,gcQuery);
if(mysql_errno(&gMysql))
tTable(gcQuery);
res=mysql_store_result(&gMysql);
if(!(field=mysql_fetch_row(res)))
{
char cMessage[256];
sprintf(cMessage,"cFKSpec contains invalid table and/or field."
" cTable=%.99s cField=%.99s cuKey=%99s",
cTable,cField,cuKey);
tField(cMessage);
}
//Now verfiy that such table and key exist
sprintf(gcQuery,"SELECT uField FROM tField,tTable"
" WHERE tField.uTable=tTable.uTable"
" AND tTable.cLabel='%s'"
" AND tField.cLabel='%s'"
" AND tField.uProject=%u"
" AND tTable.uTemplateType=%u LIMIT 1",
cTable,
cuKey,
uProject,
uTEMPLATETYPE_RAD4);
mysql_query(&gMysql,gcQuery);
if(mysql_errno(&gMysql))
tField(gcQuery);
res=mysql_store_result(&gMysql);
if(!(field=mysql_fetch_row(res)))
tField("cFKSpec contains invalid table and/or key");
*/
//tField("passed check");
}
if(!cLabel[0] || strlen(cLabel)<3)
tField("Error: No Label or too short");
char cuBSField[16]={""};
char cBSOtherOptions[100]={""};
unsigned uBSTable=0;
if(uModBSField)
{
sprintf(gcQuery,"SELECT tField.uField,tField.uTable,tField.cOtherOptions FROM tField,tTable"
" WHERE tField.uTable=tTable.uTable"
" AND tTable.cLabel='%s'"
" AND tField.cLabel='%s'"
" AND tField.uField!=%u"
" AND tTable.uTemplateType=%u LIMIT 1",
ForeignKey("tTable","cLabel",uTable),
ForeignKey("tField","cLabel",uField),
uField,
uTEMPLATETYPE_BOOTSTRAP);
mysql_query(&gMysql,gcQuery);
if(mysql_errno(&gMysql))
tField(gcQuery);
res=mysql_store_result(&gMysql);
if((field=mysql_fetch_row(res)))
{
sprintf(cuBSField,"%.15s",field[0]);
sscanf(field[1],"%u",&uBSTable);
sprintf(cBSOtherOptions,"%.99s",field[2]);
}
}
//sprintf(gcQuery,"cuBSField=%s uBSTable=%u",cuBSField,uBSTable);
//tTable(gcQuery);
guMode=0;
uModBy=guLoginClient;
if(uModBSField && cuBSField[0] && uBSTable)
{
unsigned uSaveTable=uTable;
unsigned uSaveField=uField;
char cSaveOtherOptions[100]={""};
sprintf(cSaveOtherOptions,"%.99s",cOtherOptions);
uTable=uBSTable;
sprintf(cOtherOptions,"%.99s",cBSOtherOptions);
sscanf(cuBSField,"%u",&uField);
Update_tField(cuBSField);
uTable=uSaveTable;
uField=uSaveField;
sprintf(cOtherOptions,"%.99s",cSaveOtherOptions);
}
ModtField();
}
else
tField("Error: Denied by permissions settings");
}
else if(!strcmp(gcCommand,"Reorder"))
{
ProcesstFieldVars(entries,x);
if(uAllowMod(uOwner,uCreatedBy) && uTable)
{
sprintf(gcQuery,"SELECT uField FROM tField"
" WHERE uOrder>2 AND uOrder<1000"
" AND uTable=%u"
" ORDER BY uOrder"
" LIMIT 99",uTable);
mysql_query(&gMysql,gcQuery);
if(mysql_errno(&gMysql))
tTable(mysql_error(&gMysql));
res=mysql_store_result(&gMysql);
unsigned uNewOrder=10;
while((field=mysql_fetch_row(res)))
{
sprintf(gcQuery,"UPDATE tField SET uOrder=%u WHERE uField=%s",uNewOrder+=10,field[0]);
mysql_query(&gMysql,gcQuery);
if(mysql_errno(&gMysql))
tTable(mysql_error(&gMysql));
}
mysql_free_result(res);
tField("Attempt to reorder fields has been done.");
}
else
tField("Error: Denied by permissions settings");
}
else if(!strcmp(gcCommand,"Select"))
{
ProcesstFieldVars(entries,x);
if(uAllowMod(uOwner,uCreatedBy))
{
if(uField)
{
guCookieField=uField;
SetSessionCookie();
tField("This field selected for workflow");
}
}
else
tField("Error: Denied by permissions settings");
}
else if(!strcmp(gcCommand,"Copy"))
{
ProcesstFieldVars(entries,x);
if(uAllowMod(uOwner,uCreatedBy))
{
if(uTargetTable)
{
uField=0;
uTable=uTargetTable;
Insert_tField();
tField("Field has been copied");
}
}
else
tField("Error: Denied by permissions settings");
}
}
}//void ExttFieldCommands(pentry entries[], int x)
