/* Simple calculation of backoff weights - 0.5 subtracted from each count */
static float BuildNEntry(NEntry *ne,Vector boff,float bent)
{
SEntry *cse;
AEntry *ae;
double bowt,bsum,cnt,tot,ent,prob;
ne->nse=0;
tot=cnt=0.0;
bsum=1.0;
if (ne->word[0]!=(int)exitId->aux)
for (ae=(AEntry *) ne->user; ae!=NULL; ae=ae->link) {
tot+=ae->count;
if (ae->word[0]!=0 && ae->word[0]!=(int)enterId->aux &&
ae->count>bigThresh)
cnt+=(ae->count-disCount),ne->nse++,bsum-=boff[ae->word[0]];
}
if (ne->nse==0) {
ne->se=NULL;
ne->bowt=0.0;
ent=bent;
}
else {
ne->se=(SEntry*)New(&statHeap,sizeof(SEntry)*ne->nse);
bowt = (bsum>0.0) ? (1.0-cnt/tot)/bsum : 0.0;
ent = (bowt>0.0) ? bowt*(bent-log2(bowt)) : 0.0;
for (cse=ne->se,ae=(AEntry *) ne->user; ae!=NULL; ae=ae->link)
if (ae->word[0]!=0 && ae->word[0]!=(int)enterId->aux &&
ae->count>bigThresh) {
prob=((double)ae->count-disCount)/tot;
cse->word=ae->word[0];
cse->prob=log(prob);
ent -= ent2(prob);
prob = bowt*boff[cse->word];
ent += ent2(prob);
cse++;
}
if (bowt>0.0) ne->bowt=log(bowt);
else ne->bowt=LZERO;
qsort(ne->se,ne->nse,sizeof(SEntry),se_cmp);
}
return(ent);
}
int main()
{
int ret = 0;
Operation op;
{
Task * task;
OpVector res;
if (0) {
task->TickOperation(op, res);
}
}
Entity ent1("1", 1), ent2("2", 2);
Character chr("3", 3);
{
Fell fell(chr, ent1, ent2);
fell.nextTick(1.5);
Atlas::Message::Element val;
fell.getAttr("foo", val);
assert(val.isNone());
fell.setAttr("foo", 1);
fell.getAttr("foo", val);
assert(val.isInt());
assert(!fell.obsolete());
OpVector res;
assert(res.empty());
Atlas::Objects::Operation::Generic c;
fell.initTask(c, res);
assert(!res.empty());
fell.TickOperation(op, res);
fell.irrelevant();
assert(fell.obsolete());
}
return ret;
}
/* OutputMatBigram: output matrix style bigram */
void OutputMatBigram(void)
{
LModel lm;
MatBiLM *matbi;
AEntry **aelists,*ae;
Vector vec;
double vsum,fsum,tot,scale;
double ent,bent,prob,fent;
int i,j,nf,tf=0,nu,tu=0,np,tp=0,tn=0;
lm.heap=&statHeap;
lm.type=matBigram;
matbi=CreateMatBigram(&lm,lSize);
for (i=1;i<=lSize;i++)
matbi->wdlist[i]=lTab[i].name;
aelists=(AEntry**)New(&tmpHeap,sizeof(AEntry*)*(lSize+1));
for (i=1;i<=lSize;i++) aelists[i]=NULL;
RebuildAETab(aelists); /* Un-hash hashtable */
if (trace&T_BIG) {
printf("\n BIGRAMS from MatBigram\n");
fflush(stdout);
}
bent=0.0;
fent = ent2(bigFloor);
for (i=1;i<=lSize;i++) {
vec=matbi->bigMat[i];
for (ae=aelists[i],tot=0.0; ae!=NULL; ae=ae->link)
if (ae->word[0]!=0) tot += ae->count;
fsum = (lSize-1)*bigFloor; vsum=0.0;
for (ae=aelists[i];ae!=NULL;ae=ae->link)
if (ae->count/tot > bigFloor && ae->word[0]!=0)
fsum -= bigFloor, vsum += ae->count;
else
ae->count=0;
scale = (1.0 - fsum) / vsum;
for (j=1;j<=lSize;j++) {
if (j==(int)enterId->aux) vec[j]=0.0;
else if (tot==0.0) vec[j]=1.0/(lSize-1);
else vec[j]=bigFloor;
}
for (ae=aelists[i];ae!=NULL;ae=ae->link)
if (ae->count>0)
vec[ae->word[0]]=ae->count*scale;
if (trace&T_BIG) {
nf=nu=np=0;
if (tot==0.0)
ent=-log2(1.0/(lSize-1)),prob=1.0,nu=lSize-1;
else
ent=-(lSize-1)*fent,
prob=bigFloor*(lSize-1),
nf+=lSize-1;
for (ae=aelists[i];ae!=NULL;ae=ae->link)
if (ae->count>0) {
prob += vec[ae->word[0]]-bigFloor;
ent -= ent2(vec[ae->word[0]]);
ent += fent;
nf--; np++;
}
if (i!=(int)exitId->aux){
j=lTab[i].count;
bent+=j*ent;tn+=j;
if (tot==0.0)
printf(" %-20s - %4d unis, ent %6.3f [= %6.2f] (P=%7.5f)\n",
lTab[i].name->name,nu,ent,pow(2.0,ent),prob);
else
printf(" %-20s - %4d foll, ent %6.3f [= %6.2f] (P=%7.5f)\n",
lTab[i].name->name,np,ent,pow(2.0,ent),prob);
fflush(stdout);
}
tf+=nf;tu+=nu;tp+=np;
}
}
if (trace&T_BIG) {
bent/=tn;
printf("\n BIGRAM: training data entropy %.3f (perplexity %.2f)\n",
bent,pow(2.0,bent));
printf(" Estimated %d, floored %d, unigrammed %d for %d\n",
tp,tf,tu,lSize);
fflush(stdout);
}
Dispose(&tmpHeap,aelists);
/* convert probabilities to logs */
for (i=1;i<=matbi->numWords;i++) {
vec = matbi->bigMat[i];
for (j=1; j<=matbi->numWords; j++){
vec[j] = ((vec[j]<MINLARG)?LZERO:log(vec[j]));
}
}
lm.name=CopyString(lm.heap,bigFile); /* Name and write to disk */
WriteLModel(&lm,bigFile,0);
}
/* OutputBoBigram: output ARPA/MIL-LL style back off bigram */
void OutputBoBigram(void)
{
LModel lm;
NGramLM *nglm;
NEntry *ne;
SEntry *se;
AEntry **aelists;
lmId ndx[NSIZE];
int i,tot,counts[NSIZE+1];
double uent,ent,bent;
lm.heap=&statHeap;
lm.type=boNGram;
counts[1]=lSize;counts[2]=nae;
for(i=3;i<NSIZE+1;i++)
counts[i]=0;
nglm=CreateBoNGram(&lm,lSize,counts); /* Give max size at creation */
for (i=1;i<=lSize;i++)
nglm->wdlist[i]=lTab[i].name;
aelists=(AEntry**)New(&tmpHeap,sizeof(AEntry*)*(lSize+1));
for (i=1;i<=lSize;i++) aelists[i]=NULL;
RebuildAETab(aelists); /* Un-hash hashtable */
for (i=1,tot=0.0;i<=lSize;i++) { /* Calculate unigrams first */
if (i==(int)enterId->aux)
nglm->unigrams[i]=0.0;
else if (lTab[i].count<uniFloor)
nglm->unigrams[i]=uniFloor;
else
nglm->unigrams[i]=lTab[i].count;
tot+=nglm->unigrams[i];
}
for (i=1,uent=0.0;i<=lSize;i++,se++) {
nglm->unigrams[i]=nglm->unigrams[i]/tot;
uent-=ent2(nglm->unigrams[i]);
}
nglm->counts[1]=lSize; /* Calculate real sizes during build */
nglm->counts[2]=0;
for (i=0; i<NSIZE; i++) ndx[i]=0;
if (trace&T_BIG) {
printf("\n UNIGRAM NEntry - %4d foll, ent %.3f [= %.3f]\n\n",
lSize,uent,pow(2.0,uent));
printf(" BIGRAMS NEntries\n");
fflush(stdout);
}
for (i=1,bent=0.0;i<=lSize;i++) {
ndx[0]=i;
ne=GetNEntry(nglm,ndx,TRUE);
ne->user=aelists[i];
ent = BuildNEntry(ne,nglm->unigrams,uent);
nglm->counts[2]+=ne->nse;
if (trace&T_BIG)
if (i!=(int)exitId->aux){
if (i==(int)enterId->aux)
bent+=nglm->unigrams[(int)exitId->aux]*ent;
else
bent+=nglm->unigrams[i]*ent;
printf(" %-20s - %4d foll, ent %6.3f [= %6.2f]\n",
lTab[i].name->name,ne->nse,ent,pow(2.0,ent));
fflush(stdout);
}
}
Dispose(&tmpHeap,aelists);
if (trace&T_BIG) {
printf("\n BIGRAM: training data entropy %.3f (perplexity %.2f)\n",
bent,pow(2.0,bent));
fflush(stdout);
}
ndx[0]=0; /* Set up unigram nentry separately */
ne=GetNEntry(nglm,ndx,TRUE);
ne->nse=lSize;
se=ne->se=(SEntry*)New(nglm->heap,sizeof(SEntry)*lSize);
for (i=1;i<=lSize;i++,se++) {
se->word=i;
if (nglm->unigrams[i]>0)
se->prob=nglm->unigrams[i]=log(nglm->unigrams[i]);
else
se->prob=nglm->unigrams[i]=LZERO;
}
lm.name=CopyString(lm.heap,bigFile); /* Name and write to disk */
WriteLModel(&lm,bigFile,0);
}
int main()
{
int ret = 0;
{
Entity tlve("0", 0), ent("1", 1);
ent.m_location.m_loc = &tlve;
ent.m_location.m_pos = Point3D(1, 1, 0);
ent.m_location.m_orientation = WFMath::Quaternion().identity();
Point3D relPos = relativePos(ent.m_location, ent.m_location);
std::cout << "RelPos to self: " << relPos << std::endl << std::flush;
relPos = relativePos(ent.m_location, tlve.m_location);
std::cout << "RelPos ent -> tlve: " << relPos
<< std::endl << std::flush;
relPos = relativePos(tlve.m_location, ent.m_location);
std::cout << "RelPos tlve -> ent: " << relPos
<< std::endl << std::flush;
ent.m_location.m_loc = 0;
}
{
Entity tlve("0", 0), ent1("1", 1), ent2("2", 2);
ent1.m_location.m_loc = &tlve;
ent1.m_location.m_pos = Point3D(-1, 1, 0);
ent1.m_location.m_orientation = WFMath::Quaternion().identity();
ent2.m_location.m_loc = &tlve;
ent2.m_location.m_pos = Point3D(1, 1, 0);
ent2.m_location.m_orientation = WFMath::Quaternion().identity();
Point3D relPos = relativePos(ent1.m_location, ent2.m_location);
std::cout << "RelPos ent1 -> ent2: " << relPos
<< std::endl << std::flush;
ent1.m_location.m_loc = 0;
ent2.m_location.m_loc = 0;
}
{
Entity tlve("0", 0), ent1("1", 1), ent2("2", 2),
ent3("3", 3), ent4("4", 4);
ent1.m_location.m_loc = &tlve;
ent1.m_location.m_pos = Point3D(-1, 1, 0);
ent1.m_location.m_orientation = WFMath::Quaternion().identity();
ent2.m_location.m_loc = &tlve;
ent2.m_location.m_pos = Point3D(1, 1, 0);
ent2.m_location.m_orientation = WFMath::Quaternion().identity();
ent3.m_location.m_loc = &ent1;
ent3.m_location.m_pos = Point3D(-1, 1, 0);
ent3.m_location.m_orientation = WFMath::Quaternion().identity();
ent4.m_location.m_loc = &ent2;
ent4.m_location.m_pos = Point3D(1, 1, 0);
ent4.m_location.m_orientation = WFMath::Quaternion().identity();
Point3D relPos = relativePos(ent3.m_location, ent4.m_location);
std::cout << "RelPos ent3 -> ent4: " << relPos
<< std::endl << std::flush;
ent1.m_location.m_loc = 0;
ent2.m_location.m_loc = 0;
ent3.m_location.m_loc = 0;
ent4.m_location.m_loc = 0;
}
{
Entity tlve("0", 0), ent1("1", 1), ent2("2", 2), ent3("3", 3), ent4("4", 4);
ent1.m_location.m_loc = &tlve;
ent1.m_location.m_pos = Point3D(-1, 1, 0);
ent1.m_location.m_orientation = WFMath::Quaternion().identity();
ent2.m_location.m_loc = &tlve;
ent2.m_location.m_pos = Point3D(1, 1, 0);
ent2.m_location.m_orientation = WFMath::Quaternion().identity();
ent3.m_location.m_loc = &ent1;
ent3.m_location.m_pos = Point3D(-1, 1, 0);
ent3.m_location.m_orientation = WFMath::Quaternion(2, M_PI / 2.f);
ent4.m_location.m_loc = &ent2;
ent4.m_location.m_pos = Point3D(1, 1, 0);
ent4.m_location.m_orientation = WFMath::Quaternion().identity();
Point3D relPos = relativePos(ent3.m_location, ent4.m_location);
std::cout << "RelPos ent3 -> ent4: " << relPos
<< std::endl << std::flush;
ent1.m_location.m_loc = 0;
ent2.m_location.m_loc = 0;
ent3.m_location.m_loc = 0;
ent4.m_location.m_loc = 0;
}
{
Entity tlve("0", 0), ent1("1", 1), ent2("2", 2),
ent3("3", 3), ent4("4", 4);
ent1.m_location.m_loc = &tlve;
ent1.m_location.m_pos = Point3D(-1, 1, 0);
ent1.m_location.m_orientation = WFMath::Quaternion().identity();
ent2.m_location.m_loc = &tlve;
ent2.m_location.m_pos = Point3D(1, 1, 0);
ent2.m_location.m_orientation = WFMath::Quaternion(2, -M_PI / 2.f);
ent3.m_location.m_loc = &ent1;
ent3.m_location.m_pos = Point3D(-1, 1, 0);
ent3.m_location.m_orientation = WFMath::Quaternion(2, M_PI / 2.f);
ent4.m_location.m_loc = &ent2;
ent4.m_location.m_pos = Point3D(1, 1, 0);
ent4.m_location.m_orientation = WFMath::Quaternion().identity();
Point3D relPos = relativePos(ent3.m_location, ent4.m_location);
std::cout << "RelPos ent3 -> ent4: " << relPos
<< std::endl << std::flush;
ent1.m_location.m_loc = 0;
ent2.m_location.m_loc = 0;
ent3.m_location.m_loc = 0;
ent4.m_location.m_loc = 0;
}
{
Entity tlve("0", 0), ent1("1", 1), ent2("2", 2),
ent3("3", 3), ent4("4", 4);
ent1.m_location.m_loc = &tlve;
ent1.m_location.m_pos = Point3D(-1, 1, 0);
ent1.m_location.m_orientation = WFMath::Quaternion(2, M_PI / 2.f);
ent2.m_location.m_loc = &tlve;
ent2.m_location.m_pos = Point3D(1, 1, 0);
ent2.m_location.m_orientation = WFMath::Quaternion().identity();
ent3.m_location.m_loc = &ent1;
ent3.m_location.m_pos = Point3D(-1, 1, 0);
ent3.m_location.m_orientation = WFMath::Quaternion(2, M_PI / 2.f);
ent4.m_location.m_loc = &ent2;
ent4.m_location.m_pos = Point3D(1, 1, 0);
ent4.m_location.m_orientation = WFMath::Quaternion().identity();
Point3D relPos = relativePos(ent3.m_location, ent4.m_location);
Vector3D distance = distanceTo(ent3.m_location, ent4.m_location);
std::cout << "RelPos ent3 -> ent4: " << relPos
<< " Distance ent3 -> ent4: " << distance
<< std::endl << std::flush;
ent1.m_location.m_loc = 0;
ent2.m_location.m_loc = 0;
ent3.m_location.m_loc = 0;
ent4.m_location.m_loc = 0;
}
{
Entity tlve("0", 0), ent1("1", 1), ent2("2", 2);
ent1.m_location.m_loc = &tlve;
ent1.m_location.m_pos = Point3D(1, 1, 0);
ent1.m_location.m_orientation = WFMath::Quaternion().identity();
ent2.m_location.m_loc = &ent1;
ent2.m_location.m_pos = Point3D(0, 0, 0);
ent2.m_location.m_orientation = WFMath::Quaternion().identity();
Vector3D distance = distanceTo(ent1.m_location, ent2.m_location);
std::cout << "Distance ent1 -> ent2: "
<< distance << "," << distance.isValid()
<< std::endl << std::flush;
assert(distance.isValid());
assert(distance == Vector3D(0,0,0));
ent1.m_location.m_loc = 0;
ent2.m_location.m_loc = 0;
}
return ret;
}
