void MakeMarginals(LPTable Y,LPTable dataTable)
{
int i;
int lenC = dataTable->nDimens;
int* C = new int[lenC];
if(!Y->Alloc(dataTable->Dimens,dataTable->nDimens))
{
printf("Failed to allocate marginals table.\n"); exit(1);
}
Y->GetFirst();
Y->Set(dataTable->GetGrandTotal());
while(Y->GetNext())
{
lenC = 0;
for(i=0;i<Y->nDimens;i++)
{
if(Y->Index[i])
{
C[lenC] = i;
lenC++;
}
}
LPNTable p = new NTable;
p->Create(lenC,C,dataTable);
Y->Set(p->Data[p->Total-1]);
p->Reset(); delete p; p = NULL;
}
delete[] C; C = NULL;
return;
}
LPTable CreateMin(int m, LPTable parent)
{
STable stab;
if(!stab.Create(m, parent))
{
printf("Error creating STable :: CreateS.\n");
return(NULL);
}
LPTable newtab = new Table;
if(newtab == NULL)
{
printf("Error creating STable :: CreateS.\n");
return(NULL);
}
if(!newtab->Alloc(parent->Dimens, parent->nDimens))
{
printf("Error creating STable :: CreateS.\n");
return(NULL);
}
int NotFinished = 1;
newtab->GetFirst();
while(NotFinished)
{
newtab->Set(stab.GetMin(newtab->Index));
NotFinished = newtab->GetNext();
}
stab.Reset();
return newtab;
}
LPTable Table::ReduceOne(int ind)
{
int i, j;
int dim[MAXDIMENS]; //dimensions of the new table
int ndim; //number of dimensions for the new table
int NotFinished = 1;
int* myIndex = new int[nDimens];CheckPointer(myIndex);
memset(myIndex,0,nDimens*sizeof(int));
double s;
LPTable ctab = new Table;
if(NULL == ctab)
{
printf("Error creating new table :: ReduceOne.\n");
return(NULL);
}
ndim = nDimens - 1;
j = 0;
for(i=0; i<nDimens; i++)
{
if(i != ind)
{
dim[j] = Dimens[i];
j++;
}
}
//allocate storage for the new table
ctab->Alloc(dim, ndim);
ctab->GetFirst();
while(NotFinished)
{
j = 0;
for(i=0; i<nDimens; i++)
{
if(i != ind)
{
myIndex[i] = ctab->Index[j];
j++;
}
}
myIndex[ind] = 0;
s = 0.0;
for(i=0; i<Dimens[ind]; i++)
{
s += GetI(myIndex);
myIndex[ind]++;
}
ctab->Set(s);
NotFinished = ctab->GetNext();
}
delete[] myIndex;
return(ctab);
}
void InitPriorTable(LPTable prior,LPTable data,double cPrior)
{
int i;
if(!prior->Alloc(data->Dimens,data->nDimens))
{
printf("Failed to allocate prior table.\n"); exit(1);
}
double s = cPrior;
if(s<0)
{
s = 1.0/((double)data->Total);
}
for(i=0;i<prior->Total;i++)
{
prior->Data[i] = s;
}
return;
}
void TableShotgunSearch(FILE* out,LPTable dataTable,
int ShotgunChainReplicates,double ShotgunCutoffMax,double ShotgunCutoffMin,double ShotgunProbMax, int nconfs)
{
int i;
if(dataTable->nDimens<=(2+nconfs))
{
fprintf(out,"0");
for(i=1;i<dataTable->Total;i++)
{
fprintf(out," 1");
}
fprintf(out," 1.0\n");
return;
}
int** VarSets = NULL;
int* lenVarSets = NULL;
int nVarSets = -1;
int** DownLinks = NULL;
int* nDownLinks = NULL;
int** UpLinks = NULL;
int* nUpLinks = NULL;
LPTable priorTable = new Table;
InitPriorTable(priorTable,dataTable,-1);
InitVarSets(dataTable->nDimens,
VarSets,lenVarSets,nVarSets);
InitLattice(dataTable->nDimens,
VarSets,lenVarSets,nVarSets,
DownLinks,nDownLinks,
UpLinks,nUpLinks);
LPTable datapriorTable = new Table;
if(!datapriorTable->Alloc(dataTable->Dimens,dataTable->nDimens))
{
printf("Failed to allocate data+prior table.\n"); exit(1);
}
for(i=0;i<datapriorTable->Total;i++)
{
datapriorTable->Data[i] = dataTable->Data[i]+priorTable->Data[i];
}
//this is where we store all the models we identify
CModel* models = new CModel;
models->SetCutoffs(ShotgunCutoffMax,ShotgunCutoffMax);
for(int astartpoint=1;astartpoint<=ShotgunChainReplicates;astartpoint++)
{
CModel* localmodels = new CModel;
localmodels->SetCutoffs(ShotgunCutoffMax,ShotgunCutoffMin);
doRJMCMCstart(models,localmodels,astartpoint,
dataTable,priorTable,datapriorTable,
VarSets,lenVarSets,nVarSets,
DownLinks,nDownLinks,UpLinks,nUpLinks,
mystream);
localmodels->DeleteAll();
delete localmodels; localmodels = NULL;
}
//save the best model identified
models->NormalizeWeights();
models->SaveBestModel(out);
//clean memory
models->DeleteAll();
delete models; models = NULL;
datapriorTable->Reset(); delete datapriorTable; datapriorTable = NULL;
priorTable->Reset(); delete priorTable; priorTable = NULL;
DeleteLattice(VarSets,lenVarSets,nVarSets,
DownLinks,nDownLinks,
UpLinks,nUpLinks);
DeleteVarSets(VarSets,lenVarSets,nVarSets);
return;
}
void FindHierarchicalModels(CRegression* allregs,CData& Data,char* DataFileName,int target,int nMaxRegressors)
{
int i,j;
char buffer[2048];
FILE* out = NULL;
int responseVariable = target-1;
int lenmodel = 1+nMaxRegressors + Data.NumOfConfoundingVars; // + all confounding vars
int* amodel = new int[lenmodel];
CRegression* p = allregs->Next;
int modelid = 1;
while(NULL!=p)
{
set<int>::iterator it;
lenmodel = 0;
for(it=(p->Vars).begin();it!=(p->Vars).end();it++)
{
amodel[lenmodel] = *it;
lenmodel++;
}
// Next insert all the confounding vars, followed by the response var
j = Data.NumOfConfoundingVars;
for(i=0; i<=j; i++)
{
amodel[lenmodel] = responseVariable-j+i;
lenmodel++;
}
//amodel[lenmodel] = responseVariable;
//lenmodel++;
printf("Model [%d] ::",modelid);
for(i=0;i<lenmodel;i++) printf(" %d",amodel[i]);
printf("\n");
//get the marginal table for these variables
LPTable dataTable = new Table;
int* index = new int[lenmodel];
for(j=0;j<lenmodel;j++)
{
index[j] = 2;
}
if(!dataTable->Alloc(index,lenmodel))
{
printf("Error allocating memory!\n");
exit(1);
}
for(i=0;i<Data.SampleSize;i++)
{
for(j=0;j<lenmodel;j++)
{
index[j] = (int) (Data.data[i][amodel[j]]);
}
dataTable->SetIndex(index);
dataTable->Set(dataTable->Get()+1);
}
//file where the best model will be saved
sprintf(buffer,"%s.shotgun.%d.%d.reg.model%d.txt",DataFileName,target,nMaxRegressors,modelid);
if(NULL==(out=fopen(buffer,"w")))
{
printf("Cannot open file [%s]\n",buffer);
return;
}
//do the shotgun search
TableShotgunSearch(out, dataTable, model.mnShotgunChainReplicates,
model.mdShotgunCutoffMax, model.mdShotgunCutoffMin, model.mdShotgunProbMax, Data.NumOfConfoundingVars);
fclose(out);
//clean memory
dataTable->Reset();
delete dataTable; dataTable = NULL;
//go to the next model
p = p->Next;
modelid++;
}
//clean memory
delete[] amodel; amodel = NULL;
return;
}
void ipf(int** VarSetsMarg,int* lenVarSetsMarg,int nVarSetsMarg,
LPTable P,LPTable dataTable,int* aModelGenerators,int* aModel)
{
int i,j,k;
double delta = 0.0000001;
const int PT = P->Total;
double m[PT];
double mold[PT];
double s;
int nMssShape = 0;
for(i=0;i<nVarSetsMarg;i++)
{
nMssShape += aModelGenerators[i];
}
int* lenC = new int[nMssShape];
int** C = new int*[nMssShape];
for(i=0;i<nMssShape;i++)
{
C[i] = new int[dataTable->nDimens];
}
LPNTable margin = new NTable[nMssShape];
nMssShape = 0;
for(i=0;i<nVarSetsMarg;i++)
{
if(aModelGenerators[i])
{
k = 0;
for(j=0;j<dataTable->nDimens;j++)
{
if(VarSetsMarg[i][j])
{
C[nMssShape][k]=j;
k++;
}
}
lenC[nMssShape] = k;
margin[nMssShape].Create(lenC[nMssShape],C[nMssShape],dataTable);
nMssShape++;
}
}
LPTable Theta = new Table;
if(!Theta->Alloc(dataTable->Dimens,dataTable->nDimens))
{
printf("Failed to allocate the theta's table.\n"); exit(1);
}
LPTable oldTheta = new Table;
if(!oldTheta->Alloc(dataTable->Dimens,dataTable->nDimens))
{
printf("Failed to allocate the theta's table.\n"); exit(1);
}
//generate a random theta
InitThetaTable(aModel,Theta);
PfromTheta(P,Theta);
int notdone = 1;
while(notdone)
{
for(j=0;j<PT;j++)
{
mold[j] = P->Data[j];
oldTheta->Data[j] = Theta->Data[j];
}
for(i=0;i<nMssShape;i++)
{
LPNTable p = new NTable;
p->Create(lenC[i],C[i],P);
P->GetFirst();
j=0;
m[j] = P->Data[j]*margin[i].GetI(P->Index)/p->GetI(P->Index);
while(P->GetNext())
{
j++;
m[j] = P->Data[j]*margin[i].GetI(P->Index)/p->GetI(P->Index);
}
for(j=0;j<PT;j++)
{
P->Data[j] = m[j];
}
p->Reset(); delete p ; p = NULL;
s = 0.0;
for(j=0;j<PT;j++)
{
s += P->Data[j];
}
for(j=0;j<PT;j++)
{
P->Data[j] /= s;
}
//transform in the space of thetas and impose the constraints
ThetafromP(Theta,P,aModel);
PfromTheta(P,Theta);
}
notdone = 0;
for(i=0;i<PT;i++)
{
/*
if(fabs(mold[i]-P->Data[i])>delta)
{
notdone = 1;
}
*/
//printf("\t%.5lf",Theta->Data[i]);
if(fabs(oldTheta->Data[i]-Theta->Data[i])>delta)
{
notdone = 1;
}
}
//printf("\n");
}
s = 0.0;
for(i=0;i<PT;i++)
{
s += P->Data[i];
}
for(i=0;i<PT;i++)
{
P->Data[i] /= s;
}
//clean memory
oldTheta->Reset(); delete oldTheta; oldTheta = NULL;
Theta->Reset(); delete Theta; Theta = NULL;
for(i=0;i<nMssShape;i++)
{
margin[i].Reset();
}
delete[] margin; margin = NULL;
for(i=0;i<nMssShape;i++)
{
delete[] C[i]; C[i] = NULL;
}
delete[] C; C = NULL;
delete[] lenC; lenC = NULL;
return;
}
LPTable CreateNBar(LPTable parent)
{
if(parent == NULL)
{
printf("Error :: CreateNBar.\n");
return NULL;
}
LPTable newtab = new Table;
if(newtab == NULL)
{
printf("Error :: CreateNBar\n");
return(NULL);
}
if(!newtab->Alloc(parent->Dimens, parent->nDimens))
{
printf("Error creating STable :: CreateS.\n");
return(NULL);
}
double n;
n = parent->GetGrandTotal();
int k;
k = parent->nDimens;
int m;
int plus = -1; //we begin with '-'
int first = 1; //first iteration?
int i;
for(m=1; m<k; m++)
{
LPTable smtab = CreateS(m, parent);
if(smtab == NULL)
{
printf("Error :: CreateNBar.\n");
return NULL;
}
if(first)
{
for(i=0; i<newtab->Total; i++)
{
newtab->Data[i] = 1 - smtab->Data[i]; //n
}
first = 0;
}
else //first == 0
{
if(plus == -1)
{
for(i=0; i<newtab->Total; i++)
{
newtab->Data[i] -= smtab->Data[i];
}
}
else //plus == 1
{
for(i=0; i<newtab->Total; i++)
{
newtab->Data[i] += smtab->Data[i];
}
}
}
smtab->Reset();
delete smtab;
plus = - plus;
}
if(plus == -1)
{
for(i=0; i<newtab->Total; i++)
{
newtab->Data[i] -= parent->Data[i];
}
}
else
{
for(i=0; i<newtab->Total; i++)
{
newtab->Data[i] += parent->Data[i];
}
}
return newtab;
}
int ShuttleBounds(LPTable table, int nIterations, const char* sFileName)
{
int k = table->nDimens;
LPTable UpperBound = CreateMin(k-1, table);
if(UpperBound == NULL)
{
printf("Error :: ShuttleBounds.\n");
return 0;
}
LPTable LowerBound = new Table;
if(LowerBound == NULL)
{
printf("Error :: ShuttleBounds.\n");
UpperBound->Reset(); delete UpperBound;
return 0;
}
if(!LowerBound->Alloc(table->Dimens, table->nDimens))
{
printf("Error :: ShuttleBounds.\n");
UpperBound->Reset(); delete UpperBound;
return 0;
}
int i;
for(i=0; i<table->Total; i++)
{
LowerBound->Data[i] = 0;
}
int ind;
int iteration;
for(iteration=0; iteration<nIterations; iteration++)
{
LPTable OldUpperBound = UpperBound;
LPTable OldLowerBound = LowerBound;
UpperBound = ReduceOneShuttle(table, 0, OldLowerBound);
if(UpperBound == NULL)
{
OldUpperBound->Reset(); delete OldUpperBound;
OldLowerBound->Reset(); delete OldLowerBound;
return 0;
}
for(ind=1; ind<k; ind++)
{
LPTable newtab = ReduceOneShuttle(table, ind, OldLowerBound);
if(newtab == NULL)
{
UpperBound->Reset(); delete UpperBound;
OldUpperBound->Reset(); delete OldUpperBound;
OldLowerBound->Reset(); delete OldLowerBound;
return 0;
}
for(i=0; i<table->Total; i++)
{
if(newtab->Data[i] < UpperBound->Data[i])
{
UpperBound->Data[i] = newtab->Data[i];
}
}
newtab->Reset(); delete newtab;
}
LowerBound = ReduceOneShuttle(table, 0, OldUpperBound);
if(LowerBound == NULL)
{
UpperBound->Reset(); delete UpperBound;
OldUpperBound->Reset(); delete OldUpperBound;
OldLowerBound->Reset(); delete OldLowerBound;
return 0;
}
for(ind=1; ind<k; ind++)
{
LPTable newtab = ReduceOneShuttle(table, ind, OldUpperBound);
if(newtab == NULL)
{
UpperBound->Reset(); delete UpperBound;
LowerBound->Reset(); delete LowerBound;
OldUpperBound->Reset(); delete OldUpperBound;
OldLowerBound->Reset(); delete OldLowerBound;
return 0;
}
for(i=0; i<table->Total; i++)
{
if(newtab->Data[i] > LowerBound->Data[i])
{
LowerBound->Data[i] = newtab->Data[i];
}
}
newtab->Reset(); delete newtab;
}
OldUpperBound->Reset(); delete OldUpperBound;
OldLowerBound->Reset(); delete OldLowerBound;
}
int rez = WriteBounds(UpperBound, LowerBound, sFileName);
UpperBound->Reset();
LowerBound->Reset();
delete UpperBound;
delete LowerBound;
return rez;
}
LPTable ReduceOneShuttle(LPTable tab, int ind, LPTable tabS)
{
LPTable tabR = tab->ReduceOne(ind);
if(NULL == tabR)
{
printf("Error creating new table :: ReduceOneShuttle.\n");
return(NULL);
}
LPTable rez = new Table;
if(NULL == rez)
{
printf("Error creating new table :: ReduceOneShuttle.\n");
tabR->Reset(); delete tabR;
return(NULL);
}
if(!rez->Alloc(tab->Dimens, tab->nDimens))
{
printf("Error creating new table :: ReduceOneShuttle.\n");
tabR->Reset(); delete tabR;
delete rez;
return(NULL);
}
rez->GetFirst();
int i, j;
int NotFinished = 1;
while(NotFinished)
{
j = 0;
for(i=0; i<rez->nDimens; i++)
{
if(i != ind)
{
tabR->Index[j] = rez->Index[i];
j++;
tabS->Index[i] = rez->Index[i];
}
}
double s = 0.0;
for(i=0; i<rez->Dimens[ind]; i++)
{
if(i != rez->Index[ind])
{
tabS->Index[ind] = i;
s += tabS->Get();
}
}
rez->Set(tabR->Get() - s);
if(NotFinished)
{
NotFinished = rez->GetNext();
}
}
tabR->Reset(); delete tabR;
return rez;
}
int Bonferroni(int m, LPTable table, const char* sFileName)
{
if(table == NULL)
{
printf("Error :: BonferroniBounds.\n");
return 0;
}
int i;
double nGrandTotal;
nGrandTotal = table->GetGrandTotal();
for(i=0; i<table->Total; i++)
{
table->Data[i] /= nGrandTotal;
}
table->WriteTable("tabinit.dat");
LPTable UpperBound;
LPTable LowerBound = new Table;
if(LowerBound == NULL)
{
printf("Error :: BonferroniBounds.\n");
return 0;
}
if(!LowerBound->Alloc(table->Dimens, table->nDimens))
{
printf("Error :: BonferroniBounds.\n");
return 0;
}
LPTable nBar = CreateNBar(table);
if(nBar == NULL)
{
printf("Error :: Bonferroni.\n");
return 0;
}
nBar->WriteTable("nbar.dat");
double nBarGrandTotal;
nBarGrandTotal = nBar->GetGrandTotal();
int m1;
int plus;
UpperBound = CreateS(1, nBar);
if(UpperBound == NULL)
{
printf("Error :: Bonferroni.\n");
nBar->Reset(); delete nBar;
LowerBound->Reset(); delete LowerBound;
return 0;
}
UpperBound->WriteTable("s1bar.dat");
for(i=0; i<table->Total; i++)
{
UpperBound->Data[i] = 1 - UpperBound->Data[i]; //nBarGrandTotal
}
plus = 1;
for(m1=2; m1<=m-1; m1++)
{
LPTable sm1 = CreateS(m1, nBar);
if(sm1 == NULL)
{
printf("Error :: Bonferroni.\n");
nBar->Reset(); delete nBar;
UpperBound->Reset(); delete UpperBound;
LowerBound->Reset(); delete LowerBound;
return 0;
}
if(plus == 1)
{
for(i=0; i<table->Total; i++)
{
UpperBound->Data[i] += sm1->Data[i];
}
}
else //plus == -1
{
for(i=0; i<table->Total; i++)
{
UpperBound->Data[i] -= sm1->Data[i];
}
}
sm1->Reset(); delete sm1;
plus = - plus;
}
for(i=0; i<table->Total; i++)
{
LowerBound->Data[i] = UpperBound->Data[i];
}
if(m < table->nDimens)
{
LPTable sm = CreateS(m, nBar);
if(sm == NULL)
{
printf("Error :: Bonferroni.\n");
nBar->Reset(); delete nBar;
UpperBound->Reset(); delete UpperBound;
LowerBound->Reset(); delete LowerBound;
return 0;
}
if(plus == 1) //m even
{
for(i=0; i<table->Total; i++)
{
UpperBound->Data[i] += sm->Data[i];
}
}
else //m odd
{
for(i=0; i<table->Total; i++)
{
LowerBound->Data[i] -= sm->Data[i];
}
}
sm->Reset();
delete sm;
}
else
{
if(plus == 1) //m even
{
for(i=0; i<table->Total; i++)
{
UpperBound->Data[i] += nBar->Data[i];
}
}
else //m odd
{
for(i=0; i<table->Total; i++)
{
LowerBound->Data[i] -= nBar->Data[i];
}
}
}
WriteBounds(UpperBound, LowerBound, "extra.dat");
double ratio = nGrandTotal; // /nBarGrandTotal;
printf("ratio = %lf\n", ratio);
for(i=0; i<table->Total; i++)
{
UpperBound->Data[i] = ceil(UpperBound->Data[i]*ratio);
LowerBound->Data[i] = floor(LowerBound->Data[i]*ratio);
if(LowerBound->Data[i] < 0)
{
LowerBound->Data[i] = 0;
}
}
int rez = WriteBounds(UpperBound, LowerBound, sFileName);
nBar->Reset();
UpperBound->Reset();
LowerBound->Reset();
delete nBar;
delete UpperBound;
delete LowerBound;
return rez;
}
int FrechetBounds(LPTable table, const char* sFileName)
{
if(table == NULL)
{
printf("Error :: FrechetBounds.\n");
return 0;
}
int k = table->nDimens;
LPTable UpperBound = CreateMin(k-1, table);
if(UpperBound == NULL)
{
printf("Error :: FrechetBounds.\n");
return 0;
}
LPTable LowerBound = new Table;
if(LowerBound == NULL)
{
printf("Error :: FrechetBounds.\n");
UpperBound->Reset();
delete UpperBound;
return 0;
}
if(!LowerBound->Alloc(table->Dimens, table->nDimens))
{
printf("Error :: FrechetBounds.\n");
UpperBound->Reset();
delete UpperBound;
return 0;
}
double n = table->GetGrandTotal();
int m;
int plus;
int first = 1; //first iteration?
int i;
if(k%2 == 0) //k even
{
plus = 1;
for(m=1; m<k; m++)
{
LPTable smtab = CreateS(m, table);
if(smtab == NULL)
{
printf("Error :: FrechetBounds.\n");
UpperBound->Reset();
LowerBound->Reset();
delete UpperBound;
delete LowerBound;
return 0;
}
if(first)
{
for(i=0; i<table->Total; i++)
{
LowerBound->Data[i] = smtab->Data[i] - n;
}
first = 0;
}
else
{
if(plus == -1)
{
for(i=0; i<table->Total; i++)
{
LowerBound->Data[i] -= smtab->Data[i];
}
}
else //plus == 1
{
for(i=0; i<table->Total; i++)
{
LowerBound->Data[i] += smtab->Data[i];
}
}
}
plus = - plus;
smtab->Reset();
delete smtab;
}
}
else // k odd
{
LPTable nBar = CreateNBar(table);
if(nBar == NULL)
{
printf("Error :: FrechetBounds.\n");
UpperBound->Reset();
LowerBound->Reset();
delete UpperBound;
delete LowerBound;
return 0;
}
LPTable nMinBar = CreateMin(k-1, nBar);
nBar->Reset();
delete nBar;
if(nMinBar == NULL)
{
printf("Error :: FrechetBounds.\n");
UpperBound->Reset();
LowerBound->Reset();
delete UpperBound;
delete LowerBound;
return 0;
}
plus = -1;
for(m=1; m<k; m++)
{
LPTable smtab = CreateS(m, table);
if(smtab == NULL)
{
printf("Error :: FrechetBounds.\n");
UpperBound->Reset();
LowerBound->Reset();
delete UpperBound;
delete LowerBound;
return 0;
}
if(first)
{
for(i=0; i<table->Total; i++)
{
LowerBound->Data[i] = n - smtab->Data[i];
}
first = 0;
}
else
{
if(plus == -1)
{
for(i=0; i<table->Total; i++)
{
LowerBound->Data[i] -= smtab->Data[i];
}
}
else //plus == 1
{
for(i=0; i<table->Total; i++)
{
LowerBound->Data[i] += smtab->Data[i];
}
}
}
plus = - plus;
smtab->Reset();
delete smtab;
}
for(i=0; i<table->Total; i++)
{
LowerBound->Data[i] -= nMinBar->Data[i];
}
nMinBar->Reset();
delete nMinBar;
}
for(i=0; i<table->Total; i++)
{
if(LowerBound->Data[i] < 0)
{
LowerBound->Data[i] = 0;
}
}
int rez = WriteBounds(UpperBound, LowerBound, sFileName);
UpperBound->Reset();
LowerBound->Reset();
delete UpperBound;
delete LowerBound;
return rez;
}