void parameters::Estimation(int alter, int int_discrete,int int_continue, datafile dat, model mode){
for (int it=0;it<alter;it++){
//Estimation_discrete(int_discrete, dat, mode);
Probapost(mode, dat.Get_mat_datafile());
Likelihood(dat.Get_eff_datafile());
Estimation_continue(int_continue,dat,mode);
}
Probapost(mode, dat.Get_mat_datafile());
Likelihood(dat.Get_eff_datafile());
}
double LDA::Infer(CorpusC &cor, int d, const LdaModel &m, VReal* ga, VVReal* phi) const {
VReal digamma(m.num_topics);
double likelihood_old = 0;
double c = 1;
InitVarParamter(cor, d, m, &digamma, ga, phi);
for(int it = 1; (c > var_converged_) && (it < var_max_iter_); ++it) {
for (size_t n = 0; n < cor.ULen(d); n++) {
for (int k = 0; k < m.num_topics; k++) {
(*phi)[n][k] = digamma[k] + m.log_prob_w[k][cor.Word(d, n)];
}
double log_phi_sum = LogPartition(phi->at(n));
for (int k = 0; k < m.num_topics; k++) {
(*phi)[n][k] = exp((*phi)[n][k] - log_phi_sum);
}
}
for (size_t i = 0; i < ga->size(); i++) {
ga->at(i) = m.alpha[i];
}
for (size_t n = 0; n < cor.ULen(d); n++) {
for (int k = 0; k < m.num_topics; k++) {
(*ga)[k] += cor.Count(d, n) * (*phi)[n][k];
digamma[k] = DiGamma(ga->at(k));
}
}
double likelihood = Likelihood(cor, d, m, *ga, *phi);
assert(!isnan(likelihood));
c = (likelihood_old - likelihood) / likelihood_old;
likelihood_old = likelihood;
}
return likelihood_old;
}
void parameters::Estimation_continue(int nbiter,datafile dat, model mod){
Probapost(mod, dat.Get_mat_datafile());
for (int it=0;it<nbiter;it++){
Mstep(dat, mod);
Probapost(mod, dat.Get_mat_datafile());
}
Likelihood(dat.Get_eff_datafile());
}
// Step size search for updating P_c (which is parametarized by lambda).
double TAGMFit::GetStepSizeByLineSearchForLambda(const TFltV& DeltaV, const TFltV& GradV, const double& Alpha, const double& Beta) {
double StepSize = 1.0;
double InitLikelihood = Likelihood();
IAssert(LambdaV.Len() == DeltaV.Len());
TFltV NewLambdaV(LambdaV.Len());
for (int iter = 0; ; iter++) {
for (int i = 0; i < LambdaV.Len(); i++) {
NewLambdaV[i] = LambdaV[i] + StepSize * DeltaV[i];
if (NewLambdaV[i] < MinLambda) { NewLambdaV[i] = MinLambda; }
if (NewLambdaV[i] > MaxLambda) { NewLambdaV[i] = MaxLambda; }
}
if (Likelihood(NewLambdaV) < InitLikelihood + Alpha * StepSize * TLinAlg::DotProduct(GradV, DeltaV)) {
StepSize *= Beta;
} else {
break;
}
}
return StepSize;
}
void AEMS::FindMaxApproxErrorLeaf(QNode* qnode, double likelihood,
double& bestAE, VNode*& bestNode) {
likelihood *= Likelihood(qnode);
map<OBS_TYPE, VNode*>& children = qnode->children();
for (map<OBS_TYPE, VNode*>::iterator it = children.begin();
it != children.end(); it++) {
VNode* vnode = it->second;
FindMaxApproxErrorLeaf(vnode, likelihood, bestAE, bestNode);
}
}
int TLogRegFit::MLEGradient(const double& ChangeEps, const int& MaxStep, const TStr PlotNm) {
TExeTm ExeTm;
TFltV GradV(Theta.Len());
int iter = 0;
TIntFltPrV IterLV, IterGradNormV;
double MinVal = -1e10, MaxVal = 1e10;
double GradCutOff = 100000;
for(iter = 0; iter < MaxStep; iter++) {
Gradient(GradV); //if gradient is going out of the boundary, cut off
for(int i = 0; i < Theta.Len(); i++) {
if (GradV[i] < -GradCutOff) {
GradV[i] = -GradCutOff;
}
if (GradV[i] > GradCutOff) {
GradV[i] = GradCutOff;
}
if (Theta[i] <= MinVal && GradV[i] < 0) {
GradV[i] = 0.0;
}
if (Theta[i] >= MaxVal && GradV[i] > 0) {
GradV[i] = 0.0;
}
}
double Alpha = 0.15, Beta = 0.9;
//double LearnRate = 0.1 / (0.1 * iter + 1); //GetStepSizeByLineSearch(GradV, GradV, Alpha, Beta);
double LearnRate = GetStepSizeByLineSearch(GradV, GradV, Alpha, Beta);
if (TLinAlg::Norm(GradV) < ChangeEps) {
break;
}
for(int i = 0; i < Theta.Len(); i++) {
double Change = LearnRate * GradV[i];
Theta[i] += Change;
if(Theta[i] < MinVal) {
Theta[i] = MinVal;
}
if(Theta[i] > MaxVal) {
Theta[i] = MaxVal;
}
}
if (! PlotNm.Empty()) {
double L = Likelihood();
IterLV.Add(TIntFltPr(iter, L));
IterGradNormV.Add(TIntFltPr(iter, TLinAlg::Norm(GradV)));
}
}
if (! PlotNm.Empty()) {
TGnuPlot::PlotValV(IterLV, PlotNm + ".likelihood_Q");
TGnuPlot::PlotValV(IterGradNormV, PlotNm + ".gradnorm_Q");
printf("MLE for Lambda completed with %d iterations(%s)\n",iter,ExeTm.GetTmStr());
}
return iter;
}
double Sampler::WriteLhoodUpdate(std::ostream* lhood_file, int iteration,
double elapsed) {
// double lhood = TrainLikelihoodIncrement();
double lhood = Likelihood();
(*lhood_file) << iteration;
(*lhood_file) << "\t" << lhood;
// This doesn't work anymore
// (*lhood_file) << "\t" << TestLikelihoodIncrement();
(*lhood_file) << "\t" << TrainLikelihoodIncrement();
(*lhood_file) << "\t" << elapsed;
return lhood;
}
double TLogRegFit::GetStepSizeByLineSearch(const TFltV& DeltaV, const TFltV& GradV, const double& Alpha, const double& Beta) {
double StepSize = 1.0;
double InitLikelihood = Likelihood();
IAssert(Theta.Len() == DeltaV.Len());
TFltV NewThetaV(Theta.Len());
double MinVal = -1e10, MaxVal = 1e10;
for(int iter = 0; ; iter++) {
for (int i = 0; i < Theta.Len(); i++) {
NewThetaV[i] = Theta[i] + StepSize * DeltaV[i];
if (NewThetaV[i] < MinVal) {
NewThetaV[i] = MinVal;
}
if (NewThetaV[i] > MaxVal) {
NewThetaV[i] = MaxVal;
}
}
if (Likelihood(NewThetaV) < InitLikelihood + Alpha * StepSize * TLinAlg::DotProduct(GradV, DeltaV)) {
StepSize *= Beta;
} else {
break;
}
}
return StepSize;
}
parameters::parameters(datafile dat, model nv_mod, model ref_mod,parameters ref_param,int compo,int iter){
const MatrixXi & omega=nv_mod.Get_model(),ref_omega=ref_mod.Get_model(),mat=dat.Get_mat_datafile();
const int g=omega.rows(),unique=mat.rows();
m_proba=ref_param.m_proba;
m_proba_block.resize(g);
m_param.resize(g);
for (int k=0;k<g;k++){
if (k!=compo){
m_param[k].resize(omega.rowwise().maxCoeff()(k)+1);
m_param[k]=ref_param.m_param[k];
m_proba_block[k].resize(unique,omega.rowwise().maxCoeff()(k)+1);
m_proba_block[k]=ref_param.m_proba_block[k];
for (int b=0;b<(omega.rowwise().maxCoeff()(k)+1) ;b++){
if ((omega.row(k).array()==b).any()){
m_param[k][b]=ref_param.m_param[k][b];
}
}
}else{
m_param[k].resize(omega.rowwise().maxCoeff()(k)+1);
m_proba_block[k].resize(unique,omega.rowwise().maxCoeff()(k)+1);
for (int b=0;b<(omega.rowwise().maxCoeff()(k)+1) ;b++){
if ((omega.row(k).array()==b).any()){
if ((((omega.row(k).array()==b)==(ref_omega.row(k).array()==b)).prod())==1){
m_param[k][b]=ref_param.m_param[k][b];
}else{
m_param[k][b]=param_block(k,b,dat,nv_mod,m_proba.col(k).array()/m_proba.rowwise().sum().array(),1);
if ((omega.row(k).array()==b).count()>1){
int prem=0;
while(omega(k,prem)!=b){prem++;}
if (mat.col(prem).maxCoeff()>5){
m_param[k][b]=m_param[k][b].Optimise_gamma(k,b,dat,nv_mod,5,m_proba.col(k).array()/m_proba.rowwise().sum().array(),dat.Get_eff_datafile());
}
}
}
}
}
}
}
m_propor=uniforme(g);
Probapost( nv_mod , mat );
Compte_nbparam(dat,nv_mod);
Likelihood(dat.Get_eff_datafile());
Estimation(1,0,iter,dat,nv_mod);
}
// Gradient descent for p_c while fixing community affiliation graph (CAG).
int TAGMFit::MLEGradAscentGivenCAG(const double& Thres, const int& MaxIter, const TStr PlotNm) {
int Edges = G->GetEdges();
TExeTm ExeTm;
TFltV GradV(LambdaV.Len());
int iter = 0;
TIntFltPrV IterLV, IterGradNormV;
double GradCutOff = 1000;
for (iter = 0; iter < MaxIter; iter++) {
GradLogLForLambda(GradV); //if gradient is going out of the boundary, cut off
for (int i = 0; i < LambdaV.Len(); i++) {
if (GradV[i] < -GradCutOff) { GradV[i] = -GradCutOff; }
if (GradV[i] > GradCutOff) { GradV[i] = GradCutOff; }
if (LambdaV[i] <= MinLambda && GradV[i] < 0) { GradV[i] = 0.0; }
if (LambdaV[i] >= MaxLambda && GradV[i] > 0) { GradV[i] = 0.0; }
}
double Alpha = 0.15, Beta = 0.2;
if (Edges > Kilo(100)) { Alpha = 0.00015; Beta = 0.3;}
double LearnRate = GetStepSizeByLineSearchForLambda(GradV, GradV, Alpha, Beta);
if (TLinAlg::Norm(GradV) < Thres) { break; }
for (int i = 0; i < LambdaV.Len(); i++) {
double Change = LearnRate * GradV[i];
LambdaV[i] += Change;
if(LambdaV[i] < MinLambda) { LambdaV[i] = MinLambda;}
if(LambdaV[i] > MaxLambda) { LambdaV[i] = MaxLambda;}
}
if (! PlotNm.Empty()) {
double L = Likelihood();
IterLV.Add(TIntFltPr(iter, L));
IterGradNormV.Add(TIntFltPr(iter, TLinAlg::Norm(GradV)));
}
}
if (! PlotNm.Empty()) {
TGnuPlot::PlotValV(IterLV, PlotNm + ".likelihood_Q");
TGnuPlot::PlotValV(IterGradNormV, PlotNm + ".gradnorm_Q");
printf("MLE for Lambda completed with %d iterations(%s)\n",iter,ExeTm.GetTmStr());
}
return iter;
}
parameters::parameters(datafile dat, model mod){
// TODO Auto-generated constructor stub
const MatrixXi & omega=mod.Get_model(),mat=dat.Get_mat_datafile();
const int g=omega.rows(),unique=mat.rows();
m_proba=MatrixXd::Ones(unique,g);
m_proba_block.resize(g);
m_param.resize(g);
for (int k=0;k<g;k++){
m_param[k].resize(omega.rowwise().maxCoeff()(k)+1);
m_proba_block[k].resize(unique,omega.rowwise().maxCoeff()(k)+1);
for (int b=0;b<(omega.rowwise().maxCoeff()(k)+1) ;b++){
if ((omega.row(k).array()==b).any()){
m_param[k][b]=param_block(k,b,dat,mod,VectorXd::Ones(mat.rows()),1);
}
}
}
m_propor=uniforme(g);
Probapost( mod , mat );
Compte_nbparam(dat,mod);
Likelihood(dat.Get_eff_datafile());
Estimation(1,0,6,dat,mod);
}
int TAGMFast::MLEGradAscentParallel(const double& Thres, const int& MaxIter, const int ChunkNum, const int ChunkSize, const TStr PlotNm, const double StepAlpha, const double StepBeta) {
//parallel
time_t InitTime = time(NULL);
uint64 StartTm = TSecTm::GetCurTm().GetAbsSecs();
TExeTm ExeTm, CheckTm;
double PrevL = Likelihood(true);
TIntFltPrV IterLV;
int PrevIter = 0;
int iter = 0;
TIntV NIdxV(F.Len(), 0);
for (int i = 0; i < F.Len(); i++) { NIdxV.Add(i); }
TIntV NIDOPTV(F.Len()); //check if a node needs optimization or not 1: does not require optimization
NIDOPTV.PutAll(0);
TVec<TIntFltH> NewF(ChunkNum * ChunkSize);
TIntV NewNIDV(ChunkNum * ChunkSize);
for (iter = 0; iter < MaxIter; iter++) {
NIdxV.Clr(false);
for (int i = 0; i < F.Len(); i++) {
if (NIDOPTV[i] == 0) { NIdxV.Add(i); }
}
IAssert (NIdxV.Len() <= F.Len());
NIdxV.Shuffle(Rnd);
// compute gradient for chunk of nodes
#pragma omp parallel for schedule(static, 1)
for (int TIdx = 0; TIdx < ChunkNum; TIdx++) {
TIntFltH GradV;
for (int ui = TIdx * ChunkSize; ui < (TIdx + 1) * ChunkSize; ui++) {
NewNIDV[ui] = -1;
if (ui > NIdxV.Len()) { continue; }
int u = NIdxV[ui]; //
//find set of candidate c (we only need to consider c to which a neighbor of u belongs to)
TUNGraph::TNodeI UI = G->GetNI(u);
TIntSet CIDSet(5 * UI.GetDeg());
TIntFltH CurFU = F[u];
for (int e = 0; e < UI.GetDeg(); e++) {
if (HOVIDSV[u].IsKey(UI.GetNbrNId(e))) { continue; }
TIntFltH& NbhCIDH = F[UI.GetNbrNId(e)];
for (TIntFltH::TIter CI = NbhCIDH.BegI(); CI < NbhCIDH.EndI(); CI++) {
CIDSet.AddKey(CI.GetKey());
}
}
if (CIDSet.Empty()) {
CurFU.Clr();
}
else {
for (TIntFltH::TIter CI = CurFU.BegI(); CI < CurFU.EndI(); CI++) { //remove the community membership which U does not share with its neighbors
if (! CIDSet.IsKey(CI.GetKey())) {
CurFU.DelIfKey(CI.GetKey());
}
}
GradientForRow(u, GradV, CIDSet);
if (Norm2(GradV) < 1e-4) { NIDOPTV[u] = 1; continue; }
double LearnRate = GetStepSizeByLineSearch(u, GradV, GradV, StepAlpha, StepBeta, 5);
if (LearnRate <= 1e-5) { NewNIDV[ui] = -2; continue; }
for (int ci = 0; ci < GradV.Len(); ci++) {
int CID = GradV.GetKey(ci);
double Change = LearnRate * GradV.GetDat(CID);
double NewFuc = CurFU.IsKey(CID)? CurFU.GetDat(CID) + Change : Change;
if (NewFuc <= 0.0) {
CurFU.DelIfKey(CID);
} else {
CurFU.AddDat(CID) = NewFuc;
}
}
CurFU.Defrag();
}
//store changes
NewF[ui] = CurFU;
NewNIDV[ui] = u;
}
}
int NumNoChangeGrad = 0;
int NumNoChangeStepSize = 0;
for (int ui = 0; ui < NewNIDV.Len(); ui++) {
int NewNID = NewNIDV[ui];
if (NewNID == -1) { NumNoChangeGrad++; continue; }
if (NewNID == -2) { NumNoChangeStepSize++; continue; }
for (TIntFltH::TIter CI = F[NewNID].BegI(); CI < F[NewNID].EndI(); CI++) {
SumFV[CI.GetKey()] -= CI.GetDat();
}
}
#pragma omp parallel for
for (int ui = 0; ui < NewNIDV.Len(); ui++) {
int NewNID = NewNIDV[ui];
if (NewNID < 0) { continue; }
F[NewNID] = NewF[ui];
}
for (int ui = 0; ui < NewNIDV.Len(); ui++) {
int NewNID = NewNIDV[ui];
if (NewNID < 0) { continue; }
for (TIntFltH::TIter CI = F[NewNID].BegI(); CI < F[NewNID].EndI(); CI++) {
SumFV[CI.GetKey()] += CI.GetDat();
}
}
// update the nodes who are optimal
for (int ui = 0; ui < NewNIDV.Len(); ui++) {
int NewNID = NewNIDV[ui];
if (NewNID < 0) { continue; }
TUNGraph::TNodeI UI = G->GetNI(NewNID);
NIDOPTV[NewNID] = 0;
for (int e = 0; e < UI.GetDeg(); e++) {
NIDOPTV[UI.GetNbrNId(e)] = 0;
}
}
int OPTCnt = 0;
for (int i = 0; i < NIDOPTV.Len(); i++) { if (NIDOPTV[i] == 1) { OPTCnt++; } }
if (! PlotNm.Empty()) {
printf("\r%d iterations [%s] %d secs", iter * ChunkSize * ChunkNum, ExeTm.GetTmStr(), int(TSecTm::GetCurTm().GetAbsSecs() - StartTm));
if (PrevL > TFlt::Mn) { printf(" (%f) %d g %d s %d OPT", PrevL, NumNoChangeGrad, NumNoChangeStepSize, OPTCnt); }
fflush(stdout);
}
if ((iter - PrevIter) * ChunkSize * ChunkNum >= G->GetNodes()) {
PrevIter = iter;
double CurL = Likelihood(true);
IterLV.Add(TIntFltPr(iter * ChunkSize * ChunkNum, CurL));
printf("\r%d iterations, Likelihood: %f, Diff: %f [%d secs]", iter, CurL, CurL - PrevL, int(time(NULL) - InitTime));
fflush(stdout);
if (CurL - PrevL <= Thres * fabs(PrevL)) {
break;
}
else {
PrevL = CurL;
}
}
}
if (! PlotNm.Empty()) {
printf("\nMLE completed with %d iterations(%s secs)\n", iter, int(TSecTm::GetCurTm().GetAbsSecs() - StartTm));
TGnuPlot::PlotValV(IterLV, PlotNm + ".likelihood_Q");[]
} else {
int TAGMFast::MLEGradAscent(const double& Thres, const int& MaxIter, const TStr PlotNm, const double StepAlpha, const double StepBeta) {
time_t InitTime = time(NULL);
TExeTm ExeTm, CheckTm;
int iter = 0, PrevIter = 0;
TIntFltPrV IterLV;
TUNGraph::TNodeI UI;
double PrevL = TFlt::Mn, CurL = 0.0;
TIntV NIdxV(F.Len(), 0);
for (int i = 0; i < F.Len(); i++) { NIdxV.Add(i); }
IAssert(NIdxV.Len() == F.Len());
TIntFltH GradV;
while(iter < MaxIter) {
NIdxV.Shuffle(Rnd);
for (int ui = 0; ui < F.Len(); ui++, iter++) {
int u = NIdxV[ui]; //
//find set of candidate c (we only need to consider c to which a neighbor of u belongs to)
UI = G->GetNI(u);
TIntSet CIDSet(5 * UI.GetDeg());
for (int e = 0; e < UI.GetDeg(); e++) {
if (HOVIDSV[u].IsKey(UI.GetNbrNId(e))) { continue; }
TIntFltH& NbhCIDH = F[UI.GetNbrNId(e)];
for (TIntFltH::TIter CI = NbhCIDH.BegI(); CI < NbhCIDH.EndI(); CI++) {
CIDSet.AddKey(CI.GetKey());
}
}
for (TIntFltH::TIter CI = F[u].BegI(); CI < F[u].EndI(); CI++) { //remove the community membership which U does not share with its neighbors
if (! CIDSet.IsKey(CI.GetKey())) {
DelCom(u, CI.GetKey());
}
}
if (CIDSet.Empty()) { continue; }
GradientForRow(u, GradV, CIDSet);
if (Norm2(GradV) < 1e-4) { continue; }
double LearnRate = GetStepSizeByLineSearch(u, GradV, GradV, StepAlpha, StepBeta);
if (LearnRate == 0.0) { continue; }
for (int ci = 0; ci < GradV.Len(); ci++) {
int CID = GradV.GetKey(ci);
double Change = LearnRate * GradV.GetDat(CID);
double NewFuc = GetCom(u, CID) + Change;
if (NewFuc <= 0.0) {
DelCom(u, CID);
} else {
AddCom(u, CID, NewFuc);
}
}
if (! PlotNm.Empty() && (iter + 1) % G->GetNodes() == 0) {
IterLV.Add(TIntFltPr(iter, Likelihood(false)));
}
}
printf("\r%d iterations (%f) [%lu sec]", iter, CurL, time(NULL) - InitTime);
fflush(stdout);
if (iter - PrevIter >= 2 * G->GetNodes() && iter > 10000) {
PrevIter = iter;
CurL = Likelihood();
if (PrevL > TFlt::Mn && ! PlotNm.Empty()) {
printf("\r%d iterations, Likelihood: %f, Diff: %f", iter, CurL, CurL - PrevL);
}
fflush(stdout);
if (CurL - PrevL <= Thres * fabs(PrevL)) { break; }
else { PrevL = CurL; }
}
}
printf("\n");
printf("MLE for Lambda completed with %d iterations(%s)\n", iter, ExeTm.GetTmStr());
if (! PlotNm.Empty()) {
TGnuPlot::PlotValV(IterLV, PlotNm + ".likelihood_Q");
}
return iter;
}
int TAGMFast::FindComsByCV(TIntV& ComsV, const double HOFrac, const int NumThreads, const TStr PlotLFNm, const double StepAlpha, const double StepBeta) {
if (ComsV.Len() == 0) {
int MaxComs = G->GetNodes() / 5;
ComsV.Add(2);
while(ComsV.Last() < MaxComs) { ComsV.Add(ComsV.Last() * 2); }
}
TIntPrV EdgeV(G->GetEdges(), 0);
for (TUNGraph::TEdgeI EI = G->BegEI(); EI < G->EndEI(); EI++) {
EdgeV.Add(TIntPr(EI.GetSrcNId(), EI.GetDstNId()));
}
EdgeV.Shuffle(Rnd);
int MaxIterCV = 3;
TVec<TVec<TIntSet> > HoldOutSets(MaxIterCV);
if (EdgeV.Len() > 50) { //if edges are many enough, use CV
printf("generating hold out set\n");
TIntV NIdV1, NIdV2;
G->GetNIdV(NIdV1);
G->GetNIdV(NIdV2);
for (int IterCV = 0; IterCV < MaxIterCV; IterCV++) {
// generate holdout sets
HoldOutSets[IterCV].Gen(G->GetNodes());
const int HOTotal = int(HOFrac * G->GetNodes() * (G->GetNodes() - 1) / 2.0);
int HOCnt = 0;
int HOEdges = (int) TMath::Round(HOFrac * G->GetEdges());
printf("holding out %d edges...\n", HOEdges);
for (int he = 0; he < (int) HOEdges; he++) {
HoldOutSets[IterCV][EdgeV[he].Val1].AddKey(EdgeV[he].Val2);
HoldOutSets[IterCV][EdgeV[he].Val2].AddKey(EdgeV[he].Val1);
HOCnt++;
}
printf("%d Edges hold out\n", HOCnt);
while(HOCnt++ < HOTotal) {
int SrcNID = Rnd.GetUniDevInt(G->GetNodes());
int DstNID = Rnd.GetUniDevInt(G->GetNodes());
HoldOutSets[IterCV][SrcNID].AddKey(DstNID);
HoldOutSets[IterCV][DstNID].AddKey(SrcNID);
}
}
printf("hold out set generated\n");
}
TFltV HOLV(ComsV.Len());
TIntFltPrV ComsLV;
for (int c = 0; c < ComsV.Len(); c++) {
const int Coms = ComsV[c];
printf("Try number of Coms:%d\n", Coms);
NeighborComInit(Coms);
printf("Initialized\n");
if (EdgeV.Len() > 50) { //if edges are many enough, use CV
for (int IterCV = 0; IterCV < MaxIterCV; IterCV++) {
HOVIDSV = HoldOutSets[IterCV];
if (NumThreads == 1) {
printf("MLE without parallelization begins\n");
MLEGradAscent(0.05, 10 * G->GetNodes(), "", StepAlpha, StepBeta);
} else {
printf("MLE with parallelization begins\n");
MLEGradAscentParallel(0.05, 100, NumThreads, "", StepAlpha, StepBeta);
}
double HOL = LikelihoodHoldOut();
HOL = HOL < 0? HOL: TFlt::Mn;
HOLV[c] += HOL;
}
}
else {
HOVIDSV.Gen(G->GetNodes());
MLEGradAscent(0.0001, 100 * G->GetNodes(), "");
double BIC = 2 * Likelihood() - (double) G->GetNodes() * Coms * 2.0 * log ( (double) G->GetNodes());
HOLV[c] = BIC;
}
}
int EstComs = 2;
double MaxL = TFlt::Mn;
printf("\n");
for (int c = 0; c < ComsV.Len(); c++) {
ComsLV.Add(TIntFltPr(ComsV[c].Val, HOLV[c].Val));
printf("%d(%f)\t", ComsV[c].Val, HOLV[c].Val);
if (MaxL < HOLV[c]) {
MaxL = HOLV[c];
EstComs = ComsV[c];
}
}
printf("\n");
RandomInit(EstComs);
HOVIDSV.Gen(G->GetNodes());
if (! PlotLFNm.Empty()) {
TGnuPlot::PlotValV(ComsLV, PlotLFNm, "hold-out likelihood", "communities", "likelihood");
}
return EstComs;
}
/// Newton method: DEPRECATED
int TAGMFast::MLENewton(const double& Thres, const int& MaxIter, const TStr PlotNm) {
TExeTm ExeTm;
int iter = 0, PrevIter = 0;
TIntFltPrV IterLV;
double PrevL = TFlt::Mn, CurL;
TUNGraph::TNodeI UI;
TIntV NIdxV;
G->GetNIdV(NIdxV);
int CID, UID, NewtonIter;
double Fuc, PrevFuc, Grad, H;
while(iter < MaxIter) {
NIdxV.Shuffle(Rnd);
for (int ui = 0; ui < F.Len(); ui++, iter++) {
if (! PlotNm.Empty() && iter % G->GetNodes() == 0) {
IterLV.Add(TIntFltPr(iter, Likelihood(false)));
}
UID = NIdxV[ui];
//find set of candidate c (we only need to consider c to which a neighbor of u belongs to)
TIntSet CIDSet;
UI = G->GetNI(UID);
if (UI.GetDeg() == 0) { //if the node is isolated, clear its membership and skip
if (! F[UID].Empty()) { F[UID].Clr(); }
continue;
}
for (int e = 0; e < UI.GetDeg(); e++) {
if (HOVIDSV[UID].IsKey(UI.GetNbrNId(e))) { continue; }
TIntFltH& NbhCIDH = F[UI.GetNbrNId(e)];
for (TIntFltH::TIter CI = NbhCIDH.BegI(); CI < NbhCIDH.EndI(); CI++) {
CIDSet.AddKey(CI.GetKey());
}
}
for (TIntFltH::TIter CI = F[UID].BegI(); CI < F[UID].EndI(); CI++) { //remove the community membership which U does not share with its neighbors
if (! CIDSet.IsKey(CI.GetKey())) {
DelCom(UID, CI.GetKey());
}
}
if (CIDSet.Empty()) { continue; }
for (TIntSet::TIter CI = CIDSet.BegI(); CI < CIDSet.EndI(); CI++) {
CID = CI.GetKey();
//optimize for UID, CID
//compute constants
TFltV AlphaKV(UI.GetDeg());
for (int e = 0; e < UI.GetDeg(); e++) {
if (HOVIDSV[UID].IsKey(UI.GetNbrNId(e))) { continue; }
AlphaKV[e] = (1 - PNoCom) * exp(- DotProduct(UID, UI.GetNbrNId(e)) + GetCom(UI.GetNbrNId(e), CID) * GetCom(UID, CID));
IAssertR(AlphaKV[e] <= 1.0, TStr::Fmt("AlphaKV=%f, %f, %f", AlphaKV[e].Val, PNoCom.Val, GetCom(UI.GetNbrNId(e), CID)));
}
Fuc = GetCom(UID, CID);
PrevFuc = Fuc;
Grad = GradientForOneVar(AlphaKV, UID, CID, Fuc), H = 0.0;
if (Grad <= 1e-3 && Grad >= -0.1) { continue; }
NewtonIter = 0;
while (NewtonIter++ < 10) {
Grad = GradientForOneVar(AlphaKV, UID, CID, Fuc), H = 0.0;
H = HessianForOneVar(AlphaKV, UID, CID, Fuc);
if (Fuc == 0.0 && Grad <= 0.0) { Grad = 0.0; }
if (fabs(Grad) < 1e-3) { break; }
if (H == 0.0) { Fuc = 0.0; break; }
double NewtonStep = - Grad / H;
if (NewtonStep < -0.5) { NewtonStep = - 0.5; }
Fuc += NewtonStep;
if (Fuc < 0.0) { Fuc = 0.0; }
}
if (Fuc == 0.0) {
DelCom(UID, CID);
}
else {
AddCom(UID, CID, Fuc);
}
}
}
if (iter - PrevIter >= 2 * G->GetNodes() && iter > 10000) {
PrevIter = iter;
CurL = Likelihood();
if (PrevL > TFlt::Mn && ! PlotNm.Empty()) {
printf("\r%d iterations, Likelihood: %f, Diff: %f", iter, CurL, CurL - PrevL);
}
fflush(stdout);
if (CurL - PrevL <= Thres * fabs(PrevL)) { break; }
else { PrevL = CurL; }
}
}
if (! PlotNm.Empty()) {
printf("\nMLE for Lambda completed with %d iterations(%s)\n", iter, ExeTm.GetTmStr());
TGnuPlot::PlotValV(IterLV, PlotNm + ".likelihood_Q");
}
return iter;
}
void TAGMFit::PrintSummary() {
TIntFltH CIDLambdaH(CIDNSetV.Len());
for (int c = 0; c < CIDNSetV.Len(); c++) {
CIDLambdaH.AddDat(c, LambdaV[c]);
}
CIDLambdaH.SortByDat(false);
int Coms = 0;
for (int i = 0; i < LambdaV.Len(); i++) {
int CID = CIDLambdaH.GetKey(i);
if (LambdaV[CID] <= 0.0001) { continue; }
printf("P_c : %.3f Com Sz: %d, Total Edges inside: %d \n", 1.0 - exp(- LambdaV[CID]), CIDNSetV[CID].Len(), (int) ComEdgesV[CID]);
Coms++;
}
printf("%d Communities, Total Memberships = %d, Likelihood = %.2f, Epsilon = %f\n", Coms, NIDCIDPrS.Len(), Likelihood(), PNoCom.Val);
}
void Sampler::SampleCorpusVariables(int iter, double step, int samples,
std::ostream* param_file) {
assert(step >= 0.0);
vector<double> current = hyperparameters();
vector<double> temp = hyperparameters();
int dim = current.size();
assert(dim > 0);
vector<double> l(dim);
vector<double> r(dim);
double lp = 0.0;
for (int ii = 0; ii < samples; ++ii) {
SetHyperparameters(current);
// lp = TrainLikelihoodIncrement();
lp = Likelihood();
// BOOST_ASSERT(lp == TrainLikelihoodIncrement());
cout << "OLD:";
double goal = lp + log(lib_prob::next_uniform());
for (int dd = 0; dd < dim; dd++) {
cout << exp(current[dd]) << " ";
l[dd] = current[dd] - lib_prob::next_uniform() * step;
r[dd] = l[dd] + step;
}
cout << endl << "lp = " << lp << " Goal = " << goal << endl;
while (true) {
cout << ".";
cout.flush();
for (int dd = 0; dd < dim; dd++) {
temp[dd] = l[dd] + lib_prob::next_uniform() * (r[dd] - l[dd]);
}
SetHyperparameters(temp);
// double new_prob = TrainLikelihoodIncrement();
double new_prob = Likelihood();
cout << " Hyperparamters iter" << ii << ": " << new_prob
<< " vs. " << goal << " " << lp << endl;
if (new_prob > goal) {
break;
} else {
for (int dd = 0; dd < dim; dd++) {
if (temp[dd] < current[dd]) {
l[dd] = temp[dd];
} else {
r[dd] = temp[dd];
}
assert(l[dd] <= current[dd]);
assert(r[dd] >= current[dd]);
cout << dd << ":(" << exp(l[dd]) << "-" << exp(r[dd]) << ") ";
}
cout << endl;
}
}
for (int dd = 0; dd < dim; ++dd) current[dd] = temp[dd];
}
SetHyperparameters(current);
// lp = TrainLikelihoodIncrement();
lp = Likelihood();
cout << " Final hyperparameter update " << lp << endl;
(*param_file) << iter;
for (int dd = 0; dd < dim; ++dd)
(*param_file) << "\t" << exp(current[dd]);
(*param_file) << endl;
param_file->flush();
}
double TAGMFit::Likelihood() {
return Likelihood(LambdaV);
}
double VarMGCTM::Infer(DocC &doc, MGCTMC &m, MGVar* para) const {
double c = 1;
InitVar(doc, m, para);
MGVar &p = *para;
for(int it = 1; (c > converged_.var_converged_) && (it <
converged_.var_max_iter_); ++it) {
//indicate variable eta
Vec g_theta_ep;
GThetaEp(p.g_theta, &g_theta_ep);
Mat l_theta_ep;
LThetaEp(p.l_theta, &l_theta_ep);
for (int j = 0; j < m.LTopicNum1(); j++) {
p.eta[j] = log(m.pi[j]);
int l_topic_num = m.LTopicNum2();
p.eta[j] += lgamma(l_topic_num*m.l_alpha[j]);
p.eta[j] -= l_topic_num*lgamma(m.l_alpha[j]);
for (int k = 0; k < m.LTopicNum2(); k++) {
p.eta[j] += (m.l_alpha[j] - 1)*l_theta_ep(k, j);
}
for (size_t n = 0; n < doc.ULen(); n++) {
double a = 0;
for (int k = 0; k < m.LTopicNum2(); k++) {
a += p.l_z[j](k, n) * l_theta_ep(k, j);
a += p.l_z[j](k, n) * m.l_ln_w[j](k, doc.Word(n));
}
p.eta[j] += p.delta[n]*a*doc.Count(n);
}
}
double ln_eta_sum = LogSum(p.eta);
for (int j = 0; j < m.LTopicNum1(); j++) { //normalize eta
p.eta[j] = exp(p.eta[j] - ln_eta_sum);
}
//omega
p.omega[1] = m.gamma[1];
for (size_t n = 0; n < doc.ULen(); n++) {
p.omega[1] += p.delta(n)*doc.Count(n);
}
p.omega[0] = m.gamma[0];
for (size_t n = 0; n < doc.ULen(); n++) {
p.omega[0] += (1 - p.delta(n))*doc.Count(n);
}
//local theta
for (int j = 0; j < m.LTopicNum1(); j++) {
for (int k = 0; k < m.LTopicNum2(); k++) {
p.l_theta(k, j) = p.eta[j] * m.l_alpha[j];
}
}
for (int j = 0; j < m.LTopicNum1(); j++) {
for (int k = 0; k < m.LTopicNum2(); k++) {
for (size_t n = 0; n < doc.ULen(); n++) {
p.l_theta(k, j) += doc.Count(n)* p.delta[n]*p.eta[j]* p.l_z[j](k, n);
}
p.l_theta(k, j) += 1 - p.eta[j];
}
}
//global theta
p.g_theta.setConstant(m.g_alpha);
for (size_t n = 0; n < doc.ULen(); n++) {
for (int k = 0; k < m.GTopicNum(); k++) {
p.g_theta[k] += doc.Count(n) * p.g_z(k, n) * (1 - p.delta[n]);
}
}
for (size_t n = 0; n < doc.ULen(); n++) {
//variable delta
double tmp = DiGamma(m.gamma[1]) - DiGamma(m.gamma[0]);
for (int j = 0; j < m.LTopicNum1(); j++) {
for (int k = 0; k < m.LTopicNum2(); k++) {
tmp += p.eta[j]*p.l_z[j](k, n)*l_theta_ep(k,j);
tmp += p.eta[j]*p.l_z[j](k, n)*m.l_ln_w[j](k, doc.Word(n));
}
}
for (int k = 0; k < m.GTopicNum(); k++) {
tmp -= p.g_z(k, n) * g_theta_ep[k];
tmp -= p.g_z(k, n) * m.g_ln_w(k, doc.Word(n));
}
p.delta[n] = Sigmoid(tmp);
//local z
for (int j = 0; j < m.LTopicNum1(); j++) {
for (int k = 0; k < m.LTopicNum2(); k++) {
p.l_z[j](k, n) = p.delta[n]*p.eta[j]*(l_theta_ep(k, j) +
m.l_ln_w[j](k, doc.Word(n)));
}
double ln_local_z_sum = LogSum(p.l_z[j].col(n));
for (int k = 0; k < m.LTopicNum2(); k++) {
p.l_z[j](k, n) = exp(p.l_z[j](k, n) - ln_local_z_sum);
}
}
//global z
for (int k = 0; k < m.GTopicNum(); k++) {
p.g_z(k, n) = (1 - p.delta[n])*(g_theta_ep[k] + m.g_ln_w(k, doc.Word(n)));
}
double ln_z_sum = LogSum(p.g_z.col(n));
for (int k = 0; k < m.GTopicNum(); k++) { //normalize g_z
p.g_z(k, n) = exp(p.g_z(k, n) - ln_z_sum);
}
}
}
return Likelihood(doc, p, m);
}
/// MCMC fitting
void TAGMFit::RunMCMC(const int& MaxIter, const int& EvalLambdaIter, const TStr& PlotFPrx) {
TExeTm IterTm, TotalTm;
double PrevL = Likelihood(), DeltaL = 0;
double BestL = PrevL;
printf("initial likelihood = %f\n",PrevL);
TIntFltPrV IterTrueLV, IterJoinV, IterLeaveV, IterAcceptV, IterSwitchV, IterLBV;
TIntPrV IterTotMemV;
TIntV IterV;
TFltV BestLV;
TVec<TIntSet> BestCmtySetV;
int SwitchCnt = 0, LeaveCnt = 0, JoinCnt = 0, AcceptCnt = 0, ProbBinSz;
int Nodes = G->GetNodes(), Edges = G->GetEdges();
TExeTm PlotTm;
ProbBinSz = TMath::Mx(1000, G->GetNodes() / 10); //bin to compute probabilities
IterLBV.Add(TIntFltPr(1, BestL));
for (int iter = 0; iter < MaxIter; iter++) {
IterTm.Tick();
int NID = -1;
int JoinCID = -1, LeaveCID = -1;
SampleTransition(NID, JoinCID, LeaveCID, DeltaL); //sample a move
double OptL = PrevL;
if (DeltaL > 0 || Rnd.GetUniDev() < exp(DeltaL)) { //if it is accepted
IterTm.Tick();
if (LeaveCID > -1 && LeaveCID != BaseCID) { LeaveCom(NID, LeaveCID); }
if (JoinCID > -1 && JoinCID != BaseCID) { JoinCom(NID, JoinCID); }
if (LeaveCID > -1 && JoinCID > -1 && JoinCID != BaseCID && LeaveCID != BaseCID) { SwitchCnt++; }
else if (LeaveCID > -1 && LeaveCID != BaseCID) { LeaveCnt++;}
else if (JoinCID > -1 && JoinCID != BaseCID) { JoinCnt++;}
AcceptCnt++;
if ((iter + 1) % EvalLambdaIter == 0) {
IterTm.Tick();
MLEGradAscentGivenCAG(0.01, 3);
OptL = Likelihood();
}
else{
OptL = PrevL + DeltaL;
}
if (BestL <= OptL && CIDNSetV.Len() > 0) {
BestCmtySetV = CIDNSetV;
BestLV = LambdaV;
BestL = OptL;
}
}
if (iter > 0 && (iter % ProbBinSz == 0) && PlotFPrx.Len() > 0) {
IterLBV.Add(TIntFltPr(iter, OptL));
IterSwitchV.Add(TIntFltPr(iter, (double) SwitchCnt / (double) AcceptCnt));
IterLeaveV.Add(TIntFltPr(iter, (double) LeaveCnt / (double) AcceptCnt));
IterJoinV.Add(TIntFltPr(iter, (double) JoinCnt / (double) AcceptCnt));
IterAcceptV.Add(TIntFltPr(iter, (double) AcceptCnt / (double) ProbBinSz));
SwitchCnt = JoinCnt = LeaveCnt = AcceptCnt = 0;
}
PrevL = OptL;
if ((iter + 1) % 10000 == 0) {
printf("\r%d iterations completed [%.2f]", iter, (double) iter / (double) MaxIter);
}
}
// plot the likelihood and acceptance probabilities if the plot file name is given
if (PlotFPrx.Len() > 0) {
TGnuPlot GP1;
GP1.AddPlot(IterLBV, gpwLinesPoints, "likelihood");
GP1.SetDataPlotFNm(PlotFPrx + ".likelihood.tab", PlotFPrx + ".likelihood.plt");
TStr TitleStr = TStr::Fmt(" N:%d E:%d", Nodes, Edges);
GP1.SetTitle(PlotFPrx + ".likelihood" + TitleStr);
GP1.SavePng(PlotFPrx + ".likelihood.png");
TGnuPlot GP2;
GP2.AddPlot(IterSwitchV, gpwLinesPoints, "Switch");
GP2.AddPlot(IterLeaveV, gpwLinesPoints, "Leave");
GP2.AddPlot(IterJoinV, gpwLinesPoints, "Join");
GP2.AddPlot(IterAcceptV, gpwLinesPoints, "Accept");
GP2.SetTitle(PlotFPrx + ".transition");
GP2.SetDataPlotFNm(PlotFPrx + "transition_prob.tab", PlotFPrx + "transition_prob.plt");
GP2.SavePng(PlotFPrx + "transition_prob.png");
}
CIDNSetV = BestCmtySetV;
LambdaV = BestLV;
InitNodeData();
MLEGradAscentGivenCAG(0.001, 100);
printf("\nMCMC completed (best likelihood: %.2f) [%s]\n", BestL, TotalTm.GetTmStr());
}