int TLogRegFit::MLENewton(const double& ChangeEps, const int& MaxStep, const TStr PlotNm) {
TExeTm ExeTm;
TFltV GradV(Theta.Len()), DeltaLV(Theta.Len());
TFltVV HVV(Theta.Len(), Theta.Len());
int iter = 0;
double MinVal = -1e10, MaxVal = 1e10;
for(iter = 0; iter < MaxStep; iter++) {
Gradient(GradV);
Hessian(HVV);
GetNewtonStep(HVV, GradV, DeltaLV);
double Increment = TLinAlg::DotProduct(GradV, DeltaLV);
if (Increment <= ChangeEps) {
break;
}
double LearnRate = GetStepSizeByLineSearch(DeltaLV, GradV, 0.15, 0.5);//InitLearnRate/double(0.01*(double)iter + 1);
for(int i = 0; i < Theta.Len(); i++) {
double Change = LearnRate * DeltaLV[i];
Theta[i] += Change;
if(Theta[i] < MinVal) {
Theta[i] = MinVal;
}
if(Theta[i] > MaxVal) {
Theta[i] = MaxVal;
}
}
}
if (! PlotNm.Empty()) {
printf("MLE with Newton method completed with %d iterations(%s)\n",iter,ExeTm.GetTmStr());
}
return iter;
}
int TLogRegFit::MLEGradient(const double& ChangeEps, const int& MaxStep, const TStr PlotNm) {
TExeTm ExeTm;
TFltV GradV(Theta.Len());
int iter = 0;
TIntFltPrV IterLV, IterGradNormV;
double MinVal = -1e10, MaxVal = 1e10;
double GradCutOff = 100000;
for(iter = 0; iter < MaxStep; iter++) {
Gradient(GradV); //if gradient is going out of the boundary, cut off
for(int i = 0; i < Theta.Len(); i++) {
if (GradV[i] < -GradCutOff) {
GradV[i] = -GradCutOff;
}
if (GradV[i] > GradCutOff) {
GradV[i] = GradCutOff;
}
if (Theta[i] <= MinVal && GradV[i] < 0) {
GradV[i] = 0.0;
}
if (Theta[i] >= MaxVal && GradV[i] > 0) {
GradV[i] = 0.0;
}
}
double Alpha = 0.15, Beta = 0.9;
//double LearnRate = 0.1 / (0.1 * iter + 1); //GetStepSizeByLineSearch(GradV, GradV, Alpha, Beta);
double LearnRate = GetStepSizeByLineSearch(GradV, GradV, Alpha, Beta);
if (TLinAlg::Norm(GradV) < ChangeEps) {
break;
}
for(int i = 0; i < Theta.Len(); i++) {
double Change = LearnRate * GradV[i];
Theta[i] += Change;
if(Theta[i] < MinVal) {
Theta[i] = MinVal;
}
if(Theta[i] > MaxVal) {
Theta[i] = MaxVal;
}
}
if (! PlotNm.Empty()) {
double L = Likelihood();
IterLV.Add(TIntFltPr(iter, L));
IterGradNormV.Add(TIntFltPr(iter, TLinAlg::Norm(GradV)));
}
}
if (! PlotNm.Empty()) {
TGnuPlot::PlotValV(IterLV, PlotNm + ".likelihood_Q");
TGnuPlot::PlotValV(IterGradNormV, PlotNm + ".gradnorm_Q");
printf("MLE for Lambda completed with %d iterations(%s)\n",iter,ExeTm.GetTmStr());
}
return iter;
}
int TAGMFast::MLEGradAscentParallel(const double& Thres, const int& MaxIter, const int ChunkNum, const int ChunkSize, const TStr PlotNm, const double StepAlpha, const double StepBeta) {
//parallel
time_t InitTime = time(NULL);
uint64 StartTm = TSecTm::GetCurTm().GetAbsSecs();
TExeTm ExeTm, CheckTm;
double PrevL = Likelihood(true);
TIntFltPrV IterLV;
int PrevIter = 0;
int iter = 0;
TIntV NIdxV(F.Len(), 0);
for (int i = 0; i < F.Len(); i++) { NIdxV.Add(i); }
TIntV NIDOPTV(F.Len()); //check if a node needs optimization or not 1: does not require optimization
NIDOPTV.PutAll(0);
TVec<TIntFltH> NewF(ChunkNum * ChunkSize);
TIntV NewNIDV(ChunkNum * ChunkSize);
for (iter = 0; iter < MaxIter; iter++) {
NIdxV.Clr(false);
for (int i = 0; i < F.Len(); i++) {
if (NIDOPTV[i] == 0) { NIdxV.Add(i); }
}
IAssert (NIdxV.Len() <= F.Len());
NIdxV.Shuffle(Rnd);
// compute gradient for chunk of nodes
#pragma omp parallel for schedule(static, 1)
for (int TIdx = 0; TIdx < ChunkNum; TIdx++) {
TIntFltH GradV;
for (int ui = TIdx * ChunkSize; ui < (TIdx + 1) * ChunkSize; ui++) {
NewNIDV[ui] = -1;
if (ui > NIdxV.Len()) { continue; }
int u = NIdxV[ui]; //
//find set of candidate c (we only need to consider c to which a neighbor of u belongs to)
TUNGraph::TNodeI UI = G->GetNI(u);
TIntSet CIDSet(5 * UI.GetDeg());
TIntFltH CurFU = F[u];
for (int e = 0; e < UI.GetDeg(); e++) {
if (HOVIDSV[u].IsKey(UI.GetNbrNId(e))) { continue; }
TIntFltH& NbhCIDH = F[UI.GetNbrNId(e)];
for (TIntFltH::TIter CI = NbhCIDH.BegI(); CI < NbhCIDH.EndI(); CI++) {
CIDSet.AddKey(CI.GetKey());
}
}
if (CIDSet.Empty()) {
CurFU.Clr();
}
else {
for (TIntFltH::TIter CI = CurFU.BegI(); CI < CurFU.EndI(); CI++) { //remove the community membership which U does not share with its neighbors
if (! CIDSet.IsKey(CI.GetKey())) {
CurFU.DelIfKey(CI.GetKey());
}
}
GradientForRow(u, GradV, CIDSet);
if (Norm2(GradV) < 1e-4) { NIDOPTV[u] = 1; continue; }
double LearnRate = GetStepSizeByLineSearch(u, GradV, GradV, StepAlpha, StepBeta, 5);
if (LearnRate <= 1e-5) { NewNIDV[ui] = -2; continue; }
for (int ci = 0; ci < GradV.Len(); ci++) {
int CID = GradV.GetKey(ci);
double Change = LearnRate * GradV.GetDat(CID);
double NewFuc = CurFU.IsKey(CID)? CurFU.GetDat(CID) + Change : Change;
if (NewFuc <= 0.0) {
CurFU.DelIfKey(CID);
} else {
CurFU.AddDat(CID) = NewFuc;
}
}
CurFU.Defrag();
}
//store changes
NewF[ui] = CurFU;
NewNIDV[ui] = u;
}
}
int NumNoChangeGrad = 0;
int NumNoChangeStepSize = 0;
for (int ui = 0; ui < NewNIDV.Len(); ui++) {
int NewNID = NewNIDV[ui];
if (NewNID == -1) { NumNoChangeGrad++; continue; }
if (NewNID == -2) { NumNoChangeStepSize++; continue; }
for (TIntFltH::TIter CI = F[NewNID].BegI(); CI < F[NewNID].EndI(); CI++) {
SumFV[CI.GetKey()] -= CI.GetDat();
}
}
#pragma omp parallel for
for (int ui = 0; ui < NewNIDV.Len(); ui++) {
int NewNID = NewNIDV[ui];
if (NewNID < 0) { continue; }
F[NewNID] = NewF[ui];
}
for (int ui = 0; ui < NewNIDV.Len(); ui++) {
int NewNID = NewNIDV[ui];
if (NewNID < 0) { continue; }
for (TIntFltH::TIter CI = F[NewNID].BegI(); CI < F[NewNID].EndI(); CI++) {
SumFV[CI.GetKey()] += CI.GetDat();
}
}
// update the nodes who are optimal
for (int ui = 0; ui < NewNIDV.Len(); ui++) {
int NewNID = NewNIDV[ui];
if (NewNID < 0) { continue; }
TUNGraph::TNodeI UI = G->GetNI(NewNID);
NIDOPTV[NewNID] = 0;
for (int e = 0; e < UI.GetDeg(); e++) {
NIDOPTV[UI.GetNbrNId(e)] = 0;
}
}
int OPTCnt = 0;
for (int i = 0; i < NIDOPTV.Len(); i++) { if (NIDOPTV[i] == 1) { OPTCnt++; } }
if (! PlotNm.Empty()) {
printf("\r%d iterations [%s] %d secs", iter * ChunkSize * ChunkNum, ExeTm.GetTmStr(), int(TSecTm::GetCurTm().GetAbsSecs() - StartTm));
if (PrevL > TFlt::Mn) { printf(" (%f) %d g %d s %d OPT", PrevL, NumNoChangeGrad, NumNoChangeStepSize, OPTCnt); }
fflush(stdout);
}
if ((iter - PrevIter) * ChunkSize * ChunkNum >= G->GetNodes()) {
PrevIter = iter;
double CurL = Likelihood(true);
IterLV.Add(TIntFltPr(iter * ChunkSize * ChunkNum, CurL));
printf("\r%d iterations, Likelihood: %f, Diff: %f [%d secs]", iter, CurL, CurL - PrevL, int(time(NULL) - InitTime));
fflush(stdout);
if (CurL - PrevL <= Thres * fabs(PrevL)) {
break;
}
else {
PrevL = CurL;
}
}
}
if (! PlotNm.Empty()) {
printf("\nMLE completed with %d iterations(%s secs)\n", iter, int(TSecTm::GetCurTm().GetAbsSecs() - StartTm));
TGnuPlot::PlotValV(IterLV, PlotNm + ".likelihood_Q");[]
} else {
int TAGMFast::MLEGradAscent(const double& Thres, const int& MaxIter, const TStr PlotNm, const double StepAlpha, const double StepBeta) {
time_t InitTime = time(NULL);
TExeTm ExeTm, CheckTm;
int iter = 0, PrevIter = 0;
TIntFltPrV IterLV;
TUNGraph::TNodeI UI;
double PrevL = TFlt::Mn, CurL = 0.0;
TIntV NIdxV(F.Len(), 0);
for (int i = 0; i < F.Len(); i++) { NIdxV.Add(i); }
IAssert(NIdxV.Len() == F.Len());
TIntFltH GradV;
while(iter < MaxIter) {
NIdxV.Shuffle(Rnd);
for (int ui = 0; ui < F.Len(); ui++, iter++) {
int u = NIdxV[ui]; //
//find set of candidate c (we only need to consider c to which a neighbor of u belongs to)
UI = G->GetNI(u);
TIntSet CIDSet(5 * UI.GetDeg());
for (int e = 0; e < UI.GetDeg(); e++) {
if (HOVIDSV[u].IsKey(UI.GetNbrNId(e))) { continue; }
TIntFltH& NbhCIDH = F[UI.GetNbrNId(e)];
for (TIntFltH::TIter CI = NbhCIDH.BegI(); CI < NbhCIDH.EndI(); CI++) {
CIDSet.AddKey(CI.GetKey());
}
}
for (TIntFltH::TIter CI = F[u].BegI(); CI < F[u].EndI(); CI++) { //remove the community membership which U does not share with its neighbors
if (! CIDSet.IsKey(CI.GetKey())) {
DelCom(u, CI.GetKey());
}
}
if (CIDSet.Empty()) { continue; }
GradientForRow(u, GradV, CIDSet);
if (Norm2(GradV) < 1e-4) { continue; }
double LearnRate = GetStepSizeByLineSearch(u, GradV, GradV, StepAlpha, StepBeta);
if (LearnRate == 0.0) { continue; }
for (int ci = 0; ci < GradV.Len(); ci++) {
int CID = GradV.GetKey(ci);
double Change = LearnRate * GradV.GetDat(CID);
double NewFuc = GetCom(u, CID) + Change;
if (NewFuc <= 0.0) {
DelCom(u, CID);
} else {
AddCom(u, CID, NewFuc);
}
}
if (! PlotNm.Empty() && (iter + 1) % G->GetNodes() == 0) {
IterLV.Add(TIntFltPr(iter, Likelihood(false)));
}
}
printf("\r%d iterations (%f) [%lu sec]", iter, CurL, time(NULL) - InitTime);
fflush(stdout);
if (iter - PrevIter >= 2 * G->GetNodes() && iter > 10000) {
PrevIter = iter;
CurL = Likelihood();
if (PrevL > TFlt::Mn && ! PlotNm.Empty()) {
printf("\r%d iterations, Likelihood: %f, Diff: %f", iter, CurL, CurL - PrevL);
}
fflush(stdout);
if (CurL - PrevL <= Thres * fabs(PrevL)) { break; }
else { PrevL = CurL; }
}
}
printf("\n");
printf("MLE for Lambda completed with %d iterations(%s)\n", iter, ExeTm.GetTmStr());
if (! PlotNm.Empty()) {
TGnuPlot::PlotValV(IterLV, PlotNm + ".likelihood_Q");
}
return iter;
}
