void TGnuPlot::SaveTs(const TIntKdV& KdV, const TStr& FNm, const TStr& HeadLn) {
FILE *F = fopen(FNm.CStr(), "wt");
EAssert(F);
if (! HeadLn.Empty()) fprintf(F, "# %s\n", HeadLn.CStr());
for (int i = 0; i < KdV.Len(); i++) {
fprintf(F, "%d\t%d\n", KdV[i].Key(), KdV[i].Dat()); }
fclose(F);
}
TFIn::TFIn(const TStr& FNm, bool& OpenedP):
TSBase(FNm.CStr()), TSIn(FNm), FileId(NULL), Bf(NULL), BfC(0), BfL(0){
EAssertR(!FNm.Empty(), "Empty file-name.");
FileId=fopen(FNm.CStr(), "rb");
OpenedP=(FileId!=NULL);
if (OpenedP){
Bf=new char[MxBfL]; BfC=BfL=-1; FillBf();}
}
PNmObjBs TNmObjBs::GetFromStrQuV(
const TStrQuV& IdTitleSrcHtmlQuV,
const TSwSetType& SwSetType, const TStr& CustSwSetFNm,
const TStr& NrWordBsFNm, const TStr& WordTypeBsFNm, const TStr& MteFNm,
const int& MxDocs, const bool& DumpP){
// custom-stop-words
PSIn CustSwSetSIn;
if (!CustSwSetFNm.Empty()&&(TFile::Exists(CustSwSetFNm))){
CustSwSetSIn=TFIn::New(CustSwSetFNm);}
// normalized-words
PSIn NrWordBsSIn;
if (!NrWordBsFNm.Empty()&&(TFile::Exists(NrWordBsFNm))){
NrWordBsSIn=TFIn::New(NrWordBsFNm);}
// word-types
PSIn WordTypeBsSIn;
if (!WordTypeBsFNm.Empty()&&(TFile::Exists(WordTypeBsFNm))){
WordTypeBsSIn=TFIn::New(WordTypeBsFNm);}
// create named-objects-base
PNmObjBs NmObjBs=
TNmObjBs::New(SwSetType, CustSwSetSIn, NrWordBsSIn, WordTypeBsSIn, MteFNm);
// traverse compact-documents
for (int DocN=0; DocN<IdTitleSrcHtmlQuV.Len(); DocN++){
if ((MxDocs!=-1)&&(DocN>MxDocs)){break;}
TStr DocNm=IdTitleSrcHtmlQuV[DocN].Val1;
TStr TitleStr=IdTitleSrcHtmlQuV[DocN].Val2;
TStr HtmlStr=IdTitleSrcHtmlQuV[DocN].Val4;
//if (!DocNm.IsStrIn("1998-06-18-z1b9.")){continue;}
if (DumpP){
printf("===============================================\n");
printf("%s\n", DocNm.CStr());
}
// extract candidate named-objects
TStrV CandWordStrV; NmObjBs->ExtrCandWordStrV(HtmlStr, CandWordStrV, DumpP);
// extract final named-objects from candidates
TIntV NmObjIdV; NmObjBs->FilterCandToNmObjIdV(CandWordStrV, NmObjIdV, DumpP);
// create document and add named-objects
NmObjBs->AddDoc(DocNm, "", NmObjIdV);
}
// get merged named-objects
TIntV NewNmObjIdV; NmObjBs->GetMergedNmObj(NewNmObjIdV);
// apply merged named-objects
NmObjBs->PutMergedNmObj(NewNmObjIdV);
// return named-objects-base
return NmObjBs;
}
void TWebTxtBsSrv::AddHitSetChA(
const PTxtBsRes& TxtBsRes, const TStr& RqContTypeStr,
const int& HitSetN, const int& HitSetDocs, const int& StrHitSets,
const PUrlEnv& UrlEnv, TChA& OutChA){
// get hit-set data
TStr PrevUrlStr; TStrPrV NmUrlStrPrV; TStr NextUrlStr;
TxtBsRes->GetHitSet(
HitSetN, HitSetDocs, StrHitSets, HitSetUrlFldNm, UrlEnv,
PrevUrlStr, NmUrlStrPrV, NextUrlStr);
// hit-set header
TStr HitSetHdTpl=GetTplVal(RqContTypeStr, "HitSetHd");
OutChA+=HitSetHdTpl;
// hit-set 'previous'
if (!PrevUrlStr.Empty()){
TStr HitSetPrevTpl=GetTplVal(RqContTypeStr, "HitSetPrev");
HitSetPrevTpl.ChangeStrAll(UrlMacro, PrevUrlStr);
OutChA+=HitSetPrevTpl;
}
// hit-set records
for (int NmN=0; NmN<NmUrlStrPrV.Len(); NmN++){
TStr Nm=NmUrlStrPrV[NmN].Val1;
TStr UrlStr=NmUrlStrPrV[NmN].Val2;
if (UrlStr.Empty()){
// hit-set record with name
TStr HitSetRecTpl=GetTplVal(RqContTypeStr, "HitSetRecNm");
HitSetRecTpl.ChangeStrAll(NmMacro, Nm);
OutChA+=HitSetRecTpl;
} else {
// hit-set record with linked name
TStr HitSetRecTpl=GetTplVal(RqContTypeStr, "HitSetRecLinkedNm");
HitSetRecTpl.ChangeStrAll(NmMacro, Nm);
HitSetRecTpl.ChangeStrAll(UrlMacro, UrlStr);
OutChA+=HitSetRecTpl;
}
}
// hit-set 'next'
if (!NextUrlStr.Empty()){
TStr HitSetNextTpl=GetTplVal(RqContTypeStr, "HitSetNext");
HitSetNextTpl.ChangeStrAll(UrlMacro, NextUrlStr);
OutChA+=HitSetNextTpl;
}
// hit-set footer
TStr HitSetFtTpl=GetTplVal(RqContTypeStr, "HitSetFt");
OutChA+=HitSetFtTpl;
}
/////////////////////////////////////////////////
// Connected Components
void TCnCom::Dump(const TCnComV& CnComV, const TStr& Desc) {
if (! Desc.Empty()) { printf("%s:\n", Desc.CStr()); }
for (int cc = 0; cc < CnComV.Len(); cc++) {
const TIntV& NIdV = CnComV[cc].NIdV;
printf("%d : ", NIdV.Len());
for (int i = 0; i < NIdV.Len(); i++) { printf(" %d", NIdV[i].Val); }
printf("\n");
}
}
void TGnuPlot::SaveEps(const TStr& FNm, const int& FontSz, const TStr& Comment) {
AddCmd(TStr::Fmt("set terminal postscript enhanced eps %d color", FontSz));
AddCmd(TStr::Fmt("set output '%s'", FNm.CStr()));
Pause(false);
CreatePlotFile(Comment.Empty()? Title : Comment);
RunGnuPlot();
MoreCmds.DelLast();
MoreCmds.DelLast();
}
// Inverse participation ratio: normalize EigVec to have L2=1 and then I=sum_k EigVec[i]^4
// see Spectra of "real-world" graphs: Beyond the semicircle law by Farkas, Derenyi, Barabasi and Vicsek
void PlotInvParticipRat(const PUNGraph& Graph, const int& MaxEigVecs, const int& TimeLimit, const TStr& FNmPref, TStr DescStr) {
TFltPrV EigIprV;
GetInvParticipRat(Graph, MaxEigVecs, TimeLimit, EigIprV);
if (DescStr.Empty()) { DescStr = FNmPref; }
if (EigIprV.Empty()) { DescStr+=". FAIL"; EigIprV.Add(TFltPr(-1,-1)); return; }
TGnuPlot::PlotValV(EigIprV, "eigIPR."+FNmPref, TStr::Fmt("%s. G(%d, %d). Largest eig val = %f (%d values)",
DescStr.CStr(), Graph->GetNodes(), Graph->GetEdges(), EigIprV.Last().Val1(), EigIprV.Len()),
"Eigenvalue", "Inverse Participation Ratio of corresponding Eigenvector", gpsLog10Y, false, gpwPoints);
}
void TExpHelp::GetObjHdNmV(const TStr& CatNm, TStrV& ObjHdNmV) const {
ObjHdNmV.Clr();
for (int ObjN=0; ObjN<ObjV.Len(); ObjN++){
TStr ObjCatNm=ObjV[ObjN]->GetCatNm();
TStr ObjHdNm=ObjV[ObjN]->GetHdItem()->GetNm();
if ((CatNm.Empty())||(CatNm=="All")||(CatNm==ObjCatNm)){
ObjHdNmV.AddUnique(ObjHdNm);}
}
ObjHdNmV.Sort();
}
TFFile::TFFile(const TStr& _FPath, const TStr& _FExt, const bool& _RecurseP):
FPathV(), FExtV(), FBaseWc(),
CsImpP(false), RecurseP(_RecurseP), FPathN(0-1),
FFileDesc(TFFileDesc::New()), SubFFile(), CurFNm(), CurFNmN(0-1){
FPathV.Add(TStr::GetNrFPath(_FPath));
if (!_FExt.Empty()){
FExtV.Add(TStr::GetNrFExt(_FExt));
if (!CsImpP){FExtV.Last().ToUc();}
}
}
int TSs::GetFldY(const TStr& FldNm, const TStr& NewFldNm, const int& X) const {
for (int Y=0; Y<GetYLen(); Y++){
if (GetXLen(Y)>X){
if (GetVal(X, Y).GetTrunc()==FldNm){
if (!NewFldNm.Empty()){GetVal(X, Y)=NewFldNm;}
return Y;
}
}
}
return -1;
}
void TGnuPlot::SavePng(const TStr& FNm, const int& SizeX, const int& SizeY, const TStr& Comment, const TStr& Terminal) {
if (Terminal.Empty()) {
//#ifdef GLib_WIN
//#ifndef GLib_MACOSX // The standard GNUPlot for MacOS does not support PNG (Jure: actually version 4.6 DOES!)
AddCmd(TStr::Fmt("set terminal png size %d,%d", SizeX, SizeY));
AddCmd(TStr::Fmt("set output '%s'", FNm.CStr()));
//#else // EPS
//AddCmd("set terminal postscript eps 10 enhanced color");
//AddCmd(TStr::Fmt("set output '%s%s.eps'", FNm.GetFPath().CStr(), FNm.GetFMid().CStr()));
//#endif
} else {
AddCmd(Terminal);
AddCmd(TStr::Fmt("set output '%s'", FNm.CStr()));
}
Pause(false);
CreatePlotFile(Comment.Empty()? Title : Comment);
RunGnuPlot();
MoreCmds.DelLast();
MoreCmds.DelLast();
}
bool TCategorical::Update(const TStr& Val) {
if ((Type == ctOpen) && !Val.Empty()) {
// check if new value
const bool NewP = !ValSet.IsKey(Val);
// remember if new
if (NewP) { ValSet.AddKey(Val); }
// return if we increased dimensionality
return NewP;
}
// otherwise we have fixed dimensionality
return false;
}
//GP.AddFunc("2*x**-2+4", gpwLines, "2*x^-2+4");
int TGnuPlot::AddFunc(const TStr& FuncStr, const TGpSeriesTy& SeriesTy, const TStr& Label, const TStr& Style) {
const int Id = SeriesV.Len();
TGpSeries Plot;
Plot.SeriesTy = SeriesTy;
Plot.Label = Label;
if (! FuncStr.Empty()) { Plot.DataFNm = TStr::Fmt("f%d(x)=%s, f%d(x)", Id, FuncStr.CStr(), Id); }
else { Plot.DataFNm = TStr::Fmt("f%d(x)", Id); }
Plot.XCol = -1;
Plot.WithStyle = Style;
SeriesV.Add(Plot);
return Id;
}
int main(int argc, char* argv[]) {
Env = TEnv(argc, argv, TNotify::StdNotify);
Env.PrepArgs(TStr::Fmt("Motifs. build: %s, %s. Time: %s", __TIME__, __DATE__, TExeTm::GetCurTm()));
TExeTm ExeTm;
Try
const TStr InFNm = Env.GetIfArgPrefixStr("-i:", "../as20graph.txt", "Input directed graph file (single directed edge per line)");
const int MotifSz = Env.GetIfArgPrefixInt("-m:", 3, "Motif size (has to be 3 or 4)");
const bool DrawMotifs = Env.GetIfArgPrefixBool("-d:", true, "Draw motif shapes (requires GraphViz)");
TStr OutFNm = Env.GetIfArgPrefixStr("-o:", "", "Output file prefix");
if (OutFNm.Empty()) { OutFNm = InFNm.GetFMid(); }
EAssert(MotifSz==3 || MotifSz==4);
// load graph
PNGraph G;
if (InFNm.GetFExt().GetLc()==".ungraph") {
TFIn FIn(InFNm); G=TSnap::ConvertGraph<PNGraph>(TUNGraph::Load(FIn), true); }
else if (InFNm.GetFExt().GetLc()==".ngraph") {
TFIn FIn(InFNm); G=TNGraph::Load(FIn); }
else {
G = TSnap::LoadEdgeList<PNGraph>(InFNm, 0, 1); }
bool IsOk = true;
for (int nid = 0; nid < G->GetNodes(); nid++) {
if (! G->IsNode(nid)) { IsOk=false; break; } }
if (! IsOk) {
printf("Nodes of the input graph have to be numbered 0...N-1\nRenumbering nodes...\n");
PNGraph OG = G; G = TNGraph::New();
TGraphEnumUtils::GetNormalizedGraph(OG, G);
}
// G = TSnap::GenRndGnm<PNGraph>(100, Kilo(1));
// count frequency of connected subgraphs in G that have MotifSz nodes
TD34GraphCounter GraphCounter(MotifSz);
TSubGraphEnum<TD34GraphCounter> GraphEnum;
GraphEnum.GetSubGraphs(G, MotifSz, GraphCounter);
FILE *F = fopen(TStr::Fmt("%s-counts.tab", OutFNm.CStr()).CStr(), "wt");
fprintf(F, "MotifId\tNodes\tEdges\tCount\n");
for (int i = 0; i < GraphCounter.Len(); i++) {
const int gid = GraphCounter.GetId(i);
PNGraph SG = GraphCounter.GetGraph(gid);
if (DrawMotifs) {
TGraphViz::Plot(SG, gvlNeato, TStr::Fmt("%s-motif%03d.gif", OutFNm.CStr(), i),
TStr::Fmt("GId:%d Count: %llu", gid, GraphCounter.GetCnt(gid)));
}
fprintf(F, "%d\t%d\t%d\t%llu\n", gid, SG->GetNodes(), SG->GetEdges(), GraphCounter.GetCnt(gid));
}
printf("done.");
fclose(F);
Catch
printf("\nrun time: %s (%s)\n", ExeTm.GetTmStr(), TSecTm::GetCurTm().GetTmStr().CStr());
return 0;
}
void TGnuPlot::SavePng(const TStr& FNm, const int& SizeX, const int& SizeY, const TStr& Comment, const TStr& Terminal) {
if (Terminal.Empty()) {
//#ifdef GLib_WIN
AddCmd(TStr::Fmt("set terminal png small size %d,%d", SizeX, SizeY));
AddCmd(TStr::Fmt("set output '%s'", FNm.CStr()));
/**
#else // EPS
AddCmd("set terminal postscript eps 10 enhanced color");
AddCmd(TStr::Fmt("set output '%s.eps'", FNm.GetFMid().CStr()));
#endif
*/
} else {
AddCmd(Terminal);
AddCmd(TStr::Fmt("set output '%s'", FNm.CStr()));
}
Pause(false);
CreatePlotFile(Comment.Empty()? Title : Comment);
RunGnuPlot();
MoreCmds.DelLast();
MoreCmds.DelLast();
}
void TBowFl::SaveSparseMatlabTxt(const PBowDocBs& BowDocBs,
const PBowDocWgtBs& BowDocWgtBs, const TStr& FNm,
const TStr& CatFNm, const TIntV& _DIdV) {
TIntV DIdV;
if (_DIdV.Empty()) {
BowDocBs->GetAllDIdV(DIdV);
} else {
DIdV = _DIdV;
}
// generate map of row-ids to words
TFOut WdMapSOut(TStr::PutFExt(FNm, ".row-to-word-map.dat"));
for (int WId = 0; WId < BowDocWgtBs->GetWords(); WId++) {
TStr WdStr = BowDocBs->GetWordStr(WId);
WdMapSOut.PutStrLn(TStr::Fmt("%d %s", WId+1, WdStr.CStr()));
}
WdMapSOut.Flush();
// generate map of col-ids to document names
TFOut DocMapSOut(TStr::PutFExt(FNm, ".col-to-docName-map.dat"));
for (int DocN = 0; DocN < DIdV.Len(); DocN++) {
const int DId = DIdV[DocN];
TStr DocNm = BowDocBs->GetDocNm(DId);
DocMapSOut.PutStrLn(TStr::Fmt("%d %d %s", DocN, DId, DocNm.CStr()));
}
DocMapSOut.Flush();
// save documents' sparse vectors
TFOut SOut(FNm);
for (int DocN = 0; DocN < DIdV.Len(); DocN++){
const int DId = DIdV[DocN];
PBowSpV DocSpV = BowDocWgtBs->GetSpV(DId);
const int DocWIds = DocSpV->GetWIds();
for (int DocWIdN=0; DocWIdN<DocWIds; DocWIdN++){
const int WId = DocSpV->GetWId(DocWIdN);
const double WordWgt = DocSpV->GetWgt(DocWIdN);
SOut.PutStrLn(TStr::Fmt("%d %d %.16f", WId+1, DocN+1, WordWgt));
}
}
SOut.Flush();
// save documents' category sparse vectors
if (!CatFNm.Empty()) {
TFOut CatSOut(CatFNm);
for (int DocN = 0; DocN < DIdV.Len(); DocN++){
const int DId = DIdV[DocN];
const int DocCIds = BowDocBs->GetDocCIds(DId);
for (int DocCIdN=0; DocCIdN<DocCIds; DocCIdN++){
const int CId = BowDocBs->GetDocCId(DId, DocCIdN);
const double CatWgt = 1.0;
CatSOut.PutStrLn(TStr::Fmt("%d %d %.16f", CId+1, DocN+1, CatWgt));
}
}
CatSOut.Flush();
}
}
// <last_name>_<first name innitial>
TStr TStrUtil::GetStdName(TStr AuthorName) {
TStr StdName;
AuthorName.ToLc();
AuthorName.ChangeChAll('\n', ' ');
AuthorName.ChangeChAll('.', ' ');
// if there is a number in the name, remove it and everything after it
int i, pos = 0;
while (pos<AuthorName.Len() && (AuthorName[pos]!='#' && !TCh::IsNum(AuthorName[pos]))) {
pos++;
}
if (pos < AuthorName.Len()) {
AuthorName = AuthorName.GetSubStr(0, pos-1).ToTrunc();
}
if (AuthorName.Empty()) {
return TStr::GetNullStr();
}
// replace everything after '('
int b = AuthorName.SearchCh('(');
if (b != -1) {
AuthorName = AuthorName.GetSubStr(0, b-1).ToTrunc();
}
// skip if contains ')'
if (AuthorName .SearchCh(')')!=-1) {
return TStr::GetNullStr();
}
// skip if it is not a name
if (AuthorName .SearchStr("figures")!=-1 || AuthorName .SearchStr("macros")!=-1
|| AuthorName .SearchStr("univ")!=-1 || AuthorName .SearchStr("institute")!=-1) {
return TStr::GetNullStr();
}
// remove all non-letters (latex tags, ...)
TChA NewName;
for (i = 0; i < AuthorName.Len(); i++) {
const char Ch = AuthorName[i];
if (TCh::IsAlpha(Ch) || TCh::IsWs(Ch) || Ch=='-') {
NewName += Ch;
}
}
StdName = NewName;
StdName.ToTrunc();
TStrV AuthNmV;
StdName.SplitOnWs(AuthNmV);
// too short -- not a name
if (! AuthNmV.Empty() && AuthNmV.Last() == "jr") AuthNmV.DelLast();
if (AuthNmV.Len() < 2) return TStr::GetNullStr();
const TStr LastNm = AuthNmV.Last();
if (! TCh::IsAlpha(LastNm[0]) || LastNm.Len() == 1) return TStr::GetNullStr();
IAssert(isalpha(AuthNmV[0][0]));
return TStr::Fmt("%s_%c", LastNm.CStr(), AuthNmV[0][0]);
}
PTransCorpus TTransCorpus::LoadTxt(const TStr& InOrgFNm,
const TStr& InTransFNm, const TStr& InRefTransFNm) {
// open files
PSIn OrgSIn = !InOrgFNm.Empty() ? TFIn::New(InOrgFNm) : PSIn();
PSIn TransSIn = !InTransFNm.Empty() ? TFIn::New(InTransFNm) : PSIn();
PSIn RefTransSIn = !InRefTransFNm.Empty() ? TFIn::New(InRefTransFNm) : PSIn();
// check which are given
const bool IsOrgP = !OrgSIn.Empty();
const bool IsTransP = !TransSIn.Empty();
const bool IsRefTransP = !RefTransSIn.Empty();
// print warnings
if (!IsOrgP) { printf("No original sentences!\n"); }
if (!IsTransP) { printf("No machine translation sentences!\n"); }
if (!IsRefTransP) { printf("No reference translation sentences!\n"); }
// traverse the files and add sentences
PTransCorpus TransCorpus = TTransCorpus::New();
TLnRet OrgLnRet(OrgSIn), TransLnRet(TransSIn), RefTransLnRet(RefTransSIn);
TStr OrgLnStr, TransLnStr, RefTransLnStr; int LnN = 1;
forever {
// try to read next line, otherwise break
if (IsOrgP && !OrgLnRet.NextLn(OrgLnStr)) { break; }
if (IsTransP && !TransLnRet.NextLn(TransLnStr)) { break; }
if (IsRefTransP && !RefTransLnRet.NextLn(RefTransLnStr)) { break; }
// print progress
if (LnN % 100 == 0) { printf(" %7d Sentences\r", LnN); }
// add sentence and translation(s) to the corpus
if (!IsOrgP) {
TransCorpus->AddSentenceNoOrg(LnN, TransLnStr, RefTransLnStr);
} else if (!IsTransP) {
TransCorpus->AddSentenceNoTrans(LnN, OrgLnStr, RefTransLnStr);
} else { IAssert(IsRefTransP);
TransCorpus->AddSentence(LnN, OrgLnStr, TransLnStr, RefTransLnStr);
}
// next sentence :-)
LnN++;
} printf("\n");
// finish
return TransCorpus;
}
void PlotSngVec(const PNGraph& Graph, const TStr& FNmPref, TStr DescStr) {
TFltV LeftSV, RightSV;
TSnap::GetSngVec(Graph, LeftSV, RightSV);
LeftSV.Sort(false);
RightSV.Sort(false);
TFltV BinV;
if (DescStr.Empty()) { DescStr = FNmPref; }
TGUtil::MakeExpBins(LeftSV, BinV, 1.01);
TGnuPlot::PlotValV(BinV, "sngVecL."+FNmPref, TStr::Fmt("%s. G(%d, %d). Left signular vector",
DescStr.CStr(), Graph->GetNodes(), Graph->GetEdges()), "Rank", "Component of left singular vector", gpsLog10XY, false, gpwLinesPoints);
TGnuPlot::PlotValV(BinV, "sngVecL."+FNmPref, TStr::Fmt("%s. G(%d, %d). Right signular vector",
DescStr.CStr(), Graph->GetNodes(), Graph->GetEdges()), "Rank", "Component of right singular vector", gpsLog10XY, false, gpwLinesPoints);
}
void TCnCom::SaveTxt(const TCnComV& CnComV, const TStr& FNm, const TStr& Desc) {
FILE *F = fopen(FNm.CStr(), "wt");
if (! Desc.Empty()) { fprintf(F, "# %s\n", Desc.CStr()); }
fprintf(F, "# Connected Components:\t%d\n", CnComV.Len());
fprintf(F, "# Connected components (format: <Size>\\t<NodeId1>\\t<NodeId2>...)\n");
for (int cc = 0; cc < CnComV.Len(); cc++) {
const TIntV& NIdV = CnComV[cc].NIdV;
fprintf(F, "%d", NIdV.Len());
for (int i = 0; i < NIdV.Len(); i++) { fprintf(F, "\t%d", NIdV[i].Val); }
fprintf(F, "\n");
}
fclose(F);
}
// Gradient descent for p_c while fixing community affiliation graph (CAG).
int TAGMFit::MLEGradAscentGivenCAG(const double& Thres, const int& MaxIter, const TStr PlotNm) {
int Edges = G->GetEdges();
TExeTm ExeTm;
TFltV GradV(LambdaV.Len());
int iter = 0;
TIntFltPrV IterLV, IterGradNormV;
double GradCutOff = 1000;
for (iter = 0; iter < MaxIter; iter++) {
GradLogLForLambda(GradV); //if gradient is going out of the boundary, cut off
for (int i = 0; i < LambdaV.Len(); i++) {
if (GradV[i] < -GradCutOff) { GradV[i] = -GradCutOff; }
if (GradV[i] > GradCutOff) { GradV[i] = GradCutOff; }
if (LambdaV[i] <= MinLambda && GradV[i] < 0) { GradV[i] = 0.0; }
if (LambdaV[i] >= MaxLambda && GradV[i] > 0) { GradV[i] = 0.0; }
}
double Alpha = 0.15, Beta = 0.2;
if (Edges > Kilo(100)) { Alpha = 0.00015; Beta = 0.3;}
double LearnRate = GetStepSizeByLineSearchForLambda(GradV, GradV, Alpha, Beta);
if (TLinAlg::Norm(GradV) < Thres) { break; }
for (int i = 0; i < LambdaV.Len(); i++) {
double Change = LearnRate * GradV[i];
LambdaV[i] += Change;
if(LambdaV[i] < MinLambda) { LambdaV[i] = MinLambda;}
if(LambdaV[i] > MaxLambda) { LambdaV[i] = MaxLambda;}
}
if (! PlotNm.Empty()) {
double L = Likelihood();
IterLV.Add(TIntFltPr(iter, L));
IterGradNormV.Add(TIntFltPr(iter, TLinAlg::Norm(GradV)));
}
}
if (! PlotNm.Empty()) {
TGnuPlot::PlotValV(IterLV, PlotNm + ".likelihood_Q");
TGnuPlot::PlotValV(IterGradNormV, PlotNm + ".gradnorm_Q");
printf("MLE for Lambda completed with %d iterations(%s)\n",iter,ExeTm.GetTmStr());
}
return iter;
}
TFIn::TFIn(const TStr& FNm, bool& OpenedP, const bool IgnoreBOMIfExistsP):
TSBase(FNm.CStr()), TSIn(FNm), FileId(NULL), Bf(NULL), BfC(0), BfL(0){
EAssertR(!FNm.Empty(), "Empty file-name.");
FileId=fopen(FNm.CStr(), "rb");
OpenedP=(FileId!=NULL);
if (OpenedP){
Bf=new char[MxBfL]; BfC=BfL=-1; FillBf();
if (IgnoreBOMIfExistsP && BfL >= 3) {
// https://en.wikipedia.org/wiki/Byte_order_mark
if (Bf[0] == (char)0xEF && Bf[1] == (char)0xBB && Bf[2] == (char)0xBF)
BfC = 3;
}
}
}
void TWebFetchBlocking::GetWebPg(const PHttpRq& HttpRq, bool& Ok, TStr& MsgStr,
PWebPg& WebPg, const PNotify& Notify, const TStr& ProxyStr, const TStr& UserAgentStr){
TNotify::OnNotify(Notify, ntInfo, TStr("Fetching: ")+HttpRq->GetUrl()->GetUrlStr());
TWebFetchBlocking WebFetch;
if (!UserAgentStr.Empty()) { WebFetch.PutUserAgentStr(UserAgentStr); }
WebFetch.PutProxyStr(ProxyStr);
WebFetch.FetchHttpRq(HttpRq);
TLoop::Ref();
TLoop::Run();
Ok=WebFetch.Ok; MsgStr=WebFetch.MsgStr; WebPg=WebFetch.WebPg;
if (!Ok){
TNotify::OnNotify(Notify, ntInfo, TStr("Fetching Error: ["+MsgStr+"]"));
}
}
int TSs::GetFldX(const TStr& FldNm, const TStr& NewFldNm, const int& Y) const {
if (GetYLen()>Y){
int XLen=GetXLen(Y);
for (int X=0; X<XLen; X++){
if (GetVal(X, Y).GetTrunc()==FldNm){
if (!NewFldNm.Empty()){GetVal(X, Y)=NewFldNm;}
return X;
}
}
return -1;
} else {
return -1;
}
}
int main(int argc, char* argv[]) {
Env = TEnv(argc, argv, TNotify::StdNotify);
Env.PrepArgs(TStr::Fmt("Clique Percolation Method. build: %s, %s. Time: %s", __TIME__, __DATE__, TExeTm::GetCurTm()));
TExeTm ExeTm;
Try
const TStr InFNm = Env.GetIfArgPrefixStr("-i:", "../as20graph.txt", "Input undirected graph file (single directed edge per line)");
const int OverlapSz = Env.GetIfArgPrefixInt("-k:", 2, "Min clique overlap");
TStr OutFNm = Env.GetIfArgPrefixStr("-o:", "", "Output file prefix");
if (OutFNm.Empty()) { OutFNm = InFNm.GetFMid(); }
PUNGraph G;
if (InFNm == "DEMO") { // small demo graph
G = TUNGraph::New();
for (int i = 1; i < 8; i++) { G->AddNode(i); }
G->AddEdge(1,2);
G->AddEdge(2,3); G->AddEdge(2,4);
G->AddEdge(3,4);
G->AddEdge(4,5); G->AddEdge(4,7);
G->AddEdge(5,6); G->AddEdge(5,7);
G->AddEdge(6,7);
// draw the small graph using GraphViz
TSnap::DrawGViz(G, gvlNeato, "small_graph.png", "", true);
}
// load graph
else if (InFNm.GetFExt().GetLc()==".ungraph") {
TFIn FIn(InFNm); G=TUNGraph::Load(FIn); }
else if (InFNm.GetFExt().GetLc()==".ngraph") {
TFIn FIn(InFNm); G=TSnap::ConvertGraph<PUNGraph>(TNGraph::Load(FIn), false); }
else {
G = TSnap::LoadEdgeList<PUNGraph>(InFNm, 0, 1); }
// find communities
TVec<TIntV> CmtyV;
TCliqueOverlap::GetCPMCommunities(G, OverlapSz+1, CmtyV);
// save result
FILE *F = fopen(TStr::Fmt("cpm-%s.txt", OutFNm.CStr()).CStr(), "wt");
fprintf(F, "# %d Overlapping Clique Percolation Communities (min clique overlap %d)\n", CmtyV.Len(), OverlapSz);
fprintf(F, "# Each line contains nodes belonging to the same community community\n");
for (int i = 0; i < CmtyV.Len(); i++) {
fprintf(F, "%d", CmtyV[i][0].Val);
for (int j = 1; j < CmtyV[i].Len(); j++) {
fprintf(F, "\t%d", CmtyV[i][j].Val);
}
fprintf(F, "\n");
}
Catch
printf("\nrun time: %s (%s)\n", ExeTm.GetTmStr(), TSecTm::GetCurTm().GetTmStr().CStr());
return 0;
}
PNmObjBs TNmObjBs::GetFromCpd(
const TStr& CpdFNm,
const TSwSetType& SwSetType, const TStr& CustSwSetFNm,
const TStr& NrWordBsFNm, const TStr& WordTypeBsFNm, const TStr& MteFNm,
const int& MxDocs, const bool& DumpP){
// cpd file
PSIn CpdSIn=TFIn::New(CpdFNm);
// custom-stop-words
PSIn CustSwSetSIn;
if (!CustSwSetFNm.Empty()&&(TFile::Exists(CustSwSetFNm))){
CustSwSetSIn=TFIn::New(CustSwSetFNm);}
// normalized-words
PSIn NrWordBsSIn;
if (!NrWordBsFNm.Empty()&&(TFile::Exists(NrWordBsFNm))){
NrWordBsSIn=TFIn::New(NrWordBsFNm);}
// word-types
PSIn WordTypeBsSIn;
if (!WordTypeBsFNm.Empty()&&(TFile::Exists(WordTypeBsFNm))){
WordTypeBsSIn=TFIn::New(WordTypeBsFNm);}
return TNmObjBs::GetFromCpd(
CpdSIn, SwSetType, CustSwSetSIn, NrWordBsSIn, WordTypeBsSIn, MteFNm,
MxDocs, DumpP);
}
void TGStat::Plot(const TGStatDistr& Distr, const TStr& FNmPref, TStr Desc, bool PowerFit) const {
if (Desc.Empty()) Desc = FNmPref.GetUc();
if (! HasDistr(Distr) || Distr==gsdUndef || Distr==gsdMx) { return; }
TPlotInfo Info = GetPlotInfo(Distr);
TGnuPlot GnuPlot(Info.Val1+TStr(".")+FNmPref, TStr::Fmt("%s. G(%d, %d)", Desc.CStr(), GetNodes(),GetEdges()));
GnuPlot.SetXYLabel(Info.Val2, Info.Val3);
GnuPlot.SetScale(Info.Val4);
const int plotId = GnuPlot.AddPlot(GetDistr(Distr), gpwLinesPoints, "");
if (PowerFit) { GnuPlot.AddPwrFit(plotId, gpwLines); }
#ifdef GLib_MACOSX
GnuPlot.SaveEps();
#else
GnuPlot.SavePng();
#endif
}
void TGStatVec::ImposeDistr(const TGStatDistr& Distr, const TStr& FNmPref, TStr Desc, const bool& ExpBin,
const bool& PowerFit, const TGpSeriesTy& PlotWith, const TStr& Style) const {
if (Desc.Empty()) Desc = FNmPref.GetUc();
if (! At(0)->HasDistr(Distr) || Distr==gsdUndef || Distr==gsdMx) { return; }
TGStat::TPlotInfo Info = At(0)->GetPlotInfo(Distr);
TGnuPlot GnuPlot(Info.Val1+TStr(".")+FNmPref, TStr::Fmt("%s. G(%d, %d) --> G(%d, %d)", Desc.CStr(),
At(0)->GetNodes(), At(0)->GetEdges(), Last()->GetNodes(), Last()->GetEdges()));
GnuPlot.SetXYLabel(Info.Val2, Info.Val3);
GnuPlot.SetScale(Info.Val4);
int plotId;
for (int at = 0; at < Len(); at++) {
TStr Legend = At(at)->GetNm();
if (Legend.Empty()) { Legend = At(at)->GetTmStr(); }
if (! ExpBin) {
plotId = GnuPlot.AddPlot(At(at)->GetDistr(Distr), PlotWith, Legend, Style); }
else {
TFltPrV ExpBinV;
TGnuPlot::MakeExpBins(At(at)->GetDistr(Distr), ExpBinV, 2, 0);
plotId = GnuPlot.AddPlot(ExpBinV, PlotWith, Legend, Style);
}
if (PowerFit) { GnuPlot.AddPwrFit(plotId, gpwLines); }
}
GnuPlot.SavePng();
}
void TGraphKey::SaveGViz(const TStr& OutFNm, const TStr& Desc, const TStr& NodeAttrs, const int& Size) const {
FILE *F = fopen(OutFNm.CStr(), "wt");
fprintf(F, "/*****\n");
fprintf(F, " Graph (%d, %d)\n", GetNodes(), GetEdges());
//if (! Desc.Empty()) fprintf(F, " %s\n", Desc.CStr());
fprintf(F, "*****/\n\n");
fprintf(F, "digraph G {\n");
if (Size != -1) fprintf(F, " size=\"%d,%d\";\n", Size, Size);
fprintf(F, " graph [splines=true overlap=false]\n"); //size=\"12,10\" ratio=fill
if (NodeAttrs.Empty()) fprintf(F, " node [shape=ellipse, width=0.3, height=0.3]\n");
else fprintf(F, " node [shape=ellipse, %s]\n", NodeAttrs.CStr());
if (! EdgeV.Empty()) {
for (int e = 0; e < EdgeV.Len(); e++) {
fprintf(F, " %d -> %d;\n", EdgeV[e].Val1(), EdgeV[e].Val2()); }
} else {
for (int n = 0; n < Nodes; n++) { fprintf(F, " %d;\n", n); }
}
if (! Desc.Empty()) {
fprintf(F, " label = \"\\n%s\\n\";", Desc.CStr());
fprintf(F, " fontsize=24;\n");
}
fprintf(F, "}\n");
fclose(F);
}
PAmazonFetch TAmazonFetch::New(const TStr& FNm){
// create fetch
PAmazonFetch AmazonFetch=New();
// check & load file
if (!FNm.Empty()){
PSIn XmlSIn=TFIn::New(FNm);
forever {
PXmlDoc XmlDoc=TXmlDoc::LoadTxt(XmlSIn);
if (!XmlDoc->IsOk()){break;}
TStr ItemId=XmlDoc->GetTagTok("AmazonItem|ItemId")->GetTokStr(false);
AmazonFetch->AddItem(ItemId);
if (AmazonFetch->GetItems()%1000==0){
printf("%d items loaded\r", AmazonFetch->GetItems());}
}
printf("%d items loaded\n", AmazonFetch->GetItems());
}