/// save bipartite community affiliation into gexf file
void TAGMUtil::SaveBipartiteGephi(const TStr& OutFNm, const TIntV& NIDV, const TVec<TIntV>& CmtyVV, const double MaxSz, const double MinSz, const TIntStrH& NIDNameH, const THash<TInt, TIntTr>& NIDColorH, const THash<TInt, TIntTr>& CIDColorH ) {
/// Plot bipartite graph
if (CmtyVV.Len() == 0) {
return;
}
double NXMin = 0.1, YMin = 0.1, NXMax = 250.00, YMax = 30.0;
double CXMin = 0.3 * NXMax, CXMax = 0.7 * NXMax;
double CStep = (CXMax - CXMin) / (double) CmtyVV.Len(), NStep = (NXMax - NXMin) / (double) NIDV.Len();
THash<TInt,TIntV> NIDComVH;
TAGMUtil::GetNodeMembership(NIDComVH, CmtyVV);
FILE* F = fopen(OutFNm.CStr(), "wt");
fprintf(F, "<?xml version='1.0' encoding='UTF-8'?>\n");
fprintf(F, "<gexf xmlns='http://www.gexf.net/1.2draft' xmlns:viz='http://www.gexf.net/1.1draft/viz' xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' xsi:schemaLocation='http://www.gexf.net/1.2draft http://www.gexf.net/1.2draft/gexf.xsd' version='1.2'>\n");
fprintf(F, "\t<graph mode='static' defaultedgetype='directed'>\n");
fprintf(F, "\t\t<nodes>\n");
for (int c = 0; c < CmtyVV.Len(); c++) {
int CID = c;
double XPos = c * CStep + CXMin;
TIntTr Color = CIDColorH.IsKey(CID)? CIDColorH.GetDat(CID) : TIntTr(120, 120, 120);
fprintf(F, "\t\t\t<node id='C%d' label='C%d'>\n", CID, CID);
fprintf(F, "\t\t\t\t<viz:color r='%d' g='%d' b='%d'/>\n", Color.Val1.Val, Color.Val2.Val, Color.Val3.Val);
fprintf(F, "\t\t\t\t<viz:size value='%.3f'/>\n", MaxSz);
fprintf(F, "\t\t\t\t<viz:shape value='square'/>\n");
fprintf(F, "\t\t\t\t<viz:position x='%f' y='%f' z='0.0'/>\n", XPos, YMax);
fprintf(F, "\t\t\t</node>\n");
}
for (int u = 0; u < NIDV.Len(); u++) {
int NID = NIDV[u];
TStr Label = NIDNameH.IsKey(NID)? NIDNameH.GetDat(NID): "";
double Size = MinSz;
double XPos = NXMin + u * NStep;
TIntTr Color = NIDColorH.IsKey(NID)? NIDColorH.GetDat(NID) : TIntTr(120, 120, 120);
double Alpha = 1.0;
fprintf(F, "\t\t\t<node id='%d' label='%s'>\n", NID, Label.CStr());
fprintf(F, "\t\t\t\t<viz:color r='%d' g='%d' b='%d' a='%.1f'/>\n", Color.Val1.Val, Color.Val2.Val, Color.Val3.Val, Alpha);
fprintf(F, "\t\t\t\t<viz:size value='%.3f'/>\n", Size);
fprintf(F, "\t\t\t\t<viz:shape value='square'/>\n");
fprintf(F, "\t\t\t\t<viz:position x='%f' y='%f' z='0.0'/>\n", XPos, YMin);
fprintf(F, "\t\t\t</node>\n");
}
fprintf(F, "\t\t</nodes>\n");
fprintf(F, "\t\t<edges>\n");
int EID = 0;
for (int u = 0; u < NIDV.Len(); u++) {
int NID = NIDV[u];
if (NIDComVH.IsKey(NID)) {
for (int c = 0; c < NIDComVH.GetDat(NID).Len(); c++) {
int CID = NIDComVH.GetDat(NID)[c];
fprintf(F, "\t\t\t<edge id='%d' source='C%d' target='%d'/>\n", EID++, CID, NID);
}
}
}
fprintf(F, "\t\t</edges>\n");
fprintf(F, "\t</graph>\n");
fprintf(F, "</gexf>\n");
}
// wrong reading of string attributes
void TTable::BuildGraphTopology(PNEAGraph& Graph, THash<TFlt, TInt>& FSrNodeMap, THash<TFlt, TInt>& FDsNodeMap) {
TYPE SrCT = GetColType(SrcCol);
TInt SrIdx = GetColIdx(SrcCol);
TYPE DsCT = GetColType(DstCol);
TInt DsIdx = GetColIdx(DstCol);
TInt SrcCnt = 0;
TInt DstCnt = 0;
for(TRowIterator RowI = BegRI(); RowI < EndRI(); RowI++) {
if (SrCT == INT && DsCT == INT) {
Graph->AddNode(IntCols[SrIdx][RowI.GetRowIdx()]);
Graph->AddNode(IntCols[DsIdx][RowI.GetRowIdx()]);
Graph->AddEdge(IntCols[SrIdx][RowI.GetRowIdx()], IntCols[DsIdx][RowI.GetRowIdx()], RowI.GetRowIdx());
} else if (SrCT == INT && DsCT == FLT) {
Graph->AddNode(IntCols[SrIdx][RowI.GetRowIdx()]);
TFlt val = FltCols[DsIdx][RowI.GetRowIdx()];
if (!FDsNodeMap.IsKey(val)) {
FDsNodeMap.AddDat(val, DstCnt++);
}
Graph->AddNode(FDsNodeMap.GetDat(val));
Graph->AddEdge(IntCols[SrIdx][RowI.GetRowIdx()], FDsNodeMap.GetDat(val));
} else if (SrCT == INT && DsCT == STR) {
Graph->AddNode(IntCols[SrIdx][RowI.GetRowIdx()]);
Graph->AddNode(StrColMaps[DsIdx][RowI.GetRowIdx()]);
Graph->AddEdge(IntCols[SrIdx][RowI.GetRowIdx()], StrColMaps[DsIdx][RowI.GetRowIdx()], RowI.GetRowIdx());
} else if (SrCT == FLT && DsCT == INT) {
Graph->AddNode(IntCols[DsIdx][RowI.GetRowIdx()]);
TFlt val = FltCols[SrIdx][RowI.GetRowIdx()];
if (!FSrNodeMap.IsKey(val)) {
FSrNodeMap.AddDat(val, SrcCnt++);
}
Graph->AddNode(FSrNodeMap.GetDat(val));
Graph->AddEdge(FSrNodeMap.GetDat(val), IntCols[SrIdx][RowI.GetRowIdx()], RowI.GetRowIdx());
} else if (SrCT == FLT && DsCT == STR) {
Graph->AddNode(StrColMaps[DsIdx][RowI.GetRowIdx()]);
TFlt val = FltCols[SrIdx][RowI.GetRowIdx()];
if (!FSrNodeMap.IsKey(val)) {
FSrNodeMap.AddDat(val, SrcCnt++);
}
Graph->AddNode(FSrNodeMap.GetDat(val));
Graph->AddEdge(FSrNodeMap.GetDat(val), IntCols[SrIdx][RowI.GetRowIdx()], RowI.GetRowIdx());
} else if (SrCT == FLT && DsCT == FLT) {
TFlt val = FltCols[SrIdx][RowI.GetRowIdx()];
if (!FSrNodeMap.IsKey(val)) {
FSrNodeMap.AddDat(val, SrcCnt++);
}
Graph->AddNode(FSrNodeMap.GetDat(val));
val = FltCols[DsIdx][RowI.GetRowIdx()];
if (!FDsNodeMap.IsKey(val)) {
FDsNodeMap.AddDat(val, DstCnt++);
}
Graph->AddNode(FDsNodeMap.GetDat(val));
Graph->AddEdge(FSrNodeMap.GetDat(val), FDsNodeMap.GetDat(val), RowI.GetRowIdx());
}
}
}
TFltIntIntTr FindMxQEdge() {
while (true) {
if (MxQHeap.Empty()) { break; }
const TFltIntIntTr TopQ = MxQHeap.PopHeap();
if (! CmtyQH.IsKey(TopQ.Val2) || ! CmtyQH.IsKey(TopQ.Val3)) { continue; }
if (TopQ.Val1!=CmtyQH.GetDat(TopQ.Val2).GetMxQ() && TopQ.Val1!=CmtyQH.GetDat(TopQ.Val3).GetMxQ()) { continue; }
return TopQ;
}
return TFltIntIntTr(-1, -1, -1);
}
// YES I COPIED AND PASTED CODE my section leader would be so ashamed :D
void LSH::MinHash(THash<TMd5Sig, TIntSet>& ShingleToQuoteIds,
TVec<THash<TIntV, TIntSet> >& SignatureBandBuckets) {
TRnd RandomGenerator; // TODO: make this "more random" by incorporating time
for (int i = 0; i < NumBands; ++i) {
THash < TInt, TIntV > Inverted; // (QuoteID, QuoteSignatureForBand)
THash < TIntV, TIntSet > BandBuckets; // (BandSignature, QuoteIDs)
for (int j = 0; j < BandSize; ++j) {
// Create new signature
TVec < TMd5Sig > Signature;
ShingleToQuoteIds.GetKeyV(Signature);
Signature.Shuffle(RandomGenerator);
// Place in bucket - not very efficient
int SigLen = Signature.Len();
for (int k = 0; k < SigLen; ++k) {
TIntSet CurSet = ShingleToQuoteIds.GetDat(Signature[k]);
for (TIntSet::TIter l = CurSet.BegI(); l < CurSet.EndI(); l++) {
TInt Key = l.GetKey();
if (Inverted.IsKey(Key)) {
TIntV CurSignature = Inverted.GetDat(Key);
if (CurSignature.Len() <= j) {
CurSignature.Add(k);
Inverted.AddDat(Key, CurSignature);
}
} else {
TIntV NewSignature;
NewSignature.Add(k);
Inverted.AddDat(Key, NewSignature);
}
}
}
}
TIntV InvertedKeys;
Inverted.GetKeyV(InvertedKeys);
TInt InvertedLen = InvertedKeys.Len();
for (int k = 0; k < InvertedLen; ++k) {
TIntSet Bucket;
TIntV Signature = Inverted.GetDat(InvertedKeys[k]);
if (BandBuckets.IsKey(Signature)) {
Bucket = BandBuckets.GetDat(Signature);
}
Bucket.AddKey(InvertedKeys[k]);
BandBuckets.AddDat(Signature, Bucket);
}
SignatureBandBuckets.Add(BandBuckets);
Err("%d out of %d band signatures computed\n", i + 1, NumBands);
}
Err("Minhash step complete!\n");
}
void LSH::ElCheapoHashing(TQuoteBase *QuoteBase, TInt ShingleLen,
THash<TMd5Sig, TIntSet>& ShingleToQuoteIds) {
fprintf(stderr, "Hashing shingles the el cheapo way...\n");
TIntV QuoteIds;
QuoteBase->GetAllQuoteIds(QuoteIds);
for (int qt = 0; qt < QuoteIds.Len(); qt++) {
if (qt % 1000 == 0) {
fprintf(stderr, "%d out of %d completed\n", qt, QuoteIds.Len());
}
TQuote Q;
QuoteBase->GetQuote(QuoteIds[qt], Q);
// Put x-character (or x-word) shingles into hash table; x is specified by ShingleLen parameter
TStr QContentStr;
Q.GetParsedContentString(QContentStr);
TChA QContentChA = TChA(QContentStr);
for (int i = 0; i < QContentChA.Len() - ShingleLen + 1; i++) {
TChA ShingleChA = TChA();
for (int j = 0; j < ShingleLen; j++) {
ShingleChA.AddCh(QContentChA.GetCh(i + j));
}
TStr Shingle = TStr(ShingleChA);
const TMd5Sig ShingleMd5(Shingle);
TIntSet ShingleQuoteIds;
if (ShingleToQuoteIds.IsKey(ShingleMd5)) {
ShingleQuoteIds = ShingleToQuoteIds.GetDat(ShingleMd5);
}
ShingleQuoteIds.AddKey(QuoteIds[qt]);
ShingleToQuoteIds.AddDat(ShingleMd5, ShingleQuoteIds);
}
}
Err("Done with el cheapo hashing!\n");
}
/// Shingles by words
void LSH::HashShinglesOfClusters(TQuoteBase *QuoteBase,
TClusterBase *ClusterBase, TIntV& ClusterIds, TInt ShingleLen,
THash<TMd5Sig, TIntV>& ShingleToClusterIds) {
Err("Hashing shingles of clusters...\n");
for (int i = 0; i < ClusterIds.Len(); i++) {
if (i % 1000 == 0) {
fprintf(stderr, "%d out of %d completed\n", i, ClusterIds.Len());
}
TCluster C;
ClusterBase->GetCluster(ClusterIds[i], C);
//fprintf(stderr, "%d vs. %d\n", ClusterIds[i].Val, C.GetId().Val);
// Put x-word shingles into hash table; x is specified by ShingleLen parameter
THashSet < TMd5Sig > CHashedShingles;
GetHashedShinglesOfCluster(QuoteBase, C, ShingleLen, CHashedShingles);
for (THashSet<TMd5Sig>::TIter Hash = CHashedShingles.BegI();
Hash < CHashedShingles.EndI(); Hash++) {
TIntV ShingleClusterIds;
if (ShingleToClusterIds.IsKey(*Hash)) {
ShingleClusterIds = ShingleToClusterIds.GetDat(*Hash);
}
ShingleClusterIds.Add(ClusterIds[i]);
ShingleToClusterIds.AddDat(*Hash, ShingleClusterIds);
}
}
Err("Done hashing!\n");
}
void TSockSys::DelIfSockTimer(const uint64& SockId) {
if (SockIdToTimerHndH.IsKey(SockId)) {
// get timer handle
uv_timer_t* TimerHnd = SockIdToTimerHndH.GetDat(SockId);
// stop the timer
uv_timer_stop(TimerHnd);
// remove shortcuts
SockIdToTimerHndH.DelKey(SockId);
TimerHndToSockIdH.DelKey((uint64)TimerHnd);
// remove shortcuts
}
}
void TNodeJsRf24Radio::set(const v8::FunctionCallbackInfo<v8::Value>& Args) {
v8::Isolate* Isolate = v8::Isolate::GetCurrent();
v8::HandleScope HandleScope(Isolate);
TNodeJsRf24Radio* JsRadio = ObjectWrap::Unwrap<TNodeJsRf24Radio>(Args.Holder());
if (Args.Length() == 0) { return; }
const PJsonVal ArgVal = TNodeJsUtil::GetArgJson(Args, 0);
bool Success = true;
if (ArgVal->IsArr()) {
THash<TInt, TIntPrV> NodeIdValIdValPrVH;
for (int ArgN = 0; ArgN < ArgVal->GetArrVals(); ArgN++) {
const PJsonVal& ValJson = ArgVal->GetArrVal(ArgN);
const TStr& ValNm = ValJson->GetObjStr("sensorId");
const int& Val = ValJson->GetObjInt("value");
const TIntPr& NodeIdValIdPr = JsRadio->ValNmNodeIdValIdPrH.GetDat(ValNm);
const uint16 NodeId = NodeIdValIdPr.Val1;
const int ValId = NodeIdValIdPr.Val2;
if (!NodeIdValIdValPrVH.IsKey(NodeId)) { NodeIdValIdValPrVH.AddDat(NodeId); }
TIntPrV& ValIdValPrV = NodeIdValIdValPrVH.GetDat(NodeId);
ValIdValPrV.Add(TIntPr(ValId, Val));
}
int KeyId = NodeIdValIdValPrVH.FFirstKeyId();
while (NodeIdValIdValPrVH.FNextKeyId(KeyId)) {
const uint16 NodeId = NodeIdValIdValPrVH.GetKey(KeyId);
const TIntPrV& ValIdValPrV = NodeIdValIdValPrVH[KeyId];
Success &= JsRadio->Radio.Set(NodeId, ValIdValPrV);
}
} else {
const TStr& ValueNm = ArgVal->GetObjStr("sensorId");
const int Val = ArgVal->GetObjInt("value");
const TIntPr& NodeIdValIdPr = JsRadio->ValNmNodeIdValIdPrH.GetDat(ValueNm);
const uint16 NodeId = (uint16) NodeIdValIdPr.Val1;
const int ValId = NodeIdValIdPr.Val2;
Success = JsRadio->Radio.Set(NodeId, ValId, Val);
}
Args.GetReturnValue().Set(v8::Boolean::New(Isolate, Success));
}
/// For every quote, add it to corresponding bucket for each hashed x-character shingle of the quote
// (Shingles by characters)
void LSH::HashShingles(TQuoteBase *QuoteBase, TClusterBase *CB, TInt ShingleLen,
THash<TMd5Sig, TShingleIdSet>& ShingleToQuoteIds) {
Err("Hashing shingles...\n");
TIntV QuoteIds;
QuoteBase->GetAllQuoteIds(QuoteIds);
for (int qt = 0; qt < QuoteIds.Len(); qt++) {
if (qt % 1000 == 0) {
fprintf(stderr, "%d out of %d completed\n", qt, QuoteIds.Len());
}
if (CB->IsQuoteInArchivedCluster(QuoteIds[qt]))
continue;
TQuote Q;
QuoteBase->GetQuote(QuoteIds[qt], Q);
// Put x-character (or x-word) shingles into hash table; x is specified by ShingleLen parameter
TStr QContentStr;
Q.GetParsedContentString(QContentStr);
TChA QContentChA = TChA(QContentStr);
int CurWord = 0;
for (int i = 0; i < QContentChA.Len() - ShingleLen + 1; i++) {
TChA ShingleChA = TChA();
for (int j = 0; j < ShingleLen; j++) {
ShingleChA.AddCh(QContentChA.GetCh(i + j));
}
TStr Shingle = TStr(ShingleChA);
const TMd5Sig ShingleMd5(Shingle);
TShingleIdSet ShingleQuoteIds;
if (ShingleToQuoteIds.IsKey(ShingleMd5)) {
ShingleQuoteIds = ShingleToQuoteIds.GetDat(ShingleMd5);
}
for (int j = CurWord; j > CurWord - WordWindow && j >= 0; j--) {
ShingleQuoteIds.AddKey(TShingleId(QuoteIds[qt], j));
}
ShingleToQuoteIds.AddDat(ShingleMd5, ShingleQuoteIds);
// up the current word index if we see a space
if (QContentChA.GetCh(i + ShingleLen - 1) == ' ') {
CurWord++;
}
}
}
Err("Done hashing!\n");
}
int main(int argc, char *argv[]) {
TStr BaseString = "/lfs/1/tmp/curis/week/QBDB.bin";
TFIn BaseFile(BaseString);
TQuoteBase *QB = new TQuoteBase;
TDocBase *DB = new TDocBase;
QB->Load(BaseFile);
DB->Load(BaseFile);
TIntV QuoteIds;
QB->GetAllQuoteIds(QuoteIds);
int NumQuotes = QuoteIds.Len();
THash<TInt, TStrSet> PeakCounts;
for (int i = 0; i < NumQuotes; i++) {
TQuote CurQuote;
if (QB->GetQuote(QuoteIds[i], CurQuote)) {
TVec<TSecTm> Peaks;
CurQuote.GetPeaks(DB, Peaks);
TStr QuoteString;
CurQuote.GetParsedContentString(QuoteString);
TStrSet StringSet;
if (PeakCounts.IsKey(Peaks.Len())) {
StringSet = PeakCounts.GetDat(Peaks.Len());
}
StringSet.AddKey(QuoteString);
PeakCounts.AddDat(Peaks.Len(), StringSet);
}
}
TIntV PeakCountKeys;
PeakCounts.GetKeyV(PeakCountKeys);
PeakCountKeys.Sort(true);
for (int i = 0; i < PeakCountKeys.Len(); i++) {
TStrSet CurSet = PeakCounts.GetDat(PeakCountKeys[i]);
if (CurSet.Len() > 0) {
printf("QUOTES WITH %d PEAKS\n", PeakCountKeys[i].Val);
printf("#########################################\n");
THashSet<TStr> StringSet = PeakCounts.GetDat(PeakCountKeys[i]);
for (THashSet<TStr>::TIter l = StringSet.BegI(); l < StringSet.EndI(); l++) {
printf("%s\n", l.GetKey().CStr());
}
printf("\n");
}
}
delete QB;
delete DB;
return 0;
}
/// rewire bipartite community affiliation graphs
void TAGMUtil::RewireCmtyNID(THash<TInt,TIntV >& CmtyVH, TRnd& Rnd) {
THash<TInt,TIntV > NewCmtyVH(CmtyVH.Len());
TIntV NDegV;
TIntV CDegV;
for (int i = 0; i < CmtyVH.Len(); i++) {
int CID = CmtyVH.GetKey(i);
for (int j = 0; j < CmtyVH[i].Len(); j++) {
int NID = CmtyVH[i][j];
NDegV.Add(NID);
CDegV.Add(CID);
}
}
TIntPrSet CNIDSet(CDegV.Len());
int c=0;
while (c++ < 15 && CDegV.Len() > 1) {
for (int i = 0; i < CDegV.Len(); i++) {
int u = Rnd.GetUniDevInt(CDegV.Len());
int v = Rnd.GetUniDevInt(NDegV.Len());
if (CNIDSet.IsKey(TIntPr(CDegV[u], NDegV[v]))) {
continue;
}
CNIDSet.AddKey(TIntPr(CDegV[u], NDegV[v]));
if (u == CDegV.Len() - 1) {
CDegV.DelLast();
} else {
CDegV[u] = CDegV.Last();
CDegV.DelLast();
}
if ( v == NDegV.Len() - 1) {
NDegV.DelLast();
} else {
NDegV[v] = NDegV.Last();
NDegV.DelLast();
}
}
}
for (int i = 0; i < CNIDSet.Len(); i++) {
TIntPr CNIDPr = CNIDSet[i];
IAssert(CmtyVH.IsKey(CNIDPr.Val1));
NewCmtyVH.AddDat(CNIDPr.Val1);
NewCmtyVH.GetDat(CNIDPr.Val1).Add(CNIDPr.Val2);
}
CmtyVH = NewCmtyVH;
}
void TSockSys::AddSockTimer(const uint64& SockId, const int& MSecs) {
if (SockIdToTimerHndH.IsKey(SockId)) {
// socket already has timer, stop and start with MSecs
uv_timer_t* TimerHnd = SockIdToTimerHndH.GetDat(SockId);
// stop existing count
uv_timer_stop(TimerHnd);
// start new one
uv_timer_start(TimerHnd, OnTimeOut, MSecs, 0);
} else {
// create new timer
uv_timer_t* TimerHnd = (uv_timer_t*)malloc(sizeof(uv_timer_t));
// initialize
uv_timer_init(SockSys.Loop, TimerHnd);
// start the timer
uv_timer_start(TimerHnd, OnTimeOut, MSecs, 0);
// remember handle
SockIdToTimerHndH.AddDat(SockId, TimerHnd);
TimerHndToSockIdH.AddDat((uint64)TimerHnd, SockId);
}
}
int main(int argc, char *argv[]) {
LogOutput Log;
Log.SetupNewOutputDirectory(OUTPUT_DIRECTORY);
THash<TStr, TStr> Arguments;
ArgumentParser::ParseArguments(argc, argv, Arguments, Log);
// default: compare partitioning methods
bool CompareEdgeScores = Arguments.IsKey("edgescore");
if (CompareEdgeScores) {
fprintf(stderr, "Comparing edge scores!\n");
}
// Load saved QB, DB, and graph (before edges are deleted) from file
TFIn SavedQBDBG(SAVED_QBDBGRAPH_FILE);
TQuoteBase QB;
TDocBase DB;
PNGraph QGraph;
fprintf(stderr, "Loading QB\n");
QB.Load(SavedQBDBG);
fprintf(stderr, "Loading DB\n");
DB.Load(SavedQBDBG);
fprintf(stderr, "Loading Graph\n");
QGraph = TNGraph::Load(SavedQBDBG);
if (!CompareEdgeScores) {
fprintf(stderr, "Calculating and logging percent edges deleted for different partitioning methods\n");
TEdgesDel::ComparePartitioningMethods(&QB, &DB, QGraph, Log);
} else {
fprintf(stderr, "Calculating and logging percent edges deleted for different edge scores\n");
TEdgesDel::CompareEdgeScores(&QB, &DB, QGraph, Log);
}
TSecTm EndDate(END_DATE);
Log.WriteClusteringStatisticsToFile(EndDate);
TStr Directory;
Log.GetDirectory(Directory);
Err("Done with analyzing percent edges deleted! Directory created at: %s\n", Directory.CStr());
}
void TNetInfBs::Init() {
THash<TInt, TIntV> CascPN;
Graph = TNGraph::New();
// reset vectors
EdgeGainV.Clr();
CascPerEdge.Clr();
PrecisionRecall.Clr();
for (int c = 0; c < CascV.Len(); c++) {
for (int i = 0; i < CascV[c].Len(); i++) {
if (!Graph->IsNode(CascV[c].GetNode(i))) Graph->AddNode(CascV[c].GetNode(i));
if (!CascPN.IsKey(CascV[c].GetNode(i))) CascPN.AddDat(CascV[c].GetNode(i)) = TIntV();
CascPN.GetDat(CascV[c].GetNode(i)).Add(c);
}
CascV[c].InitProb();
}
// only add edges that make sense (i.e., at least once coherent in time)
for (TNGraph::TNodeI NI = Graph->BegNI(); NI < Graph->EndNI(); NI++) {
TIntV &Cascs = CascPN.GetDat(NI.GetId());
for (int c = 0; c < Cascs.Len(); c++) {
for (int i=0; i < CascV[Cascs[c]].Len(); i++) {
if (CascV[Cascs[c]].GetNode(i)==NI.GetId())
continue;
if (CascV[Cascs[c]].GetTm(CascV[Cascs[c]].GetNode(i)) < CascV[Cascs[c]].GetTm(NI.GetId()) ) {
if (!CascPerEdge.IsKey(TIntPr(CascV[Cascs[c]].GetNode(i), NI.GetId()))) {
EdgeGainV.Add(TPair<TFlt, TIntPr>(TFlt::Mx, TIntPr(CascV[Cascs[c]].GetNode(i), NI.GetId())));
CascPerEdge.AddDat(TIntPr(CascV[Cascs[c]].GetNode(i), NI.GetId())) = TIntV();
}
// Add cascade to hash of cascades per edge (to implement localized update)
CascPerEdge.GetDat(TIntPr(CascV[Cascs[c]].GetNode(i), NI.GetId())).Add(Cascs[c]);
}
}
}
}
}
// check if timer exists
bool IsTimer(const uint64& TimerHnd) { return TimerHndToSockIdH.IsKey(TimerHnd); }
bool IsSockHnd(const uint64& SockHnd) const { return SockHndToIdH.IsKey(SockHnd); }
// socket
bool IsSock(const uint64& SockId) const { return SockIdToHndH.IsKey(SockId); }
// load from allactors.zip that was prepared by Brad Malin in 2005
PImdbNet TImdbNet::LoadTxt(const TStr& ActorFNm) {
PImdbNet Net = TImdbNet::New();
TStrV ColV;
char line [2024];
int NLines=0, DupEdge=0, Year, Position, ActorNId, MovieNId;
TIntH ActorNIdH;
THash<TIntPr, TInt> MovieNIdH;
FILE *F = fopen(ActorFNm.CStr(), "rt"); fgets(line, 2024, F);
while (! feof(F)) {
memset(line, 0, 2024);
fgets(line, 2024, F);
if (strlen(line) == 0) break;
TStr(line).SplitOnAllCh('|', ColV, false); IAssert(ColV.Len() == 7);
const int NameStrId = Net->AddStr(ColV[0].GetTrunc().GetLc()+" "+ColV[1].GetTrunc().GetLc());
const int MovieStrId = Net->AddStr(ColV[2].GetTrunc().GetLc());
TStr YearStr = ColV[3].GetTrunc();
if (YearStr.Len() > 4) YearStr = YearStr.GetSubStr(0, 3);
Year = 1; YearStr.IsInt(Year);
const TMovieTy MovieTy = TImdbNet::GetMovieTy(ColV[4]);
Position = TInt::Mx; ColV[5].GetTrunc().IsInt(Position);
IAssert(ColV[6].GetTrunc()[0] == 'M' || ColV[6].GetTrunc()[0]=='F');
const bool IsMale = ColV[6].GetTrunc()[0] == 'M';
// create nodes
if (ActorNIdH.IsKey(NameStrId)) {
ActorNId = ActorNIdH.GetDat(NameStrId); }
else {
ActorNId = Net->AddNode(-1, TImdbNode(NameStrId, Year, Position, IsMale));
ActorNIdH.AddDat(NameStrId, ActorNId);
}
if (MovieNIdH.IsKey(TIntPr(MovieStrId, Year))) {
MovieNId = MovieNIdH.GetDat(TIntPr(MovieStrId, Year)); }
else {
MovieNId = Net->AddNode(-1, TImdbNode(NameStrId, Year, MovieTy));
MovieNIdH.AddDat(TIntPr(MovieStrId, Year), MovieNId);
}
if (! Net->IsEdge(ActorNId, MovieNId)) {
Net->AddEdge(ActorNId, MovieNId); }
else { DupEdge++; }
if (++NLines % 100000 == 0) printf("\r %dk ", NLines/1000);
}
fclose(F);
printf("duplicate edges: %d\n", DupEdge);
printf("nodes: %d\n", Net->GetNodes());
printf("edges: %d\n", Net->GetEdges());
printf("actors: %d\n", ActorNIdH.Len());
printf("movies: %d\n", MovieNIdH.Len());
// set the actor year to the year of his first movie
int NUpdates=0;
for (TNet::TNodeI NI = Net->BegNI(); NI < Net->EndNI(); NI++) {
if (NI().IsActor()) {
int MinYear = NI().GetYear();
for (int e = 0; e < NI.GetOutDeg(); e++) {
const TImdbNode& NodeDat = Net->GetNDat(NI.GetOutNId(e));
if (NodeDat.IsMovie()) MinYear = TMath::Mn(MinYear, NodeDat.GetYear());
}
if (NI().Year != MinYear) NUpdates++;
NI().Year = MinYear;
}
}
printf("updated actor times: %d\n", NUpdates);
return Net;
}
bool TSockSys::IsSockEvent(const PSockEvent& SockEvent) const {
return Active ? IdToSockEventH.IsKey(SockEvent->GetSockEventId()) : false;
}
// Q: Do we want to have any gurantees in terms of order of the 0t rows - i.e.
// ordered by "this" table row idx as primary key and "Table" row idx as secondary key
// This means only keeping joint row indices (pairs of original row indices), sorting them
// and adding all rows in the end. Sorting can be expensive, but we would be able to pre-allocate
// memory for the joint table..
PTable TTable::Join(TStr Col1, const TTable& Table, TStr Col2) {
if(!ColTypeMap.IsKey(Col1)){
TExcept::Throw("no such column " + Col1);
}
if(!ColTypeMap.IsKey(Col2)){
TExcept::Throw("no such column " + Col2);
}
if (GetColType(Col1) != GetColType(Col2)) {
TExcept::Throw("Trying to Join on columns of different type");
}
// initialize result table
PTable JointTable = InitializeJointTable(Table);
// hash smaller table (group by column)
TYPE ColType = GetColType(Col1);
TBool ThisIsSmaller = (NumValidRows <= Table.NumValidRows);
const TTable& TS = ThisIsSmaller ? *this : Table;
const TTable& TB = ThisIsSmaller ? Table : *this;
TStr ColS = ThisIsSmaller ? Col1 : Col2;
TStr ColB = ThisIsSmaller ? Col2 : Col1;
// iterate over the rows of the bigger table and check for "collisions"
// with the group keys for the small table.
switch(ColType){
case INT:{
THash<TInt, TIntV> T;
TS.GroupByIntCol(Col1, T, TIntV(), true);
for(TRowIterator RowI = TB.BegRI(); RowI < TB.EndRI(); RowI++){
TInt K = RowI.GetIntAttr(ColB);
if(T.IsKey(K)){
TIntV& Group = T.GetDat(K);
for(TInt i = 0; i < Group.Len(); i++){
if(ThisIsSmaller){
JointTable->AddJointRow(*this, Table, Group[i], RowI.GetRowIdx());
} else{
JointTable->AddJointRow(*this, Table, RowI.GetRowIdx(), Group[i]);
}
}
}
}
break;
}
case FLT:{
THash<TFlt, TIntV> T;
TS.GroupByFltCol(Col1, T, TIntV(), true);
for(TRowIterator RowI = TB.BegRI(); RowI < TB.EndRI(); RowI++){
TFlt K = RowI.GetFltAttr(ColB);
if(T.IsKey(K)){
TIntV& Group = T.GetDat(K);
for(TInt i = 0; i < Group.Len(); i++){
if(ThisIsSmaller){
JointTable->AddJointRow(*this, Table, Group[i], RowI.GetRowIdx());
} else{
JointTable->AddJointRow(*this, Table, RowI.GetRowIdx(), Group[i]);
}
}
}
}
break;
}
case STR:{
THash<TStr, TIntV> T;
TS.GroupByStrCol(Col1, T, TIntV(), true);
for(TRowIterator RowI = TB.BegRI(); RowI < TB.EndRI(); RowI++){
TStr K = RowI.GetStrAttr(ColB);
if(T.IsKey(K)){
TIntV& Group = T.GetDat(K);
for(TInt i = 0; i < Group.Len(); i++){
if(ThisIsSmaller){
JointTable->AddJointRow(*this, Table, Group[i], RowI.GetRowIdx());
} else{
JointTable->AddJointRow(*this, Table, RowI.GetRowIdx(), Group[i]);
}
}
}
}
}
break;
}
return JointTable;
}
bool TSockSys::IsSockEvent(const uint64& SockEventId) const {
return Active ? IdToSockEventH.IsKey(SockEventId) : false;
}
/// save graph into a gexf file which Gephi can read
void TAGMUtil::SaveGephi(const TStr& OutFNm, const PUNGraph& G, const TVec<TIntV>& CmtyVVAtr, const double MaxSz, const double MinSz, const TIntStrH& NIDNameH, const THash<TInt, TIntTr>& NIDColorH ) {
THash<TInt,TIntV> NIDComVHAtr;
TAGMUtil::GetNodeMembership(NIDComVHAtr, CmtyVVAtr);
FILE* F = fopen(OutFNm.CStr(), "wt");
fprintf(F, "<?xml version='1.0' encoding='UTF-8'?>\n");
fprintf(F, "<gexf xmlns='http://www.gexf.net/1.2draft' xmlns:viz='http://www.gexf.net/1.1draft/viz' xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' xsi:schemaLocation='http://www.gexf.net/1.2draft http://www.gexf.net/1.2draft/gexf.xsd' version='1.2'>\n");
fprintf(F, "\t<graph mode='static' defaultedgetype='undirected'>\n");
if (CmtyVVAtr.Len() > 0) {
fprintf(F, "\t<attributes class='node'>\n");
for (int c = 0; c < CmtyVVAtr.Len(); c++) {
fprintf(F, "\t\t<attribute id='%d' title='c%d' type='boolean'>", c, c);
fprintf(F, "\t\t<default>false</default>\n");
fprintf(F, "\t\t</attribute>\n");
}
fprintf(F, "\t</attributes>\n");
}
fprintf(F, "\t\t<nodes>\n");
for (TUNGraph::TNodeI NI = G->BegNI(); NI < G->EndNI(); NI++) {
int NID = NI.GetId();
TStr Label = NIDNameH.IsKey(NID)? NIDNameH.GetDat(NID): "";
Label.ChangeChAll('<', ' ');
Label.ChangeChAll('>', ' ');
Label.ChangeChAll('&', ' ');
Label.ChangeChAll('\'', ' ');
TIntTr Color = NIDColorH.IsKey(NID)? NIDColorH.GetDat(NID) : TIntTr(120, 120, 120);
double Size = MinSz;
double SizeStep = (MaxSz - MinSz) / (double) CmtyVVAtr.Len();
if (NIDComVHAtr.IsKey(NID)) {
Size = MinSz + SizeStep * (double) NIDComVHAtr.GetDat(NID).Len();
}
double Alpha = 1.0;
fprintf(F, "\t\t\t<node id='%d' label='%s'>\n", NID, Label.CStr());
fprintf(F, "\t\t\t\t<viz:color r='%d' g='%d' b='%d' a='%.1f'/>\n", Color.Val1.Val, Color.Val2.Val, Color.Val3.Val, Alpha);
fprintf(F, "\t\t\t\t<viz:size value='%.3f'/>\n", Size);
//specify attributes
if (NIDComVHAtr.IsKey(NID)) {
fprintf(F, "\t\t\t\t<attvalues>\n");
for (int c = 0; c < NIDComVHAtr.GetDat(NID).Len(); c++) {
int CID = NIDComVHAtr.GetDat(NID)[c];
fprintf(F, "\t\t\t\t\t<attvalue for='%d' value='true'/>\n", CID);
}
fprintf(F, "\t\t\t\t</attvalues>\n");
}
fprintf(F, "\t\t\t</node>\n");
}
fprintf(F, "\t\t</nodes>\n");
//plot edges
int EID = 0;
fprintf(F, "\t\t<edges>\n");
for (TUNGraph::TNodeI NI = G->BegNI(); NI < G->EndNI(); NI++) {
for (int e = 0; e < NI.GetOutDeg(); e++) {
if (NI.GetId() > NI.GetOutNId(e)) {
continue;
}
fprintf(F, "\t\t\t<edge id='%d' source='%d' target='%d'/>\n", EID++, NI.GetId(), NI.GetOutNId(e));
}
}
fprintf(F, "\t\t</edges>\n");
fprintf(F, "\t</graph>\n");
fprintf(F, "</gexf>\n");
fclose(F);
}
