THash<TInt, TInt> * choose_seeds (const PUNGraph g, const int num, const int * infection_state, const int infect) {
THash<TInt, TInt> choices;
THash<TInt, TUNGraph::TNode> nodes;
THash<TInt, TInt> * output = new THash<TInt, TInt> ();
TInt weight = 0;
TInt num_total = 0;
for (TUNGraph::TNodeI n = g->BegNI(); n != g->EndNI(); n++) {
//cout << "nodeID: " << n.GetId() << ",\tStatus: " << infection_state[n.GetId () - 1] << endl;
if (infection_state[n.GetId () - 1] != infect) {
weight += n.GetDeg ();
choices.AddDat (num_total, weight);
nodes.AddDat (num_total, n.GetId());
num_total++;
}
}
// TRnd random ((int) time(NULL));
// TRnd random (0);
TInt num_chosen = 0;
while (num_chosen < num) {
TInt choice = my_random.GetUniDevInt (weight);
TUNGraph::TNode node_choice = nodes[find (choice, choices, 0, num_total-1)];
if (!output->IsKey(node_choice.GetId())) {
num_chosen++;
// cout << node_choice.GetId () << "\n";
output->AddDat(node_choice.GetId (), 1);
}
}
return output;
}
int ComputeKCore(const PUNGraph& G) {
int cnt = 0;
for(TUNGraph::TNodeI NI = G->BegNI(); NI < G->EndNI(); NI++)
cnt = max(cnt, NI.GetOutDeg());
THashSet <TInt> D[cnt+1];
THash <TInt, TInt> deg;
for(TUNGraph::TNodeI NI = G->BegNI(); NI < G->EndNI(); NI++) {
TInt tmp = NI.GetOutDeg() - G->IsEdge(NI.GetId(), NI.GetId() );
D[tmp.Val].AddKey(NI.GetId());
deg.AddDat(NI.GetId()) = tmp;
}
int max_k = 0;
for(int num_iters = 0;num_iters < G->GetNodes(); num_iters++)
for(int i = 0; i < cnt; i++)
if(D[i].Empty() == 0) {
max_k = max(max_k, i);
TInt a = *(D[i].BegI());
D[i].DelKey(a);
deg.AddDat(a.Val) = -1; // Hope overwriting works
TUNGraph::TNodeI NI = G->GetNI(a.Val);
for(int e = 0; e < NI.GetOutDeg(); e++) {
TInt b = NI.GetOutNId(e);
if(deg.GetDat(b) >= 0) {
int Id = deg.GetKeyId(b);
D[deg[Id].Val].DelKey(b);
deg[Id] = deg[Id] - 1; //Hope the overwriting works
D[deg[Id]].AddKey(b);
}
}
break;
}
return max_k;
}
// wrong reading of string attributes
void TTable::BuildGraphTopology(PNEAGraph& Graph, THash<TFlt, TInt>& FSrNodeMap, THash<TFlt, TInt>& FDsNodeMap) {
TYPE SrCT = GetColType(SrcCol);
TInt SrIdx = GetColIdx(SrcCol);
TYPE DsCT = GetColType(DstCol);
TInt DsIdx = GetColIdx(DstCol);
TInt SrcCnt = 0;
TInt DstCnt = 0;
for(TRowIterator RowI = BegRI(); RowI < EndRI(); RowI++) {
if (SrCT == INT && DsCT == INT) {
Graph->AddNode(IntCols[SrIdx][RowI.GetRowIdx()]);
Graph->AddNode(IntCols[DsIdx][RowI.GetRowIdx()]);
Graph->AddEdge(IntCols[SrIdx][RowI.GetRowIdx()], IntCols[DsIdx][RowI.GetRowIdx()], RowI.GetRowIdx());
} else if (SrCT == INT && DsCT == FLT) {
Graph->AddNode(IntCols[SrIdx][RowI.GetRowIdx()]);
TFlt val = FltCols[DsIdx][RowI.GetRowIdx()];
if (!FDsNodeMap.IsKey(val)) {
FDsNodeMap.AddDat(val, DstCnt++);
}
Graph->AddNode(FDsNodeMap.GetDat(val));
Graph->AddEdge(IntCols[SrIdx][RowI.GetRowIdx()], FDsNodeMap.GetDat(val));
} else if (SrCT == INT && DsCT == STR) {
Graph->AddNode(IntCols[SrIdx][RowI.GetRowIdx()]);
Graph->AddNode(StrColMaps[DsIdx][RowI.GetRowIdx()]);
Graph->AddEdge(IntCols[SrIdx][RowI.GetRowIdx()], StrColMaps[DsIdx][RowI.GetRowIdx()], RowI.GetRowIdx());
} else if (SrCT == FLT && DsCT == INT) {
Graph->AddNode(IntCols[DsIdx][RowI.GetRowIdx()]);
TFlt val = FltCols[SrIdx][RowI.GetRowIdx()];
if (!FSrNodeMap.IsKey(val)) {
FSrNodeMap.AddDat(val, SrcCnt++);
}
Graph->AddNode(FSrNodeMap.GetDat(val));
Graph->AddEdge(FSrNodeMap.GetDat(val), IntCols[SrIdx][RowI.GetRowIdx()], RowI.GetRowIdx());
} else if (SrCT == FLT && DsCT == STR) {
Graph->AddNode(StrColMaps[DsIdx][RowI.GetRowIdx()]);
TFlt val = FltCols[SrIdx][RowI.GetRowIdx()];
if (!FSrNodeMap.IsKey(val)) {
FSrNodeMap.AddDat(val, SrcCnt++);
}
Graph->AddNode(FSrNodeMap.GetDat(val));
Graph->AddEdge(FSrNodeMap.GetDat(val), IntCols[SrIdx][RowI.GetRowIdx()], RowI.GetRowIdx());
} else if (SrCT == FLT && DsCT == FLT) {
TFlt val = FltCols[SrIdx][RowI.GetRowIdx()];
if (!FSrNodeMap.IsKey(val)) {
FSrNodeMap.AddDat(val, SrcCnt++);
}
Graph->AddNode(FSrNodeMap.GetDat(val));
val = FltCols[DsIdx][RowI.GetRowIdx()];
if (!FDsNodeMap.IsKey(val)) {
FDsNodeMap.AddDat(val, DstCnt++);
}
Graph->AddNode(FDsNodeMap.GetDat(val));
Graph->AddEdge(FSrNodeMap.GetDat(val), FDsNodeMap.GetDat(val), RowI.GetRowIdx());
}
}
}
TStr GetFlagStr(const TGraphFlag& GraphFlag) {
static THash<TInt, TStr> GraphFlagToStrH;
if (GraphFlagToStrH.Empty()) {
GraphFlagToStrH.AddDat((int) gfUndef, "Undef");
GraphFlagToStrH.AddDat((int) gfDirected, "Directed");
GraphFlagToStrH.AddDat((int) gfMultiGraph, "MultiGraph");
GraphFlagToStrH.AddDat((int) gfNodeDat, "NodeDat");
GraphFlagToStrH.AddDat((int) gfEdgeDat, "EdgeDat");
GraphFlagToStrH.AddDat((int) gfSources, "Sources");
}
return GraphFlagToStrH.GetDat((int) GraphFlag);
}
/// get hash table of <Node ID, community IDs which node belongs to>. Some nodes in NIDV might belong to no community
void TAGMUtil::GetNodeMembership(THash<TInt,TIntSet >& NIDComVH, const TVec<TIntV>& CmtyVV, const TIntV& NIDV) {
NIDComVH.Clr();
for (int u = 0; u < NIDV.Len(); u++) {
NIDComVH.AddDat(NIDV[u]);
}
for (int i = 0; i < CmtyVV.Len(); i++) {
int CID = i;
for (int j = 0; j < CmtyVV[i].Len(); j++) {
int NID = CmtyVV[i][j];
NIDComVH.AddDat(NID).AddKey(CID);
}
}
}
// YES I COPIED AND PASTED CODE my section leader would be so ashamed :D
void LSH::MinHash(THash<TMd5Sig, TIntSet>& ShingleToQuoteIds,
TVec<THash<TIntV, TIntSet> >& SignatureBandBuckets) {
TRnd RandomGenerator; // TODO: make this "more random" by incorporating time
for (int i = 0; i < NumBands; ++i) {
THash < TInt, TIntV > Inverted; // (QuoteID, QuoteSignatureForBand)
THash < TIntV, TIntSet > BandBuckets; // (BandSignature, QuoteIDs)
for (int j = 0; j < BandSize; ++j) {
// Create new signature
TVec < TMd5Sig > Signature;
ShingleToQuoteIds.GetKeyV(Signature);
Signature.Shuffle(RandomGenerator);
// Place in bucket - not very efficient
int SigLen = Signature.Len();
for (int k = 0; k < SigLen; ++k) {
TIntSet CurSet = ShingleToQuoteIds.GetDat(Signature[k]);
for (TIntSet::TIter l = CurSet.BegI(); l < CurSet.EndI(); l++) {
TInt Key = l.GetKey();
if (Inverted.IsKey(Key)) {
TIntV CurSignature = Inverted.GetDat(Key);
if (CurSignature.Len() <= j) {
CurSignature.Add(k);
Inverted.AddDat(Key, CurSignature);
}
} else {
TIntV NewSignature;
NewSignature.Add(k);
Inverted.AddDat(Key, NewSignature);
}
}
}
}
TIntV InvertedKeys;
Inverted.GetKeyV(InvertedKeys);
TInt InvertedLen = InvertedKeys.Len();
for (int k = 0; k < InvertedLen; ++k) {
TIntSet Bucket;
TIntV Signature = Inverted.GetDat(InvertedKeys[k]);
if (BandBuckets.IsKey(Signature)) {
Bucket = BandBuckets.GetDat(Signature);
}
Bucket.AddKey(InvertedKeys[k]);
BandBuckets.AddDat(Signature, Bucket);
}
SignatureBandBuckets.Add(BandBuckets);
Err("%d out of %d band signatures computed\n", i + 1, NumBands);
}
Err("Minhash step complete!\n");
}
void TSockSys::NewSock(const uint64& SockId, const uint64& SockEventId) {
uv_tcp_t* SockHnd = (uv_tcp_t*)malloc(sizeof(uv_tcp_t));
int ResCd = uv_tcp_init(Loop, SockHnd);
// check all went fine
if (ResCd != 0) {
// cleanup first
free(SockHnd);
// and throw exception
throw TExcept::New("SockSys.NewSock: Error initializing TCP socket");
}
// remember handles
SockIdToHndH.AddDat(SockId, SockHnd);
SockHndToIdH.AddDat((uint64)SockHnd, SockId);
SockHndToEventIdH.AddDat((uint64)SockHnd, SockEventId);
}
// IN-OUT edges are swapped (so that the prog runs faster)
// Send message via IN edge proportional to the OUT edge weight
void TWgtNet::ReinforceEdges(const int& NIters) {
THash<TInt, TFlt> OutWgtSumH;
for (TNodeI NI = BegNI(); NI < EndNI(); NI++) {
double wgt = 0;
for (int e = 0; e < NI.GetOutDeg(); e++) {
wgt += NI.GetOutEDat(e); }
OutWgtSumH.AddDat(NI.GetId(), wgt);
}
printf("Reinforcing edges for %d iterations\n", NIters);
// iterate
TExeTm ExeTm;
for (int iter = 0; iter < NIters; iter++) {
for (TNodeI NI = BegNI(); NI < EndNI(); NI++) {
const double X = TInt::Rnd.GetUniDev() * OutWgtSumH.GetDat(NI.GetId());
double x = 0; int e = 0;
for ( ; x + NI.GetOutEDat(e) < X; e++) {
x += NI.GetOutEDat(e); }
IAssert(IsEdge(NI.GetOutNId(e), NI.GetId()));
GetEDat(NI.GetOutNId(e), NI.GetId()) += 1; // reinforce the edge
OutWgtSumH.GetDat(NI.GetOutNId(e)) += 1;
}
if (iter % (NIters/100) == 0) {
printf("\r%d [%s]", iter, ExeTm.GetStr());
}
}
printf(" done.\n");
}
void TAGM::GetNodeMembership(THash<TInt,TIntV >& NIDComVH, const TVec<TIntV>& CmtyVV) {
THash<TInt,TIntV> CmtyVH;
for(int i=0;i<CmtyVV.Len();i++) {
CmtyVH.AddDat(i,CmtyVV[i]);
}
GetNodeMembership(NIDComVH,CmtyVH);
}
void LSH::ElCheapoHashing(TQuoteBase *QuoteBase, TInt ShingleLen,
THash<TMd5Sig, TIntSet>& ShingleToQuoteIds) {
fprintf(stderr, "Hashing shingles the el cheapo way...\n");
TIntV QuoteIds;
QuoteBase->GetAllQuoteIds(QuoteIds);
for (int qt = 0; qt < QuoteIds.Len(); qt++) {
if (qt % 1000 == 0) {
fprintf(stderr, "%d out of %d completed\n", qt, QuoteIds.Len());
}
TQuote Q;
QuoteBase->GetQuote(QuoteIds[qt], Q);
// Put x-character (or x-word) shingles into hash table; x is specified by ShingleLen parameter
TStr QContentStr;
Q.GetParsedContentString(QContentStr);
TChA QContentChA = TChA(QContentStr);
for (int i = 0; i < QContentChA.Len() - ShingleLen + 1; i++) {
TChA ShingleChA = TChA();
for (int j = 0; j < ShingleLen; j++) {
ShingleChA.AddCh(QContentChA.GetCh(i + j));
}
TStr Shingle = TStr(ShingleChA);
const TMd5Sig ShingleMd5(Shingle);
TIntSet ShingleQuoteIds;
if (ShingleToQuoteIds.IsKey(ShingleMd5)) {
ShingleQuoteIds = ShingleToQuoteIds.GetDat(ShingleMd5);
}
ShingleQuoteIds.AddKey(QuoteIds[qt]);
ShingleToQuoteIds.AddDat(ShingleMd5, ShingleQuoteIds);
}
}
Err("Done with el cheapo hashing!\n");
}
void LogOutput::PrintClusterInformation(TDocBase *DB, TQuoteBase *QB, TClusterBase *CB, PNGraph& QGraph, TIntV& ClusterIds, TSecTm PresentTime, TIntV &OldTopClusters) {
if (!ShouldLog) return;
TStr CurDateString = PresentTime.GetDtYmdStr();
Err("Writing cluster information...\n");
// PREVIOUS RANKING SETUP
THash<TInt, TInt> OldRankings;
if (OldTopClusters.Len() > 0) {
for (int i = 0; i < OldTopClusters.Len(); i++) {
OldRankings.AddDat(OldTopClusters[i], i + 1);
}
}
TStrV RankStr;
TStr ClusterJSONDirectory = Directory + "/web/json/clusters/";
for (int i = 0; i < ClusterIds.Len(); i++) {
TStr OldRankStr;
ComputeOldRankString(OldRankings, ClusterIds[i], i+1, OldRankStr);
RankStr.Add(OldRankStr);
// JSON file for each cluster!
TPrintJson::PrintClusterJSON(QB, DB, CB, QGraph, ClusterJSONDirectory, ClusterIds[i], PresentTime);
}
Err("JSON Files for individual written!\n");
TStr JSONTableFileName = Directory + "/web/json/daily/" + CurDateString + ".json";
TPrintJson::PrintClusterTableJSON(QB, DB, CB, JSONTableFileName, ClusterIds, RankStr);
Err("JSON Files for the cluster table written!\n");
}
/// Shingles by words
void LSH::HashShinglesOfClusters(TQuoteBase *QuoteBase,
TClusterBase *ClusterBase, TIntV& ClusterIds, TInt ShingleLen,
THash<TMd5Sig, TIntV>& ShingleToClusterIds) {
Err("Hashing shingles of clusters...\n");
for (int i = 0; i < ClusterIds.Len(); i++) {
if (i % 1000 == 0) {
fprintf(stderr, "%d out of %d completed\n", i, ClusterIds.Len());
}
TCluster C;
ClusterBase->GetCluster(ClusterIds[i], C);
//fprintf(stderr, "%d vs. %d\n", ClusterIds[i].Val, C.GetId().Val);
// Put x-word shingles into hash table; x is specified by ShingleLen parameter
THashSet < TMd5Sig > CHashedShingles;
GetHashedShinglesOfCluster(QuoteBase, C, ShingleLen, CHashedShingles);
for (THashSet<TMd5Sig>::TIter Hash = CHashedShingles.BegI();
Hash < CHashedShingles.EndI(); Hash++) {
TIntV ShingleClusterIds;
if (ShingleToClusterIds.IsKey(*Hash)) {
ShingleClusterIds = ShingleToClusterIds.GetDat(*Hash);
}
ShingleClusterIds.Add(ClusterIds[i]);
ShingleToClusterIds.AddDat(*Hash, ShingleClusterIds);
}
}
Err("Done hashing!\n");
}
/////////////////////////////////////////////////
// TGraphEnumUtils implementation
void TGraphEnumUtils::GetNormalizedMap(const PNGraph &G, THash<TInt,TInt> &map) {
int nId=0;
for(TNGraph::TNodeI it=G->BegNI(); it<G->EndNI(); it++) {
//printf("%d -> %d\n", it.GetId(), nId);
map.AddDat(it.GetId(), nId);
nId++;
}
}
/// rewire bipartite community affiliation graphs
void TAGMUtil::RewireCmtyVV(const TVec<TIntV>& CmtyVVIn, TVec<TIntV>& CmtyVVOut, TRnd& Rnd) {
THash<TInt,TIntV> CmtyVH;
for (int i = 0; i < CmtyVVIn.Len(); i++) {
CmtyVH.AddDat(i, CmtyVVIn[i]);
}
TAGMUtil::RewireCmtyNID(CmtyVH, Rnd);
CmtyVH.GetDatV(CmtyVVOut);
}
void TAGM::GetNodeMembership(THash<TInt,TIntV >& NIDComVH, const THash<TInt,TIntV>& CmtyVH) {
for(int i=0;i<CmtyVH.Len();i++){
int CID = CmtyVH.GetKey(i);
for(int j=0;j<CmtyVH[i].Len();j++) {
int NID = CmtyVH[i][j];
NIDComVH.AddDat(NID).Add(CID);
}
}
}
void TAGMUtil::GetNodeMembership(THash<TInt,TIntSet >& NIDComVH, const TVec<TIntSet>& CmtyVV) {
for (int i = 0; i < CmtyVV.Len(); i++) {
int CID = i;
for (TIntSet::TIter SI = CmtyVV[i].BegI(); SI < CmtyVV[i].EndI(); SI++) {
int NID = SI.GetKey();
NIDComVH.AddDat(NID).AddKey(CID);
}
}
}
void TAGMUtil::GetNodeMembership(THash<TInt,TIntSet >& NIDComVH, const THash<TInt,TIntV>& CmtyVH) {
for (THash<TInt,TIntV>::TIter HI = CmtyVH.BegI(); HI < CmtyVH.EndI(); HI++) {
int CID = HI.GetKey();
for (int j = 0; j < HI.GetDat().Len(); j++) {
int NID = HI.GetDat()[j];
NIDComVH.AddDat(NID).AddKey(CID);
}
}
}
void LSH::WordHashing(TQuoteBase *QuoteBase,
THash<TMd5Sig, TIntSet>& ShingleToQuoteIds) {
fprintf(stderr, "Hashing shingles using words...\n");
TIntV QuoteIds;
QuoteBase->GetAllQuoteIds(QuoteIds);
THash<TStr, TIntSet> Temp;
for (int qt = 0; qt < QuoteIds.Len(); qt++) {
if (qt % 1000 == 0) {
fprintf(stderr, "%d out of %d completed\n", qt, QuoteIds.Len());
}
TQuote Q;
QuoteBase->GetQuote(QuoteIds[qt], Q);
TStrV Content;
Q.GetParsedContent(Content);
int ContentLen = Content.Len();
for (int i = 0; i < ContentLen; i++) {
const TMd5Sig ShingleMd5(Content[i]);
TIntSet ShingleQuoteIds;
ShingleToQuoteIds.IsKeyGetDat(ShingleMd5, ShingleQuoteIds);
ShingleQuoteIds.AddKey(QuoteIds[qt]);
ShingleToQuoteIds.AddDat(ShingleMd5, ShingleQuoteIds);
///// COMMENT OUT LATER
TIntSet TempSet;
Temp.IsKeyGetDat(Content[i], TempSet);
TempSet.AddKey(QuoteIds[qt]);
Temp.AddDat(Content[i], TempSet);
}
}
TVec<TStr> ShingleKeys;
Temp.GetKeyV(ShingleKeys);
ShingleKeys.SortCmp(TCmpSetByLen(false, &Temp));
for (int i = 0; i < 100; i++) {
TIntSet TempSet = Temp.GetDat(ShingleKeys[i]);
Err("%d: %s - %d \n", i, ShingleKeys[i].CStr(), TempSet.Len());
}
Err("Done with word hashing!\n");
}
PSwSet TSwSet::GetSwSet(const TSwSetType& SwSetType){
static THash<TInt, PSwSet> SwSetTypeToSwSetH;
PSwSet SwSet;
if (SwSetTypeToSwSetH.IsKeyGetDat(TInt(int(SwSetType)), SwSet)){
} else {
SwSet=TSwSet::New(SwSetType);
SwSetTypeToSwSetH.AddDat(TInt(int(SwSetType)), SwSet);
}
return SwSet;
}
void TAGMUtil::GetNodeMembership(THash<TInt,TInt >& NIDComVH, const TVec<TIntV>& CmtyVV) {
NIDComVH.Clr();
for (int i = 0; i < CmtyVV.Len(); i++) {
int CID = i;
for (int j = 0; j < CmtyVV[i].Len(); j++) {
int NID = CmtyVV[i][j];
NIDComVH.AddDat(NID)++;
}
}
}
void TSockSys::AddSockTimer(const uint64& SockId, const int& MSecs) {
if (SockIdToTimerHndH.IsKey(SockId)) {
// socket already has timer, stop and start with MSecs
uv_timer_t* TimerHnd = SockIdToTimerHndH.GetDat(SockId);
// stop existing count
uv_timer_stop(TimerHnd);
// start new one
uv_timer_start(TimerHnd, OnTimeOut, MSecs, 0);
} else {
// create new timer
uv_timer_t* TimerHnd = (uv_timer_t*)malloc(sizeof(uv_timer_t));
// initialize
uv_timer_init(SockSys.Loop, TimerHnd);
// start the timer
uv_timer_start(TimerHnd, OnTimeOut, MSecs, 0);
// remember handle
SockIdToTimerHndH.AddDat(SockId, TimerHnd);
TimerHndToSockIdH.AddDat((uint64)TimerHnd, SockId);
}
}
int main(int argc, char* argv[])
{
// TFltPrV v;
// v.Add(TFltPr(1,4));
// v.Add(TFltPr(5,5));
// v.Add(TFltPr(9,11));
// v.Add(TFltPr(20,8));
// v.Add(TFltPr(21,30));
// cout << "C: " << Tools::computeCorrelation(v,Pearson) << endl;
// return 0;
TExeTm ExeTm;
try
{
Env = TEnv(argc, argv, TNotify::StdNotify);
Env.PrepArgs(TStr::Fmt("\nPlotting Individually Memes-Twitter Cascades. build: %s, %s. Time: %s", __TIME__, __DATE__, TExeTm::GetCurTm()));
// URLS
THash< TStr , CascadeElementV > quotes = Tools::loadQuotes("DATA/QuotesPreprocessedData_NIFTY_RANGEFIXED_FINALFILTERED_HAVINGBOTH.rar"); // QuotesPreprocessedData_NIFTY_RANGEFIXED_FINALFILTERED_4URLS
THash< TUInt , TSecTmV > twitterUrls = Tools::loadTwitter("DATA/CascadesFullUrlsOnTwitterData_FINALFILTERED_HAVINGBOTH.rar"); // CascadesFullUrlsOnTwitterData_FINALFILTERED
// CONTENTS
//THash< TStr , CascadeElementV > quotes2 = Tools::loadQuotes("DATA/QuotesPreprocessedData_NIFTY_RANGEFIXED_FINALFILTERED_HAVINGBOTH.rar"); // QuotesPreprocessedData_NIFTY_RANGEFIXED_FINALFILTERED_4Contents
THash< TUInt , TSecTmV > twitterContents = Tools::loadTwitter("DATA/CascadesOnTwitterData_FINALFILTERED_HAVINGBOTH.rar"); // CascadesOnTwitterData_FINALFILTERED
// Plotting
THash< TUInt , TSecTmV > twitterTotal;
for(int i=0;i<twitterContents.Len();i++)
{
TSecTmV tmp;
tmp.AddV(twitterContents[i]);
tmp.AddV(twitterUrls[i]);
twitterTotal.AddDat(i,tmp);
}
plotScatterLengthOfEachCascade(quotes,twitterUrls,"Urls");
plotScatterLengthOfEachCascade(quotes,twitterContents,"Contents");
plotScatterLengthOfEachCascade(quotes,twitterTotal,"Full");
printf("\nPlots had been drawn successfully.");
}
catch(exception& ex)
{
printf("\nError1 happened, it was: %s\n\n",ex.what());
}
catch(TPt<TExcept>& ex)
{
printf("\nError2 happened: %s\n\n",ex[0].GetStr().CStr());
}
printf("\nrun time: %s (%s)\n", ExeTm.GetTmStr(), TSecTm::GetCurTm().GetTmStr().CStr());
return 0;
}
void TNodeJsRf24Radio::set(const v8::FunctionCallbackInfo<v8::Value>& Args) {
v8::Isolate* Isolate = v8::Isolate::GetCurrent();
v8::HandleScope HandleScope(Isolate);
TNodeJsRf24Radio* JsRadio = ObjectWrap::Unwrap<TNodeJsRf24Radio>(Args.Holder());
if (Args.Length() == 0) { return; }
const PJsonVal ArgVal = TNodeJsUtil::GetArgJson(Args, 0);
bool Success = true;
if (ArgVal->IsArr()) {
THash<TInt, TIntPrV> NodeIdValIdValPrVH;
for (int ArgN = 0; ArgN < ArgVal->GetArrVals(); ArgN++) {
const PJsonVal& ValJson = ArgVal->GetArrVal(ArgN);
const TStr& ValNm = ValJson->GetObjStr("sensorId");
const int& Val = ValJson->GetObjInt("value");
const TIntPr& NodeIdValIdPr = JsRadio->ValNmNodeIdValIdPrH.GetDat(ValNm);
const uint16 NodeId = NodeIdValIdPr.Val1;
const int ValId = NodeIdValIdPr.Val2;
if (!NodeIdValIdValPrVH.IsKey(NodeId)) { NodeIdValIdValPrVH.AddDat(NodeId); }
TIntPrV& ValIdValPrV = NodeIdValIdValPrVH.GetDat(NodeId);
ValIdValPrV.Add(TIntPr(ValId, Val));
}
int KeyId = NodeIdValIdValPrVH.FFirstKeyId();
while (NodeIdValIdValPrVH.FNextKeyId(KeyId)) {
const uint16 NodeId = NodeIdValIdValPrVH.GetKey(KeyId);
const TIntPrV& ValIdValPrV = NodeIdValIdValPrVH[KeyId];
Success &= JsRadio->Radio.Set(NodeId, ValIdValPrV);
}
} else {
const TStr& ValueNm = ArgVal->GetObjStr("sensorId");
const int Val = ArgVal->GetObjInt("value");
const TIntPr& NodeIdValIdPr = JsRadio->ValNmNodeIdValIdPrH.GetDat(ValueNm);
const uint16 NodeId = (uint16) NodeIdValIdPr.Val1;
const int ValId = NodeIdValIdPr.Val2;
Success = JsRadio->Radio.Set(NodeId, ValId, Val);
}
Args.GetReturnValue().Set(v8::Boolean::New(Isolate, Success));
}
static double CmtyCMN(const PUNGraph& Graph, TCnComV& CmtyV) {
TCNMQMatrix QMatrix(Graph);
// maximize modularity
while (QMatrix.MergeBestQ()) { }
// reconstruct communities
THash<TInt, TIntV> IdCmtyH;
for (TUNGraph::TNodeI NI = Graph->BegNI(); NI < Graph->EndNI(); NI++) {
IdCmtyH.AddDat(QMatrix.CmtyIdUF.Find(NI.GetId())).Add(NI.GetId());
}
CmtyV.Gen(IdCmtyH.Len());
for (int j = 0; j < IdCmtyH.Len(); j++) {
CmtyV[j].NIdV.Swap(IdCmtyH[j]);
}
return QMatrix.Q;
}
int main(int argc, char *argv[]) {
TStr BaseString = "/lfs/1/tmp/curis/week/QBDB.bin";
TFIn BaseFile(BaseString);
TQuoteBase *QB = new TQuoteBase;
TDocBase *DB = new TDocBase;
QB->Load(BaseFile);
DB->Load(BaseFile);
TIntV QuoteIds;
QB->GetAllQuoteIds(QuoteIds);
int NumQuotes = QuoteIds.Len();
THash<TInt, TStrSet> PeakCounts;
for (int i = 0; i < NumQuotes; i++) {
TQuote CurQuote;
if (QB->GetQuote(QuoteIds[i], CurQuote)) {
TVec<TSecTm> Peaks;
CurQuote.GetPeaks(DB, Peaks);
TStr QuoteString;
CurQuote.GetParsedContentString(QuoteString);
TStrSet StringSet;
if (PeakCounts.IsKey(Peaks.Len())) {
StringSet = PeakCounts.GetDat(Peaks.Len());
}
StringSet.AddKey(QuoteString);
PeakCounts.AddDat(Peaks.Len(), StringSet);
}
}
TIntV PeakCountKeys;
PeakCounts.GetKeyV(PeakCountKeys);
PeakCountKeys.Sort(true);
for (int i = 0; i < PeakCountKeys.Len(); i++) {
TStrSet CurSet = PeakCounts.GetDat(PeakCountKeys[i]);
if (CurSet.Len() > 0) {
printf("QUOTES WITH %d PEAKS\n", PeakCountKeys[i].Val);
printf("#########################################\n");
THashSet<TStr> StringSet = PeakCounts.GetDat(PeakCountKeys[i]);
for (THashSet<TStr>::TIter l = StringSet.BegI(); l < StringSet.EndI(); l++) {
printf("%s\n", l.GetKey().CStr());
}
printf("\n");
}
}
delete QB;
delete DB;
return 0;
}
/// For every quote, add it to corresponding bucket for each hashed x-character shingle of the quote
// (Shingles by characters)
void LSH::HashShingles(TQuoteBase *QuoteBase, TClusterBase *CB, TInt ShingleLen,
THash<TMd5Sig, TShingleIdSet>& ShingleToQuoteIds) {
Err("Hashing shingles...\n");
TIntV QuoteIds;
QuoteBase->GetAllQuoteIds(QuoteIds);
for (int qt = 0; qt < QuoteIds.Len(); qt++) {
if (qt % 1000 == 0) {
fprintf(stderr, "%d out of %d completed\n", qt, QuoteIds.Len());
}
if (CB->IsQuoteInArchivedCluster(QuoteIds[qt]))
continue;
TQuote Q;
QuoteBase->GetQuote(QuoteIds[qt], Q);
// Put x-character (or x-word) shingles into hash table; x is specified by ShingleLen parameter
TStr QContentStr;
Q.GetParsedContentString(QContentStr);
TChA QContentChA = TChA(QContentStr);
int CurWord = 0;
for (int i = 0; i < QContentChA.Len() - ShingleLen + 1; i++) {
TChA ShingleChA = TChA();
for (int j = 0; j < ShingleLen; j++) {
ShingleChA.AddCh(QContentChA.GetCh(i + j));
}
TStr Shingle = TStr(ShingleChA);
const TMd5Sig ShingleMd5(Shingle);
TShingleIdSet ShingleQuoteIds;
if (ShingleToQuoteIds.IsKey(ShingleMd5)) {
ShingleQuoteIds = ShingleToQuoteIds.GetDat(ShingleMd5);
}
for (int j = CurWord; j > CurWord - WordWindow && j >= 0; j--) {
ShingleQuoteIds.AddKey(TShingleId(QuoteIds[qt], j));
}
ShingleToQuoteIds.AddDat(ShingleMd5, ShingleQuoteIds);
// up the current word index if we see a space
if (QContentChA.GetCh(i + ShingleLen - 1) == ' ') {
CurWord++;
}
}
}
Err("Done hashing!\n");
}
void TSockSys::GetAsyncSockHost(const TStr& HostNm, const PSockHost& SockHost) {
// prepare address info request
uv_getaddrinfo_t* Request = (uv_getaddrinfo_t*)malloc(sizeof(uv_getaddrinfo_t));
// submit the request
int ResCd = uv_getaddrinfo(Loop, Request, TSockSys::OnGetHost, HostNm.CStr(), NULL, NULL);
// check submission went fine
if (ResCd != 0) {
// cleanup first
free(Request);
// and throw exception
throw TExcept::New("TSockSys.GetAsyncSockHost: Error requestiong resolve of hostname " + HostNm);
}
// remember SockHost for the callback
TUInt64 RequestHnd = (uint64)Request;
HndToSockHostH.AddDat(RequestHnd, SockHost);
}
void TTable::GroupAux(const TStrV& GroupBy, TInt GroupByStartIdx, THash<TInt,TIntV>& grouping, const TIntV& IndexSet, TBool All){
/* recursion base - add IndexSet as group */
if(GroupByStartIdx == GroupBy.Len()){
if(IndexSet.Len() == 0){return;}
TInt key = grouping.Len();
grouping.AddDat(key, IndexSet);
return;
}
if(!ColTypeMap.IsKey(GroupBy[GroupByStartIdx])){TExcept::Throw("no such column " + GroupBy[GroupByStartIdx]);}
switch(GetColType(GroupBy[GroupByStartIdx])){
case INT:{
// group by current column
// not sure of to estimate the size of T for constructor hinting purpose.
// It is bounded by the length of the IndexSet or the length of the grouping column if the IndexSet vector is empty
// but this bound may be way too big
THash<TInt,TIntV> T;
GroupByIntCol(GroupBy[GroupByStartIdx], T, IndexSet, All);
for(THash<TInt,TIntV>::TIter it = T.BegI(); it < T.EndI(); it++){
TIntV& CurrGroup = it->Dat;
// each group according to current column will be used as an IndexSet
// for grouping according to next column
GroupAux(GroupBy, GroupByStartIdx+1, grouping, CurrGroup, false);
}
break;
}
case FLT:{
THash<TFlt,TIntV> T;
GroupByFltCol(GroupBy[GroupByStartIdx], T, IndexSet, All);
for(THash<TFlt,TIntV>::TIter it = T.BegI(); it < T.EndI(); it++){
TIntV& CurrGroup = it->Dat;
GroupAux(GroupBy, GroupByStartIdx+1, grouping, CurrGroup, false);
}
break;
}
case STR:{
THash<TStr,TIntV> T;
GroupByStrCol(GroupBy[GroupByStartIdx], T, IndexSet, All);
for(THash<TStr,TIntV>::TIter it = T.BegI(); it < T.EndI(); it++){
TIntV& CurrGroup = it->Dat;
GroupAux(GroupBy, GroupByStartIdx+1, grouping, CurrGroup, false);
}
break;
}
}
}
void Init(const PUNGraph& Graph) {
const double M = 0.5/Graph->GetEdges(); // 1/2m
Q = 0.0;
for (TUNGraph::TNodeI NI = Graph->BegNI(); NI < Graph->EndNI(); NI++) {
CmtyIdUF.Add(NI.GetId());
const int OutDeg = NI.GetOutDeg();
if (OutDeg == 0) { continue; }
TCmtyDat& Dat = CmtyQH.AddDat(NI.GetId(), TCmtyDat(M * OutDeg, OutDeg));
for (int e = 0; e < NI.GetOutDeg(); e++) {
const int DstNId = NI.GetOutNId(e);
const double DstMod = 2 * M * (1.0 - OutDeg * Graph->GetNI(DstNId).GetOutDeg() * M);
Dat.AddQ(DstNId, DstMod);
}
Q += -1.0*TMath::Sqr(OutDeg*M);
if (NI.GetId() < Dat.GetMxQNId()) {
MxQHeap.Add(TFltIntIntTr(Dat.GetMxQ(), NI.GetId(), Dat.GetMxQNId())); }
}
MxQHeap.MakeHeap();
}
void TGStat::AvgGStat(const TGStatV& GStatV, const bool& ClipAt1) {
if (GStatV.Empty()) return;
Time = GStatV[0]->Time;
GraphNm = GStatV[0]->GraphNm;
// values
for (int statVal = 0; statVal > gsvMx; statVal++) {
const TGStatVal GStatVal = TGStatVal(statVal);
TMom Mom;
for (int i = 0; i < GStatV.Len(); i++) {
if (GStatV[i]->HasVal(GStatVal)) {
Mom.Add(GStatV[i]->GetVal(GStatVal)); }
}
Mom.Def();
if (Mom.IsUsable()) {
IAssert(Mom.GetVals() == GStatV.Len()); // all must have the value
SetVal(GStatVal, Mom.GetMean());
}
}
// distributions
for (int distr = gsdUndef; distr < gsdMx; distr++) {
const TGStatDistr GStatDistr = TGStatDistr(distr);
THash<TFlt, TFlt> ValToSumH;
int DistrCnt = 0;
for (int i = 0; i < GStatV.Len(); i++) {
if (GStatV[i]->HasDistr(GStatDistr)) {
const TFltPrV& D = GStatV[i]->GetDistr(GStatDistr);
for (int d = 0; d < D.Len(); d++) {
ValToSumH.AddDat(D[d].Val1) += D[d].Val2; }
DistrCnt++;
}
}
IAssert(DistrCnt==0 || DistrCnt==GStatV.Len()); // all must have distribution
TFltPrV AvgStatV;
ValToSumH.GetKeyDatPrV(AvgStatV); AvgStatV.Sort();
for (int i = 0; i < AvgStatV.Len(); i++) {
AvgStatV[i].Val2 /= double(DistrCnt);
if (ClipAt1 && AvgStatV[i].Val2 < 1) { AvgStatV[i].Val2 = 1; }
}
SetDistr(GStatDistr, AvgStatV);
}
}
