void TBTreeIndex<TVal>::SearchRange(const TPair<TVal, TVal>& RangeMinMax, TUInt64V& RecIdV) const {
TVec<TTreeVal> ResValRecIdV;
// execute query
BTree.RangeQuery(TTreeVal(RangeMinMax.Val1, 0), TTreeVal(RangeMinMax.Val2, TUInt64::Mx), ResValRecIdV);
// parse out record ids
RecIdV.Gen(ResValRecIdV.Len(), 0);
for (int ResN = 0; ResN < ResValRecIdV.Len(); ResN++) {
RecIdV.Add(ResValRecIdV[ResN].Val2);
}
}
PJsonVal TJsonVal::NewArr(const TUInt64V& IntV) {
PJsonVal Val = TJsonVal::NewArr();
for (int IntN = 0; IntN < IntV.Len(); IntN++) {
Val->AddToArr(TJsonVal::NewNum((double)IntV[IntN]));
}
return Val;
}
PJsonVal TGraphCascade::GetPosterior(const TStrV& NodeNmV, const TFltV& QuantileV) const {
PJsonVal Result = TJsonVal::NewObj();
TIntV NodeIdV;
if (NodeNmV.Empty()) {
// go over all zero timestamps for which samples exist
TIntV FullNodeIdV; Graph.GetNIdV(FullNodeIdV);
int Nodes = Graph.GetNodes();
for (int NodeN = 0; NodeN < Nodes; NodeN++) {
int NodeId = FullNodeIdV[NodeN];
if (Timestamps.IsKey(NodeId) && Sample.IsKey(NodeId) && !Sample.GetDat(NodeId).Empty() && Timestamps.GetDat(NodeId) == 0) {
NodeIdV.Add(NodeId);
}
}
} else {
int Nodes = NodeNmV.Len();
for (int NodeN = 0; NodeN < Nodes; NodeN++) {
if (!NodeNmIdH.IsKey(NodeNmV[NodeN])) { continue; }
int NodeId = NodeNmIdH.GetDat(NodeNmV[NodeN]);
if (Timestamps.IsKey(NodeId) && Sample.IsKey(NodeId) && !Sample.GetDat(NodeId).Empty() && Timestamps.GetDat(NodeId) == 0) {
NodeIdV.Add(NodeId);
}
}
}
EAssertR(QuantileV.Len() > 0, "TGraphCascade::GetPosterior quantiles should not be empty!");
for (int QuantileN = 0; QuantileN < QuantileV.Len(); QuantileN++) {
EAssertR((QuantileV[QuantileN] >= 0.0) && (QuantileV[QuantileN] <= 1.0), "TGraphCascade::GetPosterior quantiles should be between 0.0 and 1.0");
}
int Nodes = NodeIdV.Len();
for (int NodeN = 0; NodeN < Nodes; NodeN++) {
int NodeId = NodeIdV[NodeN];
TStr NodeNm = NodeIdNmH.GetDat(NodeId);
int Quantiles = QuantileV.Len();
TUInt64V SampleV = Sample.GetDat(NodeId);
SampleV.Sort(true);
int SampleSize = SampleV.Len();
PJsonVal QuantilesArr = TJsonVal::NewArr();
for (int QuantileN = 0; QuantileN < Quantiles; QuantileN++) {
int Idx = (int)floor(QuantileV[QuantileN] * SampleSize);
Idx = MIN(Idx, SampleSize - 1);
uint64 UnixTimestamp = TTm::GetUnixMSecsFromWinMSecs(SampleV[Idx]);
QuantilesArr->AddToArr((double)UnixTimestamp);
}
Result->AddToObj(NodeNm, QuantilesArr);
}
return Result;
}
void TJsonVal::GetArrUInt64V(TUInt64V& UInt64V) const {
EAssert(IsArr());
for (int IntN = 0; IntN < GetArrVals(); IntN++) {
PJsonVal ArrVal = GetArrVal(IntN);
EAssert(ArrVal->IsNum());
UInt64V.Add(ArrVal->GetUInt64());
}
}
TStr TStrUtil::GetStr(const TUInt64V& UInt64V, const TStr& DelimiterStr) {
TChA ResChA;
for (int N = 0; N < UInt64V.Len(); N++) {
if (!ResChA.Empty()) { ResChA += DelimiterStr; }
ResChA += UInt64V[N].GetStr();
}
return ResChA;
}
void TNodeJsSA::getOutTimestampVector(const v8::FunctionCallbackInfo<v8::Value>& Args) {
v8::Isolate* Isolate = v8::Isolate::GetCurrent();
v8::HandleScope HandleScope(Isolate);
// unwrap
TNodeJsSA* JsSA = ObjectWrap::Unwrap<TNodeJsSA>(Args.Holder());
// try to cast as IFltTmIO
TWPt<TQm::TStreamAggrOut::IFltTmIO> Aggr = dynamic_cast<TQm::TStreamAggrOut::IFltTmIO*>(JsSA->SA());
if (Aggr.Empty()) {
throw TQm::TQmExcept::New("TNodeJsSA::getOutTmV : stream aggregate does not implement IFltTmIO: " + JsSA->SA->GetAggrNm());
}
TUInt64V Res;
Aggr->GetOutTmMSecsV(Res);
int Len = Res.Len();
TFltV FltRes(Len);
for (int ElN = 0; ElN < Len; ElN++) {
FltRes[ElN] = (double)Res[ElN];
}
Args.GetReturnValue().Set(TNodeJsVec<TFlt, TAuxFltV>::New(FltRes));
}
TEST(Trie, Adding)
{
TTrie<TCh, TUInt64> TrieH;
TrieH.AddDat(TStr("testi"), 1);
TrieH.AddDat(TStr("test"), 2);
TrieH.AddDat(TStr("tes"), 3);
TrieH.AddDat(TStr("te"), 4);
TUInt64V DestV;
TrieH.SearchByPrefix(TStr("tes"), DestV, 2, true);
ASSERT_EQ(DestV.Len(), 2);
TrieH.SearchByPrefix(TStr("tes"), DestV, -1, true);
ASSERT_EQ(DestV.Len(), 3);
// test overriding the old value for the same key
TrieH.AddDat(TStr("testi"), 123);
TrieH.SearchByPrefix(TStr("testi"), DestV, -1, true);
ASSERT_EQ(DestV.Len(), 1);
ASSERT_EQ(DestV[0], 123);
TrieH.AddIfNew(TStr("testi"), 1234);
TrieH.SearchByPrefix(TStr("testi"), DestV, -1, true);
ASSERT_EQ(DestV.Len(), 1);
// we should not get 1234 since the key already existed
ASSERT_EQ(DestV[0], 123);
TrieH.SearchByPrefix(TStr("invalid"), DestV, -1, true);
ASSERT_EQ(DestV.Len(), 0);
}
/////////////////////////////////////////////////
// Online Max
void TMax::Update(const double& InVal, const uint64& InTmMSecs,
const TFltV& OutValV, const TUInt64V& OutTmMSecsV){
while (!AllValV.Empty() && AllValV[AllValV.Len() - 1].Val1 <= InVal) {
// pop back
AllValV.DelLast();
}
// push back
AllValV.Add(TFltUInt64Pr(InVal, InTmMSecs));
if (!OutTmMSecsV.Empty()) {
// find maximum timestamp of outgoing measurements
uint64 MaxOutTm = OutTmMSecsV.Last();
while (AllValV[0].Val2 <= MaxOutTm) {
// pop front
AllValV.Del(0);
}
}
TmMSecs = InTmMSecs;
Max = AllValV[0].Val1;
}
void TSkyGridBs::SaveTxt(const TStr& FNm, const uint64& CurTm){
// time-limit
TStr CurTmStr=TTm::GetTmFromMSecs(CurTm).GetWebLogDateTimeStr();
uint64 CurDateTm=TTm::GetMSecsFromTm(TTm::GetTmFromWebLogDateTimeStr(TTm::GetTmFromMSecs(CurTm).GetWebLogDateStr()));
TStr CurDateTmStr=TTm::GetTmFromMSecs(CurDateTm).GetWebLogDateTimeStr();
TUInt64V MnTmV;
MnTmV.Add(CurDateTm-0*TTmInfo::GetDayMSecs());
MnTmV.Add(CurDateTm-1*TTmInfo::GetDayMSecs());
MnTmV.Add(CurDateTm-2*TTmInfo::GetDayMSecs());
MnTmV.Add(CurDateTm-4*TTmInfo::GetDayMSecs());
MnTmV.Add(CurDateTm-8*TTmInfo::GetDayMSecs());
MnTmV.Add(CurDateTm-16*TTmInfo::GetDayMSecs());
MnTmV.Add(CurDateTm-32*TTmInfo::GetDayMSecs());
// get bow
//PBowDocBs BowDocBs=GetBowDocBs(3, 5);
PBowDocBs BowDocBs=GetBowDocBs();
PBowDocWgtBs BowDocWgtBs=GetBowDocWgtBs(BowDocBs);
// open file
TFOut FOut(FNm); FILE* fOut=FOut.GetFileId();
// get docs-entities sorted vector
TIntPrV DocsEntIdPrV; GetSorted_DocsEntIdPrV(DocsEntIdPrV);
// traverse entities
for (int EntN=0; EntN<DocsEntIdPrV.Len(); EntN++){
int EntId=DocsEntIdPrV[EntN].Val2;
TStr EntNm=GetEntNm(EntId);
int EntDocs=DocsEntIdPrV[EntN].Val1;
TSkyGridEnt& Ent=GetEnt(EntId);
int LinkEnts=Ent.GetLinkEnts();
fprintf(fOut, "'%s' [%d docs] [%d ents]\n", EntNm.CStr(), EntDocs, LinkEnts);
// output docs over dates
{TStrIntPrV DateStrDocsPrV; int _EntDocs;
Ent.GetDocsPerDateV(this, DateStrDocsPrV, _EntDocs);
fprintf(fOut, " Docs per Date (%d docs):", _EntDocs);
for (int DateN=0; DateN<DateStrDocsPrV.Len(); DateN++){
TStr DateStr=DateStrDocsPrV[DateN].Val1;
int Docs=DateStrDocsPrV[DateN].Val2;
fprintf(fOut, " [%s:%d]", DateStr.CStr(), Docs);
}
fprintf(fOut, "\n");}
fprintf(fOut, " [Now: %s]\n", CurTmStr.CStr());
TIntPrV PrevLinkWgtDstEntIdPrV;
TStrFltPrV PrevWordStrWgtPrV;
for (int MnTmN=0; MnTmN<MnTmV.Len(); MnTmN++){
uint64 MnTm=MnTmV[MnTmN];
double PastDays=(CurDateTm-MnTm)/double(TTmInfo::GetDayMSecs());
TStr MnTmStr=TTm::GetTmFromMSecs(MnTm).GetWebLogDateTimeStr();
// get linked entities
TIntPrV LinkWgtDstEntIdPrV;
Ent.GetSorted_LinkWgtDstEntIdPrV(MnTm, 0.9, LinkWgtDstEntIdPrV);
// output difference between previous and current centroid
if (MnTmN>0){
TIntPrV NegDiffLinkWgtDstEntIdPrV; TIntPrV PosDiffLinkWgtDstEntIdPrV;
GetLinkWgtDstEntIdPrVDiff(LinkWgtDstEntIdPrV, PrevLinkWgtDstEntIdPrV,
NegDiffLinkWgtDstEntIdPrV, PosDiffLinkWgtDstEntIdPrV);
// output positive change
TChA PosDiffLinkWgtDstEntIdPrVChA;
GetLinkWgtDstEntIdPrVChA(PosDiffLinkWgtDstEntIdPrV, PosDiffLinkWgtDstEntIdPrVChA);
fprintf(fOut, " Pos-Diff: %s\n", PosDiffLinkWgtDstEntIdPrVChA.CStr());
// output negative change
TChA NegDiffLinkWgtDstEntIdPrVChA;
GetLinkWgtDstEntIdPrVChA(NegDiffLinkWgtDstEntIdPrV, NegDiffLinkWgtDstEntIdPrVChA);
fprintf(fOut, " Neg-Diff: %s\n", NegDiffLinkWgtDstEntIdPrVChA.CStr());
}
PrevLinkWgtDstEntIdPrV=LinkWgtDstEntIdPrV;
// output linked entities
int TopLinkEnts=LinkWgtDstEntIdPrV.Len();
TChA LinkWgtDstEntIdPrVChA;
GetLinkWgtDstEntIdPrVChA(LinkWgtDstEntIdPrV, LinkWgtDstEntIdPrVChA);
fprintf(fOut, " Entities (%d ents): %s\n",
TopLinkEnts, LinkWgtDstEntIdPrVChA.CStr());
// get text centroid
int CtrDocs; TStrFltPrV WordStrWgtPrV;
Ent.GetDocCentroid(this, BowDocBs, BowDocWgtBs, MnTm, 150, 0.9, CtrDocs, WordStrWgtPrV);
// output difference between previous and current centroid
if (MnTmN>0){
TStrFltPrV NegDiffWordStrWgtPrV; TStrFltPrV PosDiffWordStrWgtPrV;
GetWordStrWgtPrVDiff(WordStrWgtPrV, PrevWordStrWgtPrV,
NegDiffWordStrWgtPrV, PosDiffWordStrWgtPrV);
// output positive change
TChA PosDiffWordStrWgtPrVChA; GetWordStrWgtPrVChA(PosDiffWordStrWgtPrV, PosDiffWordStrWgtPrVChA);
fprintf(fOut, " Pos-Diff: %s\n", PosDiffWordStrWgtPrVChA.CStr());
// output negative change
TChA NegDiffWordStrWgtPrVChA; GetWordStrWgtPrVChA(NegDiffWordStrWgtPrV, NegDiffWordStrWgtPrVChA);
fprintf(fOut, " Neg-Diff: %s\n", NegDiffWordStrWgtPrVChA.CStr());
}
PrevWordStrWgtPrV=WordStrWgtPrV;
// output centroid
TChA WordStrWgtPrVChA; GetWordStrWgtPrVChA(WordStrWgtPrV, WordStrWgtPrVChA);
fprintf(fOut, " Centroid (%d docs, %d words): %s\n",
CtrDocs, WordStrWgtPrV.Len(), WordStrWgtPrVChA.CStr());
// output time
fprintf(fOut, " [-%.1f days: %s]\n", PastDays, MnTmStr.CStr());
}
// entity clustering
/*TVec<TStrFltPrV> EntNmWgtPrVV;
Ent.GetEntClustV(this, MnTmV.Last(), 100, 1000, 10, EntNmWgtPrVV);
for (int ClustN=0; ClustN<EntNmWgtPrVV.Len(); ClustN++){
TStrFltPrV& EntNmWgtPrV=EntNmWgtPrVV[ClustN];
fprintf(fOut, " Clust-%d:", ClustN);
for (int EntN=0; EntN<EntNmWgtPrV.Len(); EntN++){
TStr EntNm=EntNmWgtPrV[EntN].Val1;
double Wgt=EntNmWgtPrV[EntN].Val2;
fprintf(fOut, " ['%s':%.3f]", EntNm.CStr(), Wgt);
}
fprintf(fOut, "\n");
}*/
fprintf(fOut, "\n");
}
}
