PJsonVal TOnlineHistogram::SaveJson() const {
PJsonVal Result = TJsonVal::NewObj();
PJsonVal BoundsArr = TJsonVal::NewArr();
PJsonVal CountsArr = TJsonVal::NewArr();
for (int ElN = 0; ElN < Counts.Len(); ElN++) {
BoundsArr->AddToArr(Bounds[ElN]);
CountsArr->AddToArr(Counts[ElN]);
}
BoundsArr->AddToArr(Bounds.Last());
Result->AddToObj("bounds", BoundsArr);
Result->AddToObj("counts", CountsArr);
return Result;
}
PJsonVal TDecisionTree::TNode::GetJson() const {
PJsonVal RootJson = TJsonVal::NewObj();
RootJson->AddToObj("examples", NExamples);
RootJson->AddToObj("classes", TJsonVal::NewArr(ClassHist));
RootJson->AddToObj("features", TJsonVal::NewArr(FtrHist));
if (!IsLeaf()) {
PJsonVal CutJson = TJsonVal::NewObj();
CutJson->AddToObj("id", CutFtrN);
CutJson->AddToObj("value", CutFtrVal);
if (Tree->IsCalcCorr()) {
PJsonVal FtrCorrJson = TJsonVal::NewArr();
for (int i = 0; i < CutFtrCorrFtrNPValTrV.Len(); i++) {
const TFltIntFltTr& FtrCorr = CutFtrCorrFtrNPValTrV[i];
const TFlt& Corr = FtrCorr.Val1;
const TInt& FtrN = FtrCorr.Val2;
const TFlt& PVal = FtrCorr.Val3;
PJsonVal CorrJson = TJsonVal::NewObj();
CorrJson->AddToObj("id", FtrN);
CorrJson->AddToObj("corr", Corr);
CorrJson->AddToObj("p", PVal);
FtrCorrJson->AddToArr(CorrJson);
}
CutJson->AddToObj("alternatives", FtrCorrJson);
}
RootJson->AddToObj("cut", CutJson);
}
PJsonVal ChildrenJson = TJsonVal::NewArr();
if (HasLeft()) {
ChildrenJson->AddToArr(Left->GetJson());
}
if (HasRight()) {
ChildrenJson->AddToArr(Right->GetJson());
}
RootJson->AddToObj("children", ChildrenJson);
return RootJson;
}
PJsonVal TJsonVal::NewArr(const TJsonValV& ValV) {
PJsonVal Val = TJsonVal::NewArr();
for (int ValN = 0; ValN < ValV.Len(); ValN++) {
Val->AddToArr(ValV[ValN]);
}
return Val;
}
PJsonVal TJsonVal::NewArr(const TUInt64V& IntV) {
PJsonVal Val = TJsonVal::NewArr();
for (int IntN = 0; IntN < IntV.Len(); IntN++) {
Val->AddToArr(TJsonVal::NewNum((double)IntV[IntN]));
}
return Val;
}
PJsonVal TJsonVal::NewArr(const TFltV& FltV) {
PJsonVal Val = TJsonVal::NewArr();
for (int FltN = 0; FltN < FltV.Len(); FltN++) {
Val->AddToArr(TJsonVal::NewNum(FltV[FltN]));
}
return Val;
}
PJsonVal TJsonVal::NewArr(const TIntFltKdV& IntFltKdV) {
PJsonVal Val = TJsonVal::NewArr();
for (const TIntFltKd& IntFltKd : IntFltKdV) {
Val->AddToArr(TJsonVal::NewArr((double)IntFltKd.Key, IntFltKd.Dat));
}
return Val;
}
PJsonVal TJsonVal::NewArr(const TStrV& StrV) {
PJsonVal Val = TJsonVal::NewArr();
for (int StrN = 0; StrN < StrV.Len(); StrN++) {
Val->AddToArr(TJsonVal::NewStr(StrV[StrN]));
}
return Val;
}
PJsonVal TGraphCascade::GetOrder() const {
PJsonVal OrderArr = TJsonVal::NewArr();
int Len = NIdSweep.Len();
for (int NodeN = 0; NodeN < Len; NodeN++) {
TStr NodeNm = NodeIdNmH.GetDat(NIdSweep[NodeN]);
OrderArr->AddToArr(NodeNm);
}
return OrderArr;
}
PJsonVal TNearestNeighbor::Explain(const TIntFltKdV& Vec) const {
// if not initialized, return null (JSON)
if (!IsInit()) { return TJsonVal::NewNull(); }
// find nearest neighbor
double NearDist = TFlt::Mx; int NearColN = -1;
for (int ColN = 0; ColN < Mat.Len(); ColN++) {
const double Dist = TLinAlg::Norm2(Vec) - 2 * TLinAlg::DotProduct(Vec, Mat[ColN]) + TLinAlg::Norm2(Mat[ColN]);
if (Dist < NearDist) { NearDist = Dist; NearColN = ColN; }
}
const TIntFltKdV& NearVec = Mat[NearColN];
// generate JSon explanations
PJsonVal ResVal = TJsonVal::NewObj();
// id of the nearest element
ResVal->AddToObj("nearestDat", DatV[NearColN]);
ResVal->AddToObj("distance", NearDist);
// element-wise difference
PJsonVal DiffVal = TJsonVal::NewArr();
int NearEltN = 0, EltN = 0;
while (NearEltN < NearVec.Len() || EltN < Vec.Len()) {
// get the feature ID
const int VecFtrId = EltN < Vec.Len() ? Vec[EltN].Key.Val : TInt::Mx;
const int NearFtrId = NearEltN < NearVec.Len() ? NearVec[NearEltN].Key.Val : TInt::Mx;
const int FtrId = NearFtrId < VecFtrId ? NearFtrId : VecFtrId;
// get values
const double VecVal = FtrId < VecFtrId ? 0.0 : Vec[EltN].Dat.Val;
const double NearVal = FtrId < NearFtrId ? 0.0 : NearVec[NearEltN].Dat.Val;
// get diff
const double Diff = TMath::Sqr(NearVal - VecVal) / NearDist;
// add to json result
PJsonVal FtrVal = TJsonVal::NewObj();
//avoid unnecessary fields in the explanation
if (Diff > 1e-8) {
FtrVal->AddToObj("id", FtrId);
FtrVal->AddToObj("val", VecVal);
FtrVal->AddToObj("nearVal", NearVal);
FtrVal->AddToObj("contribution", Diff);
DiffVal->AddToArr(FtrVal);
}
// move to the next feature
if (VecFtrId <= NearFtrId) {
EltN++;
}
if (NearFtrId <= VecFtrId) {
NearEltN++;
}
}
ResVal->AddToObj("features", DiffVal);
// first and last record in the buffer
ResVal->AddToObj("oldestDat", DatV[NextCol]);
int CurCol = NextCol > 0 ? NextCol - 1 : WindowSize - 1;
ResVal->AddToObj("newestDat", DatV[CurCol]);
return ResVal;
}
PJsonVal TEmaSpVec::GetJson() const {
PJsonVal arr = TJsonVal::NewArr();
for (int i = 0; i < Ema.Len(); i++) {
PJsonVal tmp = TJsonVal::NewObj();
tmp->AddToObj("Idx", Ema[i].Key);
tmp->AddToObj("Val", Ema[i].Dat);
arr->AddToArr(tmp);
}
PJsonVal res = TJsonVal::NewObj();
res->AddToObj("Sum", arr);
res->AddToObj("Tm", TmMSecs);
return res;
}
PJsonVal TGraphCascade::GetGraph() const {
PJsonVal G = TJsonVal::NewObj();
for (TNGraph::TNodeI NI = Graph.BegNI(); NI < Graph.EndNI(); NI++) {
TStr NodeNm = NodeIdNmH.GetDat(NI.GetId());
PJsonVal ParentsArr = TJsonVal::NewArr();
int InDeg = NI.GetInDeg();
for (int ParentN = 0; ParentN < InDeg; ParentN++) {
TStr ParentNm = NodeIdNmH.GetDat(NI.GetInNId(ParentN));
ParentsArr->AddToArr(ParentNm);
}
G->AddToObj(NodeNm, ParentsArr);
}
return G;
}
PJsonVal TGraphCascade::GetPosterior(const TStrV& NodeNmV, const TFltV& QuantileV) const {
PJsonVal Result = TJsonVal::NewObj();
TIntV NodeIdV;
if (NodeNmV.Empty()) {
// go over all zero timestamps for which samples exist
TIntV FullNodeIdV; Graph.GetNIdV(FullNodeIdV);
int Nodes = Graph.GetNodes();
for (int NodeN = 0; NodeN < Nodes; NodeN++) {
int NodeId = FullNodeIdV[NodeN];
if (Timestamps.IsKey(NodeId) && Sample.IsKey(NodeId) && !Sample.GetDat(NodeId).Empty() && Timestamps.GetDat(NodeId) == 0) {
NodeIdV.Add(NodeId);
}
}
} else {
int Nodes = NodeNmV.Len();
for (int NodeN = 0; NodeN < Nodes; NodeN++) {
if (!NodeNmIdH.IsKey(NodeNmV[NodeN])) { continue; }
int NodeId = NodeNmIdH.GetDat(NodeNmV[NodeN]);
if (Timestamps.IsKey(NodeId) && Sample.IsKey(NodeId) && !Sample.GetDat(NodeId).Empty() && Timestamps.GetDat(NodeId) == 0) {
NodeIdV.Add(NodeId);
}
}
}
EAssertR(QuantileV.Len() > 0, "TGraphCascade::GetPosterior quantiles should not be empty!");
for (int QuantileN = 0; QuantileN < QuantileV.Len(); QuantileN++) {
EAssertR((QuantileV[QuantileN] >= 0.0) && (QuantileV[QuantileN] <= 1.0), "TGraphCascade::GetPosterior quantiles should be between 0.0 and 1.0");
}
int Nodes = NodeIdV.Len();
for (int NodeN = 0; NodeN < Nodes; NodeN++) {
int NodeId = NodeIdV[NodeN];
TStr NodeNm = NodeIdNmH.GetDat(NodeId);
int Quantiles = QuantileV.Len();
TUInt64V SampleV = Sample.GetDat(NodeId);
SampleV.Sort(true);
int SampleSize = SampleV.Len();
PJsonVal QuantilesArr = TJsonVal::NewArr();
for (int QuantileN = 0; QuantileN < Quantiles; QuantileN++) {
int Idx = (int)floor(QuantileV[QuantileN] * SampleSize);
Idx = MIN(Idx, SampleSize - 1);
uint64 UnixTimestamp = TTm::GetUnixMSecsFromWinMSecs(SampleV[Idx]);
QuantilesArr->AddToArr((double)UnixTimestamp);
}
Result->AddToObj(NodeNm, QuantilesArr);
}
return Result;
}
PJsonVal TNearestNeighbor::Explain(const TIntFltKdV& Vec) const {
// if not initialized, return null (JSON)
if (!IsInit()) { return TJsonVal::NewNull(); }
// find nearest neighbor
double NearDist = TFlt::Mx;
int NearColN = -1;
TIntFltKdV DiffV;
for (int ColN = 0; ColN < Mat.Len(); ColN++) {
const double Dist = TLinAlg::Norm2(Vec) - 2 * TLinAlg::DotProduct(Vec, Mat[ColN]) + TLinAlg::Norm2(Mat[ColN]);
if (Dist < NearDist) { NearDist = Dist; NearColN = ColN; }
}
const TIntFltKdV& NearVec = Mat[NearColN];
// generate JSon explanations
PJsonVal ResVal = TJsonVal::NewObj();
// id of the nearest element
ResVal->AddToObj("nearestID", IDVec[NearColN]);
ResVal->AddToObj("distance", NearDist);
// element-wise difference
PJsonVal DiffVal = TJsonVal::NewArr();
int NearEltN = 0, EltN = 0;
while (NearEltN < NearVec.Len() && EltN < Vec.Len()) {
// get values
const int FtrId = (NearVec[NearEltN].Key < Vec[EltN].Key) ? NearVec[NearEltN].Key : Vec[EltN].Key;
const double Val = (NearVec[NearEltN].Key >= Vec[EltN].Key) ? Vec[EltN].Dat.Val : 0.0;
const double NearVal = (NearVec[NearEltN].Key <= Vec[EltN].Key) ? NearVec[NearEltN].Dat.Val : 0.0;
const double Diff = TMath::Sqr(NearVal - Val) / NearDist;
// add to json result
PJsonVal FtrVal = TJsonVal::NewObj();
FtrVal->AddToObj("id", FtrId);
FtrVal->AddToObj("val", Val);
FtrVal->AddToObj("nearVal", NearVal);
FtrVal->AddToObj("contribution", Diff);
DiffVal->AddToArr(FtrVal);
// move to the next feature
if (NearVec[NearEltN].Key > Vec[EltN].Key) {
EltN++;
} else if (NearVec[NearEltN].Key < Vec[EltN].Key) {
NearEltN++;
} else {
NearEltN++; EltN++;
}
}
ResVal->AddToObj("features", DiffVal);
return ResVal;
}
PJsonVal TJsonVal::NewArr(const TFltPr& FltPr) {
PJsonVal Val = TJsonVal::NewArr();
Val->AddToArr(TJsonVal::NewNum(FltPr.Val1));
Val->AddToArr(TJsonVal::NewNum(FltPr.Val2));
return Val;
}
PJsonVal TNodeJsUtil::GetObjJson(const v8::Local<v8::Value>& Val, const bool IgnoreFunc) {
AssertR(!Val->IsExternal(), "TNodeJsUtil::GetObjJson: Cannot parse v8::External!");
if (Val->IsObject()) {
// if we aren't ignoring functions and the object is a function
// then throw an exception
EAssertR(IgnoreFunc || !Val->IsFunction(), "TNodeJsUtil::GetObjJson: Cannot parse functions!");
// parse the object
if (Val->IsFunction()) {
return TJsonVal::New();
}
else if (Val->IsBooleanObject()) {
v8::Local<v8::BooleanObject> BoolObj = v8::Local<v8::BooleanObject>::Cast(Val);
return TJsonVal::NewBool(BoolObj->ValueOf());
}
else if (Val->IsNumberObject()) {
return TJsonVal::NewNum(Val->NumberValue());
}
else if (Val->IsStringObject() || Val->IsRegExp() || Val->IsDate()) {
return TJsonVal::NewStr(TStr(*v8::String::Utf8Value(Val->ToString())));
}
else if (Val->IsArray()) {
PJsonVal JsonArr = TJsonVal::NewArr();
v8::Array* Arr = v8::Array::Cast(*Val);
for (uint i = 0; i < Arr->Length(); i++) {
if (!IgnoreFunc || !Arr->Get(i)->IsFunction()) {
JsonArr->AddToArr(GetObjJson(Arr->Get(i), IgnoreFunc));
}
}
return JsonArr;
}
else { // general object with fields
PJsonVal JsonVal = TJsonVal::NewObj();
v8::Local<v8::Object> Obj = Val->ToObject();
v8::Local<v8::Array> FldNmV = Obj->GetOwnPropertyNames();
for (uint i = 0; i < FldNmV->Length(); i++) {
const TStr FldNm(*v8::String::Utf8Value(FldNmV->Get(i)->ToString()));
v8::Local<v8::Value> FldVal = Obj->Get(FldNmV->Get(i));
if (!IgnoreFunc || !FldVal->IsFunction()) {
JsonVal->AddToObj(FldNm, GetObjJson(FldVal, IgnoreFunc));
}
}
return JsonVal;
}
}
else { // primitive
if (Val->IsUndefined()) {
return TJsonVal::New();
}
else if (Val->IsNull()) {
return TJsonVal::NewNull();
}
else if (Val->IsBoolean()) {
return TJsonVal::NewBool(Val->BooleanValue());
}
else if (Val->IsNumber()) {
return TJsonVal::NewNum(Val->NumberValue());
}
else if (Val->IsString()) {
return TJsonVal::NewStr(TStr(*v8::String::Utf8Value(Val->ToString())));
}
else {
// TODO check for v8::Symbol
throw TExcept::New("TNodeJsUtil::GetObjJson: Unknown v8::Primitive type!");
}
}
}
PJsonVal TDecisionTree::TNode::ExplainLabel(const int& Label) const {
if (IsLeaf()) {
if (ClassHist[Label] <= ClassHist[1 - Label]) {
return PJsonVal();
} else {
const double Prob = ClassHist[Label];
PJsonVal Result = TJsonVal::NewArr();
PJsonVal IntersectJson = TJsonVal::NewObj();
IntersectJson->AddToObj("covered", int(NExamples*Prob));
IntersectJson->AddToObj("purity", Prob);
IntersectJson->AddToObj("terms", TJsonVal::NewArr());
Result->AddToArr(IntersectJson);
return Result;
}
}
PJsonVal Result = TJsonVal::NewArr();
if (HasLeft()) {
PJsonVal LeftUnion = Left->ExplainLabel(Label);
if (!LeftUnion.Empty()) {
if (LeftUnion->GetArrVals() == 0) {
LeftUnion->AddToArr(TJsonVal::NewArr());
}
for (int i = 0; i < LeftUnion->GetArrVals(); i++) {
PJsonVal IntersectJson = LeftUnion->GetArrVal(i);
PJsonVal TermsJson = IntersectJson->GetObjKey("terms");
bool HadFtr = false;
for (int TermN = 0; TermN < TermsJson->GetArrVals(); TermN++) {
PJsonVal TermJson = TermsJson->GetArrVal(TermN);
const int TermFtrN = TermJson->GetObjInt("ftrId");
if (TermFtrN == CutFtrN) {
HadFtr = true;
if (TermJson->GetObjNum("le") == TFlt::PInf) {
TermJson->AddToObj("le", CutFtrVal);
}
}
}
if (!HadFtr) {
PJsonVal TermJson = TJsonVal::NewObj();
TermJson->AddToObj("ftrId", CutFtrN);
TermJson->AddToObj("le", CutFtrVal);
TermJson->AddToObj("gt", TFlt::NInf);
TermsJson->AddToArr(TermJson);
}
Result->AddToArr(IntersectJson);
}
}
}
if (HasRight()) {
PJsonVal RightUnion = Right->ExplainLabel(Label);
if (!RightUnion.Empty()) {
if (RightUnion->GetArrVals() == 0) {
RightUnion->AddToArr(TJsonVal::NewArr());
}
for (int i = 0; i < RightUnion->GetArrVals(); i++) {
PJsonVal IntersectJson = RightUnion->GetArrVal(i);
PJsonVal TermsJson = IntersectJson->GetObjKey("terms");
bool HadFtr = false;
for (int TermN = 0; TermN < TermsJson->GetArrVals(); TermN++) {
PJsonVal TermJson = TermsJson->GetArrVal(TermN);
const int TermFtrN = TermJson->GetObjInt("ftrId");
if (TermFtrN == CutFtrN) {
HadFtr = true;
if (TermJson->GetObjNum("gt") == TFlt::NInf) {
TermJson->AddToObj("gt", CutFtrVal);
}
}
}
if (!HadFtr) {
PJsonVal TermJson = TJsonVal::NewObj();
TermJson->AddToObj("ftrId", CutFtrN);
TermJson->AddToObj("le", TFlt::PInf);
TermJson->AddToObj("gt", CutFtrVal);
TermsJson->AddToArr(TermJson);
}
Result->AddToArr(IntersectJson);
}
}
}
return Result->GetArrVals() > 0 ? Result : PJsonVal();
}
void TSAppSrv::OnHttpRq(const uint64& SockId, const PHttpRq& HttpRq) {
// last appropriate error code, start with bad request
int ErrStatusCd = THttp::BadRqStatusCd;
try {
// check http-request correctness - return if error
EAssertR(HttpRq->IsOk(), "Bad HTTP request!");
// check url correctness - return if error
PUrl HttpRqUrl = HttpRq->GetUrl();
EAssertR(HttpRqUrl->IsOk(), "Bad request URL!");
// extract function name
TStr FunNm = HttpRqUrl->GetPathSeg(0);
// check if we have the function registered
if (FunNm == "favicon.ico") {
PHttpResp HttpResp = THttpResp::New(THttp::OkStatusCd,
THttp::ImageIcoFldVal, false, Favicon.GetSIn());
SendHttpResp(SockId, HttpResp);
return;
} else if (!FunNm.Empty() && !FunNmToFunH.IsKey(FunNm)) {
ErrStatusCd = THttp::ErrNotFoundStatusCd;
GetNotify()->OnStatusFmt("[AppSrv] Unknown function '%s'!", FunNm.CStr());
TExcept::Throw("Unknown function '" + FunNm + "'!");
}
// extract parameters
PUrlEnv HttpRqUrlEnv = HttpRq->GetUrlEnv();
TStrKdV FldNmValPrV; HttpRqUrlEnv->GetKeyValPrV(FldNmValPrV);
// report call
if (ShowParamP) { GetNotify()->OnStatus(HttpRq->GetUrl()->GetUrlStr()); }
// request parsed well, from now on it's internal error
ErrStatusCd = THttp::InternalErrStatusCd;
// processed requested function
if (!FunNm.Empty()) {
// prepare request environment
PSAppSrvRqEnv RqEnv = TSAppSrvRqEnv::New(this, SockId, HttpRq, FunNmToFunH);
// retrieve function
PSAppSrvFun SrvFun = FunNmToFunH.GetDat(FunNm);
// call function
SrvFun->Exec(FldNmValPrV, RqEnv);
} else {
// internal SAppSrv call
if (!ListFunP) {
// we are not allowed to list functions
ErrStatusCd = THttp::ErrNotFoundStatusCd;
TExcept::Throw("Unknown page");
}
// prepare a list of registered functions
PJsonVal FunArrVal = TJsonVal::NewArr();
int KeyId = FunNmToFunH.FFirstKeyId();
while (FunNmToFunH.FNextKeyId(KeyId)) {
FunArrVal->AddToArr(TJsonVal::NewObj("name", FunNmToFunH.GetKey(KeyId)));
}
PJsonVal ResVal = TJsonVal::NewObj();
ResVal->AddToObj("port", GetPortN());
ResVal->AddToObj("connections", GetConns());
ResVal->AddToObj("functions", FunArrVal);
TStr ResStr = ResVal->SaveStr();
// prepare response
PHttpResp HttpResp = THttpResp::New(THttp::OkStatusCd,
THttp::AppJSonFldVal, false, TMIn::New(ResStr));
// send response
SendHttpResp(SockId, HttpResp);
}
} catch (PExcept Except) {
// known internal error
TNotify::StdNotify->OnNotifyFmt(ntErr, "Error: %s", Except->GetMsgStr().CStr());
TNotify::StdNotify->OnNotifyFmt(ntErr, "Error location info: %s", Except->GetLocStr().CStr());
PJsonVal ErrorVal = TJsonVal::NewObj();
ErrorVal->AddToObj("message", Except->GetMsgStr());
ErrorVal->AddToObj("location", Except->GetLocStr());
PJsonVal ResVal = TJsonVal::NewObj("error", ErrorVal);
TStr ResStr = ResVal->SaveStr();
// prepare response
PHttpResp HttpResp = THttpResp::New(ErrStatusCd,
THttp::AppJSonFldVal, false, TMIn::New(ResStr));
// send response
SendHttpResp(SockId, HttpResp);
} catch (...) {
TNotify::StdNotify->OnNotify(ntErr, "Unknown internal error");
// unknown internal error
PJsonVal ResVal = TJsonVal::NewObj("error", "Unknown internal error");
TStr ResStr = ResVal->SaveStr();
// prepare response
PHttpResp HttpResp = THttpResp::New(ErrStatusCd,
THttp::AppJSonFldVal, false, TMIn::New(ResStr));
// send response
SendHttpResp(SockId, HttpResp);
}
}
PJsonVal TJsonVal::NewArr(const double& Val1, const double& Val2) {
PJsonVal Val = TJsonVal::NewArr();
Val->AddToArr(TJsonVal::NewNum(Val1));
Val->AddToArr(TJsonVal::NewNum(Val2));
return Val;
}
