PJsonVal TBackupLogInfo::GetJson() const
{
PJsonVal RetJson = TJsonVal::NewObj("folderName", FolderName);
RetJson->AddToObj("secsNeeded", SecsNeeded);
RetJson->AddToObj("logInfo", LogInfo);
return RetJson;
}
PJsonVal TEmaSpVec::GetJson() const {
PJsonVal arr = TJsonVal::NewArr();
for (int i = 0; i < Ema.Len(); i++) {
PJsonVal tmp = TJsonVal::NewObj();
tmp->AddToObj("Idx", Ema[i].Key);
tmp->AddToObj("Val", Ema[i].Dat);
arr->AddToArr(tmp);
}
PJsonVal res = TJsonVal::NewObj();
res->AddToObj("Sum", arr);
res->AddToObj("Tm", TmMSecs);
return res;
}
PJsonVal TNearestNeighbor::Explain(const TIntFltKdV& Vec) const {
// if not initialized, return null (JSON)
if (!IsInit()) { return TJsonVal::NewNull(); }
// find nearest neighbor
double NearDist = TFlt::Mx; int NearColN = -1;
for (int ColN = 0; ColN < Mat.Len(); ColN++) {
const double Dist = TLinAlg::Norm2(Vec) - 2 * TLinAlg::DotProduct(Vec, Mat[ColN]) + TLinAlg::Norm2(Mat[ColN]);
if (Dist < NearDist) { NearDist = Dist; NearColN = ColN; }
}
const TIntFltKdV& NearVec = Mat[NearColN];
// generate JSon explanations
PJsonVal ResVal = TJsonVal::NewObj();
// id of the nearest element
ResVal->AddToObj("nearestDat", DatV[NearColN]);
ResVal->AddToObj("distance", NearDist);
// element-wise difference
PJsonVal DiffVal = TJsonVal::NewArr();
int NearEltN = 0, EltN = 0;
while (NearEltN < NearVec.Len() || EltN < Vec.Len()) {
// get the feature ID
const int VecFtrId = EltN < Vec.Len() ? Vec[EltN].Key.Val : TInt::Mx;
const int NearFtrId = NearEltN < NearVec.Len() ? NearVec[NearEltN].Key.Val : TInt::Mx;
const int FtrId = NearFtrId < VecFtrId ? NearFtrId : VecFtrId;
// get values
const double VecVal = FtrId < VecFtrId ? 0.0 : Vec[EltN].Dat.Val;
const double NearVal = FtrId < NearFtrId ? 0.0 : NearVec[NearEltN].Dat.Val;
// get diff
const double Diff = TMath::Sqr(NearVal - VecVal) / NearDist;
// add to json result
PJsonVal FtrVal = TJsonVal::NewObj();
//avoid unnecessary fields in the explanation
if (Diff > 1e-8) {
FtrVal->AddToObj("id", FtrId);
FtrVal->AddToObj("val", VecVal);
FtrVal->AddToObj("nearVal", NearVal);
FtrVal->AddToObj("contribution", Diff);
DiffVal->AddToArr(FtrVal);
}
// move to the next feature
if (VecFtrId <= NearFtrId) {
EltN++;
}
if (NearFtrId <= VecFtrId) {
NearEltN++;
}
}
ResVal->AddToObj("features", DiffVal);
// first and last record in the buffer
ResVal->AddToObj("oldestDat", DatV[NextCol]);
int CurCol = NextCol > 0 ? NextCol - 1 : WindowSize - 1;
ResVal->AddToObj("newestDat", DatV[CurCol]);
return ResVal;
}
/// Retrieve statistics for this object
PJsonVal TPgBlob::GetStats() {
int dirty = 0;
for (int i = 0; i < LoadedPages.Len(); i++) {
if (ShouldSavePage(i)) {
dirty++;
}
}
PJsonVal res = TJsonVal::NewObj();
res->AddToObj("page_size", PG_PAGE_SIZE);
res->AddToObj("loaded_pages", LoadedPages.Len());
res->AddToObj("dirty_pages", dirty);
res->AddToObj("loaded_extents", Extents.Len());
res->AddToObj("cache_size", PG_EXTENT_SIZE * Extents.Len());
return res;
}
v8::Local<v8::Value> TNodeJsYL40Adc::TReadTask::WrapResult() {
v8::Isolate* Isolate = v8::Isolate::GetCurrent();
v8::EscapableHandleScope HandleScope(Isolate);
PJsonVal ResJson = TJsonVal::NewObj();
for (int InputN = 0; InputN < Adc->InputNumNmKdV.Len(); InputN++) {
ResJson->AddToObj(Adc->InputNumNmKdV[InputN].Dat, ValV[InputN]);
}
return HandleScope.Escape(TNodeJsUtil::ParseJson(Isolate, ResJson));
}
void TNodeJsRf24Radio::OnMsgMainThread(const uint16& NodeId, const uint8& ValueId,
const int& Val) {
if (!OnValueCallback.IsEmpty()) {
v8::Isolate* Isolate = v8::Isolate::GetCurrent();
v8::HandleScope HandleScope(Isolate);
const int ValId = (int) ValueId;
Notify->OnNotifyFmt(TNotifyType::ntInfo, "Got value for value id %d", ValId);
TIntPr NodeIdValIdPr(NodeId, (int) ValId);
EAssertR(NodeIdValIdPrValNmH.IsKey(NodeIdValIdPr), "Node-valueId pair not stored in the structures!");
const TStr& ValueNm = NodeIdValIdPrValNmH.GetDat(NodeIdValIdPr);
PJsonVal JsonVal = TJsonVal::NewObj();
JsonVal->AddToObj("id", ValueNm);
JsonVal->AddToObj("value", Val);
v8::Local<v8::Function> Callback = v8::Local<v8::Function>::New(Isolate, OnValueCallback);
TNodeJsUtil::ExecuteVoid(Callback, TNodeJsUtil::ParseJson(Isolate, JsonVal));
}
}
PJsonVal TOnlineHistogram::SaveJson() const {
PJsonVal Result = TJsonVal::NewObj();
PJsonVal BoundsArr = TJsonVal::NewArr();
PJsonVal CountsArr = TJsonVal::NewArr();
for (int ElN = 0; ElN < Counts.Len(); ElN++) {
BoundsArr->AddToArr(Bounds[ElN]);
CountsArr->AddToArr(Counts[ElN]);
}
BoundsArr->AddToArr(Bounds.Last());
Result->AddToObj("bounds", BoundsArr);
Result->AddToObj("counts", CountsArr);
return Result;
}
PJsonVal TGraphCascade::GetGraph() const {
PJsonVal G = TJsonVal::NewObj();
for (TNGraph::TNodeI NI = Graph.BegNI(); NI < Graph.EndNI(); NI++) {
TStr NodeNm = NodeIdNmH.GetDat(NI.GetId());
PJsonVal ParentsArr = TJsonVal::NewArr();
int InDeg = NI.GetInDeg();
for (int ParentN = 0; ParentN < InDeg; ParentN++) {
TStr ParentNm = NodeIdNmH.GetDat(NI.GetInNId(ParentN));
ParentsArr->AddToArr(ParentNm);
}
G->AddToObj(NodeNm, ParentsArr);
}
return G;
}
PJsonVal TGraphCascade::GetPosterior(const TStrV& NodeNmV, const TFltV& QuantileV) const {
PJsonVal Result = TJsonVal::NewObj();
TIntV NodeIdV;
if (NodeNmV.Empty()) {
// go over all zero timestamps for which samples exist
TIntV FullNodeIdV; Graph.GetNIdV(FullNodeIdV);
int Nodes = Graph.GetNodes();
for (int NodeN = 0; NodeN < Nodes; NodeN++) {
int NodeId = FullNodeIdV[NodeN];
if (Timestamps.IsKey(NodeId) && Sample.IsKey(NodeId) && !Sample.GetDat(NodeId).Empty() && Timestamps.GetDat(NodeId) == 0) {
NodeIdV.Add(NodeId);
}
}
} else {
int Nodes = NodeNmV.Len();
for (int NodeN = 0; NodeN < Nodes; NodeN++) {
if (!NodeNmIdH.IsKey(NodeNmV[NodeN])) { continue; }
int NodeId = NodeNmIdH.GetDat(NodeNmV[NodeN]);
if (Timestamps.IsKey(NodeId) && Sample.IsKey(NodeId) && !Sample.GetDat(NodeId).Empty() && Timestamps.GetDat(NodeId) == 0) {
NodeIdV.Add(NodeId);
}
}
}
EAssertR(QuantileV.Len() > 0, "TGraphCascade::GetPosterior quantiles should not be empty!");
for (int QuantileN = 0; QuantileN < QuantileV.Len(); QuantileN++) {
EAssertR((QuantileV[QuantileN] >= 0.0) && (QuantileV[QuantileN] <= 1.0), "TGraphCascade::GetPosterior quantiles should be between 0.0 and 1.0");
}
int Nodes = NodeIdV.Len();
for (int NodeN = 0; NodeN < Nodes; NodeN++) {
int NodeId = NodeIdV[NodeN];
TStr NodeNm = NodeIdNmH.GetDat(NodeId);
int Quantiles = QuantileV.Len();
TUInt64V SampleV = Sample.GetDat(NodeId);
SampleV.Sort(true);
int SampleSize = SampleV.Len();
PJsonVal QuantilesArr = TJsonVal::NewArr();
for (int QuantileN = 0; QuantileN < Quantiles; QuantileN++) {
int Idx = (int)floor(QuantileV[QuantileN] * SampleSize);
Idx = MIN(Idx, SampleSize - 1);
uint64 UnixTimestamp = TTm::GetUnixMSecsFromWinMSecs(SampleV[Idx]);
QuantilesArr->AddToArr((double)UnixTimestamp);
}
Result->AddToObj(NodeNm, QuantilesArr);
}
return Result;
}
PJsonVal TNearestNeighbor::Explain(const TIntFltKdV& Vec) const {
// if not initialized, return null (JSON)
if (!IsInit()) { return TJsonVal::NewNull(); }
// find nearest neighbor
double NearDist = TFlt::Mx;
int NearColN = -1;
TIntFltKdV DiffV;
for (int ColN = 0; ColN < Mat.Len(); ColN++) {
const double Dist = TLinAlg::Norm2(Vec) - 2 * TLinAlg::DotProduct(Vec, Mat[ColN]) + TLinAlg::Norm2(Mat[ColN]);
if (Dist < NearDist) { NearDist = Dist; NearColN = ColN; }
}
const TIntFltKdV& NearVec = Mat[NearColN];
// generate JSon explanations
PJsonVal ResVal = TJsonVal::NewObj();
// id of the nearest element
ResVal->AddToObj("nearestID", IDVec[NearColN]);
ResVal->AddToObj("distance", NearDist);
// element-wise difference
PJsonVal DiffVal = TJsonVal::NewArr();
int NearEltN = 0, EltN = 0;
while (NearEltN < NearVec.Len() && EltN < Vec.Len()) {
// get values
const int FtrId = (NearVec[NearEltN].Key < Vec[EltN].Key) ? NearVec[NearEltN].Key : Vec[EltN].Key;
const double Val = (NearVec[NearEltN].Key >= Vec[EltN].Key) ? Vec[EltN].Dat.Val : 0.0;
const double NearVal = (NearVec[NearEltN].Key <= Vec[EltN].Key) ? NearVec[NearEltN].Dat.Val : 0.0;
const double Diff = TMath::Sqr(NearVal - Val) / NearDist;
// add to json result
PJsonVal FtrVal = TJsonVal::NewObj();
FtrVal->AddToObj("id", FtrId);
FtrVal->AddToObj("val", Val);
FtrVal->AddToObj("nearVal", NearVal);
FtrVal->AddToObj("contribution", Diff);
DiffVal->AddToArr(FtrVal);
// move to the next feature
if (NearVec[NearEltN].Key > Vec[EltN].Key) {
EltN++;
} else if (NearVec[NearEltN].Key < Vec[EltN].Key) {
NearEltN++;
} else {
NearEltN++; EltN++;
}
}
ResVal->AddToObj("features", DiffVal);
return ResVal;
}
void TNodeJsSA::New(const v8::FunctionCallbackInfo<v8::Value>& Args) {
v8::Isolate* Isolate = v8::Isolate::GetCurrent();
v8::HandleScope HandleScope(Isolate);
if (Args.Length() == 0) { return; } //
EAssertR(!constructor.IsEmpty(), "TNodeJsSA::New: constructor is empty. Did you call TNodeJsSA::Init(exports); in this module's init function?");
QmAssertR(Args.Length() <= 3 && Args.Length() >= 2, "stream aggregator constructor expects at least two parameters");
QmAssertR(Args[0]->IsObject() && Args[1]->IsObject(), "stream aggregator constructor expects first two arguments as objects");
// new sa(...)
if (Args.IsConstructCall()) {
TQm::PStreamAggr StreamAggr;
TNodeJsBase* JsBase = TNodeJsUtil::UnwrapCheckWatcher<TNodeJsBase>(Args[0]->ToObject());
// get aggregate type
TStr TypeNm = TNodeJsUtil::GetArgStr(Args, 1, "type", "javaScript");
if (TypeNm == "javaScript") {
// we have a javascript stream aggregate
TStr AggrName = TNodeJsUtil::GetArgStr(Args, 1, "name", "");
// we need a name, if not give just generate one
if (AggrName.Empty()) { AggrName = TGuid::GenSafeGuid(); }
// create aggregate
StreamAggr = TNodeJsStreamAggr::New(JsBase->Base, AggrName, Args[1]->ToObject());
}
else if (TypeNm == "ftrext") {
TStr AggrName = TNodeJsUtil::GetArgStr(Args, 1, "name", "");
QmAssertR(Args[1]->ToObject()->Has(v8::String::NewFromUtf8(Isolate, "featureSpace")), "addStreamAggr: featureSpace property missing!");
// we need a name, if not give just generate one
if (AggrName.Empty()) { AggrName = TGuid::GenSafeGuid(); }
throw TQm::TQmExcept::New("ftrext stream aggr not implemented yet! (needs feature space implementation)");
// TODO
//TQm::PFtrSpace FtrSpace = TJsFtrSpace::GetArgFtrSpace(Args[1]->ToObject()->Get(v8::String::NewFromUtf8(Isolate, "featureSpace")));
//StreamAggr = TStreamAggrs::TFtrExtAggr::New(JsBase->Base, AggrName, FtrSpace);
}
else if (TypeNm == "stmerger") {
// create new aggregate
PJsonVal ParamVal = TNodeJsUtil::GetArgJson(Args, 1);
StreamAggr = TQm::TStreamAggr::New(JsBase->Base, TypeNm, ParamVal);
PJsonVal FieldArrVal = ParamVal->GetObjKey("fields");
TStrV InterpNmV;
QmAssertR(ParamVal->IsObjKey("fields"), "Missing argument 'fields'!");
// automatically register the aggregate for addRec callbacks
for (int FieldN = 0; FieldN < FieldArrVal->GetArrVals(); FieldN++) {
PJsonVal FieldVal = FieldArrVal->GetArrVal(FieldN);
PJsonVal SourceVal = FieldVal->GetObjKey("source");
TStr StoreNm = "";
if (SourceVal->IsStr()) {
// we have just store name
StoreNm = SourceVal->GetStr();
}
else if (SourceVal->IsObj()) {
// get store
StoreNm = SourceVal->GetObjStr("store");
}
JsBase->Base->AddStreamAggr(JsBase->Base->GetStoreByStoreNm(StoreNm)->GetStoreId(), StreamAggr);
}
}
else {
// we have a GLib stream aggregate, translate parameters to PJsonVal
PJsonVal ParamVal = TNodeJsUtil::GetArgJson(Args, 1);
if (Args.Length() >= 3 && Args[2]->IsString()) {
ParamVal->AddToObj("store", TNodeJsUtil::GetArgStr(Args, 2));
}
// check if it's one stream aggregate or composition
if (TQm::TStreamAggrs::TCompositional::IsCompositional(TypeNm)) {
// we have a composition of aggregates, call code to assemble it
TQm::TStreamAggrs::TCompositional::Register(JsBase->Base, TypeNm, ParamVal);
}
else {
// create new aggregate
StreamAggr = TQm::TStreamAggr::New(JsBase->Base, TypeNm, ParamVal);
}
}
if (!TQm::TStreamAggrs::TCompositional::IsCompositional(TypeNm)) {
if (Args.Length() > 2) {
TStrV Stores(0);
if (Args[2]->IsString()) {
Stores.Add(TNodeJsUtil::GetArgStr(Args, 2));
}
if (Args[2]->IsArray()) {
PJsonVal StoresJson = TNodeJsUtil::GetArgJson(Args, 2);
QmAssertR(StoresJson->IsDef(), "stream aggr constructor : Args[2] should be a string (store name) or a string array (store names)");
StoresJson->GetArrStrV(Stores);
}
for (int StoreN = 0; StoreN < Stores.Len(); StoreN++) {
QmAssertR(JsBase->Base->IsStoreNm(Stores[StoreN]), "stream aggr constructor : Args[2] : store does not exist!");
JsBase->Base->AddStreamAggr(Stores[StoreN], StreamAggr);
}
}
else {
JsBase->Base->AddStreamAggr(StreamAggr);
}
// non-compositional aggregates are returned
TNodeJsSA* JsSA = new TNodeJsSA(StreamAggr);
v8::Local<v8::Object> Instance = Args.This();
JsSA->Wrap(Instance);
Args.GetReturnValue().Set(Instance);
return;
}
}
// sa(...) -> calls new sa(...)
else {
if (Args.Length() == 2) {
const int Argc = 2;
v8::Local<v8::Value> Argv[Argc] = { Args[0], Args[1] };
v8::Local<v8::Function> cons = v8::Local<v8::Function>::New(Isolate, constructor);
v8::Local<v8::Object> Instance = cons->NewInstance(Argc, Argv);
Args.GetReturnValue().Set(Instance);
return;
}
if (Args.Length() == 3) {
const int Argc = 3;
v8::Local<v8::Value> Argv[Argc] = { Args[0], Args[1], Args[2] };
v8::Local<v8::Function> cons = v8::Local<v8::Function>::New(Isolate, constructor);
v8::Local<v8::Object> Instance = cons->NewInstance(Argc, Argv);
Args.GetReturnValue().Set(Instance);
return;
}
else {
// sa()->calls new sa()
v8::Local<v8::Function> cons = v8::Local<v8::Function>::New(Isolate, constructor);
v8::Local<v8::Object> Instance = cons->NewInstance();
Args.GetReturnValue().Set(Instance);
return;
}
}
}
PJsonVal TNodeJsUtil::GetObjJson(const v8::Local<v8::Value>& Val, const bool IgnoreFunc) {
AssertR(!Val->IsExternal(), "TNodeJsUtil::GetObjJson: Cannot parse v8::External!");
if (Val->IsObject()) {
// if we aren't ignoring functions and the object is a function
// then throw an exception
EAssertR(IgnoreFunc || !Val->IsFunction(), "TNodeJsUtil::GetObjJson: Cannot parse functions!");
// parse the object
if (Val->IsFunction()) {
return TJsonVal::New();
}
else if (Val->IsBooleanObject()) {
v8::Local<v8::BooleanObject> BoolObj = v8::Local<v8::BooleanObject>::Cast(Val);
return TJsonVal::NewBool(BoolObj->ValueOf());
}
else if (Val->IsNumberObject()) {
return TJsonVal::NewNum(Val->NumberValue());
}
else if (Val->IsStringObject() || Val->IsRegExp() || Val->IsDate()) {
return TJsonVal::NewStr(TStr(*v8::String::Utf8Value(Val->ToString())));
}
else if (Val->IsArray()) {
PJsonVal JsonArr = TJsonVal::NewArr();
v8::Array* Arr = v8::Array::Cast(*Val);
for (uint i = 0; i < Arr->Length(); i++) {
if (!IgnoreFunc || !Arr->Get(i)->IsFunction()) {
JsonArr->AddToArr(GetObjJson(Arr->Get(i), IgnoreFunc));
}
}
return JsonArr;
}
else { // general object with fields
PJsonVal JsonVal = TJsonVal::NewObj();
v8::Local<v8::Object> Obj = Val->ToObject();
v8::Local<v8::Array> FldNmV = Obj->GetOwnPropertyNames();
for (uint i = 0; i < FldNmV->Length(); i++) {
const TStr FldNm(*v8::String::Utf8Value(FldNmV->Get(i)->ToString()));
v8::Local<v8::Value> FldVal = Obj->Get(FldNmV->Get(i));
if (!IgnoreFunc || !FldVal->IsFunction()) {
JsonVal->AddToObj(FldNm, GetObjJson(FldVal, IgnoreFunc));
}
}
return JsonVal;
}
}
else { // primitive
if (Val->IsUndefined()) {
return TJsonVal::New();
}
else if (Val->IsNull()) {
return TJsonVal::NewNull();
}
else if (Val->IsBoolean()) {
return TJsonVal::NewBool(Val->BooleanValue());
}
else if (Val->IsNumber()) {
return TJsonVal::NewNum(Val->NumberValue());
}
else if (Val->IsString()) {
return TJsonVal::NewStr(TStr(*v8::String::Utf8Value(Val->ToString())));
}
else {
// TODO check for v8::Symbol
throw TExcept::New("TNodeJsUtil::GetObjJson: Unknown v8::Primitive type!");
}
}
}
PJsonVal TDecisionTree::TNode::GetJson() const {
PJsonVal RootJson = TJsonVal::NewObj();
RootJson->AddToObj("examples", NExamples);
RootJson->AddToObj("classes", TJsonVal::NewArr(ClassHist));
RootJson->AddToObj("features", TJsonVal::NewArr(FtrHist));
if (!IsLeaf()) {
PJsonVal CutJson = TJsonVal::NewObj();
CutJson->AddToObj("id", CutFtrN);
CutJson->AddToObj("value", CutFtrVal);
if (Tree->IsCalcCorr()) {
PJsonVal FtrCorrJson = TJsonVal::NewArr();
for (int i = 0; i < CutFtrCorrFtrNPValTrV.Len(); i++) {
const TFltIntFltTr& FtrCorr = CutFtrCorrFtrNPValTrV[i];
const TFlt& Corr = FtrCorr.Val1;
const TInt& FtrN = FtrCorr.Val2;
const TFlt& PVal = FtrCorr.Val3;
PJsonVal CorrJson = TJsonVal::NewObj();
CorrJson->AddToObj("id", FtrN);
CorrJson->AddToObj("corr", Corr);
CorrJson->AddToObj("p", PVal);
FtrCorrJson->AddToArr(CorrJson);
}
CutJson->AddToObj("alternatives", FtrCorrJson);
}
RootJson->AddToObj("cut", CutJson);
}
PJsonVal ChildrenJson = TJsonVal::NewArr();
if (HasLeft()) {
ChildrenJson->AddToArr(Left->GetJson());
}
if (HasRight()) {
ChildrenJson->AddToArr(Right->GetJson());
}
RootJson->AddToObj("children", ChildrenJson);
return RootJson;
}
void TSAppSrv::OnHttpRq(const uint64& SockId, const PHttpRq& HttpRq) {
// last appropriate error code, start with bad request
int ErrStatusCd = THttp::BadRqStatusCd;
try {
// check http-request correctness - return if error
EAssertR(HttpRq->IsOk(), "Bad HTTP request!");
// check url correctness - return if error
PUrl HttpRqUrl = HttpRq->GetUrl();
EAssertR(HttpRqUrl->IsOk(), "Bad request URL!");
// extract function name
TStr FunNm = HttpRqUrl->GetPathSeg(0);
// check if we have the function registered
if (FunNm == "favicon.ico") {
PHttpResp HttpResp = THttpResp::New(THttp::OkStatusCd,
THttp::ImageIcoFldVal, false, Favicon.GetSIn());
SendHttpResp(SockId, HttpResp);
return;
} else if (!FunNm.Empty() && !FunNmToFunH.IsKey(FunNm)) {
ErrStatusCd = THttp::ErrNotFoundStatusCd;
GetNotify()->OnStatusFmt("[AppSrv] Unknown function '%s'!", FunNm.CStr());
TExcept::Throw("Unknown function '" + FunNm + "'!");
}
// extract parameters
PUrlEnv HttpRqUrlEnv = HttpRq->GetUrlEnv();
TStrKdV FldNmValPrV; HttpRqUrlEnv->GetKeyValPrV(FldNmValPrV);
// report call
if (ShowParamP) { GetNotify()->OnStatus(HttpRq->GetUrl()->GetUrlStr()); }
// request parsed well, from now on it's internal error
ErrStatusCd = THttp::InternalErrStatusCd;
// processed requested function
if (!FunNm.Empty()) {
// prepare request environment
PSAppSrvRqEnv RqEnv = TSAppSrvRqEnv::New(this, SockId, HttpRq, FunNmToFunH);
// retrieve function
PSAppSrvFun SrvFun = FunNmToFunH.GetDat(FunNm);
// call function
SrvFun->Exec(FldNmValPrV, RqEnv);
} else {
// internal SAppSrv call
if (!ListFunP) {
// we are not allowed to list functions
ErrStatusCd = THttp::ErrNotFoundStatusCd;
TExcept::Throw("Unknown page");
}
// prepare a list of registered functions
PJsonVal FunArrVal = TJsonVal::NewArr();
int KeyId = FunNmToFunH.FFirstKeyId();
while (FunNmToFunH.FNextKeyId(KeyId)) {
FunArrVal->AddToArr(TJsonVal::NewObj("name", FunNmToFunH.GetKey(KeyId)));
}
PJsonVal ResVal = TJsonVal::NewObj();
ResVal->AddToObj("port", GetPortN());
ResVal->AddToObj("connections", GetConns());
ResVal->AddToObj("functions", FunArrVal);
TStr ResStr = ResVal->SaveStr();
// prepare response
PHttpResp HttpResp = THttpResp::New(THttp::OkStatusCd,
THttp::AppJSonFldVal, false, TMIn::New(ResStr));
// send response
SendHttpResp(SockId, HttpResp);
}
} catch (PExcept Except) {
// known internal error
TNotify::StdNotify->OnNotifyFmt(ntErr, "Error: %s", Except->GetMsgStr().CStr());
TNotify::StdNotify->OnNotifyFmt(ntErr, "Error location info: %s", Except->GetLocStr().CStr());
PJsonVal ErrorVal = TJsonVal::NewObj();
ErrorVal->AddToObj("message", Except->GetMsgStr());
ErrorVal->AddToObj("location", Except->GetLocStr());
PJsonVal ResVal = TJsonVal::NewObj("error", ErrorVal);
TStr ResStr = ResVal->SaveStr();
// prepare response
PHttpResp HttpResp = THttpResp::New(ErrStatusCd,
THttp::AppJSonFldVal, false, TMIn::New(ResStr));
// send response
SendHttpResp(SockId, HttpResp);
} catch (...) {
TNotify::StdNotify->OnNotify(ntErr, "Unknown internal error");
// unknown internal error
PJsonVal ResVal = TJsonVal::NewObj("error", "Unknown internal error");
TStr ResStr = ResVal->SaveStr();
// prepare response
PHttpResp HttpResp = THttpResp::New(ErrStatusCd,
THttp::AppJSonFldVal, false, TMIn::New(ResStr));
// send response
SendHttpResp(SockId, HttpResp);
}
}
void TSAppSrvFun::Exec(const TStrKdV& FldNmValPrV, const PSAppSrvRqEnv& RqEnv) {
const PNotify& Notify = RqEnv->GetWebSrv()->GetNotify();
PHttpResp HttpResp;
try {
// log the call
if (NotifyOnRequest)
Notify->OnStatus(TStr::Fmt("RequestStart %s", FunNm.CStr()));
TTmStopWatch StopWatch(true);
// execute the actual function, according to the type
PSIn BodySIn; TStr ContTypeVal;
if (GetFunOutType() == saotXml) {
PXmlDoc ResXmlDoc = ExecXml(FldNmValPrV, RqEnv);
TStr ResXmlStr; ResXmlDoc->SaveStr(ResXmlStr);
BodySIn = TMIn::New(XmlHdStr + ResXmlStr);
ContTypeVal = THttp::TextXmlFldVal;
} else if (GetFunOutType() == saotJSon) {
TStr ResStr = ExecJSon(FldNmValPrV, RqEnv);
BodySIn = TMIn::New(ResStr);
ContTypeVal = THttp::AppJSonFldVal;
} else {
BodySIn = ExecSIn(FldNmValPrV, RqEnv, ContTypeVal);
}
if (ReportResponseSize)
Notify->OnStatusFmt("Response size: %.1f KB", BodySIn->Len() / (double) TInt::Kilo);
// log finish of the call
if (NotifyOnRequest)
Notify->OnStatus(TStr::Fmt("RequestFinish %s [request took %d ms]", FunNm.CStr(), StopWatch.GetMSecInt()));
// prepare response
HttpResp = THttpResp::New(THttp::OkStatusCd,
ContTypeVal, false, BodySIn);
} catch (PExcept Except) {
// known internal error
Notify->OnStatusFmt("Exception: %s", Except->GetMsgStr().CStr());
Notify->OnStatusFmt("Location: %s", Except->GetLocStr().CStr());
TStr ResStr, ContTypeVal = THttp::TextPlainFldVal;
if (GetFunOutType() == saotXml) {
PXmlTok TopTok = TXmlTok::New("error");
TopTok->AddSubTok(TXmlTok::New("message", Except->GetMsgStr()));
TopTok->AddSubTok(TXmlTok::New("location", Except->GetLocStr()));
PXmlDoc ErrorXmlDoc = TXmlDoc::New(TopTok);
ResStr = XmlHdStr + ErrorXmlDoc->SaveStr();
ContTypeVal = THttp::TextXmlFldVal;
} else if (GetFunOutType() == saotJSon) {
PJsonVal ResVal = TJsonVal::NewObj();
ResVal->AddToObj("message", Except->GetMsgStr());
ResVal->AddToObj("location", Except->GetLocStr());
ResStr = TJsonVal::NewObj("error", ResVal)->SaveStr();
ContTypeVal = THttp::AppJSonFldVal;
}
// prepare response
HttpResp = THttpResp::New(THttp::InternalErrStatusCd,
ContTypeVal, false, TMIn::New(ResStr));
} catch (...) {
// unknown internal error
TStr ResStr, ContTypeVal = THttp::TextPlainFldVal;
if (GetFunOutType() == saotXml) {
PXmlDoc ErrorXmlDoc = TXmlDoc::New(TXmlTok::New("error"));
ResStr = XmlHdStr + ErrorXmlDoc->SaveStr();
ContTypeVal = THttp::TextXmlFldVal;
} else if (GetFunOutType() == saotJSon) {
ResStr = TJsonVal::NewObj("error", "Unknown")->SaveStr();
ContTypeVal = THttp::AppJSonFldVal;
}
// prepare response
HttpResp = THttpResp::New(THttp::InternalErrStatusCd,
ContTypeVal, false, TMIn::New(ResStr));
}
if (LogRqToFile)
LogReqRes(FldNmValPrV, HttpResp);
// send response
RqEnv->GetWebSrv()->SendHttpResp(RqEnv->GetSockId(), HttpResp);
}
int main(int argc, char* argv[]) {
#ifndef NDEBUG
// report we are running with all Asserts turned on
printf("*** Running in debug mode ***\n");
#endif
try {
// initialize QMiner environment
TQm::TEnv::Init();
// create app environment
Env = TEnv(argc, argv, TNotify::StdNotify);
Env.SetNoLine(); // making output prettier
// command line parameters
Env.PrepArgs("QMiner " + TQm::TEnv::GetVersion(), 0);
// read the action
const bool ConfigP = Env.IsArgStr("config");
const bool CreateP = Env.IsArgStr("create");
const bool StartP = Env.IsArgStr("start");
const bool StopP = Env.IsArgStr("stop");
//const bool ReloadP = Env.IsArgStr("reload");
const bool DebugP = Env.IsArgStr("debug");
// stop if no action given
const bool ActionP = (ConfigP || CreateP || StartP || StopP /*|| ReloadP*/ || DebugP);
// provide basic instruction when no action given
if (!ActionP) {
printf("\n");
printf("Usage: qm ACTION [OPTION]...\n");
printf("\n");
printf("Actions: config, create, start, stop, reload, debug\n");
} else {
Env.SetSilent();
}
// configuration file
const TStr ConfFNm = Env.GetIfArgPrefixStr("-conf=", "qm.conf", "Configration file");
// read config-specific parameters
if (!Env.IsSilent()) { printf("\nConfiguration parameters:\n"); }
const int PortN = Env.GetIfArgPrefixInt("-port=", 8080, "Port number");
const int CacheSizeMB = Env.GetIfArgPrefixInt("-cache=", 1024, "Cache size");
const bool OverwriteP = Env.IsArgStr("-overwrite", "Overwrite existing configuration file");
// read create-specific parameters
if (!Env.IsSilent()) { printf("\nCreate parameters:\n"); }
const TStr SchemaFNm = Env.GetIfArgPrefixStr("-def=", "", "Store definition file");
// read start-specific parameters
if (!Env.IsSilent()) { printf("\nStart parameters:\n"); }
const bool RdOnlyP = Env.IsArgStr("-rdonly", "Open database in Read-only mode");
const bool NoLoopP = Env.IsArgStr("-noserver", "Do not start server after script execution");
TStr OnlyScriptNm = Env.GetIfArgPrefixStr("-script=", "", "Only run this script");
// read stop-specific parameters
if (!Env.IsSilent()) { printf("\nStop parameters:\n"); }
const int ReturnCode = Env.GetIfArgPrefixInt("-return=", 0, "Return code");
// read reload-specific parameters
//if (!Env.IsSilent()) { printf("\nReload parameters:\n"); }
//TStrV ReloadNmV = Env.GetIfArgPrefixStrV("-name=", "Script name");
// read debug request parameters
if (!Env.IsSilent()) { printf("\nDebug parameters:\n"); }
TStr DebugFNm = Env.GetIfArgPrefixStr("-prefix=", "Debug-", "Prefix of debug output files");
TStrV DebugTaskV = Env.GetIfArgPrefixStrV("-task=", "Debug tasks [indexvoc, index, stores, <store>, <store>_ALL]");
const int JsStatRate = Env.GetIfArgPrefixInt("-jsmemstat=", 0, "Frequency of JavaScript memory statistics");
// read logging specific parameters
if (!Env.IsSilent()) { printf("\nLogging parameters:\n"); }
TStr LogFPath = Env.GetIfArgPrefixStr("-log=", "std", "Log Folder (std for standard output, null for silent)");
const bool Verbose = Env.IsArgStr("-v", "Verbose output (used for debugging)");
if (!Env.IsSilent()) { printf("\nPre-run file:\n"); }
const TStr PreRunFNm = Env.GetIfArgPrefixStr("-prerun=", "", "Pre-run file name");
if (!Env.IsSilent()) { printf("\n"); }
// execute pre-run command when provided
if (!PreRunFNm.Empty()) {
const int ReturnCd = system(PreRunFNm.CStr());
if (ReturnCd != 0) {
TQm::ErrorLog(TStr::Fmt("Error running prerun script: %d", ReturnCd));
}
}
// stop if no action specified
if (!ActionP) { return 0; }
// initialize notifier
TQm::TEnv::InitLogger(Verbose ? 2 : 1, LogFPath, true);
printf("\n");
// Create directory structure with basic qm.conf file
if (ConfigP) {
// check so we don't overwrite any existing configuration file
if (TFile::Exists(ConfFNm) && ! OverwriteP) {
TQm::InfoLog("Configuration file already exists (" + ConfFNm + ")");
TQm::InfoLog("Use -overwrite to force overwrite");
return 2;
}
// create configuration file
PJsonVal ConfigVal = TJsonVal::NewObj();
ConfigVal->AddToObj("port", PortN);
PJsonVal CacheVal = TJsonVal::NewObj();
CacheVal->AddToObj("index", CacheSizeMB);
CacheVal->AddToObj("store", CacheSizeMB);
ConfigVal->AddToObj("cache", CacheVal);
// save configuration file
ConfigVal->SaveStr().SaveTxt(ConfFNm);
// make folders if needed
if (!TFile::Exists("db")) { TDir::GenDir("db"); }
if (!TFile::Exists("src")) { TDir::GenDir("src"); }
if (!TFile::Exists("src/lib")) { TDir::GenDir("src/lib"); }
if (!TFile::Exists("sandbox")) { TDir::GenDir("sandbox"); }
}
// parse configuration file
TQmParam Param(ConfFNm);
// prepare lock
TFileLock Lock(Param.LockFNm);
// Initialize empty database
if (CreateP) {
// do not mess with folders with existing running qminer instance
Lock.Lock();
{
// parse schema (if no given, create an empty array)
PJsonVal SchemaVal = SchemaFNm.Empty() ? TJsonVal::NewArr() :
TJsonVal::GetValFromStr(TStr::LoadTxt(SchemaFNm));
// initialize base
TQm::PBase Base = TQm::TStorage::NewBase(Param.DbFPath, SchemaVal, 16, 16);
// save base
TQm::TStorage::SaveBase(Base);
}
// remove lock
Lock.Unlock();
}
// Start QMiner engine
if (StartP) {
// do not mess with folders with running qminer instance
Lock.Lock();
// load database and start the server
{
// resolve access type
TFAccess FAccess = RdOnlyP ? faRdOnly : faUpdate;
// load base
TQm::PBase Base = TQm::TStorage::LoadBase(Param.DbFPath, FAccess,
Param.IndexCacheSize, Param.DefStoreCacheSize, Param.StoreNmCacheSizeH);
// initialize javascript contexts
TQm::TJsUtil::SetObjStatRate(JsStatRate);
TVec<TQm::PScript> ScriptV; InitJs(Param, Base, OnlyScriptNm, ScriptV);
// start server
if (!NoLoopP) {
// prepare server functions
TSAppSrvFunV SrvFunV;
// used to stop the server
SrvFunV.Add(TSASFunExit::New());
// admin webservices
TQm::TSrvFun::RegDefFun(Base, SrvFunV);
// initialize static content serving thingies
for (int WwwRootN = 0; WwwRootN < Param.WwwRootV.Len(); WwwRootN++) {
const TStrPr& WwwRoot = Param.WwwRootV[WwwRootN];
const TStr& UrlPath = WwwRoot.Val1, FPath = WwwRoot.Val2;
TQm::TEnv::Logger->OnStatusFmt("Registering '%s' at '/%s/'", FPath.CStr(), UrlPath.CStr());
SrvFunV.Add(TSASFunFPath::New(UrlPath, FPath));
}
// register admin services
SrvFunV.Add(TQm::TJsAdminSrvFun::New(ScriptV, "qm_status"));
// register javascript contexts
for (int ScriptN = 0; ScriptN < ScriptV.Len(); ScriptN++) {
// register server function
ScriptV[ScriptN]->RegSrvFun(SrvFunV);
}
// start server
PWebSrv WebSrv = TSAppSrv::New(Param.PortN, SrvFunV, TQm::TEnv::Logger, true, true);
// report we started
TQm::TEnv::Logger->OnStatusFmt("Server started on port %d", Param.PortN);
// wait for the end
TLoop::Run();
}
// save base
TQm::TStorage::SaveBase(Base);
}
// remove lock
Lock.Unlock();
}
// Stop QMiner engine
if (StopP) {
ExecUrl(TStr::Fmt("http://127.0.0.1:%d/exit?return=%d", Param.PortN, ReturnCode),
"Server stop procedure initiated", "Error stopping server: ");
}
// Reload QMiner script
//if (ReloadP) {
// for (int ReloadNmN = 0; ReloadNmN < ReloadNmV.Len(); ReloadNmN++) {
// TStr ScriptNm = ReloadNmV[ReloadNmN];
// ExecUrl(TStr::Fmt("http://127.0.0.1:%d/%s/admin/reload", Param.PortN, ScriptNm.CStr()),
// "Initializing reload of script '" + ScriptNm + "'",
// "Error reloading script '" + ScriptNm + "': ");
// }
//}
// Debug dumps of database and index
if (DebugP) {
// do not mess with folders with existing running qminer instance
Lock.Lock();
{
// load base
TQm::PBase Base = TQm::TStorage::LoadBase(Param.DbFPath, faRdOnly,
Param.IndexCacheSize, Param.DefStoreCacheSize, Param.StoreNmCacheSizeH);
// go over task lists and prepare outputs
for (int TaskN = 0; TaskN < DebugTaskV.Len(); TaskN++) {
TStr Task = DebugTaskV[TaskN];
if (Task == "index") {
Base->PrintIndex(DebugFNm + "index.txt", true);
} else if (Task == "indexvoc") {
Base->PrintIndexVoc(DebugFNm + "indexvoc.txt");
} else if (Task == "stores") {
Base->PrintStores(DebugFNm + "stores.txt");
} else if (Task.IsSuffix("_ALL")) {
TStr StoreNm = Task.LeftOfLast('_');
Base->GetStoreByStoreNm(StoreNm)->PrintAll(Base, DebugFNm + Task + ".txt");
} else if (Base->IsStoreNm(Task)) {
Base->GetStoreByStoreNm(Task)->PrintTypes(Base, DebugFNm + Task + ".txt");
} else {
TQm::InfoLog("Unkown debug task '" + Task + "'");
}
}
}
// remove lock
Lock.Unlock();
}
} catch (const PExcept& Except) {
// GLib exception
TQm::ErrorLog("Error: " + Except->GetMsgStr());
return 2;
} catch (...) {
// other exceptions
TQm::ErrorLog("Unknown error");
return 1;
}
return TQm::TEnv::ReturnCode.Val;
}
PJsonVal TDecisionTree::TNode::ExplainLabel(const int& Label) const {
if (IsLeaf()) {
if (ClassHist[Label] <= ClassHist[1 - Label]) {
return PJsonVal();
} else {
const double Prob = ClassHist[Label];
PJsonVal Result = TJsonVal::NewArr();
PJsonVal IntersectJson = TJsonVal::NewObj();
IntersectJson->AddToObj("covered", int(NExamples*Prob));
IntersectJson->AddToObj("purity", Prob);
IntersectJson->AddToObj("terms", TJsonVal::NewArr());
Result->AddToArr(IntersectJson);
return Result;
}
}
PJsonVal Result = TJsonVal::NewArr();
if (HasLeft()) {
PJsonVal LeftUnion = Left->ExplainLabel(Label);
if (!LeftUnion.Empty()) {
if (LeftUnion->GetArrVals() == 0) {
LeftUnion->AddToArr(TJsonVal::NewArr());
}
for (int i = 0; i < LeftUnion->GetArrVals(); i++) {
PJsonVal IntersectJson = LeftUnion->GetArrVal(i);
PJsonVal TermsJson = IntersectJson->GetObjKey("terms");
bool HadFtr = false;
for (int TermN = 0; TermN < TermsJson->GetArrVals(); TermN++) {
PJsonVal TermJson = TermsJson->GetArrVal(TermN);
const int TermFtrN = TermJson->GetObjInt("ftrId");
if (TermFtrN == CutFtrN) {
HadFtr = true;
if (TermJson->GetObjNum("le") == TFlt::PInf) {
TermJson->AddToObj("le", CutFtrVal);
}
}
}
if (!HadFtr) {
PJsonVal TermJson = TJsonVal::NewObj();
TermJson->AddToObj("ftrId", CutFtrN);
TermJson->AddToObj("le", CutFtrVal);
TermJson->AddToObj("gt", TFlt::NInf);
TermsJson->AddToArr(TermJson);
}
Result->AddToArr(IntersectJson);
}
}
}
if (HasRight()) {
PJsonVal RightUnion = Right->ExplainLabel(Label);
if (!RightUnion.Empty()) {
if (RightUnion->GetArrVals() == 0) {
RightUnion->AddToArr(TJsonVal::NewArr());
}
for (int i = 0; i < RightUnion->GetArrVals(); i++) {
PJsonVal IntersectJson = RightUnion->GetArrVal(i);
PJsonVal TermsJson = IntersectJson->GetObjKey("terms");
bool HadFtr = false;
for (int TermN = 0; TermN < TermsJson->GetArrVals(); TermN++) {
PJsonVal TermJson = TermsJson->GetArrVal(TermN);
const int TermFtrN = TermJson->GetObjInt("ftrId");
if (TermFtrN == CutFtrN) {
HadFtr = true;
if (TermJson->GetObjNum("gt") == TFlt::NInf) {
TermJson->AddToObj("gt", CutFtrVal);
}
}
}
if (!HadFtr) {
PJsonVal TermJson = TJsonVal::NewObj();
TermJson->AddToObj("ftrId", CutFtrN);
TermJson->AddToObj("le", TFlt::PInf);
TermJson->AddToObj("gt", CutFtrVal);
TermsJson->AddToArr(TermJson);
}
Result->AddToArr(IntersectJson);
}
}
}
return Result->GetArrVals() > 0 ? Result : PJsonVal();
}
