PTokenizer THtml::New(const PJsonVal& ParamVal) {
// get stopwords
PSwSet SwSet = ParamVal->IsObjKey("stopwords") ?
TSwSet::ParseJson(ParamVal->GetObjKey("stopwords")) :
TSwSet::New(swstNone);
// get stemmer
PStemmer Stemmer = ParamVal->IsObjKey("stemmer") ?
TStemmer::ParseJson(ParamVal->GetObjKey("stemmer"), false) :
TStemmer::New(stmtNone, false);
const bool ToUcP = ParamVal->GetObjBool("uppercase", true);
return new THtml(SwSet, Stemmer, ToUcP);
}
//
// TBackupProfile
//
TBackupProfile::TBackupProfile(const PJsonVal& SettingsJson, const TStr& Destination_, const TStr& ProfileName_)
{
Destination = Destination_;
if (Destination.Len() > 0 && (Destination.LastCh() != '\\' || Destination.LastCh() != '/'))
Destination += "/";
ProfileName = ProfileName_;
if (!TDir::Exists(Destination))
TDir::GenDir(Destination);
VersionsToKeep = SettingsJson->GetObjInt("versionsToKeep", 1);
PJsonVal FoldersJson = SettingsJson->GetObjKey("folders");
EAssertR(FoldersJson->IsArr(), "Expected to get an array of folders");
for (int N = 0; N < FoldersJson->GetArrVals(); N++) {
PJsonVal FolderJson = FoldersJson->GetArrVal(N);
TBackupFolderInfo FolderInfo;
FolderInfo.Folder = FolderJson->GetObjStr("folder");
if (FolderJson->IsObjKey("extensions"))
FolderJson->GetObjStrV("extensions", FolderInfo.Extensions);
if (FolderInfo.Extensions.IsIn("*"))
FolderInfo.Extensions.Clr();
FolderInfo.IncludeSubfolders = FolderJson->GetObjBool("includeSubfolders");
if (FolderJson->IsObjKey("skipIfContaining"))
FolderJson->GetObjStrV("skipIfContaining", FolderInfo.SkipIfContainingV);
FolderV.Add(FolderInfo);
}
// load logs of the previous backups
ProfileLogFile = Destination + ProfileName + "/backupInfo.json";
if (TFile::Exists(ProfileLogFile)) {
PJsonVal LogJson = TJsonVal::GetValFromStr(TStr::LoadTxt(ProfileLogFile));
if (LogJson->IsArr()) {
for (int N = 0; N < LogJson->GetArrVals(); N++) {
PJsonVal Log = LogJson->GetArrVal(N);
LogV.Add(TBackupLogInfo(Log));
}
}
}
}
TStayPointDetector::TStayPointDetector(const TWPt<TBase>& Base, const PJsonVal& ParamVal) : TStreamAggr(Base, ParamVal) {
// TODO parse constructor arguments
if (ParamVal->IsObjKey("params")) {
Params = ParamVal->GetObjKey("params");
}
TStr StoreNm = ParamVal->GetObjStr("store");
Store = Base->GetStoreByStoreNm(StoreNm);
TStr TimeFieldName = ParamVal->GetObjStr("timeField");
TimeFieldId = Store->GetFieldId(TimeFieldName);
TStr LocationFieldName = ParamVal->GetObjStr("locationField");
LocationFieldId = Store->GetFieldId(LocationFieldName);
TStr AccuracyFieldName = ParamVal->GetObjStr("accuracyField");
AccuracyFieldId = Store->GetFieldId(AccuracyFieldName);
}
// parse from json configuration file
TJsParam(const TStr& RootFNm, const PJsonVal& JsVal) {
EAssertR(JsVal->IsObj(), "Unsupported type: " + TJsonVal::GetStrFromVal(JsVal));
// we must have at least the script name
EAssert(JsVal->IsObjKey("file"));
// get script name
FNm = JsVal->GetObjStr("file");
// get namespace (get from script name if not available)
Nm = JsVal->IsObjKey("name") ? JsVal->GetObjStr("name") : FNm.GetFMid();
// get initialization parameters (if available)
InitVal = JsVal->IsObjKey("init") ? JsVal->GetObjKey("init") : TJsonVal::NewObj();
// get library include folders
if (JsVal->IsObjKey("include")) {
PJsonVal IncludesVal = JsVal->GetObjKey("include");
EAssertR(IncludesVal->IsArr(), "Expected array of strings, not: " + TJsonVal::GetStrFromVal(IncludesVal));
for (int IncludeN = 0; IncludeN < IncludesVal->GetArrVals(); IncludeN++) {
PJsonVal IncludeVal = IncludesVal->GetArrVal(IncludeN);
EAssertR(IncludeVal->IsStr(), "Expected string, not: " + TJsonVal::GetStrFromVal(IncludeVal));
IncludeFPathV.Add(IncludeVal->GetStr());
}
}
// handle default includes
AddLocalLibFPath();
AddQMinerLibFPath();
// get folders with access permissions
if (JsVal->IsObjKey("dirs")) {
PJsonVal DirsVal = JsVal->GetObjKey("dirs");
EAssertR(DirsVal->IsArr(), "Expected array of strings, not: " + TJsonVal::GetStrFromVal(DirsVal));
for (int DirN = 0; DirN < DirsVal->GetArrVals(); DirN++) {
PJsonVal DirVal = DirsVal->GetArrVal(DirN);
EAssertR(DirVal->IsStr(), "Expected string, not: " + TJsonVal::GetStrFromVal(DirVal));
AccessFPathV.Add(DirVal->GetStr());
}
}
// add sandbox access
AddSandboxAccessFPath(RootFNm);
}
TOnlineHistogram::TOnlineHistogram(const PJsonVal& ParamVal) {
EAssertR(ParamVal->IsObjKey("lowerBound"), "TOnlineHistogram: lowerBound key missing!");
EAssertR(ParamVal->IsObjKey("upperBound"), "TOnlineHistogram: upperBound key missing!");
// bounded lowest point
TFlt LBound = ParamVal->GetObjNum("lowerBound");
// bounded highest point
TFlt UBound = ParamVal->GetObjNum("upperBound");
EAssertR(LBound < UBound, "TOnlineHistogram: Lower bound should be smaller than upper bound");
// number of equal bins ? (not counting possibly infinite ones)
TInt Bins = ParamVal->GetObjInt("bins", 5);
EAssertR(Bins > 0, "TOnlineHistogram: Number of bins should be greater than 0");
// include infinities in the bounds?
TBool AddNegInf = ParamVal->GetObjBool("addNegInf", false);
TBool AddPosInf = ParamVal->GetObjBool("addPosInf", false);
MinCount = ParamVal->GetObjInt("initMinCount", 0);
Init(LBound, UBound, Bins, AddNegInf, AddPosInf);
};
void TNodeJsSA::New(const v8::FunctionCallbackInfo<v8::Value>& Args) {
v8::Isolate* Isolate = v8::Isolate::GetCurrent();
v8::HandleScope HandleScope(Isolate);
if (Args.Length() == 0) { return; } //
EAssertR(!constructor.IsEmpty(), "TNodeJsSA::New: constructor is empty. Did you call TNodeJsSA::Init(exports); in this module's init function?");
QmAssertR(Args.Length() <= 3 && Args.Length() >= 2, "stream aggregator constructor expects at least two parameters");
QmAssertR(Args[0]->IsObject() && Args[1]->IsObject(), "stream aggregator constructor expects first two arguments as objects");
// new sa(...)
if (Args.IsConstructCall()) {
TQm::PStreamAggr StreamAggr;
TNodeJsBase* JsBase = TNodeJsUtil::UnwrapCheckWatcher<TNodeJsBase>(Args[0]->ToObject());
// get aggregate type
TStr TypeNm = TNodeJsUtil::GetArgStr(Args, 1, "type", "javaScript");
if (TypeNm == "javaScript") {
// we have a javascript stream aggregate
TStr AggrName = TNodeJsUtil::GetArgStr(Args, 1, "name", "");
// we need a name, if not give just generate one
if (AggrName.Empty()) { AggrName = TGuid::GenSafeGuid(); }
// create aggregate
StreamAggr = TNodeJsStreamAggr::New(JsBase->Base, AggrName, Args[1]->ToObject());
}
else if (TypeNm == "ftrext") {
TStr AggrName = TNodeJsUtil::GetArgStr(Args, 1, "name", "");
QmAssertR(Args[1]->ToObject()->Has(v8::String::NewFromUtf8(Isolate, "featureSpace")), "addStreamAggr: featureSpace property missing!");
// we need a name, if not give just generate one
if (AggrName.Empty()) { AggrName = TGuid::GenSafeGuid(); }
throw TQm::TQmExcept::New("ftrext stream aggr not implemented yet! (needs feature space implementation)");
// TODO
//TQm::PFtrSpace FtrSpace = TJsFtrSpace::GetArgFtrSpace(Args[1]->ToObject()->Get(v8::String::NewFromUtf8(Isolate, "featureSpace")));
//StreamAggr = TStreamAggrs::TFtrExtAggr::New(JsBase->Base, AggrName, FtrSpace);
}
else if (TypeNm == "stmerger") {
// create new aggregate
PJsonVal ParamVal = TNodeJsUtil::GetArgJson(Args, 1);
StreamAggr = TQm::TStreamAggr::New(JsBase->Base, TypeNm, ParamVal);
PJsonVal FieldArrVal = ParamVal->GetObjKey("fields");
TStrV InterpNmV;
QmAssertR(ParamVal->IsObjKey("fields"), "Missing argument 'fields'!");
// automatically register the aggregate for addRec callbacks
for (int FieldN = 0; FieldN < FieldArrVal->GetArrVals(); FieldN++) {
PJsonVal FieldVal = FieldArrVal->GetArrVal(FieldN);
PJsonVal SourceVal = FieldVal->GetObjKey("source");
TStr StoreNm = "";
if (SourceVal->IsStr()) {
// we have just store name
StoreNm = SourceVal->GetStr();
}
else if (SourceVal->IsObj()) {
// get store
StoreNm = SourceVal->GetObjStr("store");
}
JsBase->Base->AddStreamAggr(JsBase->Base->GetStoreByStoreNm(StoreNm)->GetStoreId(), StreamAggr);
}
}
else {
// we have a GLib stream aggregate, translate parameters to PJsonVal
PJsonVal ParamVal = TNodeJsUtil::GetArgJson(Args, 1);
if (Args.Length() >= 3 && Args[2]->IsString()) {
ParamVal->AddToObj("store", TNodeJsUtil::GetArgStr(Args, 2));
}
// check if it's one stream aggregate or composition
if (TQm::TStreamAggrs::TCompositional::IsCompositional(TypeNm)) {
// we have a composition of aggregates, call code to assemble it
TQm::TStreamAggrs::TCompositional::Register(JsBase->Base, TypeNm, ParamVal);
}
else {
// create new aggregate
StreamAggr = TQm::TStreamAggr::New(JsBase->Base, TypeNm, ParamVal);
}
}
if (!TQm::TStreamAggrs::TCompositional::IsCompositional(TypeNm)) {
if (Args.Length() > 2) {
TStrV Stores(0);
if (Args[2]->IsString()) {
Stores.Add(TNodeJsUtil::GetArgStr(Args, 2));
}
if (Args[2]->IsArray()) {
PJsonVal StoresJson = TNodeJsUtil::GetArgJson(Args, 2);
QmAssertR(StoresJson->IsDef(), "stream aggr constructor : Args[2] should be a string (store name) or a string array (store names)");
StoresJson->GetArrStrV(Stores);
}
for (int StoreN = 0; StoreN < Stores.Len(); StoreN++) {
QmAssertR(JsBase->Base->IsStoreNm(Stores[StoreN]), "stream aggr constructor : Args[2] : store does not exist!");
JsBase->Base->AddStreamAggr(Stores[StoreN], StreamAggr);
}
}
else {
JsBase->Base->AddStreamAggr(StreamAggr);
}
// non-compositional aggregates are returned
TNodeJsSA* JsSA = new TNodeJsSA(StreamAggr);
v8::Local<v8::Object> Instance = Args.This();
JsSA->Wrap(Instance);
Args.GetReturnValue().Set(Instance);
return;
}
}
// sa(...) -> calls new sa(...)
else {
if (Args.Length() == 2) {
const int Argc = 2;
v8::Local<v8::Value> Argv[Argc] = { Args[0], Args[1] };
v8::Local<v8::Function> cons = v8::Local<v8::Function>::New(Isolate, constructor);
v8::Local<v8::Object> Instance = cons->NewInstance(Argc, Argv);
Args.GetReturnValue().Set(Instance);
return;
}
if (Args.Length() == 3) {
const int Argc = 3;
v8::Local<v8::Value> Argv[Argc] = { Args[0], Args[1], Args[2] };
v8::Local<v8::Function> cons = v8::Local<v8::Function>::New(Isolate, constructor);
v8::Local<v8::Object> Instance = cons->NewInstance(Argc, Argv);
Args.GetReturnValue().Set(Instance);
return;
}
else {
// sa()->calls new sa()
v8::Local<v8::Function> cons = v8::Local<v8::Function>::New(Isolate, constructor);
v8::Local<v8::Object> Instance = cons->NewInstance();
Args.GetReturnValue().Set(Instance);
return;
}
}
}
TQmParam(const TStr& FNm) {
EAssertR(TFile::Exists(FNm), "Missing configuration file " + FNm);
// load configuration file
PJsonVal ConfigVal = TJsonVal::GetValFromSIn(TFIn::New(FNm));
EAssertR(ConfigVal->IsObj(), "Invalid setting file - not valid JSON");
// parse out common stuff
RootFPath = TStr::GetNrFPath(ConfigVal->GetObjStr("directory", TDir::GetCurDir()));
LockFNm = RootFPath + "./lock";
DbFPath = ConfigVal->GetObjStr("database", "./db/");
PortN = TFlt::Round(ConfigVal->GetObjNum("port"));
// parse out unicode definition file
TStr UnicodeFNm = ConfigVal->GetObjStr("unicode", TQm::TEnv::QMinerFPath + "./UnicodeDef.Bin");
if (!TUnicodeDef::IsDef()) { TUnicodeDef::Load(UnicodeFNm); }
// parse cache
if (ConfigVal->IsObjKey("cache")) {
PJsonVal CacheVal = ConfigVal->GetObjKey("cache");
// parse out index and default store cache sizes
IndexCacheSize = int64(CacheVal->GetObjNum("index", 1024)) * int64(TInt::Mega);
DefStoreCacheSize = int64(CacheVal->GetObjNum("store", 1024)) * int64(TInt::Mega);
// prase out store specific sizes, when available
if (CacheVal->IsObjKey("stores")) {
PJsonVal StoreCacheVals = CacheVal->GetObjKey("stores");
for (int StoreN = 0; StoreN < StoreCacheVals->GetArrVals(); StoreN++) {
PJsonVal StoreCacheVal = StoreCacheVals->GetArrVal(StoreN);
TStr StoreName = StoreCacheVal->GetObjStr("name");
uint64 StoreCacheSize = int64(StoreCacheVal->GetObjNum("size")) * int64(TInt::Mega);
StoreNmCacheSizeH.AddDat(StoreName, StoreCacheSize);
}
}
} else {
// default sizes are set to 1GB for index and stores
IndexCacheSize = int64(1024) * int64(TInt::Mega);
DefStoreCacheSize = int64(1024) * int64(TInt::Mega);
}
// load scripts
if (ConfigVal->IsObjKey("script")) {
// we have configuration file, read it
PJsonVal JsVals = ConfigVal->GetObjKey("script");
if (JsVals->IsArr()) {
for (int JsValN = 0; JsValN < JsVals->GetArrVals(); JsValN++) {
JsParamV.Add(TJsParam(RootFPath, JsVals->GetArrVal(JsValN)));
}
} else {
JsParamV.Add(TJsParam(RootFPath, JsVals));
}
} else {
// no settings for scripts, assume default setting
TStr SrcFPath = TStr::GetNrAbsFPath("src", RootFPath);
TFFile File(SrcFPath, ".js", false); TStr SrcFNm;
while (File.Next(SrcFNm)) {
JsParamV.Add(TJsParam(RootFPath, SrcFNm));
}
}
// load serving folders
//TODO: Add to qm config ability to edit this
if (ConfigVal->IsObjKey("wwwroot")) {
PJsonVal WwwVals = ConfigVal->GetObjKey("wwwroot");
if (WwwVals->IsArr()) {
for (int WwwValN = 0; WwwValN < WwwVals->GetArrVals(); WwwValN++) {
AddWwwRoot(WwwVals->GetArrVal(WwwValN));
}
} else {
AddWwwRoot(WwwVals);
}
}
// check for folder with admin GUI
TStr GuiFPath = TStr::GetNrAbsFPath("gui", TQm::TEnv::QMinerFPath);
if (TDir::Exists(GuiFPath)) {
WwwRootV.Add(TStrPr("admin", GuiFPath));
}
// check for any default wwwroot
TStr DefaultWwwRootFPath = TStr::GetNrAbsFPath("www", RootFPath);
if (TDir::Exists(DefaultWwwRootFPath)) {
WwwRootV.Add(TStrPr("www", DefaultWwwRootFPath));
}
}
TChiSquare::TChiSquare(const PJsonVal& ParamVal): P(TFlt::PInf) {
// P value is set to infinity by default (null hypothesis is not rejected)
EAssertR(ParamVal->IsObjKey("degreesOfFreedom"), "TChiSquare: degreesOfFreedom key missing!");
// degrees of freedom
DegreesOfFreedom = ParamVal->GetObjInt("degreesOfFreedom");
}
