// save the logs of the backups to the log file
void TBackupProfile::SaveLogs() const
{
TJsonValV ArrV;
for (int N = 0; N < LogV.Len(); N++)
ArrV.Add(LogV[N].GetJson());
PJsonVal JsonArr = TJsonVal::NewArr(ArrV);
TStr JsonStr = JsonArr->SaveStr();
JsonStr.SaveTxt(ProfileLogFile);
}
PStemmer TStemmer::ParseJson(const PJsonVal& StemmerVal, const bool& RealWordP) {
if (StemmerVal->IsBool()) {
return TStemmer::New(StemmerVal->GetBool() ? stmtPorter : stmtNone, RealWordP);
} else if (StemmerVal->IsObj()) {
TStr StemmerType = StemmerVal->GetObjStr("type", "none");
const bool RealWordP = StemmerVal->GetObjBool("realWord", RealWordP);
return TStemmer::New((StemmerType == "porter") ? stmtPorter : stmtNone, RealWordP);
}
throw TExcept::New("Unknown stemmer definiton " + StemmerVal->SaveStr());
}
void TNodeJsFOut::writeJson(const v8::FunctionCallbackInfo<v8::Value>& Args) {
v8::Isolate* Isolate = v8::Isolate::GetCurrent();
v8::HandleScope HandleScope(Isolate);
PJsonVal JsonVal = TNodeJsUtil::GetArgJson(Args, 0);
TStr JsonStr = JsonVal->SaveStr();
TNodeJsFOut* JsFOut = ObjectWrap::Unwrap<TNodeJsFOut>(Args.This());
EAssertR(!JsFOut->SOut.Empty(), "Output stream already closed!");
JsonStr.Save(*JsFOut->SOut);
Args.GetReturnValue().Set(Args.Holder());
}
void TSAppSrv::OnHttpRq(const uint64& SockId, const PHttpRq& HttpRq) {
// last appropriate error code, start with bad request
int ErrStatusCd = THttp::BadRqStatusCd;
try {
// check http-request correctness - return if error
EAssertR(HttpRq->IsOk(), "Bad HTTP request!");
// check url correctness - return if error
PUrl HttpRqUrl = HttpRq->GetUrl();
EAssertR(HttpRqUrl->IsOk(), "Bad request URL!");
// extract function name
TStr FunNm = HttpRqUrl->GetPathSeg(0);
// check if we have the function registered
if (FunNm == "favicon.ico") {
PHttpResp HttpResp = THttpResp::New(THttp::OkStatusCd,
THttp::ImageIcoFldVal, false, Favicon.GetSIn());
SendHttpResp(SockId, HttpResp);
return;
} else if (!FunNm.Empty() && !FunNmToFunH.IsKey(FunNm)) {
ErrStatusCd = THttp::ErrNotFoundStatusCd;
GetNotify()->OnStatusFmt("[AppSrv] Unknown function '%s'!", FunNm.CStr());
TExcept::Throw("Unknown function '" + FunNm + "'!");
}
// extract parameters
PUrlEnv HttpRqUrlEnv = HttpRq->GetUrlEnv();
TStrKdV FldNmValPrV; HttpRqUrlEnv->GetKeyValPrV(FldNmValPrV);
// report call
if (ShowParamP) { GetNotify()->OnStatus(HttpRq->GetUrl()->GetUrlStr()); }
// request parsed well, from now on it's internal error
ErrStatusCd = THttp::InternalErrStatusCd;
// processed requested function
if (!FunNm.Empty()) {
// prepare request environment
PSAppSrvRqEnv RqEnv = TSAppSrvRqEnv::New(this, SockId, HttpRq, FunNmToFunH);
// retrieve function
PSAppSrvFun SrvFun = FunNmToFunH.GetDat(FunNm);
// call function
SrvFun->Exec(FldNmValPrV, RqEnv);
} else {
// internal SAppSrv call
if (!ListFunP) {
// we are not allowed to list functions
ErrStatusCd = THttp::ErrNotFoundStatusCd;
TExcept::Throw("Unknown page");
}
// prepare a list of registered functions
PJsonVal FunArrVal = TJsonVal::NewArr();
int KeyId = FunNmToFunH.FFirstKeyId();
while (FunNmToFunH.FNextKeyId(KeyId)) {
FunArrVal->AddToArr(TJsonVal::NewObj("name", FunNmToFunH.GetKey(KeyId)));
}
PJsonVal ResVal = TJsonVal::NewObj();
ResVal->AddToObj("port", GetPortN());
ResVal->AddToObj("connections", GetConns());
ResVal->AddToObj("functions", FunArrVal);
TStr ResStr = ResVal->SaveStr();
// prepare response
PHttpResp HttpResp = THttpResp::New(THttp::OkStatusCd,
THttp::AppJSonFldVal, false, TMIn::New(ResStr));
// send response
SendHttpResp(SockId, HttpResp);
}
} catch (PExcept Except) {
// known internal error
TNotify::StdNotify->OnNotifyFmt(ntErr, "Error: %s", Except->GetMsgStr().CStr());
TNotify::StdNotify->OnNotifyFmt(ntErr, "Error location info: %s", Except->GetLocStr().CStr());
PJsonVal ErrorVal = TJsonVal::NewObj();
ErrorVal->AddToObj("message", Except->GetMsgStr());
ErrorVal->AddToObj("location", Except->GetLocStr());
PJsonVal ResVal = TJsonVal::NewObj("error", ErrorVal);
TStr ResStr = ResVal->SaveStr();
// prepare response
PHttpResp HttpResp = THttpResp::New(ErrStatusCd,
THttp::AppJSonFldVal, false, TMIn::New(ResStr));
// send response
SendHttpResp(SockId, HttpResp);
} catch (...) {
TNotify::StdNotify->OnNotify(ntErr, "Unknown internal error");
// unknown internal error
PJsonVal ResVal = TJsonVal::NewObj("error", "Unknown internal error");
TStr ResStr = ResVal->SaveStr();
// prepare response
PHttpResp HttpResp = THttpResp::New(ErrStatusCd,
THttp::AppJSonFldVal, false, TMIn::New(ResStr));
// send response
SendHttpResp(SockId, HttpResp);
}
}
int main(int argc, char* argv[]) {
#ifndef NDEBUG
// report we are running with all Asserts turned on
printf("*** Running in debug mode ***\n");
#endif
try {
// initialize QMiner environment
TQm::TEnv::Init();
// create app environment
Env = TEnv(argc, argv, TNotify::StdNotify);
Env.SetNoLine(); // making output prettier
// command line parameters
Env.PrepArgs("QMiner " + TQm::TEnv::GetVersion(), 0);
// read the action
const bool ConfigP = Env.IsArgStr("config");
const bool CreateP = Env.IsArgStr("create");
const bool StartP = Env.IsArgStr("start");
const bool StopP = Env.IsArgStr("stop");
//const bool ReloadP = Env.IsArgStr("reload");
const bool DebugP = Env.IsArgStr("debug");
// stop if no action given
const bool ActionP = (ConfigP || CreateP || StartP || StopP /*|| ReloadP*/ || DebugP);
// provide basic instruction when no action given
if (!ActionP) {
printf("\n");
printf("Usage: qm ACTION [OPTION]...\n");
printf("\n");
printf("Actions: config, create, start, stop, reload, debug\n");
} else {
Env.SetSilent();
}
// configuration file
const TStr ConfFNm = Env.GetIfArgPrefixStr("-conf=", "qm.conf", "Configration file");
// read config-specific parameters
if (!Env.IsSilent()) { printf("\nConfiguration parameters:\n"); }
const int PortN = Env.GetIfArgPrefixInt("-port=", 8080, "Port number");
const int CacheSizeMB = Env.GetIfArgPrefixInt("-cache=", 1024, "Cache size");
const bool OverwriteP = Env.IsArgStr("-overwrite", "Overwrite existing configuration file");
// read create-specific parameters
if (!Env.IsSilent()) { printf("\nCreate parameters:\n"); }
const TStr SchemaFNm = Env.GetIfArgPrefixStr("-def=", "", "Store definition file");
// read start-specific parameters
if (!Env.IsSilent()) { printf("\nStart parameters:\n"); }
const bool RdOnlyP = Env.IsArgStr("-rdonly", "Open database in Read-only mode");
const bool NoLoopP = Env.IsArgStr("-noserver", "Do not start server after script execution");
TStr OnlyScriptNm = Env.GetIfArgPrefixStr("-script=", "", "Only run this script");
// read stop-specific parameters
if (!Env.IsSilent()) { printf("\nStop parameters:\n"); }
const int ReturnCode = Env.GetIfArgPrefixInt("-return=", 0, "Return code");
// read reload-specific parameters
//if (!Env.IsSilent()) { printf("\nReload parameters:\n"); }
//TStrV ReloadNmV = Env.GetIfArgPrefixStrV("-name=", "Script name");
// read debug request parameters
if (!Env.IsSilent()) { printf("\nDebug parameters:\n"); }
TStr DebugFNm = Env.GetIfArgPrefixStr("-prefix=", "Debug-", "Prefix of debug output files");
TStrV DebugTaskV = Env.GetIfArgPrefixStrV("-task=", "Debug tasks [indexvoc, index, stores, <store>, <store>_ALL]");
const int JsStatRate = Env.GetIfArgPrefixInt("-jsmemstat=", 0, "Frequency of JavaScript memory statistics");
// read logging specific parameters
if (!Env.IsSilent()) { printf("\nLogging parameters:\n"); }
TStr LogFPath = Env.GetIfArgPrefixStr("-log=", "std", "Log Folder (std for standard output, null for silent)");
const bool Verbose = Env.IsArgStr("-v", "Verbose output (used for debugging)");
if (!Env.IsSilent()) { printf("\nPre-run file:\n"); }
const TStr PreRunFNm = Env.GetIfArgPrefixStr("-prerun=", "", "Pre-run file name");
if (!Env.IsSilent()) { printf("\n"); }
// execute pre-run command when provided
if (!PreRunFNm.Empty()) {
const int ReturnCd = system(PreRunFNm.CStr());
if (ReturnCd != 0) {
TQm::ErrorLog(TStr::Fmt("Error running prerun script: %d", ReturnCd));
}
}
// stop if no action specified
if (!ActionP) { return 0; }
// initialize notifier
TQm::TEnv::InitLogger(Verbose ? 2 : 1, LogFPath, true);
printf("\n");
// Create directory structure with basic qm.conf file
if (ConfigP) {
// check so we don't overwrite any existing configuration file
if (TFile::Exists(ConfFNm) && ! OverwriteP) {
TQm::InfoLog("Configuration file already exists (" + ConfFNm + ")");
TQm::InfoLog("Use -overwrite to force overwrite");
return 2;
}
// create configuration file
PJsonVal ConfigVal = TJsonVal::NewObj();
ConfigVal->AddToObj("port", PortN);
PJsonVal CacheVal = TJsonVal::NewObj();
CacheVal->AddToObj("index", CacheSizeMB);
CacheVal->AddToObj("store", CacheSizeMB);
ConfigVal->AddToObj("cache", CacheVal);
// save configuration file
ConfigVal->SaveStr().SaveTxt(ConfFNm);
// make folders if needed
if (!TFile::Exists("db")) { TDir::GenDir("db"); }
if (!TFile::Exists("src")) { TDir::GenDir("src"); }
if (!TFile::Exists("src/lib")) { TDir::GenDir("src/lib"); }
if (!TFile::Exists("sandbox")) { TDir::GenDir("sandbox"); }
}
// parse configuration file
TQmParam Param(ConfFNm);
// prepare lock
TFileLock Lock(Param.LockFNm);
// Initialize empty database
if (CreateP) {
// do not mess with folders with existing running qminer instance
Lock.Lock();
{
// parse schema (if no given, create an empty array)
PJsonVal SchemaVal = SchemaFNm.Empty() ? TJsonVal::NewArr() :
TJsonVal::GetValFromStr(TStr::LoadTxt(SchemaFNm));
// initialize base
TQm::PBase Base = TQm::TStorage::NewBase(Param.DbFPath, SchemaVal, 16, 16);
// save base
TQm::TStorage::SaveBase(Base);
}
// remove lock
Lock.Unlock();
}
// Start QMiner engine
if (StartP) {
// do not mess with folders with running qminer instance
Lock.Lock();
// load database and start the server
{
// resolve access type
TFAccess FAccess = RdOnlyP ? faRdOnly : faUpdate;
// load base
TQm::PBase Base = TQm::TStorage::LoadBase(Param.DbFPath, FAccess,
Param.IndexCacheSize, Param.DefStoreCacheSize, Param.StoreNmCacheSizeH);
// initialize javascript contexts
TQm::TJsUtil::SetObjStatRate(JsStatRate);
TVec<TQm::PScript> ScriptV; InitJs(Param, Base, OnlyScriptNm, ScriptV);
// start server
if (!NoLoopP) {
// prepare server functions
TSAppSrvFunV SrvFunV;
// used to stop the server
SrvFunV.Add(TSASFunExit::New());
// admin webservices
TQm::TSrvFun::RegDefFun(Base, SrvFunV);
// initialize static content serving thingies
for (int WwwRootN = 0; WwwRootN < Param.WwwRootV.Len(); WwwRootN++) {
const TStrPr& WwwRoot = Param.WwwRootV[WwwRootN];
const TStr& UrlPath = WwwRoot.Val1, FPath = WwwRoot.Val2;
TQm::TEnv::Logger->OnStatusFmt("Registering '%s' at '/%s/'", FPath.CStr(), UrlPath.CStr());
SrvFunV.Add(TSASFunFPath::New(UrlPath, FPath));
}
// register admin services
SrvFunV.Add(TQm::TJsAdminSrvFun::New(ScriptV, "qm_status"));
// register javascript contexts
for (int ScriptN = 0; ScriptN < ScriptV.Len(); ScriptN++) {
// register server function
ScriptV[ScriptN]->RegSrvFun(SrvFunV);
}
// start server
PWebSrv WebSrv = TSAppSrv::New(Param.PortN, SrvFunV, TQm::TEnv::Logger, true, true);
// report we started
TQm::TEnv::Logger->OnStatusFmt("Server started on port %d", Param.PortN);
// wait for the end
TLoop::Run();
}
// save base
TQm::TStorage::SaveBase(Base);
}
// remove lock
Lock.Unlock();
}
// Stop QMiner engine
if (StopP) {
ExecUrl(TStr::Fmt("http://127.0.0.1:%d/exit?return=%d", Param.PortN, ReturnCode),
"Server stop procedure initiated", "Error stopping server: ");
}
// Reload QMiner script
//if (ReloadP) {
// for (int ReloadNmN = 0; ReloadNmN < ReloadNmV.Len(); ReloadNmN++) {
// TStr ScriptNm = ReloadNmV[ReloadNmN];
// ExecUrl(TStr::Fmt("http://127.0.0.1:%d/%s/admin/reload", Param.PortN, ScriptNm.CStr()),
// "Initializing reload of script '" + ScriptNm + "'",
// "Error reloading script '" + ScriptNm + "': ");
// }
//}
// Debug dumps of database and index
if (DebugP) {
// do not mess with folders with existing running qminer instance
Lock.Lock();
{
// load base
TQm::PBase Base = TQm::TStorage::LoadBase(Param.DbFPath, faRdOnly,
Param.IndexCacheSize, Param.DefStoreCacheSize, Param.StoreNmCacheSizeH);
// go over task lists and prepare outputs
for (int TaskN = 0; TaskN < DebugTaskV.Len(); TaskN++) {
TStr Task = DebugTaskV[TaskN];
if (Task == "index") {
Base->PrintIndex(DebugFNm + "index.txt", true);
} else if (Task == "indexvoc") {
Base->PrintIndexVoc(DebugFNm + "indexvoc.txt");
} else if (Task == "stores") {
Base->PrintStores(DebugFNm + "stores.txt");
} else if (Task.IsSuffix("_ALL")) {
TStr StoreNm = Task.LeftOfLast('_');
Base->GetStoreByStoreNm(StoreNm)->PrintAll(Base, DebugFNm + Task + ".txt");
} else if (Base->IsStoreNm(Task)) {
Base->GetStoreByStoreNm(Task)->PrintTypes(Base, DebugFNm + Task + ".txt");
} else {
TQm::InfoLog("Unkown debug task '" + Task + "'");
}
}
}
// remove lock
Lock.Unlock();
}
} catch (const PExcept& Except) {
// GLib exception
TQm::ErrorLog("Error: " + Except->GetMsgStr());
return 2;
} catch (...) {
// other exceptions
TQm::ErrorLog("Unknown error");
return 1;
}
return TQm::TEnv::ReturnCode.Val;
}
