size_t TFormatOutput::skipword(
const char *str,
size_t pos,
size_t len,
int &NextLine,
Stroka &Line) {
for(size_t i = pos; i < len; i++) {
if(str[i] == '~' && str[i + 1] == '+') {
Level++;
Line += Stroka(str, pos, i - pos);
i += 1;
pos = i + 1;
continue;
}
if(str[i] == '~' && str[i + 1] == '-') {
Level--;
Line += Stroka(str, pos, i - pos);
i += 1;
pos = i + 1;
continue;
}
if(str[i] > 0 && isspace(str[i])) {
NextLine = 0;
Line += Stroka(str, pos, i - pos);
return i;
}
}
NextLine = 0;
Line += Stroka(str, pos, len - pos);
return len;
}
FreeEChemMatterMultiV::FreeEChemMatterMultiV()
: FreeEIOFind("material.cfg", "MatterChemH2Met"), FreeMinim(new DissMultiV()) {
SubstMap[Stroka("M1")] = NamedSubst("M1", new FreeEIdeal(), 2, 1, 1, 1);
SubstMap[Stroka("M2")] = NamedSubst("M2", new FreeEIdeal(), 2, 0, 1, 1);
ChemEq eq(Stroka("M1"), -1);
eq.AddSubst(Stroka("M2"), 1);
ChemEqs.push_back(eq);
ClcStartParams();
}
Stroka GetEnv(const Stroka& key) {
#ifdef _win_
char buf[1001];
Zero(buf);
int len = GetEnvironmentVariable(~key, &buf[0], 1000);
return Stroka(&buf[0], len);
#else
const char* env = getenv(~key);
return env ? Stroka(env) : Stroka();
#endif
}
static inline Stroka FreeBSDGetExecPath() {
int mib[] = { CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, -1 };
TTempBuf buf;
int r = FreeBSDSysCtl(mib, ARRAY_SIZE(mib), buf);
if (r == 0) {
return Stroka(buf.Data(), buf.Filled()-1);
} else if (r == ENOTSUP) { // older FreeBSD version
return ReadLink("/proc/" + ToString(getpid()) + "/file");
} else {
return Stroka();
}
}
static inline Stroka FreeBSDGetArgv0() {
int mib[] = { CTL_KERN, KERN_PROC, KERN_PROC_ARGS, getpid() };
TTempBuf buf;
int r = FreeBSDSysCtl(mib, ARRAY_SIZE(mib), buf);
if (r == 0) {
return Stroka(buf.Data());
} else if (r == ENOTSUP) {
return Stroka();
} else {
ythrow yexception() << "FreeBSDGetArgv0() failed: " << LastSystemErrorText();
}
}
TFile& TFile::AddAlias(const Stroka& name) {
AddAliasImpl(name);
const TStringBuf proto = STRINGBUF(".proto");
const TStringBuf gztproto = STRINGBUF(".gztproto");
// also add complementing .gztproto or .proto
if (name.has_suffix(proto))
AddAliasImpl(Stroka(TStringBuf(name).Chop(proto.size()), gztproto));
else if (name.has_suffix(gztproto))
AddAliasImpl(Stroka(TStringBuf(name).Chop(gztproto.size()), proto));
return *this;
}
Stroka GetCwd() {
TTempBuf result;
#ifdef _win_
int r = GetCurrentDirectory(result.Size(), result.Data());
if (r == 0)
throw TIoSystemError() << "failed to GetCurrentDirectory";
return Stroka(result.Data(), r);
#else
char* r = getcwd(result.Data(), result.Size());
if (r == 0)
throw TIoSystemError() << "failed to getcwd";
return Stroka(result.Data());
#endif
}
Stroka CCommonParm::GetSourceFormat() const {
if (NULL != Config.Get() && Config->input().has_format())
switch (Config->input().GetFormat()) {
case TTextMinerConfig::TInputParams::plain:
return Stroka("text");
case TTextMinerConfig::TInputParams::html:
return Stroka("html");
default:
ythrow yexception() << "Source format unknown.";
}
return Stroka("text");
}
Stroka EncodeXML(const char* str, int qEncodeEntities) {
Stroka strout, tmp;
if (!str)
return Stroka();
const char* s;
if (qEncodeEntities) {
tmp = EncodeHtmlPcdata(str, 1);
s = ~tmp;
} else {
s = str;
}
int prev = 0;
int i;
for (i = 0; s[i]; i++) {
// remove all "irregular" chars
// XML 1.0 specification, section 2.2:
// Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF]
if (s[i] > 0 && s[i] < 32
&& s[i] != 0x09 && s[i] != 0x0A && s[i] != 0x0D) {
strout.append(s + prev, 0, i-prev, i-prev);
prev = i + 1;
}
}
strout.append(s + prev, 0, i-prev, i-prev);
return strout;
}
void WriteFile(
const char *file,
const vector<vector<double>> &vec,
const char *head,
const char *mes) {
FILE *out = open(file, "w", mes);
if(head) {
Stroka tmp = Stroka(head) + "\n";
fputs(tmp.c_str(), out);
}
if(vec.size() == 0)
return;
char Buf[256];
for(size_t i = 0; i < vec[0].size(); i++) {
for(size_t j = 0; j < vec.size(); j++) {
if(vec[j][i] == NullNumber)
fputs(" \"\" ", out);
else {
sprintf(Buf, "%10.9g ", vec[j][i]);
fputs(Buf, out);
}
}
fputs("\x0d\x0a", out);
}
close(out);
}
inline void TestFile() {
Stroka s = NUnitTest::RandomString(1000000, 1);
const Stroka tmpFile = "tmp";
{
TBufferedFileOutput fo(tmpFile);
fo.Write(~s, +s);
}
char buf1[100];
char buf2[100];
UNIT_ASSERT_EQUAL(Stroka(MD5::File(~tmpFile, buf1)), Stroka(MD5::Data((const unsigned char*)~s, +s, buf2)));
NFs::Remove(~tmpFile);
}
double FromStringImpl<double>(const char* data, size_t len) {
char* se;
double d = StrToD(data, data + len, &se);
if (se != data + len) {
ythrow TFromStringException() << "cannot parse float(" << Stroka(data, len) << ")";
}
return d;
}
TSimpleLemmer::TSimpleLemmer(const Stroka& path) {
Stroka libPath = path;
if (path.length() == 0) {
Stroka prgDir = GetProgramDir();
if ((prgDir.length() > 1 && prgDir[1] == ':') || (prgDir.length() > 0 && prgDir[0] == '/'))
libPath = prgDir;
else
libPath = GetCwd() + Stroka("/") + prgDir;
SlashFolderLocal(libPath);
libPath += Stroka("libmystem_c_binding.so");
//Cerr << "libPath == " << libPath << Endl;
}
Lib.Open(~libPath, RTLD_NOW | RTLD_DEEPBIND | RTLD_NODELETE);
if (!IsInitialized())
yexception() << "Can't load lemmer from \"" << path << "\"";
}
static Stroka ReadLink(const Stroka& path) {
TTempBuf buf;
for (;;) {
ssize_t r = readlink(~path, buf.Data(), buf.Size());
if (r < 0)
ythrow yexception() << "can't read link " << path;
if (r < (ssize_t)buf.Size())
return Stroka(buf.Data(), r);
buf = TTempBuf(buf.Size() * 2);
}
}
Stroka CCommonParm::GetOutputFileName() const {
if (NULL != Config.Get() && Config->has_output() && Config->output().has_file()) {
Stroka fn = Config->output().GetFile();
fn.to_lower();
if ("stdout" == fn)
return "-";
return Config->output().GetFile();
}
return Stroka("-");
}
void TStreamStackOutputFileFactory
::DoOpen(TStreamStackOutput& stack, const TStringBuf& file)
{
if (file.empty())
stack.Push(Cout);
else {
// this is unbuffered
stack.Push(new TFileOutput(Stroka(file)));
if (!ExtensionMap.Push(stack, file))
stack.Push(new TBufferedOutput(stack));
}
}
Stroka MatterStable::TwoPhaseBoundary::MakeSpline(
CurveSpline &spl,
int Nspl,
double Misf,
double *y,
double *x,
int NumX,
const char *err_name) {
Stroka res;
if(!spl.Generate(Nspl, Misf, 0, x, y, NumX))
throw info_except(
"GenerateTwoPhaseBnd SaveBound error Generating %s spline\n", err_name);
res += Stroka(" MakeSpline Control ") + err_name + Stroka("\n");
for(int k = 0; k < NumX - 1; k++) {
// double tmp=(x[k]+x[k+1])/2;
//char a[256];
//res<<x[k]<<" "<<y[k]<<" "<<spl.Evaluate(x[k])<<"\n";
//res<<(y[k]+y[k+1])/2<<" "<<spl.Evaluate(tmp)<<"\n";
}
return res;
}
NLastGetopt::TEasySetup MakeTomitaParserOpts() {
NLastGetopt::TEasySetup opts;
opts('b', OPT_BINARY_DIR, " ", "Alternative directory for compiled grammars.", false)
('v', "version", &PrintSvnVersionAndExit0, "Print version information");
opts.SetFreeArgsNum(1, 1);
opts.SetFreeArgTitle(0, "config.proto", "parser configuration file name");
opts.SetTitle(Stroka("Yandex Tomita-parser ") + GetProgramVersionTag() + ". Open-source edition.\nhttp://api.yandex.ru/tomita");
return opts;
}
inline void TestMD5() {
// echo -n 'qwertyuiopqwertyuiopasdfghjklasdfghjkl' | md5sum
char b[] = "qwertyuiopqwertyuiopasdfghjklasdfghjkl";
MD5 r;
r.Update((const unsigned char*)b, 15);
r.Update((const unsigned char*)b + 15, strlen(b) - 15);
char rs[33];
Stroka s(r.End(rs));
s.to_lower();
UNIT_ASSERT_EQUAL(s, Stroka("3ac00dd696b966fd74deee3c35a59d8f"));
}
Stroka CCommonParm::GetPrettyOutputFileName() const {
if (NULL != Config.Get() && Config->has_prettyoutput()) {
Stroka fn = Config->GetPrettyOutput();
fn.to_lower();
if ("stdout" == fn || "stderr" == fn)
return fn;
if ("-" == fn)
return "stdout";
if (fn.size() > 0)
return Config->GetPrettyOutput();
}
return Stroka("");
}
void THashTest::TestHMap1() {
typedef yhash_map<char, Stroka, THash<char>, TEqualTo<char> > maptype;
maptype m;
// Store mappings between roman numerals and decimals.
m['l'] = "50";
m['x'] = "20"; // Deliberate mistake.
m['v'] = "5";
m['i'] = "1";
UNIT_ASSERT(!strcmp(m['x'].c_str(),"20"));
m['x'] = "10"; // Correct mistake.
UNIT_ASSERT(!strcmp(m['x'].c_str(),"10"));
UNIT_ASSERT(!m.has('z'));
UNIT_ASSERT(!strcmp(m['z'].c_str(),""));
UNIT_ASSERT(m.has('z'));
UNIT_ASSERT(m.count('z') == 1);
TPair<maptype::iterator, bool> p = m.insert(TPair<const char, Stroka>('c', Stroka("100")));
UNIT_ASSERT(p.second);
p = m.insert(TPair<const char, Stroka>('c', Stroka("100")));
UNIT_ASSERT(!p.second);
//Some iterators compare check, really compile time checks
maptype::iterator ite(m.begin());
maptype::const_iterator cite(m.begin());
cite = m.begin();
maptype const& cm = m;
cite = cm.begin();
UNIT_ASSERT((maptype::const_iterator)ite == cite);
UNIT_ASSERT(!((maptype::const_iterator)ite != cite));
UNIT_ASSERT(cite == (maptype::const_iterator)ite);
UNIT_ASSERT(!(cite != (maptype::const_iterator)ite));
}
static inline Stroka GetFile(const Stroka& s) {
const char* e = s.end();
const char* b = s.begin();
const char* c = e - 1;
while (c != b && !IsDelim(*c)) {
--c;
}
if (c != e && IsDelim(*c)) {
++c;
}
return Stroka(c, e - c);
}
int TCommandParse::SimpleRun(int argc, const char *argv[]) {
int formattedLength = 90;
if(argc < 2) {
Stroka descr = "Program build date is :" + Stroka(__DATE__) + "\t" +
Stroka(__TIME__) + "\n\n";
Stroka str =
descr + "General help for the program is:~+\n" + MainHelp + "~-\n";
str += "Parameters defined are: ";
for(int i = 1; i < min(argc, 5); i++)
str += Stroka("<") + argv[i] + "> ";
str += "\nHave to define 2 paramteres: mode \"mode parameters\"\n" +
MakeHelp() + "\n";
std::cout << TFormatOutput(" ", 0, formattedLength).Format(~str);
std::cout.flush();
exit(1);
}
Stroka par;
if(argc == 3)
par = argv[2];
else
for(int i = 2; i < argc; i++)
par += Stroka(argv[i]) + " ";
map<Stroka, TKeyData>::iterator it = Params.find(argv[1]);
if(it == Params.end()) {
std::cout << "Mode <" << argv[1] << "> is not defined.\n"
<< TFormatOutput(" ", 0, formattedLength).Format(~MakeHelp())
<< "\n";
exit(1);
}
if(it->second.GivenDefaults())
it->second.Func(Str::ReadDefinedParams(
~par, it->second.GetParams2Defaults(), "TOBEDEFINED"));
else
it->second.Func(Str::ReadParams(~par, ~it->second.GetParamsNames()));
return 1;
}
Stroka GetSystemTempDir() {
#ifdef _win_
char buffer[1024];
DWORD size = GetTempPath(1024, buffer);
if (!size) {
ythrow TSystemError() << "failed to get system temporary directory";
}
return Stroka(buffer, size);
#else
const char* var = "TMPDIR";
const char* def = "/tmp";
const char* r = getenv(var);
const char* result = r ? r : def;
return result[0] == '/' ? result : ResolveDir(result);
#endif
}
bool FromStringImpl<bool>(const char* data, size_t len) {
if (len == 1) {
if (data[0] == '0') {
return false;
} else if (data[0] == '1') {
return true;
}
}
TStringBuf buf(data, len);
if (istrue(buf)) {
return true;
} else if (isfalse(buf)) {
return false;
}
ythrow TFromStringException() << "cannot parse bool(" << Stroka(data, len) << ")";
}
Stroka CommandLineExe::MakeHelp() {
//Stroka ret(Format->Format(MainHelp.c_str()));
Stroka ret = "Main help:~+\n" + MainHelp + "~-\n" + "Available keys are:~+\n";
//Format->SetLevel(Format->GetLevel()+1);
for(map<Stroka, KeyData>::iterator iter = Params.begin(); iter != Params.end();
iter++) {
ret += Stroka(CommandPrefix) + iter->second.Key + "~+\n" + iter->second.Help +
"~-\n";
//Stroka Add("option:");
//Add += CommandPrefix;
//Add += Stroka(iter->second.Key)+iter->second.Help;//+"\n";
//ret += Format->Format(Add.c_str());
}
//Format->SetLevel(Format->GetLevel()-1);
return TFormatOutput(" ", 0, 70).Format(~ret);
//return ret;
}
Stroka EncodeXMLString(const char* str) {
Stroka strout;
if (!str)
return Stroka();
char num[15];
num[14] = 0;
num[13] = ';';
for (const char* s = str, *s1; *s;) {
for (s1 = s; safe_chars[(unsigned char)(*s1)]; s1++)
;
if (s1 != s)
strout.replace(+strout, 0, s, 0, s1 - s, s1 - s);
s = s1;
if (*s == 0)
break;
strout.replace(+strout, 0, "&#", 0, 2, 2);
char *p = &num[13];
for (int code = csYandex.unicode[(unsigned char)(*s++)]; code > 0; code /= 10)
*(--p) = '0' + (code % 10);
strout.append(p);
}
return strout;
}
TCommandParse::TKeyData::TKeyData(
const char *mode,
const char *help,
const char *params,
MAINFUNC *func,
int givenDefaultsMode)
: Mode(mode), Help(help), GivenDefaultsMode(givenDefaultsMode), Func(func) {
vector<Stroka> vec = Str::SplitLine(params, 0, '\n');
Par2Descr.clear();
Params2Defaults.clear();
ParamsSettings = "";
ParamsNames = "";
for(size_t i = 0; i < vec.size(); i++) {
vector<Stroka> par = Str::SplitLine(vec[i], 0);
if(!par.size())
continue;
Par2Descr[par[0]] = Str::JoinLine(par, ' ', 2);
Stroka defaultValue = par.size() > 1 ? par[1] : " ";
Params2Defaults[par[0]] = defaultValue;
Stroka prefix = (ParamsSettings.size() != 0) ? " " : "";
ParamsSettings += prefix + par[0] + Stroka(" ") + defaultValue;
ParamsNames += prefix + par[0];
}
};
static inline T ParseFlt(const char* data, size_t len) {
/*
* TODO
*/
if (len > 256) {
len = 256;
}
char* c = (char*)alloca(len + 1);
memcpy(c, data, len);
c[len] = 0;
T ret;
char ec;
// try to read a value and an extra character in order to catch cases when
// the string start with a valid float but is followed by unexpected characters
if (sscanf(c, TFltModifiers<T>::ModifierReadAndChar, &ret, &ec) == 1) {
return ret;
}
ythrow TFromStringException() << "cannot parse float(" << Stroka(data, len) << ")";
}
Stroka CCommonParm::GetOutputFormat() const {
if (NULL != Config.Get() && Config->output().has_format())
switch (Config->output().GetFormat()) {
case TTextMinerConfig::TOutputParams::text:
return Stroka("text");
case TTextMinerConfig::TOutputParams::xml:
return Stroka("xml");
case TTextMinerConfig::TOutputParams::proto:
return Stroka("proto");
case TTextMinerConfig::TOutputParams::json:
return Stroka("json");
case TTextMinerConfig::TOutputParams::mapreduce:
return Stroka("mapreduce");
default:
ythrow yexception() << "This type of input isn't supported";
}
return Stroka("xml");
}