bool TDependencyCollectorInputStream::TryConsumeImportFileName(const NProtoBuf::io::Tokenizer::Token& token)
{
ImportProcessor.NextState(token);
if (!ImportProcessor.Found())
return false;
YASSERT(token.type == NProtoBuf::io::Tokenizer::TYPE_STRING);
// original imported file (unquoted)
Stroka original;
Tokenizer_.ParseString(token.text, &original);
// canonic file
const Stroka canonic = SourceTree->CanonicName(original);
if (canonic.empty() || canonic == original) {
// make a hint for this special case
if (original == STRINGBUF("base.proto"))
ErrorCollector->AddErrorAtCurrentFile(token.line, token.column,
"Importing base.proto by shortcut name is not allowed from arcadia. You should use full path instead: import \"kernel/gazetteer/base.proto\";");
return false; // just proceed with original file name
}
// restore quotes and replace original with canonic in further pipeline.
CanonicImportedFile.clear();
CanonicImportedFile.append('"');
EscapeC(canonic, CanonicImportedFile);
CanonicImportedFile.append('"');
return true;
}
static inline Stroka Fix(Stroka f) {
if (!f.empty() && IsDelim(f[+f - 1])) {
f.pop_back();
}
return f;
}
void WriteFile(
const char *file,
const vector<vector<double>> &vec,
const char *head,
const char *mes) {
FILE *out = open(file, "w", mes);
if(head) {
Stroka tmp = Stroka(head) + "\n";
fputs(tmp.c_str(), out);
}
if(vec.size() == 0)
return;
char Buf[256];
for(size_t i = 0; i < vec[0].size(); i++) {
for(size_t j = 0; j < vec.size(); j++) {
if(vec[j][i] == NullNumber)
fputs(" \"\" ", out);
else {
sprintf(Buf, "%10.9g ", vec[j][i]);
fputs(Buf, out);
}
}
fputs("\x0d\x0a", out);
}
close(out);
}
void CLRCollectionSet::SaveStateCollection(Stroka GrammarFileName, const CWorkGrammar* p_WorkGrammar) const
{
if (GrammarFileName.empty()) return;
FILE * fp = fopen(GrammarFileName.c_str(), "wb");
for (int i = 0; i < (int)m_ItemSets.size(); i++) {
fprintf(fp, "I%i =\n", i);
for (yset<CLRItem>::const_iterator it = m_ItemSets[i].begin(); it != m_ItemSets[i].end(); it++) {
fprintf(fp, "\t%s -> ", p_WorkGrammar->m_UniqueGrammarItems[it->m_pRule->m_LeftPart].m_ItemStrId.c_str());
for (int j = 0; j < (int)it->m_pRule->m_RightPart.m_Items.size(); j++) {
if (j == (int)it->m_DotSymbolNo)
fprintf(fp, "; ");
fprintf(fp, "%s ", p_WorkGrammar->m_UniqueGrammarItems[it->m_pRule->m_RightPart.m_Items[j]].m_ItemStrId.c_str());
}
if (it->m_DotSymbolNo == it->m_pRule->m_RightPart.m_Items.size())
fprintf(fp, "; ");
fprintf(fp, "\n");
}
fprintf(fp, "\n");
}
fprintf(fp, "\n");
for (ymap< CStateAndSymbol, size_t>::const_iterator it_goto = m_GotoFunction.begin(); it_goto != m_GotoFunction.end(); it_goto++)
fprintf(fp, "GOTO( I%" PRISZT ", %s ) = I%" PRISZT "\n", it_goto->first.m_StateNo, p_WorkGrammar->m_UniqueGrammarItems[it_goto->first.m_SymbolNo].m_ItemStrId.c_str(), it_goto->second);
fclose(fp);
}
bool CAgencyInfoRetriver::Init(const Stroka& strDoc2AgFile)
{
if (!strDoc2AgFile.empty() && isexist(strDoc2AgFile.c_str()))
m_DocId2AgNameFile = TFile(strDoc2AgFile.c_str(), OpenExisting | RdOnly);
return true;
}
Stroka StripFileComponent(const Stroka& fileName)
{
Stroka dir = IsDir(fileName) ? fileName : GetDirName(fileName);
if (!dir.empty() && dir.back() != GetDirectorySeparator()) {
dir.append(GetDirectorySeparator());
}
return dir;
}
void PrintItemSet(const CWorkGrammar* pWorkGrammar, const yset<CLRItem>& ItemSet)
{
for (yset<CLRItem>::const_iterator it = ItemSet.begin(); it != ItemSet.end(); it++) {
Stroka s = pWorkGrammar->GetRuleStr(*it->m_pRule, it->m_DotSymbolNo);
printf ("%s\n", s.c_str());
};
};
// Like Stroka::Quote(), but does not quote digits-only string
static Stroka QuoteForHelp(const Stroka& str) {
if (str.empty())
return str.Quote();
for (size_t i = 0; i < str.size(); ++i) {
if (!isdigit(str[i]))
return str.Quote();
}
return str;
}
Stroka BitsetToString(const yvector<TGramBitSet>& bitset, const Stroka& delim /*= ", "*/, const Stroka& groupdelim /*= ", "*/) {
Stroka s;
for (size_t i = 0; i < bitset.size(); ++i) {
if (!s.empty())
s += groupdelim;
s += bitset[i].ToString(~delim);
}
return s;
}
Stroka TFileCollection::DebugString() const {
Stroka ret;
for (size_t i = 0; i < Files.size(); ++i) {
ret.append(Files[i]->Name()).append(" ->");
for (size_t j = 0; j < Files[i]->Aliases().size(); ++j)
ret.append(' ').append(Files[i]->Aliases()[j]);
ret.append('\n');
}
return ret;
}
Stroka CCommonParm::GetOutputFileName() const {
if (NULL != Config.Get() && Config->has_output() && Config->output().has_file()) {
Stroka fn = Config->output().GetFile();
fn.to_lower();
if ("stdout" == fn)
return "-";
return Config->output().GetFile();
}
return Stroka("-");
}
Wtroka CProcessor::GetInterviewFio(Stroka strUrl) const
{
ymap<Stroka, Wtroka>::const_iterator it = InterviewUrl2Fio.find(strUrl);
if (it != InterviewUrl2Fio.end())
return it->second;
static const Stroka kHTTP = "http://";
if (strUrl.has_prefix(kHTTP)) {
strUrl = strUrl.substr(kHTTP.size());
return GetInterviewFio(strUrl);
}
return Wtroka();
}
void SlashFolderLocal(Stroka &folder)
{
if (!folder)
return;
#ifdef _win32_
size_t pos;
while ((pos = folder.find('/')) != Stroka::npos)
folder.replace(pos, 1, LOCSLASH_S);
#endif
if (folder[+folder-1] != LOCSLASH_C)
folder.append(LOCSLASH_S);
}
inline void TestRaise1() {
try {
Throw2DontMove();
UNIT_ASSERT(false);
} catch (...) {
Stroka err = CurrentExceptionMessage();
char *ptr = err.begin();
while ((ptr = strchr(ptr, '\\')) != 0)
*ptr = '/';
UNIT_ASSERT_VALUES_EQUAL(err, "util/generic/yexception_ut.cpp:8: 1 qw 12.1");
}
}
void CParserOptions::ParseListOfFacts(TXmlNodePtrBase piNode) {
TXmlNodePtrBase piFacts = piNode->children;
m_ParserOutputOptions.m_bShowFactsWithEmptyField = piNode.HasAttr("addEmpty");
for (; piFacts.Get() != NULL; piFacts = piFacts->next) {
if (!piFacts.HasName("fact"))
continue;
Stroka str = piFacts.GetAttr("name");
if (!str.empty())
AddFactToShow(str, !piFacts.HasAttr("noequality"));
}
}
bool TGztParser::ParseChainedFieldValues(TFieldValueDescriptorProto* value)
{
DO(ParseSingleFieldValue(value));
// try read several more values, interleaved with "+" or "|"
if (!LookingAtListDelimiter())
return true;
// What was previously read into @value was actually a first item of chained list,
// not a single value. So transform @value to a list and place its current content
// as first sub-value of this list.
// Re-use previuosly allocated items
THolder<TFieldValueDescriptorProto> sub_value;
if (value->mutable_list()->mutable_value()->ClearedCount() > 0) {
sub_value.Reset(value->mutable_list()->mutable_value()->ReleaseCleared());
sub_value->CopyFrom(*value);
//sub_value->Swap(value); -- Swap is unsafe here because it creates cycles for some reason!
}
else
sub_value.Reset(new TFieldValueDescriptorProto(*value));
value->Clear();
value->set_type(TFieldValueDescriptorProto::TYPE_LIST);
value->mutable_list()->mutable_value()->AddAllocated(sub_value.Release());
// only single kind of separating token is allowed at single chained list.
// so next we only accept a delimiters of same level which are equal to the first one.
Stroka delimiter = CurrentToken().text;
if (delimiter == "|")
value->mutable_list()->set_type(TValuesListDescriptorProto::PIPE_DELIMITED);
else if (delimiter == "+")
value->mutable_list()->set_type(TValuesListDescriptorProto::PLUS_DELIMITED);
else
YASSERT(false);
const char* delim_text = delimiter.c_str();
while (TryConsume(delim_text))
DO(ParseSingleFieldValue(value->mutable_list()->add_value()));
// it is an error to meet any list delimiter here (as it will be mixed with previuos delimiters).
if (!LookingAtListDelimiter())
return true;
else {
AddError(Substitute("Distinct kinds of delimiters (\"$0\" and \"$1\") "
"should not be mixed in a single chained list at same level.",
delimiter, CurrentToken().text));
return false;
}
}
void CHomonym::Print(TOutputStream& stream, ECharset encoding) const
{
Stroka s;
s = GetStrKWType(encoding);
if (s.size())
s = Substitute(" (<b>$0</b>) ", s);
if (IsDeleted())
s += "[deleted]";
stream << NStr::Encode(GetShortLemma(), encoding) << " ";
PrintGrammems(Grammems.All(), stream, encoding);
stream << s << GetLabelsString(encoding);
}
Stroka NUnitTest::RandomString(size_t len, ui32 seed) {
TRand rand;
Stroka ret;
rand.srandom(seed);
ret.reserve(len);
for (size_t i = 0; i < len; ++i) {
ret.push_back(char(1 + (rand.random() % 126)));
}
return ret;
}
// make vector of columns (for tableData)
// else - make vector of rows
int ReadFile(
const char *file,
vector<vector<double>> &vec,
const char *mes,
int makeVectorOfColumns) {
if(!file)
throw info_except("File name is null?\n%s\n", mes);
FILE *in = open(file, "r", mes);
vec.clear();
if(!in) {
if(mes != NULL)
throw info_except("Cannot open file %s.\n%s\n", file, mes);
else
return 0;
}
std::vector<Stroka> line;
vector<double> cur;
Stroka err = "";
if(mes)
err = mes;
while(GetLine(in, line)) {
cur.clear();
for(size_t i = 0; i < line.size(); i++) {
double f;
if(!IsDouble(line[i].c_str(), f, 1)) {
cur.clear();
break;
}
cur.push_back(f);
}
if(cur.size() == 0)
continue;
if(makeVectorOfColumns) {
if(vec.size() == 0)
vec.resize(cur.size());
if(cur.size() != vec.size())
throw info_except(
"File <%s>\nstr <%s>\nsize of str %i vector size %i\n%s",
file,
Str::JoinLine(line).c_str(),
cur.size(),
vec.size(),
err.c_str());
for(size_t i = 0; i < line.size(); i++)
vec[i].push_back(cur[i]);
} else
vec.push_back(cur);
}
close(in);
return 1;
}
Stroka CCommonParm::GetPrettyOutputFileName() const {
if (NULL != Config.Get() && Config->has_prettyoutput()) {
Stroka fn = Config->GetPrettyOutput();
fn.to_lower();
if ("stdout" == fn || "stderr" == fn)
return fn;
if ("-" == fn)
return "stdout";
if (fn.size() > 0)
return Config->GetPrettyOutput();
}
return Stroka("");
}
static inline Stroka GetFile(const Stroka& s) {
const char* e = s.end();
const char* b = s.begin();
const char* c = e - 1;
while (c != b && !IsDelim(*c)) {
--c;
}
if (c != e && IsDelim(*c)) {
++c;
}
return Stroka(c, e - c);
}
TGramBitSet HumanGrammemsToGramBitSet(const TWtringBuf& strGrammems)
{
Stroka str = WideToChar(~strGrammems, +strGrammems, CODES_WIN);
VectorStrok gramTokens = splitStrokuBySet(str.c_str(), ",");
TGramBitSet grammars;
for (size_t j = 0; j < gramTokens.size(); j++) {
if (gramTokens[j].empty())
continue;
TGrammar gr = TGrammarIndex::GetCode(gramTokens[j]);
if (gr != gInvalid)
grammars.Set(gr);
}
return grammars;
}
size_t CWorkGrammar::GetCountOfRoots() const
{
size_t Result = 0;
Stroka Dump;
for (size_t i=0; i<m_UniqueGrammarItems.size(); i++)
if (m_UniqueGrammarItems[i].m_bGrammarRoot) {
Result++;
Dump += m_UniqueGrammarItems[i].GetDumpString();
Dump += ",";
}
if (Result > 1)
fprintf (stderr, "Roots : %s\n",Dump.c_str());
return Result;
};
size_t TFormatOutput::FormLine(const char *str, size_t pos, Stroka &Line) {
Line = "";
size_t len = strlen(str);
for(size_t i = 0; i < Level; i++)
Line += Ident;
int NextLine;
size_t pos1 = pos;
while(1) {
pos1 = skipspace(str, pos1, len, NextLine, Line);
if(NextLine || Line.size() >= MaxLen || pos1 >= len)
return pos1;
pos1 = skipword(str, pos1, len, NextLine, Line);
if(NextLine || Line.size() >= MaxLen || pos1 >= len)
return pos1;
}
}
static Stroka cutFirstSlash(const Stroka& fileName) {
if (fileName[0] == '/') {
return fileName.substr(1);
} else {
return fileName;
}
}
int ut_real(Stroka args, bool err_exp, const char *A_exp, int b_exp, bool a_exp, const char *p1_exp, const char *p2_exp) {
char *argv[32];
int argc = sf(' ', argv, args.begin());
Opt2 opt(argc, argv, ut_optspec, 2, "option-1=A,option-2=a,");
const char *A = opt.Arg('A', "<qqq> - blah");
int b = opt.Int('b', "<rrr> - blah", 2);
bool a = opt.Has('a', "- blah");
/*const char *C = */opt.Arg('C', "<ccc> - blah", 0);
if (opt_ut_verbose)
opt.AutoUsage("");
if (opt.HasErrors != err_exp)
return 1;
if (err_exp)
return false;
if (!A && A_exp || A && !A_exp || A && A_exp && strcmp(A, A_exp))
return 2;
if (b != b_exp)
return 3;
if (a != a_exp)
return 4;
if (strcmp(opt.Pos[0], p1_exp))
return 5;
if (strcmp(opt.Pos[1], p2_exp))
return 6;
return false;
}
void WriteToLogFile(const Stroka& sGrammarFileLog, Stroka& str, bool bRW)
{
if (sGrammarFileLog.empty())
return;
str += '\n';
THolder<TFile> f;
if (bRW)
f.Reset(new TFile(sGrammarFileLog, CreateAlways | WrOnly | Seq));
else
f.Reset(new TFile(sGrammarFileLog, OpenAlways | WrOnly | Seq | ForAppend));
TFileOutput out(*f);
out.Write(str.c_str());
};
static inline TAutoPtr<TOutputStream> OpenOutput(const Stroka& url) {
if (url.empty()) {
return new TBuffered<TFileOutput>(8192, Duplicate(1));
} else {
return new TBuffered<TFileOutput>(8192, url);
}
}
inline void TFile::AddAliasImpl(const Stroka& name) {
if (!name.empty() && !Is(name)) {
Aliases_.push_back(name);
if (!GeneratedDescriptor)
GeneratedDescriptor = NBuiltin::FindInGeneratedPool(name);
}
}
bool CWorkGrammar::SaveFirstAndFollowSets(Stroka GrammarFileName) const
{
FILE * fp = fopen(GrammarFileName.c_str(), "wb");
fprintf(fp, "FIRST sets:\n\n");
for (int i = 0; i < (int)m_FirstSets.size(); i++) {
fprintf(fp, "%s = { ", m_UniqueGrammarItems[i].m_ItemStrId.c_str());
for (yset<size_t>::const_iterator it_term = m_FirstSets[i].begin(); it_term != m_FirstSets[i].end(); it_term++)
fprintf(fp, "%s ", m_UniqueGrammarItems[*it_term].m_ItemStrId.c_str());
fprintf(fp, "}\n");
}
fprintf(fp, "\n\nFOLLOW sets:\n\n");
for (int i = 0; i < (int)m_FollowSets.size(); i++) {
fprintf(fp, "%s = { ", m_UniqueGrammarItems[i].m_ItemStrId.c_str());
for (yset<size_t>::const_iterator it_term = m_FollowSets[i].begin(); it_term != m_FollowSets[i].end(); it_term++)
fprintf(fp, "%s ", m_UniqueGrammarItems[*it_term].m_ItemStrId.c_str());
fprintf(fp, "}\n");
}
fclose(fp);
return true;
};
