void skipComments(Lexer &Lex, Token &Tok) {
while (Tok.is(tok::comment))
if (Lex.LexFromRawLexer(Tok))
return;
}
void Template::Format::readFormat(Lexer & lex)
{
enum {
FO_PRODUCT = 1,
FO_UPDATEFORMAT,
FO_UPDATERESULT,
FO_REQUIREMENT,
FO_OPTION,
FO_PREAMBLE,
FO_TRANSFORMCOMMAND,
FO_TRANSFORMOPTION,
FO_REFERENCEDFILE,
FO_END
};
LexerKeyword formattags[] = {
{ "formatend", FO_END },
{ "option", FO_OPTION },
{ "preamble", FO_PREAMBLE },
{ "product", FO_PRODUCT },
{ "referencedfile", FO_REFERENCEDFILE },
{ "requirement", FO_REQUIREMENT },
{ "transformcommand", FO_TRANSFORMCOMMAND },
{ "transformoption", FO_TRANSFORMOPTION },
{ "updateformat", FO_UPDATEFORMAT },
{ "updateresult", FO_UPDATERESULT }
};
PushPopHelper pph(lex, formattags);
while (lex.isOK()) {
switch (lex.lex()) {
case FO_PRODUCT:
lex.next(true);
product = lex.getString();
break;
case FO_UPDATEFORMAT:
lex.next(true);
updateFormat = lex.getString();
break;
case FO_UPDATERESULT:
lex.next(true);
updateResult = lex.getString();
break;
case FO_REQUIREMENT:
lex.next(true);
requirements.push_back(lex.getString());
break;
case FO_PREAMBLE:
lex.next(true);
preambleNames.push_back(lex.getString());
break;
case FO_TRANSFORMCOMMAND: {
lex.next(true);
string const name = lex.getString();
lex.next(true);
setCommandFactory(*this, name, lex.getString());
break;
}
case FO_TRANSFORMOPTION: {
lex.next(true);
string const name = lex.getString();
lex.next(true);
setOptionFactory(*this, name, lex.getString());
break;
}
case FO_OPTION: {
lex.next(true);
string const name = lex.getString();
lex.next(true);
string const opt = lex.getString();
options.push_back(Option(name, opt));
break;
}
case FO_REFERENCEDFILE: {
lex.next(true);
string const format = lex.getString();
lex.next(true);
string const file = lex.getString();
referencedFiles[format].push_back(file);
break;
}
case FO_END:
return;
}
}
}
void Layout::readLabelType(Lexer & lex)
{
enum {
LA_NO_LABEL = 1,
LA_MANUAL,
LA_ABOVE,
LA_CENTERED,
LA_STATIC,
LA_SENSITIVE,
LA_ENUMERATE,
LA_ITEMIZE,
LA_BIBLIO
};
LexerKeyword labelTypeTags[] = {
{ "above", LA_ABOVE },
{ "bibliography", LA_BIBLIO },
{ "centered", LA_CENTERED },
{ "enumerate", LA_ENUMERATE },
{ "itemize", LA_ITEMIZE },
{ "manual", LA_MANUAL },
{ "no_label", LA_NO_LABEL },
{ "sensitive", LA_SENSITIVE },
{ "static", LA_STATIC }
};
PushPopHelper pph(lex, labelTypeTags);
int le = lex.lex();
switch (le) {
case Lexer::LEX_UNDEF:
lex.printError("Unknown labeltype tag `$$Token'");
return;
default: break;
}
switch (le) {
case LA_NO_LABEL:
labeltype = LABEL_NO_LABEL;
break;
case LA_MANUAL:
labeltype = LABEL_MANUAL;
break;
case LA_ABOVE:
labeltype = LABEL_ABOVE;
break;
case LA_CENTERED:
labeltype = LABEL_CENTERED;
break;
case LA_STATIC:
labeltype = LABEL_STATIC;
break;
case LA_SENSITIVE:
labeltype = LABEL_SENSITIVE;
break;
case LA_ENUMERATE:
labeltype = LABEL_ENUMERATE;
break;
case LA_ITEMIZE:
labeltype = LABEL_ITEMIZE;
break;
case LA_BIBLIO:
labeltype = LABEL_BIBLIO;
break;
}
}
double Calculator::prim(Lexer& lexer)
{
Token tok = lexer.peek();
switch(tok.Type)
{
case Token::FUNC:
lexer.pop();
return func(lexer);
case Token::NUMBER:
return lexer.pop().NumberValue;
case Token::NAME:
{
string name = tok.StringValue;
lexer.pop();
if(lexer.peek().Type == Token::LP)
{
lexer.pop();
return call(lexer, name);
}
else if (lexer.peek().Type == Token::ASSIGN)
{
lexer.pop();
SymEntry* sym = symbols.Get(name);
if(sym && sym->type != SymEntry::VAR) throw Error::SyntaxError("not a variable: "+name);
double v = expr(lexer);
symbols.Set(name, new SymVariable(v));
return v;
}
else
{
SymEntry* sym = symbols.Get(name);
if(!sym) throw Error::SyntaxError("undefined symbol: "+name);
if(sym->type != SymEntry::VAR) throw Error::SyntaxError("not a variable: "+name);
SymVariable* sVar = static_cast<SymVariable*>(sym);
return sVar->value;
}
}
case Token::MINUS:
lexer.pop();
return -prim(lexer);
case Token::LP:
{
lexer.pop();
double e = expr(lexer);
Token tok = lexer.pop();
if(tok.Type != Token::RP) throw Error::SyntaxError("')' expected");
return e;
}
default:
throw Error::SyntaxError("primary expected");
}
}
// Expand __has_include and __has_include_next if possible. If there's no
// definitive answer return false.
bool InclusionRewriter::HandleHasInclude(
FileID FileId, Lexer &RawLex, const DirectoryLookup *Lookup, Token &Tok,
bool &FileExists) {
// Lex the opening paren.
RawLex.LexFromRawLexer(Tok);
if (Tok.isNot(tok::l_paren))
return false;
RawLex.LexFromRawLexer(Tok);
SmallString<128> FilenameBuffer;
StringRef Filename;
// Since the raw lexer doesn't give us angle_literals we have to parse them
// ourselves.
// FIXME: What to do if the file name is a macro?
if (Tok.is(tok::less)) {
RawLex.LexFromRawLexer(Tok);
FilenameBuffer += '<';
do {
if (Tok.is(tok::eod)) // Sanity check.
return false;
if (Tok.is(tok::raw_identifier))
PP.LookUpIdentifierInfo(Tok);
// Get the string piece.
SmallVector<char, 128> TmpBuffer;
bool Invalid = false;
StringRef TmpName = PP.getSpelling(Tok, TmpBuffer, &Invalid);
if (Invalid)
return false;
FilenameBuffer += TmpName;
RawLex.LexFromRawLexer(Tok);
} while (Tok.isNot(tok::greater));
FilenameBuffer += '>';
Filename = FilenameBuffer;
} else {
if (Tok.isNot(tok::string_literal))
return false;
bool Invalid = false;
Filename = PP.getSpelling(Tok, FilenameBuffer, &Invalid);
if (Invalid)
return false;
}
// Lex the closing paren.
RawLex.LexFromRawLexer(Tok);
if (Tok.isNot(tok::r_paren))
return false;
// Now ask HeaderInfo if it knows about the header.
// FIXME: Subframeworks aren't handled here. Do we care?
bool isAngled = PP.GetIncludeFilenameSpelling(Tok.getLocation(), Filename);
const DirectoryLookup *CurDir;
const FileEntry *File = PP.getHeaderSearchInfo().LookupFile(
Filename, SourceLocation(), isAngled, nullptr, CurDir,
PP.getSourceManager().getFileEntryForID(FileId), nullptr, nullptr,
nullptr, false);
FileExists = File != nullptr;
return true;
}
bool Parser::parseExpression (Lexer& lexer, Lookup& lookup, Int32u* slot, const Extractor** output)
{
BinaryOp binaryOp;
stack<BinaryOp> binaryOps;
stack<const Extractor*> operands;
const Extractor* value;
while (true)
{
switch (lexer.getType ())
{
case Lexer::PLUS:
lexer.next ();
if (!this->parseExpression (lexer, lookup, slot, &value))
return false;
break;
case Lexer::MINUS:
lexer.next ();
if (!this->parseExpression (lexer, lookup, slot, &value))
return false;
value = new NumberUnaryExtractor (value, [] (Float64 number)
{
return Variant (-number);
});
this->extractors.push_back (value);
break;
case Lexer::NOT:
lexer.next ();
if (!this->parseExpression (lexer, lookup, slot, &value))
return false;
value = new BooleanUnaryExtractor (value, [] (bool value)
{
return Variant (!value);
});
this->extractors.push_back (value);
break;
case Lexer::PARENTHESIS_BEGIN:
lexer.next ();
if (!this->parseExpression (lexer, lookup, slot, &value) ||
!this->parseType (lexer, Lexer::PARENTHESIS_END, "closing parenthesis"))
return false;
break;
default:
if (!this->parseValue (lexer, lookup, slot, &value))
return false;
break;
}
operands.push (value);
switch (lexer.getType ())
{
case Lexer::AMPERSAND:
binaryOp = make_pair (1, [] (const Extractor* lhs, const Extractor* rhs) -> Extractor*
{
return new AndLogicalExtractor (lhs, rhs);
});
break;
case Lexer::DIFFERENT:
binaryOp = make_pair (2, [] (const Extractor* lhs, const Extractor* rhs) -> Extractor*
{
return new VariantBinaryExtractor (lhs, rhs, [] (const Variant& a, const Variant& b)
{
return Variant (a != b);
});
});
break;
case Lexer::DIVIDE:
binaryOp = make_pair (4, [] (const Extractor* lhs, const Extractor* rhs) -> Extractor*
{
return new NumberBinaryExtractor (lhs, rhs, [] (Float64 a, Float64 b)
{
return b != 0 ? Variant (a / b) : Variant::empty;
});
});
break;
case Lexer::EQUAL:
binaryOp = make_pair (2, [] (const Extractor* lhs, const Extractor* rhs) -> Extractor*
{
return new VariantBinaryExtractor (lhs, rhs, [] (const Variant& a, const Variant& b)
{
return Variant (a == b);
});
});
break;
case Lexer::GREATER_EQUAL:
binaryOp = make_pair (2, [] (const Extractor* lhs, const Extractor* rhs) -> Extractor*
{
return new VariantBinaryExtractor (lhs, rhs, [] (const Variant& a, const Variant& b)
{
return Variant (a >= b);
});
});
break;
case Lexer::GREATER_THAN:
binaryOp = make_pair (2, [] (const Extractor* lhs, const Extractor* rhs) -> Extractor*
{
return new VariantBinaryExtractor (lhs, rhs, [] (const Variant& a, const Variant& b)
{
return Variant (a > b);
});
});
break;
case Lexer::LOWER_EQUAL:
binaryOp = make_pair (2, [] (const Extractor* lhs, const Extractor* rhs) -> Extractor*
{
return new VariantBinaryExtractor (lhs, rhs, [] (const Variant& a, const Variant& b)
{
return Variant (a <= b);
});
});
break;
case Lexer::LOWER_THAN:
binaryOp = make_pair (2, [] (const Extractor* lhs, const Extractor* rhs) -> Extractor*
{
return new VariantBinaryExtractor (lhs, rhs, [] (const Variant& a, const Variant& b)
{
return Variant (a < b);
});
});
break;
case Lexer::MINUS:
binaryOp = make_pair (3, [] (const Extractor* lhs, const Extractor* rhs) -> Extractor*
{
return new NumberBinaryExtractor (lhs, rhs, [] (Float64 a, Float64 b)
{
return Variant (a - b);
});
});
break;
case Lexer::MODULO:
binaryOp = make_pair (4, [] (const Extractor* lhs, const Extractor* rhs) -> Extractor*
{
return new NumberBinaryExtractor (lhs, rhs, [] (Float64 a, Float64 b)
{
return b != 0 ? Variant (fmod (a, b)) : Variant::empty;
});
});
break;
case Lexer::MULTIPLY:
binaryOp = make_pair (4, [] (const Extractor* lhs, const Extractor* rhs) -> Extractor*
{
return new NumberBinaryExtractor (lhs, rhs, [] (Float64 a, Float64 b)
{
return Variant (a * b);
});
});
break;
case Lexer::PIPE:
binaryOp = make_pair (1, [] (const Extractor* lhs, const Extractor* rhs) -> Extractor*
{
return new OrLogicalExtractor (lhs, rhs);
});
break;
case Lexer::PLUS:
binaryOp = make_pair (3, [] (const Extractor* lhs, const Extractor* rhs) -> Extractor*
{
return new NumberBinaryExtractor (lhs, rhs, [] (Float64 a, Float64 b)
{
return Variant (a + b);
});
});
break;
default:
for (; !binaryOps.empty (); binaryOps.pop ())
{
value = operands.top ();
operands.pop ();
value = binaryOps.top ().second (operands.top (), value);
operands.pop ();
operands.push (value);
this->extractors.push_back (value);
}
if (operands.top ()->getFlags () == 0)
return this->fail (lexer, "inconsistent expression");
*output = operands.top ();
return true;
}
lexer.next ();
for (; !binaryOps.empty () && binaryOps.top ().first >= binaryOp.first; binaryOps.pop ())
{
value = operands.top ();
operands.pop ();
value = binaryOps.top ().second (operands.top (), value);
operands.pop ();
operands.push (value);
this->extractors.push_back (value);
}
binaryOps.push (binaryOp);
}
}
int main(int argc, const char** argv) {
auto params = getParams(argc, argv);
auto exit_with_errors = [](int error_count) {
cout << "Exiting with " << error_count << (error_count == 1 ? " error" : " errors") << endl;
};
Lexer lexer;
vector<Token> tokens;
int error_count = 0;
if (paramIsSet(params, "evaluate")) {
auto str = params["evaluate"][0];
istringstream stream {str};
tie(tokens, error_count) = lexer.tokenize(stream, "(command line)");
} else if (paramIsSet(params, "files")) {
auto filename = params["files"][0];
ifstream file {filename};
if (file.is_open()) {
tie(tokens, error_count) = lexer.tokenize(file, filename);
} else {
++error_count;
cerr << "ERROR: " << filename << " not found" << endl;
}
} else {
tie(tokens, error_count) = lexer.tokenize(cin, "(stdin)");
}
bool ignore_errors = paramIsSet(params, "ignore-errors");
if (!ignore_errors && error_count > 0) {
exit_with_errors(error_count);
return -1;
}
if (tokens.empty()) {
return 0;
}
if (paramIsSet(params, "print-tokens")) {
print_tokens(tokens);
}
TokenStream token_stream {begin(tokens), end(tokens), true};
Parser parser;
shared_ptr<ASTNode> tree;
setupGlobalScope();
tie(tree, error_count) = parser.parse(token_stream);
if (!ignore_errors && error_count > 0) {
exit_with_errors(error_count);
return -1;
}
bool print_ast = paramIsSet(params, "print-ast");
if (tree) {
if (print_ast) {
cout << "\nOutput: " << endl;
tree->output(cout, 0);
cout << endl;
}
if (print_ast || ignore_errors) {
cout << "\nEvaluate: " << endl;
}
try {
auto eval = tree->evaluate();
if (eval) {
eval->output(cout);
}
} catch (const exception& ex) {
++error_count;
cerr << ex.what() << endl;
}
cout << endl;
} else if (print_ast) {
cout << "No parse tree produced" << endl;
}
return 0;
}
void ConverterCache::Impl::readIndex()
{
time_t const now = current_time();
FileName const index(addName(cache_dir.absFileName(), "index"));
ifstream is(index.toFilesystemEncoding().c_str());
Lexer lex;
lex.setStream(is);
while (lex.isOK()) {
if (!lex.next(true))
break;
string const orig_from = lex.getString();
if (!lex.next())
break;
string const to_format = lex.getString();
if (!lex.next())
break;
time_t const timestamp =
convert<unsigned long>(lex.getString());
if (!lex.next())
break;
unsigned long const checksum =
convert<unsigned long>(lex.getString());
FileName const orig_from_name(orig_from);
CacheItem item(orig_from_name, to_format, timestamp, checksum);
// Don't cache files that do not exist anymore
if (!orig_from_name.exists()) {
LYXERR(Debug::FILES, "Not caching file `"
<< orig_from << "' (does not exist anymore).");
item.cache_name.removeFile();
continue;
}
// Don't add items that are not in the cache anymore
// This can happen if two instances of LyX are running
// at the same time and update the index file independantly.
if (!item.cache_name.exists()) {
LYXERR(Debug::FILES, "Not caching file `" << orig_from
<< "' (cached copy does not exist anymore).");
continue;
}
// Delete the cached file if it is too old
if (difftime(now, item.cache_name.lastModified())
> lyxrc.converter_cache_maxage) {
LYXERR(Debug::FILES, "Not caching file `"
<< orig_from << "' (too old).");
item.cache_name.removeFile();
continue;
}
FormatCache & format_cache = cache[orig_from_name];
if (format_cache.from_format.empty())
format_cache.from_format =
formats.getFormatFromFile(orig_from_name);
format_cache.cache[to_format] = item;
}
is.close();
}
int main() {
system("mode con cols=100 lines=30");
system("COLOR 0A");
while(1){
dat.init();
lexer.init();
buf.clear();
deb.init();
ending=0;
cout << "请将要运行的程序放在与本程序相同的目录下" << endl;
cout << "输入1运行程序，输入2调试程序，输入3退出程序" << endl;
int type;
while(cin >> type){
if (type == 1) {
cout << "输入要运行的程序文件名:" << endl;
break;
}
else if (type == 2) {
cout << "输入要调试的程序文件名:" << endl;
break;
}
else if(type==3) exit(0);
else {
cout << "输入不合法!请重新输入:" << endl;
}
}
string filename;
//cin >> filename;
ifstream file;
string s;
while(cin >> filename){
file.open(filename);
if(!file) cout << "输入文件名错误!重新输入:" << endl;
else break;
}
while (!file.eof()) {
getline(file, s);
buf.push_back(s);
}
if(type==2){
cout << "进入调试模式\n以下为调试器的命令，输入h即可再次查看" << endl;
cout << "列出所有代码:l\n查看变量的值:p 变量名\n设置断点:b 行号\n取消断点:nb 行号\n查看当前所有的断点的行号:wb\n下一步:n\n运行程序:r\n" << endl;
deb.wait();
}
/*buf.push_back("begin");
buf.push_back("out 123;");
buf.push_back("end");*/
int x = 0, y = 0;
bool flag=0;
while (1) {
if (x == buf.size()) break;
if (buf[x].size() == 0) {
x++;
y = 0;
continue;
}
if(buf[x][y]=='/' && y+1<buf[x].size() && buf[x][y]=='/'){
x++;
y=0;
}
if (y + 4<buf[x].size()) {
if (buf[x].substr(y, 5) == "begin") {
lexer.curx = x + 1;
lexer.cury = 0;
if(deb.next) {
cout << lexer.curx << " -> " << buf[lexer.curx] << endl;
deb.wait();
}
if(deb.is_bp(lexer.curx)){
cout << lexer.curx << " -> " << buf[lexer.curx] << endl;
deb.wait();
}
flag=1;
lexer.domain();
break;
}
}
y++;
if (y == buf[x].size()) {
x++;
y = 0;
}
}
if(!flag){
cout << "没有检测到begin" <<endl;
system("pause");
exit(0);
}
}
return 0;
}
Parser::Parser(Lexer &m_lexer): lexer_ptr(nullptr), lexer( m_lexer ), meta(m_lexer.size())
{
parseToken( lexer, meta );
}
bool Font::fromString(string const & data, bool & toggle)
{
istringstream is(data);
Lexer lex;
lex.setStream(is);
int nset = 0;
while (lex.isOK()) {
string token;
if (lex.next())
token = lex.getString();
if (token.empty() || !lex.next())
break;
if (token == "family") {
int const next = lex.getInteger();
bits_.setFamily(FontFamily(next));
} else if (token == "series") {
int const next = lex.getInteger();
bits_.setSeries(FontSeries(next));
} else if (token == "shape") {
int const next = lex.getInteger();
bits_.setShape(FontShape(next));
} else if (token == "size") {
int const next = lex.getInteger();
bits_.setSize(FontSize(next));
} else if (token == "emph" || token == "underbar" ||
token == "noun" || token == "number" ||
token == "uuline" || token == "uwave" ||
token == "strikeout") {
int const next = lex.getInteger();
FontState const misc = FontState(next);
if (token == "emph")
bits_.setEmph(misc);
else if (token == "underbar")
bits_.setUnderbar(misc);
else if (token == "strikeout")
bits_.setStrikeout(misc);
else if (token == "uuline")
bits_.setUuline(misc);
else if (token == "uwave")
bits_.setUwave(misc);
else if (token == "noun")
bits_.setNoun(misc);
else if (token == "number")
bits_.setNumber(misc);
} else if (token == "color") {
int const next = lex.getInteger();
bits_.setColor(ColorCode(next));
/**
} else if (token == "background") {
int const next = lex.getInteger();
bits_.setBackground(ColorCode(next));
*/
} else if (token == "language") {
string const next = lex.getString();
setLanguage(languages.getLanguage(next));
} else if (token == "toggleall") {
toggle = lex.getBool();
} else {
// Unrecognised token
break;
}
++nset;
}
return (nset > 0);
}
MinimalAction::MinimalAction(Lexer &l)
: Idents(l.getIdentifierTable()) {
TypeNameInfoTablePtr = new TypeNameInfoTable();
}
/*
================
ReadMesh
================
*/
static void ReadMesh( Lexer &lexer, aseMesh *inMesh ) {
int idx;
Vec3 temp;
lexer.ExpectToken("{");
const char *p;
const Token *token;
while ( (token = lexer.ReadToken()) != OG_NULL ) {
p = token->GetString();
if ( !p || !*p )
continue;
if ( String::Icmp( p, "}" ) == 0 )
return;
if ( String::Icmp( p, "*" ) != 0 )
lexer.Error( Format("expected *, got '$*'") << p );
if ( lexer.CheckToken( "MESH_NUMVERTEX" ) ) {
inMesh->numVerts = lexer.ReadInt();
inMesh->vertices = new aseVertex[inMesh->numVerts];
}
else if ( lexer.CheckToken( "MESH_NUMFACES" ) ) {
inMesh->numTris = lexer.ReadInt();
inMesh->triangles = new aseTriangle[inMesh->numTris];
}
else if ( lexer.CheckToken( "MESH_VERTEX_LIST" ) ) {
lexer.ExpectToken("{");
for( int i=0; i<inMesh->numVerts; i++ ) {
lexer.ExpectToken("*");
lexer.ExpectToken("MESH_VERTEX");
idx = lexer.ReadInt();
inMesh->vertices[idx].origin.x = lexer.ReadFloat() * ASE_MODEL_SCALE;
inMesh->vertices[idx].origin.y = lexer.ReadFloat() * ASE_MODEL_SCALE;
inMesh->vertices[idx].origin.z = lexer.ReadFloat() * ASE_MODEL_SCALE;
}
lexer.ExpectToken("}");
}
else if ( lexer.CheckToken( "MESH_NORMALS" ) ) {
lexer.ExpectToken("{");
for( ;; ) {
if ( lexer.CheckToken("}") )
break;
lexer.ExpectToken("*");
// don't need the face normal
if ( lexer.CheckToken("MESH_FACENORMAL") ) {
lexer.GotoNextLine();
continue;
}
lexer.ExpectToken("MESH_VERTEXNORMAL");
idx = lexer.ReadInt();
inMesh->vertices[idx].normal.x = lexer.ReadFloat();
inMesh->vertices[idx].normal.y = lexer.ReadFloat();
inMesh->vertices[idx].normal.z = lexer.ReadFloat();
}
}
else if ( lexer.CheckToken( "MESH_FACE_LIST" ) ) {
lexer.ExpectToken("{");
for( int i=0; i<inMesh->numTris; i++ ) {
lexer.ExpectToken("*");
lexer.ExpectToken("MESH_FACE");
idx = lexer.ReadInt();
lexer.CheckToken(":"); // might or might not be there
lexer.ExpectToken("A");
lexer.ExpectToken(":");
inMesh->triangles[idx].v[0] = lexer.ReadInt();
lexer.ExpectToken("B");
lexer.ExpectToken(":");
inMesh->triangles[idx].v[1] = lexer.ReadInt();
lexer.ExpectToken("C");
lexer.ExpectToken(":");
inMesh->triangles[idx].v[2] = lexer.ReadInt();
lexer.GotoNextLine();
}
lexer.ExpectToken("}");
}
else if ( lexer.CheckToken( "MESH_NUMTVERTEX" ) ) {
inMesh->numTVerts = lexer.ReadInt();
inMesh->texCoords = new Vec2[inMesh->numTVerts];
}
else if ( lexer.CheckToken( "MESH_TVERTLIST" ) ) {
lexer.ExpectToken("{");
for( int i=0; i<inMesh->numTVerts; i++ ) {
lexer.ExpectToken("*");
lexer.ExpectToken("MESH_TVERT");
idx = lexer.ReadInt();
inMesh->texCoords[idx].x = lexer.ReadFloat();
inMesh->texCoords[idx].y = 1.0f-lexer.ReadFloat();
lexer.ReadFloat();// don't need 3rd component
}
lexer.ExpectToken("}");
}
else if ( lexer.CheckToken( "MESH_NUMTVFACES" ) )
lexer.ExpectToken( Format() << inMesh->numTris );
else if ( lexer.CheckToken( "MESH_TFACELIST" ) ) {
lexer.ExpectToken("{");
for( int i=0; i<inMesh->numTris; i++ ) {
lexer.ExpectToken("*");
lexer.ExpectToken("MESH_TFACE");
idx = lexer.ReadInt();
inMesh->triangles[idx].t[0] = lexer.ReadInt();
inMesh->triangles[idx].t[1] = lexer.ReadInt();
inMesh->triangles[idx].t[2] = lexer.ReadInt();
}
lexer.ExpectToken("}");
}
else if ( lexer.CheckToken( "MATERIAL_REF" ) )
inMesh->materialId = lexer.ReadInt();
else if ( !SkipUnknown(lexer) )
break;
}
}
inline expr_ptr get_root_node( Lexer & lex )
{
return convert_token_to_expression( lex.get_token() );
}
bool InsetLayout::read(Lexer & lex, TextClass const & tclass)
{
enum {
IL_BGCOLOR,
IL_CONTENTASLABEL,
IL_COPYSTYLE,
IL_COUNTER,
IL_CUSTOMPARS,
IL_DECORATION,
IL_FONT,
IL_FORCELTR,
IL_FORCEPLAIN,
IL_FREESPACING,
IL_HTMLTAG,
IL_HTMLATTR,
IL_HTMLFORCECSS,
IL_HTMLINNERTAG,
IL_HTMLINNERATTR,
IL_HTMLISBLOCK,
IL_HTMLLABEL,
IL_HTMLSTYLE,
IL_HTMLPREAMBLE,
IL_INTOC,
IL_LABELFONT,
IL_LABELSTRING,
IL_LATEXNAME,
IL_LATEXPARAM,
IL_LATEXTYPE,
IL_LYXTYPE,
IL_KEEPEMPTY,
IL_MULTIPAR,
IL_NEEDPROTECT,
IL_PASSTHRU,
IL_PARBREAKISNEWLINE,
IL_PREAMBLE,
IL_REQUIRES,
IL_SPELLCHECK,
IL_REFPREFIX,
IL_END
};
LexerKeyword elementTags[] = {
{ "bgcolor", IL_BGCOLOR },
{ "contentaslabel", IL_CONTENTASLABEL },
{ "copystyle", IL_COPYSTYLE },
{ "counter", IL_COUNTER},
{ "custompars", IL_CUSTOMPARS },
{ "decoration", IL_DECORATION },
{ "end", IL_END },
{ "font", IL_FONT },
{ "forceltr", IL_FORCELTR },
{ "forceplain", IL_FORCEPLAIN },
{ "freespacing", IL_FREESPACING },
{ "htmlattr", IL_HTMLATTR },
{ "htmlforcecss", IL_HTMLFORCECSS },
{ "htmlinnerattr", IL_HTMLINNERATTR},
{ "htmlinnertag", IL_HTMLINNERTAG},
{ "htmlisblock", IL_HTMLISBLOCK},
{ "htmllabel", IL_HTMLLABEL },
{ "htmlpreamble", IL_HTMLPREAMBLE },
{ "htmlstyle", IL_HTMLSTYLE },
{ "htmltag", IL_HTMLTAG },
{ "intoc", IL_INTOC },
{ "keepempty", IL_KEEPEMPTY },
{ "labelfont", IL_LABELFONT },
{ "labelstring", IL_LABELSTRING },
{ "latexname", IL_LATEXNAME },
{ "latexparam", IL_LATEXPARAM },
{ "latextype", IL_LATEXTYPE },
{ "lyxtype", IL_LYXTYPE },
{ "multipar", IL_MULTIPAR },
{ "needprotect", IL_NEEDPROTECT },
{ "parbreakisnewline", IL_PARBREAKISNEWLINE },
{ "passthru", IL_PASSTHRU },
{ "preamble", IL_PREAMBLE },
{ "refprefix", IL_REFPREFIX },
{ "requires", IL_REQUIRES },
{ "spellcheck", IL_SPELLCHECK }
};
lex.pushTable(elementTags);
FontInfo font = inherit_font;
labelfont_ = inherit_font;
bgcolor_ = Color_none;
bool getout = false;
// whether we've read the CustomPars or ForcePlain tag
// for issuing a warning in case MultiPars comes later
bool readCustomOrPlain = false;
string tmp;
while (!getout && lex.isOK()) {
int le = lex.lex();
switch (le) {
case Lexer::LEX_UNDEF:
lex.printError("Unknown InsetLayout tag");
continue;
default:
break;
}
switch (le) {
// FIXME
// Perhaps a more elegant way to deal with the next two would be the
// way this sort of thing is handled in Layout::read(), namely, by
// using the Lexer.
case IL_LYXTYPE: {
string lt;
lex >> lt;
lyxtype_ = translateLyXType(lt);
if (lyxtype_ == NOLYXTYPE)
LYXERR0("Unknown LyXType `" << lt << "'.");
if (lyxtype_ == CHARSTYLE)
multipar_ = false;
break;
}
case IL_LATEXTYPE: {
string lt;
lex >> lt;
latextype_ = translateLaTeXType(lt);
if (latextype_ == ILT_ERROR)
LYXERR0("Unknown LaTeXType `" << lt << "'.");
break;
}
case IL_LABELSTRING:
lex >> labelstring_;
break;
case IL_DECORATION:
lex >> tmp;
decoration_ = translateDecoration(tmp);
break;
case IL_LATEXNAME:
lex >> latexname_;
break;
case IL_LATEXPARAM:
lex >> tmp;
latexparam_ = support::subst(tmp, """, "\"");
break;
case IL_LABELFONT:
labelfont_ = lyxRead(lex, inherit_font);
break;
case IL_FORCELTR:
lex >> forceltr_;
break;
case IL_INTOC:
lex >> intoc_;
break;
case IL_MULTIPAR:
lex >> multipar_;
// the defaults for these depend upon multipar_
if (readCustomOrPlain)
LYXERR0("Warning: Read MultiPar after CustomPars or ForcePlain. "
"Previous value may be overwritten!");
readCustomOrPlain = false;
custompars_ = multipar_;
forceplain_ = !multipar_;
break;
case IL_COUNTER:
lex >> counter_;
break;
case IL_CUSTOMPARS:
lex >> custompars_;
readCustomOrPlain = true;
break;
case IL_FORCEPLAIN:
lex >> forceplain_;
readCustomOrPlain = true;
break;
case IL_PASSTHRU:
lex >> passthru_;
break;
case IL_PARBREAKISNEWLINE:
lex >> parbreakisnewline_;
break;
case IL_KEEPEMPTY:
lex >> keepempty_;
break;
case IL_FREESPACING:
lex >> freespacing_;
break;
case IL_NEEDPROTECT:
lex >> needprotect_;
break;
case IL_CONTENTASLABEL:
lex >> contentaslabel_;
break;
case IL_COPYSTYLE: { // initialize with a known style
docstring style;
lex >> style;
style = support::subst(style, '_', ' ');
// We don't want to apply the algorithm in DocumentClass::insetLayout()
// here. So we do it the long way.
TextClass::InsetLayouts::const_iterator it =
tclass.insetLayouts().find(style);
if (it != tclass.insetLayouts().end()) {
docstring const tmpname = name_;
this->operator=(it->second);
name_ = tmpname;
} else {
LYXERR0("Cannot copy unknown InsetLayout `"
<< style << "'\n"
<< "All InsetLayouts so far:");
TextClass::InsetLayouts::const_iterator lit =
tclass.insetLayouts().begin();
TextClass::InsetLayouts::const_iterator len =
tclass.insetLayouts().end();
for (; lit != len; ++lit)
lyxerr << lit->second.name() << "\n";
}
break;
}
case IL_FONT: {
font_ = lyxRead(lex, inherit_font);
// If you want to define labelfont, you need to do so after
// font is defined.
labelfont_ = font_;
break;
}
case IL_BGCOLOR:
lex >> tmp;
bgcolor_ = lcolor.getFromLyXName(tmp);
break;
case IL_PREAMBLE:
preamble_ = from_utf8(lex.getLongString("EndPreamble"));
break;
case IL_REFPREFIX:
lex >> refprefix_;
break;
case IL_HTMLTAG:
lex >> htmltag_;
break;
case IL_HTMLATTR:
lex >> htmlattr_;
break;
case IL_HTMLFORCECSS:
lex >> htmlforcecss_;
break;
case IL_HTMLINNERTAG:
lex >> htmlinnertag_;
break;
case IL_HTMLINNERATTR:
lex >> htmlinnerattr_;
break;
case IL_HTMLLABEL:
lex >> htmllabel_;
break;
case IL_HTMLISBLOCK:
lex >> htmlisblock_;
break;
case IL_HTMLSTYLE:
htmlstyle_ = from_utf8(lex.getLongString("EndHTMLStyle"));
break;
case IL_HTMLPREAMBLE:
htmlpreamble_ = from_utf8(lex.getLongString("EndPreamble"));
break;
case IL_REQUIRES: {
lex.eatLine();
vector<string> const req
= support::getVectorFromString(lex.getString());
requires_.insert(req.begin(), req.end());
break;
}
case IL_SPELLCHECK:
lex >> spellcheck_;
break;
case IL_END:
getout = true;
break;
}
}
// Here add element to list if getout == true
if (!getout)
return false;
// The label font is generally used as-is without
// any realization against a given context.
labelfont_.realize(sane_font);
lex.popTable();
return true;
}
long double Calculation::Resolve(const string& str) const
throw(invalid_argument)
{
if (!CheckParenthesis(str))
throw invalid_argument("The number of left parenthesis is not equal with"
" the number of right parenthesis");
/*
* TODO
* To test special cases.
* 5 + * 2
* 5 + 2 *
* * 5 + 2
*
* 5 (2 + 3)
* (2 + 3) 5
*
* 5 2 * 3
* 5 sin(90)
*
* sin(90) cos(90)
* sin 90 cos(45)
*
* To test if there are the functions that are called.
*/
Lexer l;
list<Token*> tokens = l.Split(str);
long double r;
if (l.PrintErrors())
{
throw invalid_argument("Invalid input");
}
if (tokens.size() == 0)
{
throw invalid_argument("No tokens");
}
AddZero(tokens);
try
{
r = Resolve(tokens);
for (auto it = tokens.begin(); it != tokens.end(); ++it)
delete (*it);
return r;
}
catch (invalid_argument& e)
{
for (auto it = tokens.begin(); it != tokens.end(); ++it)
delete (*it);
throw e;
}
}
bool Parser::parseValue (Lexer& lexer, Lookup& lookup, Int32u* slot, const Extractor** output)
{
const Extractor* argument;
Extractors arguments;
const Constant* constant;
const Function* function;
string name;
Float64 number;
switch (lexer.getType ())
{
case Lexer::IDENTIFIER:
name = lexer.getCurrent ();
lexer.next ();
if (lexer.getType () == Lexer::PARENTHESIS_BEGIN)
{
function = 0;
for (Int32u i = 0; Function::functions[i].name; ++i)
{
if (Function::functions[i].name == name)
{
function = &Function::functions[i];
break;
}
}
if (!function)
return this->fail (lexer, string ("unknown function name '") + lexer.getCurrent () + "'");
lexer.next ();
while (lexer.getType () != Lexer::PARENTHESIS_END)
{
if (arguments.size () > 0 && !this->parseType (lexer, Lexer::COMMA, "argument separator"))
return false;
if (!this->parseExpression (lexer, lookup, slot, &argument))
return false;
arguments.push_back (argument);
}
lexer.next ();
if (arguments.size () < function->min || (function->max > 0 && arguments.size () > function->max))
return this->fail (lexer, "wrong number of arguments");
*output = function->builder (arguments, slot);
}
else
{
constant = 0;
for (Int32u i = 0; Constant::constants[i].name; ++i)
{
if (Constant::constants[i].name == name)
{
constant = &Constant::constants[i];
break;
}
}
if (constant)
*output = new ConstantExtractor (constant->value);
else
*output = new FieldExtractor (lookup.store (name));
}
break;
case Lexer::NUMBER:
if (!Convert::toFloat (&number, lexer.getCurrent ().data (), lexer.getCurrent ().length ()))
return this->fail (lexer, "invalid number");
*output = new ConstantExtractor (Variant (number));
lexer.next ();
break;
case Lexer::STRING:
*output = new ConstantExtractor (Variant (lexer.getCurrent ()));
lexer.next ();
break;
default:
this->fail (lexer, "unexpected character");
return false;
}
this->extractors.push_back (*output);
return true;
}
Parser::Parser(Lexer lexer, Tokenizer tokenizer, const char *output_file, const char *processing) : lexer_tokens(lexer.get_tokens()),
lexer_parser_pos(0), look(this->scan()), document_tokens(tokenizer.get_tokens()) {
view v = view("Document");
col c = col("text");
span s = span(tokenizer.get_text(), 0, tokenizer.get_text().length());
c.spans.push_back(s);
v.cols.push_back(c);
this->views.push_back(v);
this->output_file = fopen(output_file, "w+");
fprintf(this->output_file, "Processing %s\n", processing);
}
void Interface::instrument_emit_post(PragmaCustomConstruct ctr)
{
ObjectList<std::string> arguments;
if (!ctr.is_parameterized()
|| (arguments = ctr.get_parameter_arguments(ExpressionTokenizerTrim())).size() != 2)
{
invalid_instrument_emit(ctr);
ctr.get_ast().remove_in_list();
return;
}
Lexer l = Lexer::get_current_lexer();
ObjectList<Lexer::pair_token> tokens_key = l.lex_string(arguments[0]);
ObjectList<Lexer::pair_token> tokens_descr = l.lex_string(arguments[1]);
if (tokens_key.size() != 1
|| tokens_descr.size() != 1)
{
invalid_instrument_emit(ctr);
return;
}
if ((IS_C_LANGUAGE && (tokens_key[0].first != TokensC::IDENTIFIER))
|| (IS_CXX_LANGUAGE && (tokens_key[0].first != TokensCXX::IDENTIFIER)))
{
std::cerr << ctr.get_ast().get_locus() << ": warning: first argument must be an identifier" << std::endl;
invalid_instrument_emit(ctr);
return;
}
if ((IS_C_LANGUAGE && (tokens_descr[0].first != TokensC::STRING_LITERAL))
|| (IS_CXX_LANGUAGE && (tokens_descr[0].first != TokensCXX::STRING_LITERAL)))
{
std::cerr << ctr.get_ast().get_locus() << ": warning: second argument must be a string-literal" << std::endl;
invalid_instrument_emit(ctr);
return;
}
if (_map_events.find(arguments[0]) == _map_events.end())
{
std::cerr << ctr.get_ast().get_locus() << ": warning: event key '" << arguments[0] << "' has not been previously declared" << std::endl;
invalid_instrument_emit(ctr);
ctr.get_ast().remove_in_list();
return;
}
Source src;
src
<< "{"
<< "static int nanos_funct_id_init = 0;"
<< "static nanos_event_key_t nanos_instr_name_key = 0;"
<< "static nanos_event_value_t nanos_instr_name_value = 0;"
<< "if (nanos_funct_id_init == 0)"
<< "{"
<< "nanos_err_t err = nanos_instrument_get_key(\"" << arguments[0] << "\", &nanos_instr_name_key);"
<< "if (err != NANOS_OK) nanos_handle_error(err);"
<< "err = nanos_instrument_register_value (&nanos_instr_name_value, "
<< "\"" << arguments[0] << "\", "
<< arguments[1] << ", "
<< "\"" << ctr.get_ast().get_locus() << "\","
<< "/* abort_if_registered */ 0);"
<< "if (err != NANOS_OK) nanos_handle_error(err);"
<< "nanos_funct_id_init = 1;"
<< "}"
<< "nanos_event_t _events[1];"
<< "_events[0].type = NANOS_POINT;"
<< "_events[0].info.point.nkvs = 1;"
<< "_events[0].info.point.keys = &nanos_instr_name_key;"
<< "_events[0].info.point.values = &nanos_instr_name_value;"
<< "nanos_instrument_events(1, _events);"
<< "}"
;
AST_t tree = src.parse_statement(ctr.get_ast(), ctr.get_scope_link());
ctr.get_ast().replace(tree);
}
ASTNode::ASTNode(Lexer& l) {
_lineNumber = l.getLineNumber();
_charNumber = l.getCharNumber();
}
static void compile_structure( Lexer& lexer, types::VariantStructDescription& result, const types::NormalizeFunctionMap* typemap, const types::keymap<types::FormDescriptionR>& formmap, std::vector<std::string>& unresolvedSymbols)
{
bool seenCloseStruct = false;
Lexem lx;
for (lx = lexer.next2(false); lx.id() != Lexem::CloseStruct; lx=lexer.next2(false))
{
if (lx.id() != Lexem::Identifier && lx.id() != Lexem::String)
{
throw std::runtime_error( std::string("unexpected ") + lexer.curtoken() + ", structure element name as identifier or string expected");
}
std::string elementname( lx.value()); //< name of the element
bool isOptional = false; //< element is optional (also in strict validation)
bool isMandatory = false; //< element is mandatory (also in relaxed validation)
bool isIndirection = false; //< element is an indirection (means it is expanded only when it exists)
bool isArray = false; //< element is an array
bool isAttribute = false; //< element is an attribute (for languages that know attributes like XML)
bool isInherited = (elementname == "_"); //< element is inherited (means it has no name and it acts as content of the enclosing element)
for (lx=lexer.next2(true);; lx=lexer.next2(true))
{
if (lx.id() == Lexem::Separator)
{
for (lx=lexer.next2(true); lx.id() == Lexem::Separator; lx=lexer.next2(true)){}
if (lx.id() != Lexem::OpenStruct)
{
throw std::runtime_error("unexpected end of line in the middle of an atomic element definition");
}
}
if (lx.id() == Lexem::OpenStruct)
{
types::VariantStructDescription substruct;
compile_structure( lexer, substruct, typemap, formmap, unresolvedSymbols);
int elementidx = result.addStructure( elementname, substruct);
if (isArray) result.at( elementidx)->makeArray();
if (isOptional) result.at( elementidx)->setOptional();
if (isMandatory) result.at( elementidx)->setMandatory();
if (isIndirection) throw std::runtime_error("cannot declare embedded substructure as indirection ('^')");
if (isAttribute) throw std::runtime_error("cannot declare structure as attribute ('@')");
if (isInherited) throw std::runtime_error("cannot declare embedded substructure with name \"_\" used to declare inheritance");
lx = lexer.next2(true);
if (lx.id() == Lexem::CloseStruct)
{
seenCloseStruct = true;
}
else if (lx.id() != Lexem::Separator)
{
throw std::runtime_error("expected close structure '}' or end of line or comma ',' as separator after close struct");
}
break;
}
else if (lx.id() == Lexem::Identifier)
{
std::string elementtype( lx.value());
std::string defaultvalue;
bool hasDefault = false;
lx = lexer.next2(true);
for (; lx.id() != Lexem::Separator && lx.id() != Lexem::CloseStruct; lx = lexer.next2(true))
{
if (lx.id() == Lexem::ArrayTag)
{
if (hasDefault) throw std::runtime_error("unexpected array marker after default value assignment");
if (isArray) throw std::runtime_error("cannot handle duplicate array tag '[]'");
isArray = true;
}
else if (lx.id() == Lexem::Assign)
{
if (isArray) throw std::runtime_error("cannot handle default value assignment for array");
if (hasDefault) throw std::runtime_error("cannot handle duplicate default value assignment");
lx = lexer.next2(true);
if (lx.id() != Lexem::String && lx.id() != Lexem::Identifier)
{
throw std::runtime_error( "string or identifier expected for default value declaration after '='");
}
defaultvalue = lx.value();
hasDefault = true;
}
else
{
throw std::runtime_error( std::string("unexpected token ") + lexer.curtoken() + ", default value assignment or array marker or following structure element declaration expected");
}
}
if (lx.id() == Lexem::CloseStruct)
{
seenCloseStruct = true;
}
// Resolve type:
types::keymap<types::FormDescriptionR>::const_iterator fmi;
fmi = formmap.find( elementtype);
if (fmi != formmap.end())
{
// ... type referenced is a sub structure
if (isAttribute) throw std::runtime_error("cannot declare structure as attribute ('@')");
// Sub structure referenced by 'elementtype':
const types::FormDescription* substruct = fmi->second.get();
if (isIndirection)
{
// ... resolved indirection
int elementidx = result.addIndirection( elementname, substruct);
if (isArray) result.at( elementidx)->makeArray();
}
else if (isInherited)
{
// ... resolved inheritance
result.inherit( *substruct);
}
else
{
// ... resolved sub structure
int elementidx = result.addStructure( elementname, *substruct);
if (isArray) result.at( elementidx)->makeArray();
if (isOptional) result.at( elementidx)->setOptional();
if (isMandatory) result.at( elementidx)->setMandatory();
}
}
else if (boost::algorithm::iequals( elementtype, "string"))
{
// ... type referenced is the reserved type string (no normalizer defined)
if (isIndirection) throw std::runtime_error("cannot declare atomic string type as indirection ('^')");
int elementidx; //< index of atomic element created
if (isAttribute)
{
elementidx = result.addAttribute( elementname, hasDefault?types::Variant(defaultvalue):types::Variant(types::Variant::String));
//... atomic attribute value
}
else if (isInherited)
{
elementidx = result.addAtom( "", hasDefault?types::Variant(defaultvalue):types::Variant(types::Variant::String));
//... atomic unnamed (inherited) content value
}
else
{
elementidx = result.addAtom( elementname, hasDefault?types::Variant(defaultvalue):types::Variant(types::Variant::String));
//... atomic named content value (non attribute)
}
if (isArray) result.at( elementidx)->makeArray();
if (isOptional) result.at( elementidx)->setOptional();
if (isMandatory) result.at( elementidx)->setMandatory();
}
else
{
// ... type referenced must be an atomic type defined in a .wnmp file or a custom data type
const types::NormalizeFunction* atomictype;
if (isIndirection)
{
// ... unresolved indirection
int elementidx = result.addUnresolved( elementname, elementtype);
if (isArray) result.at( elementidx)->makeArray();
unresolvedSymbols.push_back( elementtype);
}
else if (0!=(atomictype=typemap->get( elementtype)))
{
// ... type referenced is an atomic type defined in a .wnmp file or a custom data type
int elementidx; //< index of atomic element created
if (isIndirection) throw std::runtime_error("cannot declare atomic type as indirection ('^')");
if (isAttribute)
{
elementidx = result.addAttribute( elementname, atomictype->execute(hasDefault?types::Variant(defaultvalue):types::Variant()), atomictype);
//... atomic attribute value
}
else if (isInherited)
{
elementidx = result.addAtom( "", atomictype->execute(hasDefault?types::Variant(defaultvalue):types::Variant()), atomictype);
//... atomic unnamed (inherited) content value
}
else
{
elementidx = result.addAtom( elementname, atomictype->execute(hasDefault?types::Variant(defaultvalue):types::Variant()), atomictype);
//... atomic named content value (non attribute)
}
if (isArray) result.at( elementidx)->makeArray();
if (isOptional) result.at( elementidx)->setOptional();
if (isMandatory) result.at( elementidx)->setMandatory();
}
else if (isInherited)
{
// ... unresolved inheritance = error
throw std::runtime_error( std::string("undefined atomic data type or inherited structure '") + elementtype + "'");
}
else
{
throw std::runtime_error( std::string("cannot find type '") + elementtype + "'");
}
}
break;
}
else if (lx.id() == Lexem::ArrayTag)
{
if (isInherited) throw std::runtime_error("contradicting array/inheritance tag '[]' and '_'");
if (isArray) throw std::runtime_error("cannot handle duplicate array tag '[]'");
isArray = true;
}
else if (lx.id() == Lexem::AttributeTag)
{
if (isInherited) throw std::runtime_error("contradicting attribute/inheritance tag '@' and '_'");
if (isAttribute) throw std::runtime_error("cannot handle duplicate attribute tag '@'");
isAttribute = true;
}
else if (lx.id() == Lexem::MandatoryTag)
{
if (isInherited) throw std::runtime_error("contradicting mandatory/inheritance tag '!' and '_'");
if (isOptional) throw std::runtime_error("contradicting optional/mandatory tag '?' and '!'");
if (isMandatory) throw std::runtime_error("cannot handle duplicate mandatory tag '!'");
isMandatory = true;
}
else if (lx.id() == Lexem::OptionalTag)
{
if (isInherited) throw std::runtime_error("contradicting optional/inheritance tag '?' and '_'");
if (isMandatory) throw std::runtime_error("contradicting optional/mandatory tag '?' and '!'");
if (isOptional) throw std::runtime_error("cannot handle duplicate optional tag '?'");
isOptional = true;
}
else if (lx.id() == Lexem::Indirection)
{
if (isInherited) throw std::runtime_error("contradicting indirection/inheritance tag '^' and '_'");
if (isOptional) throw std::runtime_error("contradicting indirection/optional tag '^' and '?'");
if (isMandatory) throw std::runtime_error("contradicting indirection/mandatory tag '^' and '!'");
if (isIndirection) throw std::runtime_error("cannot handle duplicate indirection tag '^'");
isIndirection = true;
}
else
{
throw std::runtime_error( std::string("unexpected token ") + lexer.curtoken() + ", attribute or type name or substructure expected");
}
}
if (seenCloseStruct)
{
break;
}
}
}
/*
* DROP TABLE <name>
*/
void DropTable::Parse(Lexer& lex)
{
Token t = lex.NextToken(TokenType::Identifier, false);
SetName(t.strData);
}
int main(int argc, char *argv[])
{
QCoreApplication app(argc, argv);
try {
if(qApp->arguments().length() != 3) {
throw std::runtime_error("Incorrect arguments (thriftDir generatedDir)");
}
QString thriftDir = qApp->arguments().at(1);
QString generatedDir = qApp->arguments().at(2);
// QString thriftDir = "D:/sources/lib/QEverCloud/src/QEverCloud/thrift/src";
// QString generatedDir = "D:/tmp/2";
Lexer* lexer = new Lexer(&app);
QDir dir(thriftDir);
for(QString thriftFile : dir.entryList(QStringList() << "*.thrift", QDir::Files, QDir::Name)) {
lexer->feedFile(dir.absoluteFilePath(thriftFile));
}
/*QFile outFile("D:/tmp/2/lexer.txt");
outFile.open(QIODevice::WriteOnly);
QTextStream out(&outFile);
out.setCodec("UTF-8");
for(const Lexer::TerminalSymbol& term : lexer->terminals()) {
out << "[" << term.line << "] " << static_cast<int>(term.type) << " " << term.data << endl;
}*/
Parser* parser = new Parser(&app);
for(const Lexer::TerminalSymbol& term : lexer->terminals()) {
parser->setFile(term.file);
//qDebug() << term.file << term.line << term.data;
parser->feed(term.type, term.data);
if(parser->isError()) {
throw std::runtime_error(QString("%4 in file %1 at line %2: %3\ndetected token type: %5").arg(term.file).arg(term.line).arg(term.data)
.arg(parser->errorMessage()).arg(static_cast<int>(term.type)).toStdString());
}
}
parser->complete();
if(parser->isError()) {
throw std::runtime_error(QString("Parser error at completion: %1").arg(parser->errorMessage()).toStdString());
}
// qDebug() << "=========== NAMESPACES ==============";
// for(const Parser::Namespace& n : parser->namespaces()) {
// qDebug() << n.name << n.scope << n.file;
// }
// qDebug() << "=========== INCLUDES ==============";
// for(const Parser::Include& i : parser->includes()) {
// qDebug() << i.name << i.file;
// }
// qDebug() << "=========== TYPEDEFS ==============";
// for(const Parser::TypeDefinition& td : parser->typedefs()) {
// QString s;
// if(td.type.dynamicCast<Parser::BaseType>()) {
// s = td.type.dynamicCast<Parser::BaseType>()->basetype;
// }
// qDebug() << td.name << s << td.file;
// }
// qDebug() << "=========== CONSTANTS ==============";
// for(const Parser::Constant& c : parser->constants()) {
// QString s;
// if(c.type.dynamicCast<Parser::BaseType>()) {
// s = c.type.dynamicCast<Parser::BaseType>()->basetype;
// }
// if(c.type.dynamicCast<Parser::IdentifierType>()) {
// s = c.type.dynamicCast<Parser::IdentifierType>()->identifier;
// }
// if(c.type.dynamicCast<Parser::MapType>()) {
// s = "MAP";
// }
// if(c.type.dynamicCast<Parser::SetType>()) {
// s = "SET";
// }
// if(c.type.dynamicCast<Parser::ListType>()) {
// s = "LIST";
// }
// QString v;
// if(c.value && c.value.dynamicCast<Parser::LiteralValue>()) {
// v = c.value.dynamicCast<Parser::LiteralValue>()->value;
// }
// qDebug() << c.name << s << v << c.file;
// }
// qDebug() << "=========== STRUCTURES ==============";
// for(const Parser::Structure& s : parser->structures()) {
// QString fields;
// for(const Parser::Field& f: s.fields) {
// fields += f.name + (f.required == Parser::Field::RequiredFlag::Optional ? "*" : "") + " ";
// }
// qDebug() << s.name << fields << s.file;
// }
// qDebug() << "=========== UNIONS ==============";
// for(const Parser::Structure& s : parser->unions()) {
// QString fields;
// for(const Parser::Field& f: s.fields) {
// fields += f.name + (f.required == Parser::Field::RequiredFlag::Optional ? "*" : "") + " ";
// }
// qDebug() << s.name << fields << s.file;
// }
// qDebug() << "=========== EXCEPTIONS ==============";
// for(const Parser::Structure& s : parser->exceptions()) {
// QString fields;
// for(const Parser::Field& f: s.fields) {
// fields += f.name + (f.required == Parser::Field::RequiredFlag::Optional ? "*" : "") + " ";
// }
// qDebug() << s.name << fields << s.file;
// }
// qDebug() << "=========== SERVICES ==============";
// for(const Parser::Service& s : parser->services()) {
// qDebug() << "******** " << s.name << " " << s.file;
// for(const Parser::Function& f: s.functions) {
// qDebug() << f.name << f.isOneway;
// }
// }
// qDebug() << "=========== ENUMS ==============";
// for(const Parser::Enumeration& e : parser->enumerations()) {
// QString values;
// for(auto v: e.values) {
// values += v.first + "=" + v.second + " ";
// }
// qDebug() << e.name << values << e.file;
// }
// qDebug() << "=========== FILES ==============";
// for(QString file : parser->files()) {
// qDebug() << file;
// }
generateSources(parser, generatedDir);
} catch(const std::exception& e) {
qDebug() << "\nFAILURE!!!";
qDebug() << e.what();
return 1;
} catch(...) {
qDebug() << "\nFAILURE!!! Unknown exception\n";
return 2;
}
return 0;
}
bool Counter::read(Lexer & lex)
{
enum {
CT_WITHIN = 1,
CT_LABELSTRING,
CT_LABELSTRING_APPENDIX,
CT_PRETTYFORMAT,
CT_END
};
LexerKeyword counterTags[] = {
{ "end", CT_END },
{ "labelstring", CT_LABELSTRING },
{ "labelstringappendix", CT_LABELSTRING_APPENDIX },
{ "prettyformat", CT_PRETTYFORMAT },
{ "within", CT_WITHIN }
};
lex.pushTable(counterTags);
bool getout = false;
while (!getout && lex.isOK()) {
int le = lex.lex();
switch (le) {
case Lexer::LEX_UNDEF:
lex.printError("Unknown counter tag `$$Token'");
continue;
default:
break;
}
switch (le) {
case CT_WITHIN:
lex.next();
master_ = lex.getDocString();
if (master_ == "none")
master_.erase();
break;
case CT_PRETTYFORMAT:
lex.next();
prettyformat_ = lex.getDocString();
break;
case CT_LABELSTRING:
lex.next();
labelstring_ = lex.getDocString();
labelstringappendix_ = labelstring_;
break;
case CT_LABELSTRING_APPENDIX:
lex.next();
labelstringappendix_ = lex.getDocString();
break;
case CT_END:
getout = true;
break;
}
}
// Here if have a full counter if getout == true
if (!getout)
LYXERR0("No End tag found for counter!");
lex.popTable();
return getout;
}
void Template::readTemplate(Lexer & lex)
{
enum {
TO_GUINAME = 1,
TO_HELPTEXT,
TO_INPUTFORMAT,
TO_FILTER,
TO_AUTOMATIC,
TO_PREVIEW,
TO_TRANSFORM,
TO_FORMAT,
TO_END
};
LexerKeyword templateoptiontags[] = {
{ "automaticproduction", TO_AUTOMATIC },
{ "filefilter", TO_FILTER },
{ "format", TO_FORMAT },
{ "guiname", TO_GUINAME },
{ "helptext", TO_HELPTEXT },
{ "inputformat", TO_INPUTFORMAT },
{ "preview", TO_PREVIEW },
{ "templateend", TO_END },
{ "transform", TO_TRANSFORM }
};
PushPopHelper pph(lex, templateoptiontags);
lex.setContext("Template::readTemplate");
string token;
while (lex.isOK()) {
switch (lex.lex()) {
case TO_GUINAME:
lex.next(true);
guiName = lex.getString();
break;
case TO_HELPTEXT:
helpText = lex.getLongString("HelpTextEnd");
break;
case TO_INPUTFORMAT:
lex.next(true);
inputFormat = lex.getString();
break;
case TO_FILTER:
lex.next(true);
fileRegExp = lex.getString();
break;
case TO_AUTOMATIC:
lex.next();
automaticProduction = lex.getBool();
break;
case TO_PREVIEW:
lex >> token;
if (token == "InstantPreview")
preview_mode = PREVIEW_INSTANT;
else if (token == "Graphics")
preview_mode = PREVIEW_GRAPHICS;
else
preview_mode = PREVIEW_OFF;
break;
case TO_TRANSFORM: {
lex >> token;
TransformID id = transformIDTranslator().find(token);
if (int(id) == -1)
LYXERR0("Transform " << token << " is not recognized");
else
transformIds.push_back(id);
break;
}
case TO_FORMAT:
lex.next(true);
formats[lex.getString()].readFormat(lex);
break;
case TO_END:
return;
default:
lex.printError("external::Template::readTemplate: "
"Wrong tag: $$Token");
LASSERT(false, /**/);
break;
}
}
}
int main (int, char**)
{
UnitTest t (1208);
std::vector <std::pair <std::string, Lexer::Type>> tokens;
std::string token;
Lexer::Type type;
// Feed in some attributes and types, so that the Lexer knows what a DOM
// reference is.
Lexer::attributes["due"] = "date";
Lexer::attributes["tags"] = "string";
Lexer::attributes["description"] = "string";
// White space detection.
t.notok (Lexer::isWhitespace (0x0041), "U+0041 (A) ! isWhitespace");
t.ok (Lexer::isWhitespace (0x0020), "U+0020 isWhitespace");
t.ok (Lexer::isWhitespace (0x0009), "U+0009 isWhitespace");
t.ok (Lexer::isWhitespace (0x000A), "U+000A isWhitespace");
t.ok (Lexer::isWhitespace (0x000B), "U+000B isWhitespace");
t.ok (Lexer::isWhitespace (0x000C), "U+000C isWhitespace");
t.ok (Lexer::isWhitespace (0x000D), "U+000D isWhitespace");
t.ok (Lexer::isWhitespace (0x0085), "U+0085 isWhitespace");
t.ok (Lexer::isWhitespace (0x00A0), "U+00A0 isWhitespace");
t.ok (Lexer::isWhitespace (0x1680), "U+1680 isWhitespace"); // 10
t.ok (Lexer::isWhitespace (0x180E), "U+180E isWhitespace");
t.ok (Lexer::isWhitespace (0x2000), "U+2000 isWhitespace");
t.ok (Lexer::isWhitespace (0x2001), "U+2001 isWhitespace");
t.ok (Lexer::isWhitespace (0x2002), "U+2002 isWhitespace");
t.ok (Lexer::isWhitespace (0x2003), "U+2003 isWhitespace");
t.ok (Lexer::isWhitespace (0x2004), "U+2004 isWhitespace");
t.ok (Lexer::isWhitespace (0x2005), "U+2005 isWhitespace");
t.ok (Lexer::isWhitespace (0x2006), "U+2006 isWhitespace");
t.ok (Lexer::isWhitespace (0x2007), "U+2007 isWhitespace");
t.ok (Lexer::isWhitespace (0x2008), "U+2008 isWhitespace"); // 20
t.ok (Lexer::isWhitespace (0x2009), "U+2009 isWhitespace");
t.ok (Lexer::isWhitespace (0x200A), "U+200A isWhitespace");
t.ok (Lexer::isWhitespace (0x2028), "U+2028 isWhitespace");
t.ok (Lexer::isWhitespace (0x2029), "U+2029 isWhitespace");
t.ok (Lexer::isWhitespace (0x202F), "U+202F isWhitespace");
t.ok (Lexer::isWhitespace (0x205F), "U+205F isWhitespace");
t.ok (Lexer::isWhitespace (0x3000), "U+3000 isWhitespace");
// static bool Lexer::isBoundary (int, int);
t.ok (Lexer::isBoundary (' ', 'a'), "' ' --> 'a' = isBoundary");
t.ok (Lexer::isBoundary ('a', ' '), "'a' --> ' ' = isBoundary");
t.ok (Lexer::isBoundary (' ', '+'), "' ' --> '+' = isBoundary");
t.ok (Lexer::isBoundary (' ', ','), "' ' --> ',' = isBoundary");
t.notok (Lexer::isBoundary ('3', '4'), "'3' --> '4' = isBoundary");
t.ok (Lexer::isBoundary ('(', '('), "'(' --> '(' = isBoundary");
t.notok (Lexer::isBoundary ('r', 'd'), "'r' --> 'd' = isBoundary");
// static bool Lexer::wasQuoted (const std::string&);
t.notok (Lexer::wasQuoted (""), "'' --> !wasQuoted");
t.notok (Lexer::wasQuoted ("foo"), "'foo' --> !wasQuoted");
t.ok (Lexer::wasQuoted ("a b"), "'a b' --> wasQuoted");
t.ok (Lexer::wasQuoted ("(a)"), "'(a)' --> wasQuoted");
// static bool Lexer::dequote (std::string&, const std::string& quotes = "'\"");
token = "foo";
Lexer::dequote (token);
t.is (token, "foo", "dequote foo --> foo");
token = "'foo'";
Lexer::dequote (token);
t.is (token, "foo", "dequote 'foo' --> foo");
token = "'o\\'clock'";
Lexer::dequote (token);
t.is (token, "o\\'clock", "dequote 'o\\'clock' --> o\\'clock");
token = "abba";
Lexer::dequote (token, "a");
t.is (token, "bb", "dequote 'abba' (a) --> bb");
// Should result in no tokens.
Lexer l0 ("");
t.notok (l0.token (token, type), "'' --> no tokens");
// Should result in no tokens.
Lexer l1 (" \t ");
t.notok (l1.token (token, type), "' \\t ' --> no tokens");
// \u20ac = Euro symbol.
Lexer l2 (" one 'two \\'three\\''+456-(1.3*2 - 0x12) 1.2e-3.4 foo.bar and '\\u20ac'");
tokens.clear ();
while (l2.token (token, type))
{
std::cout << "# «" << token << "» " << Lexer::typeName (type) << "\n";
tokens.push_back (std::pair <std::string, Lexer::Type> (token, type));
}
t.is (tokens[0].first, "one", "tokens[0] = 'one'"); // 30
t.is (Lexer::typeName (tokens[0].second), "identifier", "tokens[0] = identifier");
t.is (tokens[1].first, "'two 'three''", "tokens[1] = 'two 'three''");
t.is (Lexer::typeName (tokens[1].second), "string", "tokens[1] = string");
t.is (tokens[2].first, "+", "tokens[2] = '+'");
t.is (Lexer::typeName (tokens[2].second), "op", "tokens[2] = op");
t.is (tokens[3].first, "456", "tokens[3] = '456'");
t.is (Lexer::typeName (tokens[3].second), "number", "tokens[3] = number");
t.is (tokens[4].first, "-", "tokens[4] = '-'");
t.is (Lexer::typeName (tokens[4].second), "op", "tokens[4] = op");
t.is (tokens[5].first, "(", "tokens[5] = '('"); // 40
t.is (Lexer::typeName (tokens[5].second), "op", "tokens[5] = op");
t.is (tokens[6].first, "1.3", "tokens[6] = '1.3'");
t.is (Lexer::typeName (tokens[6].second), "number", "tokens[6] = number");
t.is (tokens[7].first, "*", "tokens[7] = '*'");
t.is (Lexer::typeName (tokens[7].second), "op", "tokens[7] = op");
t.is (tokens[8].first, "2", "tokens[8] = '2'");
t.is (Lexer::typeName (tokens[8].second), "number", "tokens[8] = number");
t.is (tokens[9].first, "-", "tokens[9] = '-'");
t.is (Lexer::typeName (tokens[9].second), "op", "tokens[9] = op");
t.is (tokens[10].first, "0x12", "tokens[10] = '0x12'"); // 50
t.is (Lexer::typeName (tokens[10].second), "hex", "tokens[10] = hex");
t.is (tokens[11].first, ")", "tokens[11] = ')'");
t.is (Lexer::typeName (tokens[11].second), "op", "tokens[11] = op");
t.is (tokens[12].first, "1.2e-3.4", "tokens[12] = '1.2e-3.4'");
t.is (Lexer::typeName (tokens[12].second), "number", "tokens[12] = number");
t.is (tokens[13].first, "foo.bar", "tokens[13] = 'foo.bar'");
t.is (Lexer::typeName (tokens[13].second), "identifier", "tokens[13] = identifier");
t.is (tokens[14].first, "and", "tokens[14] = 'and'"); // 60
t.is (Lexer::typeName (tokens[14].second), "op", "tokens[14] = op");
t.is (tokens[15].first, "'€'", "tokens[15] = \\u20ac --> ''€''");
t.is (Lexer::typeName (tokens[15].second), "string", "tokens[15] = string");
// Test for numbers that are no longer ISO-8601 dates.
Lexer l3 ("1 12 123 1234 12345 123456 1234567");
tokens.clear ();
while (l3.token (token, type))
{
std::cout << "# «" << token << "» " << Lexer::typeName (type) << "\n";
tokens.push_back (std::pair <std::string, Lexer::Type> (token, type));
}
t.is ((int)tokens.size (), 7, "7 tokens");
t.is (tokens[0].first, "1", "tokens[0] == '1'");
t.is ((int) tokens[0].second, (int) Lexer::Type::number, "tokens[0] == Type::number");
t.is (tokens[1].first, "12", "tokens[1] == '12'");
t.is ((int) tokens[1].second, (int) Lexer::Type::number, "tokens[1] == Type::date");
t.is (tokens[2].first, "123", "tokens[2] == '123'");
t.is ((int) tokens[2].second, (int) Lexer::Type::number, "tokens[2] == Type::number"); // 70
t.is (tokens[3].first, "1234", "tokens[3] == '1234'");
t.is ((int) tokens[3].second, (int) Lexer::Type::number, "tokens[3] == Type::date");
t.is (tokens[4].first, "12345", "tokens[4] == '12345'");
t.is ((int) tokens[4].second, (int) Lexer::Type::number, "tokens[4] == Type::number");
t.is (tokens[5].first, "123456", "tokens[5] == '123456'");
t.is ((int) tokens[5].second, (int) Lexer::Type::number, "tokens[5] == Type::date");
t.is (tokens[6].first, "1234567", "tokens[6] == '1234567'");
t.is ((int) tokens[6].second, (int) Lexer::Type::number, "tokens[6] == Type::number");
// void split (std::vector<std::string>&, const std::string&);
std::string unsplit = " ( A or B ) ";
std::vector <std::string> items;
items = Lexer::split (unsplit);
t.is (items.size (), (size_t) 5, "split ' ( A or B ) '");
t.is (items[0], "(", "split ' ( A or B ) ' -> [0] '('");
t.is (items[1], "A", "split ' ( A or B ) ' -> [1] 'A'");
t.is (items[2], "or", "split ' ( A or B ) ' -> [2] 'or'");
t.is (items[3], "B", "split ' ( A or B ) ' -> [3] 'B'");
t.is (items[4], ")", "split ' ( A or B ) ' -> [4] ')'");
// Test simple mode with contrived tokens that ordinarily split.
unsplit = " +-* a+b 12.3e4 'c d'";
items = Lexer::split (unsplit);
t.is (items.size (), (size_t) 8, "split ' +-* a+b 12.3e4 'c d''");
t.is (items[0], "+", "split ' +-* a+b 12.3e4 'c d'' -> [0] '+'");
t.is (items[1], "-", "split ' +-* a+b 12.3e4 'c d'' -> [1] '-'");
t.is (items[2], "*", "split ' +-* a+b 12.3e4 'c d'' -> [2] '*'");
t.is (items[3], "a", "split ' +-* a+b 12.3e4 'c d'' -> [3] 'a'");
t.is (items[4], "+", "split ' +-* a+b 12.3e4 'c d'' -> [4] '+'");
t.is (items[5], "b", "split ' +-* a+b 12.3e4 'c d'' -> [5] 'b'");
t.is (items[6], "12.3e4", "split ' +-* a+b 12.3e4 'c d'' -> [6] '12.3e4'");
t.is (items[7], "'c d'", "split ' +-* a+b 12.3e4 'c d'' -> [7] ''c d''");
// static bool decomposePair (const std::string&, std::string&, std::string&, std::string&, std::string&);
// 2 * 4 * 2 * 5 = 80 tests.
std::string outName, outMod, outValue, outSep;
for (auto& name : {"name"})
{
for (auto& mod : {"", "mod"})
{
for (auto& sep : {":", "=", "::", ":="})
{
for (auto& value : {"", "value", "a:b", "a::b", "a=b", "a:=b"})
{
std::string input = std::string ("name") + (strlen (mod) ? "." : "") + mod + sep + value;
t.ok (Lexer::decomposePair (input, outName, outMod, outSep, outValue), "decomposePair '" + input + "' --> true");
t.is (name, outName, " '" + input + "' --> name '" + name + "'");
t.is (mod, outMod, " '" + input + "' --> mod '" + mod + "'");
t.is (value, outValue, " '" + input + "' --> value '" + value + "'");
t.is (sep, outSep, " '" + input + "' --> sep '" + sep + "'");
}
}
}
}
// static bool readWord (const std::string&, const std::string&, std::string::size_type&, std::string&);
std::string::size_type cursor = 0;
std::string word;
t.ok (Lexer::readWord ("'one two'", "'\"", cursor, word), "readWord ''one two'' --> true");
t.is (word, "'one two'", " word '" + word + "'");
t.is ((int)cursor, 9, " cursor");
// Unterminated quoted string is invalid.
cursor = 0;
t.notok (Lexer::readWord ("'one", "'\"", cursor, word), "readWord ''one' --> false");
// static bool readWord (const std::string&, std::string::size_type&, std::string&);
cursor = 0;
t.ok (Lexer::readWord ("input", cursor, word), "readWord 'input' --> true");
t.is (word, "input", " word '" + word + "'");
t.is ((int)cursor, 5, " cursor");
cursor = 0;
t.ok (Lexer::readWord ("one\\ two", cursor, word), "readWord 'one\\ two' --> true");
t.is (word, "one two", " word '" + word + "'");
t.is ((int)cursor, 8, " cursor");
cursor = 0;
t.ok (Lexer::readWord ("\\u20A43", cursor, word), "readWord '\\u20A43' --> true");
t.is (word, "₤3", " word '" + word + "'");
t.is ((int)cursor, 7, " cursor");
cursor = 0;
t.ok (Lexer::readWord ("U+20AC4", cursor, word), "readWord '\\u20AC4' --> true");
t.is (word, "€4", " word '" + word + "'");
t.is ((int)cursor, 7, " cursor");
std::string text = "one 'two' three\\ four";
cursor = 0;
t.ok (Lexer::readWord (text, cursor, word), "readWord \"one 'two' three\\ four\" --> true");
t.is (word, "one", " word '" + word + "'");
cursor++;
t.ok (Lexer::readWord (text, cursor, word), "readWord \"one 'two' three\\ four\" --> true");
t.is (word, "'two'", " word '" + word + "'");
cursor++;
t.ok (Lexer::readWord (text, cursor, word), "readWord \"one 'two' three\\ four\" --> true");
t.is (word, "three four", " word '" + word + "'");
text = "one ";
cursor = 0;
t.ok (Lexer::readWord (text, cursor, word), "readWord \"one \" --> true");
t.is (word, "one", " word '" + word + "'");
// bool isLiteral (const std::string&, bool, bool);
Lexer l4 ("one.two");
t.notok (l4.isLiteral("zero", false, false), "isLiteral 'one.two' --> false");
t.ok (l4.isLiteral("one", false, false), "isLiteral 'one.two' --> 'one'");
t.ok (l4.isLiteral(".", false, false), "isLiteral 'one.two' --> '.'");
t.ok (l4.isLiteral("two", false, true), "isLiteral 'one.two' --> 'two'");
Lexer l5 ("wonder");
t.notok (l5.isLiteral ("wonderful", false, false), "isLiteral 'wonderful' != 'wonder' without abbreviation");
t.ok (l5.isLiteral ("wonderful", true, false), "isLiteral 'wonderful' == 'wonder' with abbreviation");
// bool isOneOf (const std::string&, bool, bool);
Lexer l6 ("Grumpy.");
std::vector <std::string> dwarves = {"Sneezy", "Doc", "Bashful", "Grumpy", "Happy", "Sleepy", "Dopey"};
t.notok (l6.isOneOf (dwarves, false, true), "isOneof ('Grumpy', true) --> false");
t.ok (l6.isOneOf (dwarves, false, false), "isOneOf ('Grumpy', false) --> true");
// static std::string::size_type commonLength (const std::string&, const std::string&);
t.is ((int)Lexer::commonLength ("", ""), 0, "commonLength '' : '' --> 0");
t.is ((int)Lexer::commonLength ("a", "a"), 1, "commonLength 'a' : 'a' --> 1");
t.is ((int)Lexer::commonLength ("abcde", "abcde"), 5, "commonLength 'abcde' : 'abcde' --> 5");
t.is ((int)Lexer::commonLength ("abc", ""), 0, "commonLength 'abc' : '' --> 0");
t.is ((int)Lexer::commonLength ("abc", "def"), 0, "commonLength 'abc' : 'def' --> 0");
t.is ((int)Lexer::commonLength ("foobar", "foo"), 3, "commonLength 'foobar' : 'foo' --> 3");
t.is ((int)Lexer::commonLength ("foo", "foobar"), 3, "commonLength 'foo' : 'foobar' --> 3");
// static std::string::size_type commonLength (const std::string&, std::string::size_type, const std::string&, std::string::size_type);
t.is ((int)Lexer::commonLength ("wonder", 0, "prowonderbread", 3), 6, "'wonder'+0 : 'prowonderbread'+3 --> 6");
// Test all Lexer types.
#define NO {"",Lexer::Type::word}
struct
{
const char* input;
struct
{
const char* token;
Lexer::Type type;
} results[5];
} lexerTests[] =
{
// Pattern
{ "/foo/", { { "/foo/", Lexer::Type::pattern }, NO, NO, NO, NO }, },
{ "/a\\/b/", { { "/a\\/b/", Lexer::Type::pattern }, NO, NO, NO, NO }, },
{ "/'/", { { "/'/", Lexer::Type::pattern }, NO, NO, NO, NO }, },
// Substitution
{ "/from/to/g", { { "/from/to/g", Lexer::Type::substitution }, NO, NO, NO, NO }, },
{ "/from/to/", { { "/from/to/", Lexer::Type::substitution }, NO, NO, NO, NO }, },
// Tag
{ "+tag", { { "+tag", Lexer::Type::tag }, NO, NO, NO, NO }, },
{ "-tag", { { "-tag", Lexer::Type::tag }, NO, NO, NO, NO }, },
{ "+@tag", { { "+@tag", Lexer::Type::tag }, NO, NO, NO, NO }, },
// Path
{ "/long/path/to/file.txt", { { "/long/path/to/file.txt", Lexer::Type::path }, NO, NO, NO, NO }, },
// Word
{ "1.foo.bar", { { "1.foo.bar", Lexer::Type::word }, NO, NO, NO, NO }, },
// Identifier
{ "foo", { { "foo", Lexer::Type::identifier }, NO, NO, NO, NO }, },
{ "Çirçös", { { "Çirçös", Lexer::Type::identifier }, NO, NO, NO, NO }, },
{ "☺", { { "☺", Lexer::Type::identifier }, NO, NO, NO, NO }, },
{ "name", { { "name", Lexer::Type::identifier }, NO, NO, NO, NO }, },
{ "f1", { { "f1", Lexer::Type::identifier }, NO, NO, NO, NO }, },
{ "foo.bar", { { "foo.bar", Lexer::Type::identifier }, NO, NO, NO, NO }, },
{ "a1a1a1a1_a1a1_a1a1_a1a1_a1a1a1a1a1a1", { { "a1a1a1a1_a1a1_a1a1_a1a1_a1a1a1a1a1a1", Lexer::Type::identifier }, NO, NO, NO, NO }, },
// Word that starts wih 'or', which is an operator, but should be ignored.
{ "ordinary", { { "ordinary", Lexer::Type::identifier }, NO, NO, NO, NO }, },
// DOM
{ "due", { { "due", Lexer::Type::dom }, NO, NO, NO, NO }, },
{ "123.tags", { { "123.tags", Lexer::Type::dom }, NO, NO, NO, NO }, },
{ "123.tags.PENDING", { { "123.tags.PENDING", Lexer::Type::dom }, NO, NO, NO, NO }, },
{ "123.description", { { "123.description", Lexer::Type::dom }, NO, NO, NO, NO }, },
{ "123.annotations.1.description", { { "123.annotations.1.description", Lexer::Type::dom }, NO, NO, NO, NO }, },
{ "123.annotations.1.entry", { { "123.annotations.1.entry", Lexer::Type::dom }, NO, NO, NO, NO }, },
{ "123.annotations.1.entry.year", { { "123.annotations.1.entry.year", Lexer::Type::dom }, NO, NO, NO, NO }, },
{ "a360fc44-315c-4366-b70c-ea7e7520b749.due", { { "a360fc44-315c-4366-b70c-ea7e7520b749.due", Lexer::Type::dom }, NO, NO, NO, NO }, },
{ "12345678-1234-1234-1234-123456789012.due", { { "12345678-1234-1234-1234-123456789012.due", Lexer::Type::dom }, NO, NO, NO, NO }, },
{ "system.os", { { "system.os", Lexer::Type::dom }, NO, NO, NO, NO }, },
{ "rc.foo", { { "rc.foo", Lexer::Type::dom }, NO, NO, NO, NO }, },
// URL
{ "http://tasktools.org", { { "http://tasktools.org", Lexer::Type::url }, NO, NO, NO, NO }, },
{ "https://bug.tasktools.org", { { "https://bug.tasktools.org", Lexer::Type::url }, NO, NO, NO, NO }, },
// String
{ "'one two'", { { "'one two'", Lexer::Type::string }, NO, NO, NO, NO }, },
{ "\"three\"", { { "\"three\"", Lexer::Type::string }, NO, NO, NO, NO }, },
{ "'\\''", { { "'''", Lexer::Type::string }, NO, NO, NO, NO }, },
{ "\"\\\"\"", { { "\"\"\"", Lexer::Type::string }, NO, NO, NO, NO }, },
{ "\"\tfoo\t\"", { { "\"\tfoo\t\"", Lexer::Type::string }, NO, NO, NO, NO }, },
{ "\"\\u20A43\"", { { "\"₤3\"", Lexer::Type::string }, NO, NO, NO, NO }, },
{ "\"U+20AC4\"", { { "\"€4\"", Lexer::Type::string }, NO, NO, NO, NO }, },
// Number
{ "1", { { "1", Lexer::Type::number }, NO, NO, NO, NO }, },
{ "3.14", { { "3.14", Lexer::Type::number }, NO, NO, NO, NO }, },
{ "6.02217e23", { { "6.02217e23", Lexer::Type::number }, NO, NO, NO, NO }, },
{ "1.2e-3.4", { { "1.2e-3.4", Lexer::Type::number }, NO, NO, NO, NO }, },
{ "0x2f", { { "0x2f", Lexer::Type::hex }, NO, NO, NO, NO }, },
// Set (1,2,4-7,9)
{ "1,2", { { "1,2", Lexer::Type::set }, NO, NO, NO, NO }, },
{ "1-2", { { "1-2", Lexer::Type::set }, NO, NO, NO, NO }, },
{ "1-2,4", { { "1-2,4", Lexer::Type::set }, NO, NO, NO, NO }, },
{ "1-2,4,6-8", { { "1-2,4,6-8", Lexer::Type::set }, NO, NO, NO, NO }, },
{ "1-2,4,6-8,10-12", { { "1-2,4,6-8,10-12", Lexer::Type::set }, NO, NO, NO, NO }, },
// Pair
{ "name:value", { { "name:value", Lexer::Type::pair }, NO, NO, NO, NO }, },
{ "name=value", { { "name=value", Lexer::Type::pair }, NO, NO, NO, NO }, },
{ "name:=value", { { "name:=value", Lexer::Type::pair }, NO, NO, NO, NO }, },
{ "name.mod:value", { { "name.mod:value", Lexer::Type::pair }, NO, NO, NO, NO }, },
{ "name.mod=value", { { "name.mod=value", Lexer::Type::pair }, NO, NO, NO, NO }, },
{ "name:", { { "name:", Lexer::Type::pair }, NO, NO, NO, NO }, },
{ "name=", { { "name=", Lexer::Type::pair }, NO, NO, NO, NO }, },
{ "name.mod:", { { "name.mod:", Lexer::Type::pair }, NO, NO, NO, NO }, },
{ "name.mod=", { { "name.mod=", Lexer::Type::pair }, NO, NO, NO, NO }, },
{ "pro:'P 1'", { { "pro:'P 1'", Lexer::Type::pair }, NO, NO, NO, NO }, },
{ "rc:x", { { "rc:x", Lexer::Type::pair }, NO, NO, NO, NO }, },
{ "rc.name:value", { { "rc.name:value", Lexer::Type::pair }, NO, NO, NO, NO }, },
{ "rc.name=value", { { "rc.name=value", Lexer::Type::pair }, NO, NO, NO, NO }, },
{ "rc.name:=value", { { "rc.name:=value", Lexer::Type::pair }, NO, NO, NO, NO }, },
{ "due:='eow - 2d'", { { "due:='eow - 2d'", Lexer::Type::pair }, NO, NO, NO, NO }, },
// Operator - complete set
{ "^", { { "^", Lexer::Type::op }, NO, NO, NO, NO }, },
{ "!", { { "!", Lexer::Type::op }, NO, NO, NO, NO }, },
{ "_neg_", { { "_neg_", Lexer::Type::op }, NO, NO, NO, NO }, },
{ "_pos_", { { "_pos_", Lexer::Type::op }, NO, NO, NO, NO }, },
{ "_hastag_", { { "_hastag_", Lexer::Type::op }, NO, NO, NO, NO }, },
{ "_notag_", { { "_notag_", Lexer::Type::op }, NO, NO, NO, NO }, },
{ "*", { { "*", Lexer::Type::op }, NO, NO, NO, NO }, },
{ "/", { { "/", Lexer::Type::op }, NO, NO, NO, NO }, },
{ "%", { { "%", Lexer::Type::op }, NO, NO, NO, NO }, },
{ "+", { { "+", Lexer::Type::op }, NO, NO, NO, NO }, },
{ "-", { { "-", Lexer::Type::op }, NO, NO, NO, NO }, },
{ "<=", { { "<=", Lexer::Type::op }, NO, NO, NO, NO }, },
{ ">=", { { ">=", Lexer::Type::op }, NO, NO, NO, NO }, },
{ ">", { { ">", Lexer::Type::op }, NO, NO, NO, NO }, },
{ "<", { { "<", Lexer::Type::op }, NO, NO, NO, NO }, },
{ "=", { { "=", Lexer::Type::op }, NO, NO, NO, NO }, },
{ "==", { { "==", Lexer::Type::op }, NO, NO, NO, NO }, },
{ "!=", { { "!=", Lexer::Type::op }, NO, NO, NO, NO }, },
{ "!==", { { "!==", Lexer::Type::op }, NO, NO, NO, NO }, },
{ "~", { { "~", Lexer::Type::op }, NO, NO, NO, NO }, },
{ "!~", { { "!~", Lexer::Type::op }, NO, NO, NO, NO }, },
{ "and", { { "and", Lexer::Type::op }, NO, NO, NO, NO }, },
{ "or", { { "or", Lexer::Type::op }, NO, NO, NO, NO }, },
{ "xor", { { "xor", Lexer::Type::op }, NO, NO, NO, NO }, },
{ "(", { { "(", Lexer::Type::op }, NO, NO, NO, NO }, },
{ ")", { { ")", Lexer::Type::op }, NO, NO, NO, NO }, },
// UUID
{ "ffffffff-ffff-ffff-ffff-ffffffffffff", { { "ffffffff-ffff-ffff-ffff-ffffffffffff", Lexer::Type::uuid }, NO, NO, NO, NO }, },
{ "00000000-0000-0000-0000-0000000", { { "00000000-0000-0000-0000-0000000", Lexer::Type::uuid }, NO, NO, NO, NO }, },
{ "00000000-0000-0000-0000", { { "00000000-0000-0000-0000", Lexer::Type::uuid }, NO, NO, NO, NO }, },
{ "00000000-0000-0000", { { "00000000-0000-0000", Lexer::Type::uuid }, NO, NO, NO, NO }, },
{ "00000000-0000", { { "00000000-0000", Lexer::Type::uuid }, NO, NO, NO, NO }, },
{ "00000000", { { "00000000", Lexer::Type::uuid }, NO, NO, NO, NO }, },
{ "a360fc44-315c-4366-b70c-ea7e7520b749", { { "a360fc44-315c-4366-b70c-ea7e7520b749", Lexer::Type::uuid }, NO, NO, NO, NO }, },
{ "a360fc44-315c-4366-b70c-ea7e752", { { "a360fc44-315c-4366-b70c-ea7e752", Lexer::Type::uuid }, NO, NO, NO, NO }, },
{ "a360fc44-315c-4366-b70c", { { "a360fc44-315c-4366-b70c", Lexer::Type::uuid }, NO, NO, NO, NO }, },
{ "a360fc44-315c-4366", { { "a360fc44-315c-4366", Lexer::Type::uuid }, NO, NO, NO, NO }, },
{ "a360fc44-315c", { { "a360fc44-315c", Lexer::Type::uuid }, NO, NO, NO, NO }, },
{ "a360fc44", { { "a360fc44", Lexer::Type::uuid }, NO, NO, NO, NO }, },
// Date
{ "2015-W01", { { "2015-W01", Lexer::Type::date }, NO, NO, NO, NO }, },
{ "2015-02-17", { { "2015-02-17", Lexer::Type::date }, NO, NO, NO, NO }, },
{ "2013-11-29T22:58:00Z", { { "2013-11-29T22:58:00Z", Lexer::Type::date }, NO, NO, NO, NO }, },
{ "20131129T225800Z", { { "20131129T225800Z", Lexer::Type::date }, NO, NO, NO, NO }, },
{ "9th", { { "9th", Lexer::Type::date }, NO, NO, NO, NO }, },
{ "10th", { { "10th", Lexer::Type::date }, NO, NO, NO, NO }, },
{ "today", { { "today", Lexer::Type::date }, NO, NO, NO, NO }, },
// Duration
{ "year", { { "year", Lexer::Type::duration }, NO, NO, NO, NO }, },
{ "4weeks", { { "4weeks", Lexer::Type::duration }, NO, NO, NO, NO }, },
{ "PT23H", { { "PT23H", Lexer::Type::duration }, NO, NO, NO, NO }, },
{ "1second", { { "1second", Lexer::Type::duration }, NO, NO, NO, NO }, },
{ "1s", { { "1s", Lexer::Type::duration }, NO, NO, NO, NO }, },
{ "1minute", { { "1minute", Lexer::Type::duration }, NO, NO, NO, NO }, },
{ "2hour", { { "2hour", Lexer::Type::duration }, NO, NO, NO, NO }, },
{ "3 days", { { "3 days", Lexer::Type::duration }, NO, NO, NO, NO }, },
{ "4w", { { "4w", Lexer::Type::duration }, NO, NO, NO, NO }, },
{ "5mo", { { "5mo", Lexer::Type::duration }, NO, NO, NO, NO }, },
{ "6 years", { { "6 years", Lexer::Type::duration }, NO, NO, NO, NO }, },
{ "P1Y", { { "P1Y", Lexer::Type::duration }, NO, NO, NO, NO }, },
{ "PT1H", { { "PT1H", Lexer::Type::duration }, NO, NO, NO, NO }, },
{ "P1Y1M1DT1H1M1S", { { "P1Y1M1DT1H1M1S", Lexer::Type::duration }, NO, NO, NO, NO }, },
// Misc
{ "--", { { "--", Lexer::Type::separator }, NO, NO, NO, NO }, },
// Expression
// due:eom-2w
// due < eom + 1w + 1d
// ( /pattern/ or 8ad2e3db-914d-4832-b0e6-72fa04f6e331,3b6218f9-726a-44fc-aa63-889ff52be442 )
{ "(1+2)", { { "(", Lexer::Type::op },
{ "1", Lexer::Type::number },
{ "+", Lexer::Type::op },
{ "2", Lexer::Type::number },
{ ")", Lexer::Type::op }, }, },
{ "description~pattern", { { "description", Lexer::Type::dom },
{ "~", Lexer::Type::op },
{ "pattern", Lexer::Type::identifier }, NO, NO }, },
{ "(+tag)", { { "(", Lexer::Type::op },
{ "+tag", Lexer::Type::tag },
{ ")", Lexer::Type::op }, NO, NO }, },
{ "(name:value)", { { "(", Lexer::Type::op },
{ "name:value", Lexer::Type::pair },
{ ")", Lexer::Type::op }, NO, NO }, },
};
#define NUM_TESTS (sizeof (lexerTests) / sizeof (lexerTests[0]))
for (unsigned int i = 0; i < NUM_TESTS; i++)
{
// The isolated test puts the input string directly into the Lexer.
Lexer isolated (lexerTests[i].input);
for (int j = 0; j < 5; j++)
{
if (lexerTests[i].results[j].token[0])
{
// Isolated: "<token>"
t.ok (isolated.token (token, type), "Isolated Lexer::token(...) --> true");
t.is (token, lexerTests[i].results[j].token, " token --> " + token);
t.is ((int)type, (int)lexerTests[i].results[j].type, " type --> Lexer::Type::" + Lexer::typeToString (type));
}
}
// The embedded test surrounds the input string with a space.
Lexer embedded (std::string (" ") + lexerTests[i].input + " ");
for (int j = 0; j < 5; j++)
{
if (lexerTests[i].results[j].token[0])
{
// Embedded: "<token>"
t.ok (embedded.token (token, type), "Embedded Lexer::token(...) --> true");
t.is (token, lexerTests[i].results[j].token, " token --> " + token);
t.is ((int)type, (int)lexerTests[i].results[j].type, " type --> Lexer::Type::" + Lexer::typeToString (type));
}
}
}
return 0;
}
PTHEntry PTHWriter::LexTokens(Lexer& L) {
// Pad 0's so that we emit tokens to a 4-byte alignment.
// This speed up reading them back in.
using namespace llvm::support;
endian::Writer<little> LE(Out);
uint32_t TokenOff = Out.tell();
for (uint64_t N = llvm::OffsetToAlignment(TokenOff, 4); N; --N, ++TokenOff)
LE.write<uint8_t>(0);
// Keep track of matching '#if' ... '#endif'.
typedef std::vector<std::pair<Offset, unsigned> > PPCondTable;
PPCondTable PPCond;
std::vector<unsigned> PPStartCond;
bool ParsingPreprocessorDirective = false;
Token Tok;
do {
L.LexFromRawLexer(Tok);
NextToken:
if ((Tok.isAtStartOfLine() || Tok.is(tok::eof)) &&
ParsingPreprocessorDirective) {
// Insert an eod token into the token cache. It has the same
// position as the next token that is not on the same line as the
// preprocessor directive. Observe that we continue processing
// 'Tok' when we exit this branch.
Token Tmp = Tok;
Tmp.setKind(tok::eod);
Tmp.clearFlag(Token::StartOfLine);
Tmp.setIdentifierInfo(nullptr);
EmitToken(Tmp);
ParsingPreprocessorDirective = false;
}
if (Tok.is(tok::raw_identifier)) {
PP.LookUpIdentifierInfo(Tok);
EmitToken(Tok);
continue;
}
if (Tok.is(tok::hash) && Tok.isAtStartOfLine()) {
// Special processing for #include. Store the '#' token and lex
// the next token.
assert(!ParsingPreprocessorDirective);
Offset HashOff = (Offset) Out.tell();
// Get the next token.
Token NextTok;
L.LexFromRawLexer(NextTok);
// If we see the start of line, then we had a null directive "#". In
// this case, discard both tokens.
if (NextTok.isAtStartOfLine())
goto NextToken;
// The token is the start of a directive. Emit it.
EmitToken(Tok);
Tok = NextTok;
// Did we see 'include'/'import'/'include_next'?
if (Tok.isNot(tok::raw_identifier)) {
EmitToken(Tok);
continue;
}
IdentifierInfo* II = PP.LookUpIdentifierInfo(Tok);
tok::PPKeywordKind K = II->getPPKeywordID();
ParsingPreprocessorDirective = true;
switch (K) {
case tok::pp_not_keyword:
// Invalid directives "#foo" can occur in #if 0 blocks etc, just pass
// them through.
default:
break;
case tok::pp_include:
case tok::pp_import:
case tok::pp_include_next: {
// Save the 'include' token.
EmitToken(Tok);
// Lex the next token as an include string.
L.setParsingPreprocessorDirective(true);
L.LexIncludeFilename(Tok);
L.setParsingPreprocessorDirective(false);
assert(!Tok.isAtStartOfLine());
if (Tok.is(tok::raw_identifier))
PP.LookUpIdentifierInfo(Tok);
break;
}
case tok::pp_if:
case tok::pp_ifdef:
case tok::pp_ifndef: {
// Add an entry for '#if' and friends. We initially set the target
// index to 0. This will get backpatched when we hit #endif.
PPStartCond.push_back(PPCond.size());
PPCond.push_back(std::make_pair(HashOff, 0U));
break;
}
case tok::pp_endif: {
// Add an entry for '#endif'. We set the target table index to itself.
// This will later be set to zero when emitting to the PTH file. We
// use 0 for uninitialized indices because that is easier to debug.
unsigned index = PPCond.size();
// Backpatch the opening '#if' entry.
assert(!PPStartCond.empty());
assert(PPCond.size() > PPStartCond.back());
assert(PPCond[PPStartCond.back()].second == 0);
PPCond[PPStartCond.back()].second = index;
PPStartCond.pop_back();
// Add the new entry to PPCond.
PPCond.push_back(std::make_pair(HashOff, index));
EmitToken(Tok);
// Some files have gibberish on the same line as '#endif'.
// Discard these tokens.
do
L.LexFromRawLexer(Tok);
while (Tok.isNot(tok::eof) && !Tok.isAtStartOfLine());
// We have the next token in hand.
// Don't immediately lex the next one.
goto NextToken;
}
case tok::pp_elif:
case tok::pp_else: {
// Add an entry for #elif or #else.
// This serves as both a closing and opening of a conditional block.
// This means that its entry will get backpatched later.
unsigned index = PPCond.size();
// Backpatch the previous '#if' entry.
assert(!PPStartCond.empty());
assert(PPCond.size() > PPStartCond.back());
assert(PPCond[PPStartCond.back()].second == 0);
PPCond[PPStartCond.back()].second = index;
PPStartCond.pop_back();
// Now add '#elif' as a new block opening.
PPCond.push_back(std::make_pair(HashOff, 0U));
PPStartCond.push_back(index);
break;
}
}
}
EmitToken(Tok);
}
while (Tok.isNot(tok::eof));
assert(PPStartCond.empty() && "Error: imblanced preprocessor conditionals.");
// Next write out PPCond.
Offset PPCondOff = (Offset) Out.tell();
// Write out the size of PPCond so that clients can identifer empty tables.
Emit32(PPCond.size());
for (unsigned i = 0, e = PPCond.size(); i!=e; ++i) {
Emit32(PPCond[i].first - TokenOff);
uint32_t x = PPCond[i].second;
assert(x != 0 && "PPCond entry not backpatched.");
// Emit zero for #endifs. This allows us to do checking when
// we read the PTH file back in.
Emit32(x == i ? 0 : x);
}
return PTHEntry(TokenOff, PPCondOff);
}
Parser::stmt_t Parser::p_stmt_toks(Lexer &lex,const Context &ctx,std::vector<Lexer::Token> *toks){
Lexer::Token tok,tok1,tok2;
lex >> tok;
stmt_t res_stmt = stmt_t::nop();
std::vector<Lexer::Token> mytoks;
switch(tok.type){
case Lexer::NOP:
ppush(&mytoks,tok);
res_stmt = stmt_t::nop(tok.pos,mytoks);
break;
case Lexer::READ:
{
ppush(&mytoks,tok);
force_toks(lex,Lexer::COLON,&mytoks);
lex >> tok1;
if(tok1.type == Lexer::REG){
ppush(&mytoks,tok1);
force_toks(lex,Lexer::ASSIGNMENT,&mytoks);
memloc_or_pointer_t ml = p_memloc_toks(lex,ctx,&mytoks);
res_stmt = resolve_pointer(ml,[&tok1,&tok](const memloc_t &ml){
return stmt_t::read_assign(tok1.value,ml,tok.pos);
},ctx,mytoks);
}else{
lex.putback(tok1);
Parser::memloc_or_pointer_t ml = p_memloc_toks(lex,ctx,&mytoks);
force_toks(lex,Lexer::EQ,&mytoks);
expr_t e = p_expr_toks(lex,&mytoks);
res_stmt = resolve_pointer(ml,[&e,&tok](const memloc_t &ml){
return stmt_t::read_assert(ml,e,tok.pos);
},ctx,mytoks);
}
break;
}
case Lexer::SYNCRD:
{
ppush(&mytoks,tok);
force_toks(lex,Lexer::COLON,&mytoks);
lex >> tok1;
if(tok1.type == Lexer::REG){
ppush(&mytoks,tok1);
force_toks(lex,Lexer::ASSIGNMENT,&mytoks);
memloc_or_pointer_t ml = p_memloc_toks(lex,ctx,&mytoks);
res_stmt = resolve_pointer(ml,[&tok1,&tok](const memloc_t &ml){
return stmt_t::syncrd_assign(tok1.value,ml,tok.pos);
},ctx,mytoks);
}else{
lex.putback(tok1);
Parser::memloc_or_pointer_t ml = p_memloc_toks(lex,ctx,&mytoks);
force_toks(lex,Lexer::EQ,&mytoks);
expr_t e = p_expr_toks(lex,&mytoks);
res_stmt = resolve_pointer(ml,[&e,&tok](const memloc_t &ml){
return stmt_t::syncrd_assert(ml,e,tok.pos);
},ctx,mytoks);
}
break;
}
case Lexer::WRITE:
{
ppush(&mytoks,tok);
force_toks(lex,Lexer::COLON,&mytoks);
Parser::memloc_or_pointer_t ml = p_memloc_toks(lex,ctx,&mytoks);
force_toks(lex,Lexer::ASSIGNMENT,&mytoks);
expr_t e = p_expr_toks(lex,&mytoks);
res_stmt = resolve_pointer(ml,[&e,&tok](const memloc_t &ml){
return stmt_t::write(ml,e,tok.pos);
},ctx,mytoks);
break;
}
case Lexer::SYNCWR:
{
ppush(&mytoks,tok);
force_toks(lex,Lexer::COLON,&mytoks);
Parser::memloc_or_pointer_t ml = p_memloc_toks(lex,ctx,&mytoks);
force_toks(lex,Lexer::ASSIGNMENT,&mytoks);
expr_t e = p_expr_toks(lex,&mytoks);
res_stmt = resolve_pointer(ml,[&e,&tok](const memloc_t &ml){
return stmt_t::syncwr(ml,e,tok.pos);
},ctx,mytoks);
break;
}
case Lexer::FENCE:
{
ppush(&mytoks,tok);
res_stmt = stmt_t::full_fence(tok.pos,mytoks);
break;
}
case Lexer::SSFENCE:
{
ppush(&mytoks,tok);
res_stmt = stmt_t::ss_fence(tok.pos,mytoks);
break;
}
case Lexer::LLFENCE:
{
ppush(&mytoks,tok);
res_stmt = stmt_t::ll_fence(tok.pos,mytoks);
break;
}
case Lexer::CAS:
{
ppush(&mytoks,tok);
force_toks(lex,Lexer::LPAREN,&mytoks);
Parser::memloc_or_pointer_t ml = p_memloc_toks(lex,ctx,&mytoks);
force_toks(lex,Lexer::COMMA,&mytoks);
expr_t v0(p_expr_toks(lex,&mytoks));
force_toks(lex,Lexer::COMMA,&mytoks);
expr_t v1(p_expr_toks(lex,&mytoks));
force_toks(lex,Lexer::RPAREN,&mytoks);
res_stmt = resolve_pointer(ml,[&v0,&v1,&tok](const memloc_t &ml){
return stmt_t::cas(ml,v0,v1,tok.pos);
},ctx,mytoks);
break;
}
case Lexer::REG:
{
ppush(&mytoks,tok);
force_toks(lex,Lexer::ASSIGNMENT,&mytoks);
expr_t e = p_expr_toks(lex,&mytoks);
res_stmt = stmt_t::assignment(tok.value,e,tok.pos,mytoks);
break;
}
case Lexer::IF:
{
ppush(&mytoks,tok);
bexpr_t b(p_bexpr_toks(lex,&mytoks));
force_toks(lex,Lexer::THEN,&mytoks);
stmt_t::labeled_stmt_t s0 = p_lstmt(lex,ctx,&mytoks);
lex >> tok1;
if(tok1.type == Lexer::ELSE){
ppush(&mytoks,tok1);
stmt_t::labeled_stmt_t s1 = p_lstmt(lex,ctx,&mytoks);
res_stmt = stmt_t::if_stmt(b,s0,s1,tok.pos,mytoks);
}else{
lex.putback(tok1);
res_stmt = stmt_t::if_stmt(b,s0,tok.pos,mytoks);
}
break;
}
case Lexer::WHILE:
{
ppush(&mytoks,tok);
bexpr_t b(p_bexpr_toks(lex,&mytoks));
force_toks(lex,Lexer::DO,&mytoks);
stmt_t::labeled_stmt_t s = p_lstmt(lex,ctx,&mytoks);
res_stmt = stmt_t::while_stmt(b,s,tok.pos,mytoks);
break;
}
case Lexer::GOTO:
{
ppush(&mytoks,tok);
Lang::label_t lbl = p_label(lex,&mytoks);
res_stmt = stmt_t::goto_stmt(lbl,tok.pos,mytoks);
break;
}
case Lexer::EITHER:
{
ppush(&mytoks,tok);
Lexer::TokenPos pos0 = tok.pos;
std::vector<stmt_t> seq;
Lexer::Token lctok;
Lexer::Token tok;
lex >> lctok;
lex.putback(lctok);
force_toks(lex,Lexer::LCURL,&mytoks);
seq.push_back(p_stmt_list(lex,ctx,&mytoks));
lex >> tok;
while(tok.type == Lexer::EITHER_OR){
ppush(&mytoks,tok);
seq.push_back(p_stmt_list(lex,ctx,&mytoks));
lex >> tok;
}
lex.putback(tok);
force_toks(lex,Lexer::RCURL,&mytoks,"Expected '}' at "," to match '{' at "+
lctok.pos.to_long_string()+".");
res_stmt = stmt_t::either(seq,pos0);
break;
}
case Lexer::LOCKED:
{
ppush(&mytoks,tok);
Lexer::TokenPos pos0 = tok.pos;
std::vector<stmt_t> seq;
Lexer::Token lctok;
Lexer::Token tok;
lex >> lctok;
if(lctok.type == Lexer::WRITE){
ppush(&mytoks,lctok);
force_toks(lex,Lexer::COLON,&mytoks);
Parser::memloc_or_pointer_t ml = p_memloc_toks(lex,ctx,&mytoks);
force_toks(lex,Lexer::ASSIGNMENT,&mytoks);
expr_t e = p_expr_toks(lex,&mytoks);
res_stmt = resolve_pointer(ml,[&e,&pos0](const memloc_t &ml){
return stmt_t::locked_write(ml,e,pos0);
},ctx,mytoks);
}else{
lex.putback(lctok);
force_toks(lex,Lexer::LCURL,&mytoks);
stmt_t s = p_stmt_list(lex,ctx,&mytoks);
std::string cmt;
if(!stmt_t::check_locked_invariant(s,&cmt)){
throw new SyntaxError("Error in locked block at "+lctok.pos.to_long_string()+": "+cmt,lctok.pos);
}
seq.push_back(s);
lex >> tok;
while(tok.type == Lexer::EITHER_OR){
ppush(&mytoks,tok);
s = p_stmt_list(lex,ctx,&mytoks);
if(!stmt_t::check_locked_invariant(s,&cmt)){
throw new SyntaxError("Error in locked block at "+tok.pos.to_long_string()+": "+cmt,tok.pos);
}
seq.push_back(s);
lex >> tok;
}
lex.putback(tok);
force_toks(lex,Lexer::RCURL,&mytoks,"Expected '}' at "," to match '{' at "+
lctok.pos.to_long_string()+".");
res_stmt = stmt_t::locked_block(seq,pos0);
}
break;
}
case Lexer::SLOCKED:
{
Lexer::TokenPos pos0 = tok.pos;
force(lex,Lexer::WRITE);
force(lex,Lexer::COLON);
Parser::memloc_or_pointer_t ml = p_memloc(lex,ctx);
force(lex,Lexer::ASSIGNMENT);
expr_t e = p_expr(lex);
return resolve_pointer(ml,[&e,&pos0](const memloc_t &ml){
return stmt_t::slocked_write(ml,e,pos0);
},ctx,mytoks);
}
case Lexer::LCURL:
{
ppush(&mytoks,tok);
stmt_t sl = p_stmt_list(lex,ctx,&mytoks);
force_toks(lex,Lexer::RCURL,&mytoks,"Expected '}' at "," to match '{' at "+
tok.pos.to_long_string()+".");
res_stmt = sl;
break;
}
case Lexer::ASSUME:
{
ppush(&mytoks,tok);
force_toks(lex,Lexer::COLON,&mytoks);
bexpr_t cnd = p_bexpr_toks(lex,&mytoks);
res_stmt = stmt_t::assume(cnd,tok.pos,mytoks);
break;
}
default:
throw new SyntaxError("Expected statement at "+tok.pos.to_long_string()+".",tok.pos);
}
if(toks) toks->insert(toks->end(),mytoks.begin(),mytoks.end());
return res_stmt;
}
void Layout::readArgument(Lexer & lex)
{
latexarg arg;
// writeArgument() makes use of these default values
arg.mandatory = false;
arg.autoinsert = false;
bool error = false;
bool finished = false;
arg.font = inherit_font;
arg.labelfont = inherit_font;
string id;
lex >> id;
bool const itemarg = prefixIs(id, "item:");
bool const postcmd = prefixIs(id, "post:");
while (!finished && lex.isOK() && !error) {
lex.next();
string const tok = ascii_lowercase(lex.getString());
if (tok.empty()) {
continue;
} else if (tok == "endargument") {
finished = true;
} else if (tok == "labelstring") {
lex.next();
arg.labelstring = lex.getDocString();
} else if (tok == "menustring") {
lex.next();
arg.menustring = lex.getDocString();
} else if (tok == "mandatory") {
lex.next();
arg.mandatory = lex.getBool();
} else if (tok == "autoinsert") {
lex.next();
arg.autoinsert = lex.getBool();
} else if (tok == "leftdelim") {
lex.next();
arg.ldelim = lex.getDocString();
arg.ldelim = support::subst(arg.ldelim, from_ascii("<br/>"),
from_ascii("\n"));
} else if (tok == "rightdelim") {
lex.next();
arg.rdelim = lex.getDocString();
arg.rdelim = support::subst(arg.rdelim, from_ascii("<br/>"),
from_ascii("\n"));
} else if (tok == "defaultarg") {
lex.next();
arg.defaultarg = lex.getDocString();
} else if (tok == "presetarg") {
lex.next();
arg.presetarg = lex.getDocString();
} else if (tok == "tooltip") {
lex.next();
arg.tooltip = lex.getDocString();
} else if (tok == "requires") {
lex.next();
arg.requires = lex.getString();
} else if (tok == "decoration") {
lex.next();
arg.decoration = lex.getString();
} else if (tok == "font") {
arg.font = lyxRead(lex, arg.font);
} else if (tok == "labelfont") {
arg.labelfont = lyxRead(lex, arg.labelfont);
} else {
lex.printError("Unknown tag");
error = true;
}
}
if (arg.labelstring.empty())
LYXERR0("Incomplete Argument definition!");
else if (itemarg)
itemargs_[id] = arg;
else if (postcmd)
postcommandargs_[id] = arg;
else
latexargs_[id] = arg;
}
ExpPtr parseiPut (Lexer& lex)
{
throw lex.current().span.die("^put unimplemented");
}
