Token DefTokenEmitter::doEmit(Tokenizer& tokenizer) { while (!tokenizer.eof()) { size_t line = tokenizer.line(); size_t column = tokenizer.column(); const char* c = tokenizer.nextChar(); switch (*c) { case '/': { const char* begin = c; if (tokenizer.peekChar() == '*') { // eat all chars immediately after the '*' because it's often followed by QUAKE while (!isWhitespace(*tokenizer.nextChar()) && !tokenizer.eof()); return Token(ODefinition, begin, c, tokenizer.offset(begin), line, column); } else if (tokenizer.peekChar() == '/') { // eat everything up to and including the next newline while (*tokenizer.nextChar() != '\n'); break; } error(line, column, *c); break; } case '*': { const char* begin = c; if (tokenizer.peekChar() == '/') { tokenizer.nextChar(); return Token(ODefinition, begin, c, tokenizer.offset(begin), line, column); } error(line, column, *c); break; } case '(': return Token(OParenthesis, c, c + 1, tokenizer.offset(c), line, column); case ')': return Token(CParenthesis, c, c + 1, tokenizer.offset(c), line, column); case '{': return Token(OBrace, c, c + 1, tokenizer.offset(c), line, column); case '}': return Token(CBrace, c, c + 1, tokenizer.offset(c), line, column); case '=': return Token(Equality, c, c + 1, tokenizer.offset(c), line, column); case ';': return Token(Semicolon, c, c + 1, tokenizer.offset(c), line, column); case '?': return Token(Question, c, c + 1, tokenizer.offset(c), line, column); case '\r': if (tokenizer.peekChar() == '\n') { tokenizer.nextChar(); } case '\n': return Token(Newline, c, c + 1, tokenizer.offset(c), line, column); case ',': return Token(Comma, c, c + 1, tokenizer.offset(c), line, column); case ' ': case '\t': break; case '"': { // quoted string const char* begin = c; const char* end; tokenizer.quotedString(begin, end); return Token(QuotedString, begin, end, tokenizer.offset(begin), line, column); } default: { // integer, decimal or word const char* begin = c; // try to read a number if (*c == '-' || isDigit(*c)) { while (isDigit(*(c = tokenizer.nextChar()))); if (isDelimiter(*c)) { if (!tokenizer.eof()) tokenizer.pushChar(); return Token(Integer, begin, c, tokenizer.offset(begin), line, column); } } // try to read a decimal (may start with '.') if (*c == '.') { while (isDigit(*(c = tokenizer.nextChar()))); if (isDelimiter(*c)) { if (!tokenizer.eof()) tokenizer.pushChar(); return Token(Decimal, begin, c, tokenizer.offset(begin), line, column); } } // read a word while (!tokenizer.eof() && !isDelimiter(*(c = tokenizer.nextChar()))); if (!tokenizer.eof()) tokenizer.pushChar(); return Token(Word, begin, c, tokenizer.offset(begin), line, column); } } } return Token(Eof, NULL, NULL, 0, tokenizer.line(), tokenizer.column()); }
/* ParseTreeNode::parse * Parses formatted text data. Current valid formatting is: * (type) child = value; * (type) child = value1, value2, ...; * (type) child = { value1, value2, ... } * (type) child { grandchild = value; etc... } * (type) child : inherited { ... } * All values are read as strings, but can be retrieved as string, * int, bool or float. *******************************************************************/ bool ParseTreeNode::parse(Tokenizer& tz) { // Get first token string token = tz.getToken(); // Keep parsing until final } is reached (or end of file) while (!(S_CMP(token, "}")) && !token.IsEmpty()) { // Check for preprocessor stuff if (parser) { // #define if (S_CMPNOCASE(token, "#define")) { parser->define(tz.getToken()); token = tz.getToken(); continue; } // #if(n)def if (S_CMPNOCASE(token, "#ifdef") || S_CMPNOCASE(token, "#ifndef")) { bool test = true; if (S_CMPNOCASE(token, "#ifndef")) test = false; string define = tz.getToken(); if (parser->defined(define) == test) { // Continue token = tz.getToken(); continue; } else { // Skip section int skip = 0; while (true) { token = tz.getToken(); if (S_CMPNOCASE(token, "#endif")) skip--; else if (S_CMPNOCASE(token, "#ifdef")) skip++; else if (S_CMPNOCASE(token, "#ifndef")) skip++; if (skip < 0) break; if (token.IsEmpty()) { wxLogMessage("Error: found end of file within #if(n)def block"); break; } } continue; } } // #include (ignore) if (S_CMPNOCASE(token, "#include")) { tz.skipToken(); // Skip include path token = tz.getToken(); continue; } // #endif (ignore) if (S_CMPNOCASE(token, "#endif")) { token = tz.getToken(); continue; } } // If it's a special character (ie not a valid name), parsing fails if (tz.isSpecialCharacter(token.at(0))) { wxLogMessage("Parsing error: Unexpected special character '%s' in %s (line %d)", CHR(token), CHR(tz.getName()), tz.lineNo()); return false; } // So we have either a node or property name string name = token; // Check next token to determine what we're doing string next = tz.peekToken(); // Check for type+name pair string type = ""; if (next != "=" && next != "{" && next != ";" && next != ":") { type = name; name = tz.getToken(); next = tz.peekToken(); } // Assignment if (S_CMP(next, "=")) { // Skip = tz.skipToken(); // Create item node ParseTreeNode* child = (ParseTreeNode*)addChild(name); child->type = type; // Check type of assignment list token = tz.getToken(); string list_end = ";"; if (token == "{") { list_end = "}"; token = tz.getToken(); } // Parse until ; or } while (1) { // Check for list end if (S_CMP(token, list_end) && !tz.quotedString()) break; // Setup value Property value; // Detect value type if (tz.quotedString()) // Quoted string value = token; else if (S_CMPNOCASE(token, "true")) // Boolean (true) value = true; else if (S_CMPNOCASE(token, "false")) // Boolean (false) value = false; else if (re_int1.Matches(token) || // Integer re_int2.Matches(token)) { long val; token.ToLong(&val); value = (int)val; } else if (re_int3.Matches(token)) // Hex (0xXXXXXX) { long val; token.ToLong(&val, 0); value = (int)val; //wxLogMessage("%s: %s is hex %d", CHR(name), CHR(token), value.getIntValue()); } else if (re_float.Matches(token)) // Floating point { double val; token.ToDouble(&val); value = (double)val; //LOG_MESSAGE(3, S_FMT("%s: %s is float %1.3f", CHR(name), CHR(token), val)); } else // Unknown, just treat as string value = token; // Add value child->values.push_back(value); // Check for , if (S_CMP(tz.peekToken(), ",")) tz.skipToken(); // Skip it else if (!(S_CMP(tz.peekToken(), list_end))) { string t = tz.getToken(); string n = tz.getName(); wxLogMessage("Parsing error: Expected \",\" or \"%s\", got \"%s\" in %s (line %d)", CHR(list_end), CHR(t), CHR(n), tz.lineNo()); return false; } token = tz.getToken(); } } // Child node else if (S_CMP(next, "{")) { // Add child node ParseTreeNode* child = (ParseTreeNode*)addChild(name); child->type = type; // Skip { tz.skipToken(); // Parse child node if (!child->parse(tz)) return false; } // Child node (with no values/children) else if (S_CMP(next, ";")) { // Add child node ParseTreeNode* child = (ParseTreeNode*)addChild(name); child->type = type; // Skip ; tz.skipToken(); } // Child node + inheritance else if (S_CMP(next, ":")) { // Skip : tz.skipToken(); // Read inherited name string inherit = tz.getToken(); // Check for opening brace if (tz.checkToken("{")) { // Add child node ParseTreeNode* child = (ParseTreeNode*)addChild(name); child->type = type; // Set its inheritance child->inherit = inherit; // Parse child node if (!child->parse(tz)) return false; } } // Unexpected token else { wxLogMessage("Parsing error: \"%s\" unexpected in %s (line %d)", CHR(next), CHR(tz.getName()), tz.lineNo()); return false; } // Continue parsing token = tz.getToken(); } return true; }
Token MapTokenEmitter::doEmit(Tokenizer& tokenizer) { while (!tokenizer.eof()) { size_t line = tokenizer.line(); size_t column = tokenizer.column(); const char* c = tokenizer.nextChar(); switch (*c) { case '/': if (tokenizer.peekChar() == '/') { tokenizer.nextChar(); if (tokenizer.peekChar() == '/') { tokenizer.nextChar(); // it's a TB comment } else { // eat everything up to and including the next newline while (*tokenizer.nextChar() != '\n'); } } break; case '{': return Token(TokenType::OBrace, c, c + 1, tokenizer.offset(c), line, column); case '}': return Token(TokenType::CBrace, c, c + 1, tokenizer.offset(c), line, column); case '(': return Token(TokenType::OParenthesis, c, c + 1, tokenizer.offset(c), line, column); case ')': return Token(TokenType::CParenthesis, c, c + 1, tokenizer.offset(c), line, column); case '[': return Token(TokenType::OBracket, c, c + 1, tokenizer.offset(c), line, column); case ']': return Token(TokenType::CBracket, c, c + 1, tokenizer.offset(c), line, column); case '"': { // quoted string const char* begin = c; const char* end; tokenizer.quotedString(begin, end); return Token(TokenType::String, begin, end, tokenizer.offset(begin), line, column); } default: { // whitespace, integer, decimal or word if (isWhitespace(*c)) break; const char* begin = c; // try to read a number if (*c == '-' || isDigit(*c)) { while (isDigit(*(c = tokenizer.nextChar()))); if (isDelimiter(*c)) { if (!tokenizer.eof()) tokenizer.pushChar(); return Token(TokenType::Integer, begin, c, tokenizer.offset(begin), line, column); } } // try to read a decimal (may start with '.') if (*c == '.') { while (isDigit(*(c = tokenizer.nextChar()))); if (isDelimiter(*c)) { if (!tokenizer.eof()) tokenizer.pushChar(); return Token(TokenType::Decimal, begin, c, tokenizer.offset(begin), line, column); } } // try to read decimal in scientific notation if (*c == 'e') { c = tokenizer.nextChar(); if (isDigit(*c) || *c == '+' || *c == '-') { while (isDigit(*(c = tokenizer.nextChar()))); if (isDelimiter(*c)) { if (!tokenizer.eof()) tokenizer.pushChar(); return Token(TokenType::Decimal, begin, c, tokenizer.offset(begin), line, column); } } } // read a word while (!tokenizer.eof() && !isDelimiter(*(c = tokenizer.nextChar()))); if (!tokenizer.eof()) tokenizer.pushChar(); return Token(TokenType::String, begin, c, tokenizer.offset(begin), line, column); } } } return Token(TokenType::Eof, NULL, NULL, 0, tokenizer.line(), tokenizer.column()); }