Token DefTokenEmitter::doEmit(Tokenizer& tokenizer) {
while (!tokenizer.eof()) {
size_t line = tokenizer.line();
size_t column = tokenizer.column();
const char* c = tokenizer.nextChar();
switch (*c) {
case '/': {
const char* begin = c;
if (tokenizer.peekChar() == '*') {
// eat all chars immediately after the '*' because it's often followed by QUAKE
while (!isWhitespace(*tokenizer.nextChar()) && !tokenizer.eof());
return Token(ODefinition, begin, c, tokenizer.offset(begin), line, column);
} else if (tokenizer.peekChar() == '/') {
// eat everything up to and including the next newline
while (*tokenizer.nextChar() != '\n');
break;
}
error(line, column, *c);
break;
}
case '*': {
const char* begin = c;
if (tokenizer.peekChar() == '/') {
tokenizer.nextChar();
return Token(ODefinition, begin, c, tokenizer.offset(begin), line, column);
}
error(line, column, *c);
break;
}
case '(':
return Token(OParenthesis, c, c + 1, tokenizer.offset(c), line, column);
case ')':
return Token(CParenthesis, c, c + 1, tokenizer.offset(c), line, column);
case '{':
return Token(OBrace, c, c + 1, tokenizer.offset(c), line, column);
case '}':
return Token(CBrace, c, c + 1, tokenizer.offset(c), line, column);
case '=':
return Token(Equality, c, c + 1, tokenizer.offset(c), line, column);
case ';':
return Token(Semicolon, c, c + 1, tokenizer.offset(c), line, column);
case '?':
return Token(Question, c, c + 1, tokenizer.offset(c), line, column);
case '\r':
if (tokenizer.peekChar() == '\n') {
tokenizer.nextChar();
}
case '\n':
return Token(Newline, c, c + 1, tokenizer.offset(c), line, column);
case ',':
return Token(Comma, c, c + 1, tokenizer.offset(c), line, column);
case ' ':
case '\t':
break;
case '"': { // quoted string
const char* begin = c;
const char* end;
tokenizer.quotedString(begin, end);
return Token(QuotedString, begin, end, tokenizer.offset(begin), line, column);
}
default: { // integer, decimal or word
const char* begin = c;
// try to read a number
if (*c == '-' || isDigit(*c)) {
while (isDigit(*(c = tokenizer.nextChar())));
if (isDelimiter(*c)) {
if (!tokenizer.eof())
tokenizer.pushChar();
return Token(Integer, begin, c, tokenizer.offset(begin), line, column);
}
}
// try to read a decimal (may start with '.')
if (*c == '.') {
while (isDigit(*(c = tokenizer.nextChar())));
if (isDelimiter(*c)) {
if (!tokenizer.eof())
tokenizer.pushChar();
return Token(Decimal, begin, c, tokenizer.offset(begin), line, column);
}
}
// read a word
while (!tokenizer.eof() && !isDelimiter(*(c = tokenizer.nextChar())));
if (!tokenizer.eof())
tokenizer.pushChar();
return Token(Word, begin, c, tokenizer.offset(begin), line, column);
}
}
}
return Token(Eof, NULL, NULL, 0, tokenizer.line(), tokenizer.column());
}
/* ParseTreeNode::parse
* Parses formatted text data. Current valid formatting is:
* (type) child = value;
* (type) child = value1, value2, ...;
* (type) child = { value1, value2, ... }
* (type) child { grandchild = value; etc... }
* (type) child : inherited { ... }
* All values are read as strings, but can be retrieved as string,
* int, bool or float.
*******************************************************************/
bool ParseTreeNode::parse(Tokenizer& tz)
{
// Get first token
string token = tz.getToken();
// Keep parsing until final } is reached (or end of file)
while (!(S_CMP(token, "}")) && !token.IsEmpty())
{
// Check for preprocessor stuff
if (parser)
{
// #define
if (S_CMPNOCASE(token, "#define"))
{
parser->define(tz.getToken());
token = tz.getToken();
continue;
}
// #if(n)def
if (S_CMPNOCASE(token, "#ifdef") || S_CMPNOCASE(token, "#ifndef"))
{
bool test = true;
if (S_CMPNOCASE(token, "#ifndef"))
test = false;
string define = tz.getToken();
if (parser->defined(define) == test)
{
// Continue
token = tz.getToken();
continue;
}
else
{
// Skip section
int skip = 0;
while (true)
{
token = tz.getToken();
if (S_CMPNOCASE(token, "#endif"))
skip--;
else if (S_CMPNOCASE(token, "#ifdef"))
skip++;
else if (S_CMPNOCASE(token, "#ifndef"))
skip++;
if (skip < 0)
break;
if (token.IsEmpty())
{
wxLogMessage("Error: found end of file within #if(n)def block");
break;
}
}
continue;
}
}
// #include (ignore)
if (S_CMPNOCASE(token, "#include"))
{
tz.skipToken(); // Skip include path
token = tz.getToken();
continue;
}
// #endif (ignore)
if (S_CMPNOCASE(token, "#endif"))
{
token = tz.getToken();
continue;
}
}
// If it's a special character (ie not a valid name), parsing fails
if (tz.isSpecialCharacter(token.at(0)))
{
wxLogMessage("Parsing error: Unexpected special character '%s' in %s (line %d)", CHR(token), CHR(tz.getName()), tz.lineNo());
return false;
}
// So we have either a node or property name
string name = token;
// Check next token to determine what we're doing
string next = tz.peekToken();
// Check for type+name pair
string type = "";
if (next != "=" && next != "{" && next != ";" && next != ":")
{
type = name;
name = tz.getToken();
next = tz.peekToken();
}
// Assignment
if (S_CMP(next, "="))
{
// Skip =
tz.skipToken();
// Create item node
ParseTreeNode* child = (ParseTreeNode*)addChild(name);
child->type = type;
// Check type of assignment list
token = tz.getToken();
string list_end = ";";
if (token == "{")
{
list_end = "}";
token = tz.getToken();
}
// Parse until ; or }
while (1)
{
// Check for list end
if (S_CMP(token, list_end) && !tz.quotedString())
break;
// Setup value
Property value;
// Detect value type
if (tz.quotedString()) // Quoted string
value = token;
else if (S_CMPNOCASE(token, "true")) // Boolean (true)
value = true;
else if (S_CMPNOCASE(token, "false")) // Boolean (false)
value = false;
else if (re_int1.Matches(token) || // Integer
re_int2.Matches(token))
{
long val;
token.ToLong(&val);
value = (int)val;
}
else if (re_int3.Matches(token)) // Hex (0xXXXXXX)
{
long val;
token.ToLong(&val, 0);
value = (int)val;
//wxLogMessage("%s: %s is hex %d", CHR(name), CHR(token), value.getIntValue());
}
else if (re_float.Matches(token)) // Floating point
{
double val;
token.ToDouble(&val);
value = (double)val;
//LOG_MESSAGE(3, S_FMT("%s: %s is float %1.3f", CHR(name), CHR(token), val));
}
else // Unknown, just treat as string
value = token;
// Add value
child->values.push_back(value);
// Check for ,
if (S_CMP(tz.peekToken(), ","))
tz.skipToken(); // Skip it
else if (!(S_CMP(tz.peekToken(), list_end)))
{
string t = tz.getToken();
string n = tz.getName();
wxLogMessage("Parsing error: Expected \",\" or \"%s\", got \"%s\" in %s (line %d)", CHR(list_end), CHR(t), CHR(n), tz.lineNo());
return false;
}
token = tz.getToken();
}
}
// Child node
else if (S_CMP(next, "{"))
{
// Add child node
ParseTreeNode* child = (ParseTreeNode*)addChild(name);
child->type = type;
// Skip {
tz.skipToken();
// Parse child node
if (!child->parse(tz))
return false;
}
// Child node (with no values/children)
else if (S_CMP(next, ";"))
{
// Add child node
ParseTreeNode* child = (ParseTreeNode*)addChild(name);
child->type = type;
// Skip ;
tz.skipToken();
}
// Child node + inheritance
else if (S_CMP(next, ":"))
{
// Skip :
tz.skipToken();
// Read inherited name
string inherit = tz.getToken();
// Check for opening brace
if (tz.checkToken("{"))
{
// Add child node
ParseTreeNode* child = (ParseTreeNode*)addChild(name);
child->type = type;
// Set its inheritance
child->inherit = inherit;
// Parse child node
if (!child->parse(tz))
return false;
}
}
// Unexpected token
else
{
wxLogMessage("Parsing error: \"%s\" unexpected in %s (line %d)", CHR(next), CHR(tz.getName()), tz.lineNo());
return false;
}
// Continue parsing
token = tz.getToken();
}
return true;
}
Token MapTokenEmitter::doEmit(Tokenizer& tokenizer) {
while (!tokenizer.eof()) {
size_t line = tokenizer.line();
size_t column = tokenizer.column();
const char* c = tokenizer.nextChar();
switch (*c) {
case '/':
if (tokenizer.peekChar() == '/') {
tokenizer.nextChar();
if (tokenizer.peekChar() == '/') {
tokenizer.nextChar(); // it's a TB comment
} else {
// eat everything up to and including the next newline
while (*tokenizer.nextChar() != '\n');
}
}
break;
case '{':
return Token(TokenType::OBrace, c, c + 1, tokenizer.offset(c), line, column);
case '}':
return Token(TokenType::CBrace, c, c + 1, tokenizer.offset(c), line, column);
case '(':
return Token(TokenType::OParenthesis, c, c + 1, tokenizer.offset(c), line, column);
case ')':
return Token(TokenType::CParenthesis, c, c + 1, tokenizer.offset(c), line, column);
case '[':
return Token(TokenType::OBracket, c, c + 1, tokenizer.offset(c), line, column);
case ']':
return Token(TokenType::CBracket, c, c + 1, tokenizer.offset(c), line, column);
case '"': { // quoted string
const char* begin = c;
const char* end;
tokenizer.quotedString(begin, end);
return Token(TokenType::String, begin, end, tokenizer.offset(begin), line, column);
}
default: { // whitespace, integer, decimal or word
if (isWhitespace(*c))
break;
const char* begin = c;
// try to read a number
if (*c == '-' || isDigit(*c)) {
while (isDigit(*(c = tokenizer.nextChar())));
if (isDelimiter(*c)) {
if (!tokenizer.eof())
tokenizer.pushChar();
return Token(TokenType::Integer, begin, c, tokenizer.offset(begin), line, column);
}
}
// try to read a decimal (may start with '.')
if (*c == '.') {
while (isDigit(*(c = tokenizer.nextChar())));
if (isDelimiter(*c)) {
if (!tokenizer.eof())
tokenizer.pushChar();
return Token(TokenType::Decimal, begin, c, tokenizer.offset(begin), line, column);
}
}
// try to read decimal in scientific notation
if (*c == 'e') {
c = tokenizer.nextChar();
if (isDigit(*c) || *c == '+' || *c == '-') {
while (isDigit(*(c = tokenizer.nextChar())));
if (isDelimiter(*c)) {
if (!tokenizer.eof())
tokenizer.pushChar();
return Token(TokenType::Decimal, begin, c, tokenizer.offset(begin), line, column);
}
}
}
// read a word
while (!tokenizer.eof() && !isDelimiter(*(c = tokenizer.nextChar())));
if (!tokenizer.eof())
tokenizer.pushChar();
return Token(TokenType::String, begin, c, tokenizer.offset(begin), line, column);
}
}
}
return Token(TokenType::Eof, NULL, NULL, 0, tokenizer.line(), tokenizer.column());
}
