Token DefTokenEmitter::doEmit(Tokenizer& tokenizer) { while (!tokenizer.eof()) { size_t line = tokenizer.line(); size_t column = tokenizer.column(); const char* c = tokenizer.nextChar(); switch (*c) { case '/': { const char* begin = c; if (tokenizer.peekChar() == '*') { // eat all chars immediately after the '*' because it's often followed by QUAKE while (!isWhitespace(*tokenizer.nextChar()) && !tokenizer.eof()); return Token(ODefinition, begin, c, tokenizer.offset(begin), line, column); } else if (tokenizer.peekChar() == '/') { // eat everything up to and including the next newline while (*tokenizer.nextChar() != '\n'); break; } error(line, column, *c); break; } case '*': { const char* begin = c; if (tokenizer.peekChar() == '/') { tokenizer.nextChar(); return Token(ODefinition, begin, c, tokenizer.offset(begin), line, column); } error(line, column, *c); break; } case '(': return Token(OParenthesis, c, c + 1, tokenizer.offset(c), line, column); case ')': return Token(CParenthesis, c, c + 1, tokenizer.offset(c), line, column); case '{': return Token(OBrace, c, c + 1, tokenizer.offset(c), line, column); case '}': return Token(CBrace, c, c + 1, tokenizer.offset(c), line, column); case '=': return Token(Equality, c, c + 1, tokenizer.offset(c), line, column); case ';': return Token(Semicolon, c, c + 1, tokenizer.offset(c), line, column); case '?': return Token(Question, c, c + 1, tokenizer.offset(c), line, column); case '\r': if (tokenizer.peekChar() == '\n') { tokenizer.nextChar(); } case '\n': return Token(Newline, c, c + 1, tokenizer.offset(c), line, column); case ',': return Token(Comma, c, c + 1, tokenizer.offset(c), line, column); case ' ': case '\t': break; case '"': { // quoted string const char* begin = c; const char* end; tokenizer.quotedString(begin, end); return Token(QuotedString, begin, end, tokenizer.offset(begin), line, column); } default: { // integer, decimal or word const char* begin = c; // try to read a number if (*c == '-' || isDigit(*c)) { while (isDigit(*(c = tokenizer.nextChar()))); if (isDelimiter(*c)) { if (!tokenizer.eof()) tokenizer.pushChar(); return Token(Integer, begin, c, tokenizer.offset(begin), line, column); } } // try to read a decimal (may start with '.') if (*c == '.') { while (isDigit(*(c = tokenizer.nextChar()))); if (isDelimiter(*c)) { if (!tokenizer.eof()) tokenizer.pushChar(); return Token(Decimal, begin, c, tokenizer.offset(begin), line, column); } } // read a word while (!tokenizer.eof() && !isDelimiter(*(c = tokenizer.nextChar()))); if (!tokenizer.eof()) tokenizer.pushChar(); return Token(Word, begin, c, tokenizer.offset(begin), line, column); } } } return Token(Eof, NULL, NULL, 0, tokenizer.line(), tokenizer.column()); }
Token MapTokenEmitter::doEmit(Tokenizer& tokenizer) { while (!tokenizer.eof()) { size_t line = tokenizer.line(); size_t column = tokenizer.column(); const char* c = tokenizer.nextChar(); switch (*c) { case '/': if (tokenizer.peekChar() == '/') { tokenizer.nextChar(); if (tokenizer.peekChar() == '/') { tokenizer.nextChar(); // it's a TB comment } else { // eat everything up to and including the next newline while (*tokenizer.nextChar() != '\n'); } } break; case '{': return Token(TokenType::OBrace, c, c + 1, tokenizer.offset(c), line, column); case '}': return Token(TokenType::CBrace, c, c + 1, tokenizer.offset(c), line, column); case '(': return Token(TokenType::OParenthesis, c, c + 1, tokenizer.offset(c), line, column); case ')': return Token(TokenType::CParenthesis, c, c + 1, tokenizer.offset(c), line, column); case '[': return Token(TokenType::OBracket, c, c + 1, tokenizer.offset(c), line, column); case ']': return Token(TokenType::CBracket, c, c + 1, tokenizer.offset(c), line, column); case '"': { // quoted string const char* begin = c; const char* end; tokenizer.quotedString(begin, end); return Token(TokenType::String, begin, end, tokenizer.offset(begin), line, column); } default: { // whitespace, integer, decimal or word if (isWhitespace(*c)) break; const char* begin = c; // try to read a number if (*c == '-' || isDigit(*c)) { while (isDigit(*(c = tokenizer.nextChar()))); if (isDelimiter(*c)) { if (!tokenizer.eof()) tokenizer.pushChar(); return Token(TokenType::Integer, begin, c, tokenizer.offset(begin), line, column); } } // try to read a decimal (may start with '.') if (*c == '.') { while (isDigit(*(c = tokenizer.nextChar()))); if (isDelimiter(*c)) { if (!tokenizer.eof()) tokenizer.pushChar(); return Token(TokenType::Decimal, begin, c, tokenizer.offset(begin), line, column); } } // try to read decimal in scientific notation if (*c == 'e') { c = tokenizer.nextChar(); if (isDigit(*c) || *c == '+' || *c == '-') { while (isDigit(*(c = tokenizer.nextChar()))); if (isDelimiter(*c)) { if (!tokenizer.eof()) tokenizer.pushChar(); return Token(TokenType::Decimal, begin, c, tokenizer.offset(begin), line, column); } } } // read a word while (!tokenizer.eof() && !isDelimiter(*(c = tokenizer.nextChar()))); if (!tokenizer.eof()) tokenizer.pushChar(); return Token(TokenType::String, begin, c, tokenizer.offset(begin), line, column); } } } return Token(TokenType::Eof, NULL, NULL, 0, tokenizer.line(), tokenizer.column()); }