Token DefTokenEmitter::doEmit(Tokenizer& tokenizer) { while (!tokenizer.eof()) { size_t line = tokenizer.line(); size_t column = tokenizer.column(); const char* c = tokenizer.nextChar(); switch (*c) { case '/': { const char* begin = c; if (tokenizer.peekChar() == '*') { // eat all chars immediately after the '*' because it's often followed by QUAKE while (!isWhitespace(*tokenizer.nextChar()) && !tokenizer.eof()); return Token(ODefinition, begin, c, tokenizer.offset(begin), line, column); } else if (tokenizer.peekChar() == '/') { // eat everything up to and including the next newline while (*tokenizer.nextChar() != '\n'); break; } error(line, column, *c); break; } case '*': { const char* begin = c; if (tokenizer.peekChar() == '/') { tokenizer.nextChar(); return Token(ODefinition, begin, c, tokenizer.offset(begin), line, column); } error(line, column, *c); break; } case '(': return Token(OParenthesis, c, c + 1, tokenizer.offset(c), line, column); case ')': return Token(CParenthesis, c, c + 1, tokenizer.offset(c), line, column); case '{': return Token(OBrace, c, c + 1, tokenizer.offset(c), line, column); case '}': return Token(CBrace, c, c + 1, tokenizer.offset(c), line, column); case '=': return Token(Equality, c, c + 1, tokenizer.offset(c), line, column); case ';': return Token(Semicolon, c, c + 1, tokenizer.offset(c), line, column); case '?': return Token(Question, c, c + 1, tokenizer.offset(c), line, column); case '\r': if (tokenizer.peekChar() == '\n') { tokenizer.nextChar(); } case '\n': return Token(Newline, c, c + 1, tokenizer.offset(c), line, column); case ',': return Token(Comma, c, c + 1, tokenizer.offset(c), line, column); case ' ': case '\t': break; case '"': { // quoted string const char* begin = c; const char* end; tokenizer.quotedString(begin, end); return Token(QuotedString, begin, end, tokenizer.offset(begin), line, column); } default: { // integer, decimal or word const char* begin = c; // try to read a number if (*c == '-' || isDigit(*c)) { while (isDigit(*(c = tokenizer.nextChar()))); if (isDelimiter(*c)) { if (!tokenizer.eof()) tokenizer.pushChar(); return Token(Integer, begin, c, tokenizer.offset(begin), line, column); } } // try to read a decimal (may start with '.') if (*c == '.') { while (isDigit(*(c = tokenizer.nextChar()))); if (isDelimiter(*c)) { if (!tokenizer.eof()) tokenizer.pushChar(); return Token(Decimal, begin, c, tokenizer.offset(begin), line, column); } } // read a word while (!tokenizer.eof() && !isDelimiter(*(c = tokenizer.nextChar()))); if (!tokenizer.eof()) tokenizer.pushChar(); return Token(Word, begin, c, tokenizer.offset(begin), line, column); } } } return Token(Eof, NULL, NULL, 0, tokenizer.line(), tokenizer.column()); }
Token MapTokenEmitter::doEmit(Tokenizer& tokenizer, size_t line, size_t column) { const size_t startPosition = tokenizer.position(); while (!tokenizer.eof()) { char c = tokenizer.nextChar(); switch (c) { case '/': if (tokenizer.peekChar() == '/') { // eat everything up to and including the next newline while (tokenizer.nextChar() != '\n'); } break; case '{': return Token(TokenType::OBrace, "", startPosition, tokenizer.position() - startPosition, line, column); case '}': return Token(TokenType::CBrace, "", startPosition, tokenizer.position() - startPosition, line, column); case '(': return Token(TokenType::OParenthesis, "", startPosition, tokenizer.position() - startPosition, line, column); case ')': return Token(TokenType::CParenthesis, "", startPosition, tokenizer.position() - startPosition, line, column); case '[': return Token(TokenType::OBracket, "", startPosition, tokenizer.position() - startPosition, line, column); case ']': return Token(TokenType::CBracket, "", startPosition, tokenizer.position() - startPosition, line, column); case '"': // quoted string m_buffer.str(String()); while (!tokenizer.eof() && (c = tokenizer.nextChar()) != '"') m_buffer << c; return Token(TokenType::String, m_buffer.str(), startPosition, tokenizer.position() - startPosition, line, column); default: // whitespace, integer, decimal or word if (isWhitespace(c)) break; // clear the buffer m_buffer.str(String()); // try to read a number if (c == '-' || isDigit(c)) { m_buffer << c; while (isDigit((c = tokenizer.nextChar()))) m_buffer << c; if (isDelimiter(c)) { if (!tokenizer.eof()) tokenizer.pushChar(); return Token(TokenType::Integer, m_buffer.str(), startPosition, tokenizer.position() - startPosition, line, column); } } // try to read a decimal (may start with '.') if (c == '.') { m_buffer << c; while (isDigit((c = tokenizer.nextChar()))) m_buffer << c; if (isDelimiter(c)) { if (!tokenizer.eof()) tokenizer.pushChar(); return Token(TokenType::Decimal, m_buffer.str(), startPosition, tokenizer.position() - startPosition, line, column); } } // read a word m_buffer << c; while (!tokenizer.eof() && !isDelimiter(c = tokenizer.nextChar())) m_buffer << c; if (!tokenizer.eof()) tokenizer.pushChar(); return Token(TokenType::String, m_buffer.str(), startPosition, tokenizer.position() - startPosition, line, column); } } return Token(TokenType::Eof, "", startPosition, tokenizer.position() - startPosition, line, column); }
Token MapTokenEmitter::doEmit(Tokenizer& tokenizer) { while (!tokenizer.eof()) { size_t line = tokenizer.line(); size_t column = tokenizer.column(); const char* c = tokenizer.nextChar(); switch (*c) { case '/': if (tokenizer.peekChar() == '/') { tokenizer.nextChar(); if (tokenizer.peekChar() == '/') { tokenizer.nextChar(); // it's a TB comment } else { // eat everything up to and including the next newline while (*tokenizer.nextChar() != '\n'); } } break; case '{': return Token(TokenType::OBrace, c, c + 1, tokenizer.offset(c), line, column); case '}': return Token(TokenType::CBrace, c, c + 1, tokenizer.offset(c), line, column); case '(': return Token(TokenType::OParenthesis, c, c + 1, tokenizer.offset(c), line, column); case ')': return Token(TokenType::CParenthesis, c, c + 1, tokenizer.offset(c), line, column); case '[': return Token(TokenType::OBracket, c, c + 1, tokenizer.offset(c), line, column); case ']': return Token(TokenType::CBracket, c, c + 1, tokenizer.offset(c), line, column); case '"': { // quoted string const char* begin = c; const char* end; tokenizer.quotedString(begin, end); return Token(TokenType::String, begin, end, tokenizer.offset(begin), line, column); } default: { // whitespace, integer, decimal or word if (isWhitespace(*c)) break; const char* begin = c; // try to read a number if (*c == '-' || isDigit(*c)) { while (isDigit(*(c = tokenizer.nextChar()))); if (isDelimiter(*c)) { if (!tokenizer.eof()) tokenizer.pushChar(); return Token(TokenType::Integer, begin, c, tokenizer.offset(begin), line, column); } } // try to read a decimal (may start with '.') if (*c == '.') { while (isDigit(*(c = tokenizer.nextChar()))); if (isDelimiter(*c)) { if (!tokenizer.eof()) tokenizer.pushChar(); return Token(TokenType::Decimal, begin, c, tokenizer.offset(begin), line, column); } } // try to read decimal in scientific notation if (*c == 'e') { c = tokenizer.nextChar(); if (isDigit(*c) || *c == '+' || *c == '-') { while (isDigit(*(c = tokenizer.nextChar()))); if (isDelimiter(*c)) { if (!tokenizer.eof()) tokenizer.pushChar(); return Token(TokenType::Decimal, begin, c, tokenizer.offset(begin), line, column); } } } // read a word while (!tokenizer.eof() && !isDelimiter(*(c = tokenizer.nextChar()))); if (!tokenizer.eof()) tokenizer.pushChar(); return Token(TokenType::String, begin, c, tokenizer.offset(begin), line, column); } } } return Token(TokenType::Eof, NULL, NULL, 0, tokenizer.line(), tokenizer.column()); }