Token DefTokenEmitter::doEmit(Tokenizer& tokenizer) {
while (!tokenizer.eof()) {
size_t line = tokenizer.line();
size_t column = tokenizer.column();
const char* c = tokenizer.nextChar();
switch (*c) {
case '/': {
const char* begin = c;
if (tokenizer.peekChar() == '*') {
// eat all chars immediately after the '*' because it's often followed by QUAKE
while (!isWhitespace(*tokenizer.nextChar()) && !tokenizer.eof());
return Token(ODefinition, begin, c, tokenizer.offset(begin), line, column);
} else if (tokenizer.peekChar() == '/') {
// eat everything up to and including the next newline
while (*tokenizer.nextChar() != '\n');
break;
}
error(line, column, *c);
break;
}
case '*': {
const char* begin = c;
if (tokenizer.peekChar() == '/') {
tokenizer.nextChar();
return Token(ODefinition, begin, c, tokenizer.offset(begin), line, column);
}
error(line, column, *c);
break;
}
case '(':
return Token(OParenthesis, c, c + 1, tokenizer.offset(c), line, column);
case ')':
return Token(CParenthesis, c, c + 1, tokenizer.offset(c), line, column);
case '{':
return Token(OBrace, c, c + 1, tokenizer.offset(c), line, column);
case '}':
return Token(CBrace, c, c + 1, tokenizer.offset(c), line, column);
case '=':
return Token(Equality, c, c + 1, tokenizer.offset(c), line, column);
case ';':
return Token(Semicolon, c, c + 1, tokenizer.offset(c), line, column);
case '?':
return Token(Question, c, c + 1, tokenizer.offset(c), line, column);
case '\r':
if (tokenizer.peekChar() == '\n') {
tokenizer.nextChar();
}
case '\n':
return Token(Newline, c, c + 1, tokenizer.offset(c), line, column);
case ',':
return Token(Comma, c, c + 1, tokenizer.offset(c), line, column);
case ' ':
case '\t':
break;
case '"': { // quoted string
const char* begin = c;
const char* end;
tokenizer.quotedString(begin, end);
return Token(QuotedString, begin, end, tokenizer.offset(begin), line, column);
}
default: { // integer, decimal or word
const char* begin = c;
// try to read a number
if (*c == '-' || isDigit(*c)) {
while (isDigit(*(c = tokenizer.nextChar())));
if (isDelimiter(*c)) {
if (!tokenizer.eof())
tokenizer.pushChar();
return Token(Integer, begin, c, tokenizer.offset(begin), line, column);
}
}
// try to read a decimal (may start with '.')
if (*c == '.') {
while (isDigit(*(c = tokenizer.nextChar())));
if (isDelimiter(*c)) {
if (!tokenizer.eof())
tokenizer.pushChar();
return Token(Decimal, begin, c, tokenizer.offset(begin), line, column);
}
}
// read a word
while (!tokenizer.eof() && !isDelimiter(*(c = tokenizer.nextChar())));
if (!tokenizer.eof())
tokenizer.pushChar();
return Token(Word, begin, c, tokenizer.offset(begin), line, column);
}
}
}
return Token(Eof, NULL, NULL, 0, tokenizer.line(), tokenizer.column());
}
Token MapTokenEmitter::doEmit(Tokenizer& tokenizer, size_t line, size_t column) {
const size_t startPosition = tokenizer.position();
while (!tokenizer.eof()) {
char c = tokenizer.nextChar();
switch (c) {
case '/':
if (tokenizer.peekChar() == '/') {
// eat everything up to and including the next newline
while (tokenizer.nextChar() != '\n');
}
break;
case '{':
return Token(TokenType::OBrace, "", startPosition, tokenizer.position() - startPosition, line, column);
case '}':
return Token(TokenType::CBrace, "", startPosition, tokenizer.position() - startPosition, line, column);
case '(':
return Token(TokenType::OParenthesis, "", startPosition, tokenizer.position() - startPosition, line, column);
case ')':
return Token(TokenType::CParenthesis, "", startPosition, tokenizer.position() - startPosition, line, column);
case '[':
return Token(TokenType::OBracket, "", startPosition, tokenizer.position() - startPosition, line, column);
case ']':
return Token(TokenType::CBracket, "", startPosition, tokenizer.position() - startPosition, line, column);
case '"': // quoted string
m_buffer.str(String());
while (!tokenizer.eof() && (c = tokenizer.nextChar()) != '"')
m_buffer << c;
return Token(TokenType::String, m_buffer.str(), startPosition, tokenizer.position() - startPosition, line, column);
default: // whitespace, integer, decimal or word
if (isWhitespace(c))
break;
// clear the buffer
m_buffer.str(String());
// try to read a number
if (c == '-' || isDigit(c)) {
m_buffer << c;
while (isDigit((c = tokenizer.nextChar())))
m_buffer << c;
if (isDelimiter(c)) {
if (!tokenizer.eof())
tokenizer.pushChar();
return Token(TokenType::Integer, m_buffer.str(), startPosition, tokenizer.position() - startPosition, line, column);
}
}
// try to read a decimal (may start with '.')
if (c == '.') {
m_buffer << c;
while (isDigit((c = tokenizer.nextChar())))
m_buffer << c;
if (isDelimiter(c)) {
if (!tokenizer.eof())
tokenizer.pushChar();
return Token(TokenType::Decimal, m_buffer.str(), startPosition, tokenizer.position() - startPosition, line, column);
}
}
// read a word
m_buffer << c;
while (!tokenizer.eof() && !isDelimiter(c = tokenizer.nextChar()))
m_buffer << c;
if (!tokenizer.eof())
tokenizer.pushChar();
return Token(TokenType::String, m_buffer.str(), startPosition, tokenizer.position() - startPosition, line, column);
}
}
return Token(TokenType::Eof, "", startPosition, tokenizer.position() - startPosition, line, column);
}
Token MapTokenEmitter::doEmit(Tokenizer& tokenizer) {
while (!tokenizer.eof()) {
size_t line = tokenizer.line();
size_t column = tokenizer.column();
const char* c = tokenizer.nextChar();
switch (*c) {
case '/':
if (tokenizer.peekChar() == '/') {
tokenizer.nextChar();
if (tokenizer.peekChar() == '/') {
tokenizer.nextChar(); // it's a TB comment
} else {
// eat everything up to and including the next newline
while (*tokenizer.nextChar() != '\n');
}
}
break;
case '{':
return Token(TokenType::OBrace, c, c + 1, tokenizer.offset(c), line, column);
case '}':
return Token(TokenType::CBrace, c, c + 1, tokenizer.offset(c), line, column);
case '(':
return Token(TokenType::OParenthesis, c, c + 1, tokenizer.offset(c), line, column);
case ')':
return Token(TokenType::CParenthesis, c, c + 1, tokenizer.offset(c), line, column);
case '[':
return Token(TokenType::OBracket, c, c + 1, tokenizer.offset(c), line, column);
case ']':
return Token(TokenType::CBracket, c, c + 1, tokenizer.offset(c), line, column);
case '"': { // quoted string
const char* begin = c;
const char* end;
tokenizer.quotedString(begin, end);
return Token(TokenType::String, begin, end, tokenizer.offset(begin), line, column);
}
default: { // whitespace, integer, decimal or word
if (isWhitespace(*c))
break;
const char* begin = c;
// try to read a number
if (*c == '-' || isDigit(*c)) {
while (isDigit(*(c = tokenizer.nextChar())));
if (isDelimiter(*c)) {
if (!tokenizer.eof())
tokenizer.pushChar();
return Token(TokenType::Integer, begin, c, tokenizer.offset(begin), line, column);
}
}
// try to read a decimal (may start with '.')
if (*c == '.') {
while (isDigit(*(c = tokenizer.nextChar())));
if (isDelimiter(*c)) {
if (!tokenizer.eof())
tokenizer.pushChar();
return Token(TokenType::Decimal, begin, c, tokenizer.offset(begin), line, column);
}
}
// try to read decimal in scientific notation
if (*c == 'e') {
c = tokenizer.nextChar();
if (isDigit(*c) || *c == '+' || *c == '-') {
while (isDigit(*(c = tokenizer.nextChar())));
if (isDelimiter(*c)) {
if (!tokenizer.eof())
tokenizer.pushChar();
return Token(TokenType::Decimal, begin, c, tokenizer.offset(begin), line, column);
}
}
}
// read a word
while (!tokenizer.eof() && !isDelimiter(*(c = tokenizer.nextChar())));
if (!tokenizer.eof())
tokenizer.pushChar();
return Token(TokenType::String, begin, c, tokenizer.offset(begin), line, column);
}
}
}
return Token(TokenType::Eof, NULL, NULL, 0, tokenizer.line(), tokenizer.column());
}
