Token& Tokenizer::_ParseHexOperand() { const char* begin = fCurrentChar; fCurrentChar += 2; // skip "0x" if (!_IsHexDigit(*fCurrentChar)) throw ParseException("expected hex digit", _CurrentPos()); fCurrentChar++; while (_IsHexDigit(*fCurrentChar)) fCurrentChar++; int32 length = fCurrentChar - begin; fCurrentToken = Token(begin, length, _CurrentPos() - length, TOKEN_CONSTANT); if (length <= 10) { // including the leading 0x, a 32-bit constant will be at most // 10 characters. Anything larger, and 64 is necessary. fCurrentToken.value.SetTo((uint32)strtoul( fCurrentToken.string.String(), NULL, 16)); } else { fCurrentToken.value.SetTo((uint64)strtoull( fCurrentToken.string.String(), NULL, 16)); } return fCurrentToken; }
const Token& Tokenizer::NextToken() { if (fCurrentToken.type == TOKEN_END_OF_LINE) return fCurrentToken; if (fReuseToken) { fReuseToken = false; return fCurrentToken; } while (*fCurrentChar != 0 && isspace(*fCurrentChar)) fCurrentChar++; if (*fCurrentChar == 0) { return fCurrentToken = Token("", 0, _CurrentPos(), TOKEN_END_OF_LINE); } bool decimal = *fCurrentChar == '.'; if (decimal || isdigit(*fCurrentChar)) { if (*fCurrentChar == '0' && fCurrentChar[1] == 'x') return _ParseHexOperand(); BString temp; const char* begin = fCurrentChar; // optional digits before the comma while (isdigit(*fCurrentChar)) { temp << *fCurrentChar; fCurrentChar++; } // optional post decimal part // (required if there are no digits before the decimal) if (*fCurrentChar == '.') { decimal = true; temp << '.'; fCurrentChar++; // optional post decimal digits while (isdigit(*fCurrentChar)) { temp << *fCurrentChar; fCurrentChar++; } } int32 length = fCurrentChar - begin; if (length == 1 && decimal) { // check for . operator fCurrentChar = begin; if (!_ParseOperator()) throw ParseException("unexpected character", _CurrentPos()); return fCurrentToken; } BString test = temp; test << "&_"; double value; char t[2]; int32 matches = sscanf(test.String(), "%lf&%s", &value, t); if (matches != 2) throw ParseException("error in constant", _CurrentPos() - length); fCurrentToken = Token(begin, length, _CurrentPos() - length, TOKEN_CONSTANT); if (decimal) fCurrentToken.value.SetTo(value); else fCurrentToken.value.SetTo((int64)strtoll(temp.String(), NULL, 10)); } else if (isalpha(*fCurrentChar) || *fCurrentChar == '_') { const char* begin = fCurrentChar; while (*fCurrentChar != 0 && (isalpha(*fCurrentChar) || isdigit(*fCurrentChar) || *fCurrentChar == '_')) { fCurrentChar++; } int32 length = fCurrentChar - begin; fCurrentToken = Token(begin, length, _CurrentPos() - length, TOKEN_IDENTIFIER); } else if (*fCurrentChar == '"' || *fCurrentChar == '\'') { const char* begin = fCurrentChar++; while (*fCurrentChar != 0) { if (*fCurrentChar == '\\') { if (*(fCurrentChar++) != 0) fCurrentChar++; } else if (*(fCurrentChar++) == *begin) break; } int32 length = fCurrentChar - begin; fCurrentToken = Token(begin, length, _CurrentPos() - length, TOKEN_STRING_LITERAL); } else { if (!_ParseOperator()) { int32 type = TOKEN_NONE; switch (*fCurrentChar) { case '\n': type = TOKEN_END_OF_LINE; break; case '(': type = TOKEN_OPENING_PAREN; break; case ')': type = TOKEN_CLOSING_PAREN; break; case '[': type = TOKEN_OPENING_SQUARE_BRACKET; break; case ']': type = TOKEN_CLOSING_SQUARE_BRACKET; break; case '{': type = TOKEN_OPENING_CURLY_BRACE; break; case '}': type = TOKEN_CLOSING_CURLY_BRACE; break; case '\\': type = TOKEN_BACKSLASH; break; case ':': type = TOKEN_COLON; break; case ';': type = TOKEN_SEMICOLON; break; case ',': type = TOKEN_COMMA; break; case '.': type = TOKEN_PERIOD; break; case '#': type = TOKEN_POUND; break; default: throw ParseException("unexpected character", _CurrentPos()); } fCurrentToken = Token(fCurrentChar, 1, _CurrentPos(), type); fCurrentChar++; } } return fCurrentToken; }