Token&
Tokenizer::_ParseHexOperand()
{
const char* begin = fCurrentChar;
fCurrentChar += 2;
// skip "0x"
if (!_IsHexDigit(*fCurrentChar))
throw ParseException("expected hex digit", _CurrentPos());
fCurrentChar++;
while (_IsHexDigit(*fCurrentChar))
fCurrentChar++;
int32 length = fCurrentChar - begin;
fCurrentToken = Token(begin, length, _CurrentPos() - length,
TOKEN_CONSTANT);
if (length <= 10) {
// including the leading 0x, a 32-bit constant will be at most
// 10 characters. Anything larger, and 64 is necessary.
fCurrentToken.value.SetTo((uint32)strtoul(
fCurrentToken.string.String(), NULL, 16));
} else {
fCurrentToken.value.SetTo((uint64)strtoull(
fCurrentToken.string.String(), NULL, 16));
}
return fCurrentToken;
}
const Token&
Tokenizer::NextToken()
{
if (fCurrentToken.type == TOKEN_END_OF_LINE)
return fCurrentToken;
if (fReuseToken) {
fReuseToken = false;
return fCurrentToken;
}
while (*fCurrentChar != 0 && isspace(*fCurrentChar))
fCurrentChar++;
if (*fCurrentChar == 0) {
return fCurrentToken = Token("", 0, _CurrentPos(),
TOKEN_END_OF_LINE);
}
bool decimal = *fCurrentChar == '.';
if (decimal || isdigit(*fCurrentChar)) {
if (*fCurrentChar == '0' && fCurrentChar[1] == 'x')
return _ParseHexOperand();
BString temp;
const char* begin = fCurrentChar;
// optional digits before the comma
while (isdigit(*fCurrentChar)) {
temp << *fCurrentChar;
fCurrentChar++;
}
// optional post decimal part
// (required if there are no digits before the decimal)
if (*fCurrentChar == '.') {
decimal = true;
temp << '.';
fCurrentChar++;
// optional post decimal digits
while (isdigit(*fCurrentChar)) {
temp << *fCurrentChar;
fCurrentChar++;
}
}
int32 length = fCurrentChar - begin;
if (length == 1 && decimal) {
// check for . operator
fCurrentChar = begin;
if (!_ParseOperator())
throw ParseException("unexpected character", _CurrentPos());
return fCurrentToken;
}
BString test = temp;
test << "&_";
double value;
char t[2];
int32 matches = sscanf(test.String(), "%lf&%s", &value, t);
if (matches != 2)
throw ParseException("error in constant", _CurrentPos() - length);
fCurrentToken = Token(begin, length, _CurrentPos() - length,
TOKEN_CONSTANT);
if (decimal)
fCurrentToken.value.SetTo(value);
else
fCurrentToken.value.SetTo((int64)strtoll(temp.String(), NULL, 10));
} else if (isalpha(*fCurrentChar) || *fCurrentChar == '_') {
const char* begin = fCurrentChar;
while (*fCurrentChar != 0 && (isalpha(*fCurrentChar)
|| isdigit(*fCurrentChar) || *fCurrentChar == '_')) {
fCurrentChar++;
}
int32 length = fCurrentChar - begin;
fCurrentToken = Token(begin, length, _CurrentPos() - length,
TOKEN_IDENTIFIER);
} else if (*fCurrentChar == '"' || *fCurrentChar == '\'') {
const char* begin = fCurrentChar++;
while (*fCurrentChar != 0) {
if (*fCurrentChar == '\\') {
if (*(fCurrentChar++) != 0)
fCurrentChar++;
} else if (*(fCurrentChar++) == *begin)
break;
}
int32 length = fCurrentChar - begin;
fCurrentToken = Token(begin, length, _CurrentPos() - length,
TOKEN_STRING_LITERAL);
} else {
if (!_ParseOperator()) {
int32 type = TOKEN_NONE;
switch (*fCurrentChar) {
case '\n':
type = TOKEN_END_OF_LINE;
break;
case '(':
type = TOKEN_OPENING_PAREN;
break;
case ')':
type = TOKEN_CLOSING_PAREN;
break;
case '[':
type = TOKEN_OPENING_SQUARE_BRACKET;
break;
case ']':
type = TOKEN_CLOSING_SQUARE_BRACKET;
break;
case '{':
type = TOKEN_OPENING_CURLY_BRACE;
break;
case '}':
type = TOKEN_CLOSING_CURLY_BRACE;
break;
case '\\':
type = TOKEN_BACKSLASH;
break;
case ':':
type = TOKEN_COLON;
break;
case ';':
type = TOKEN_SEMICOLON;
break;
case ',':
type = TOKEN_COMMA;
break;
case '.':
type = TOKEN_PERIOD;
break;
case '#':
type = TOKEN_POUND;
break;
default:
throw ParseException("unexpected character",
_CurrentPos());
}
fCurrentToken = Token(fCurrentChar, 1, _CurrentPos(),
type);
fCurrentChar++;
}
}
return fCurrentToken;
}