const Token& _UnquotedString()
{
const char* begin = fCurrentChar;
while (*fCurrentChar != 0 && !_IsUnquotedDelimitingChar(*fCurrentChar))
fCurrentChar++;
int32 length = fCurrentChar - begin;
fCurrentToken.SetTo(begin, length, _CurrentPos() - length,
TOKEN_UNKNOWN);
return fCurrentToken;
}
const Token& _NextTokenCommand()
{
switch (*fCurrentChar) {
case '(':
case ')':
case '[':
case ']':
case '|':
case ';':
fCurrentToken.SetTo(fCurrentChar, 1, _CurrentPos(),
*fCurrentChar);
fCurrentChar++;
return fCurrentToken;
case '"':
return _QuotedString();
default:
return _UnquotedString();
}
}
const Token& NextToken()
{
if (fCurrentToken.type == TOKEN_END_OF_LINE)
return fCurrentToken;
if (fReuseToken) {
fReuseToken = false;
return fCurrentToken;
}
while (*fCurrentChar != 0 && isspace(*fCurrentChar))
fCurrentChar++;
if (*fCurrentChar == 0) {
fCurrentToken.SetTo("", 0, _CurrentPos(), TOKEN_END_OF_LINE);
return fCurrentToken;
}
return (fCommandMode ? _NextTokenCommand() : _NextTokenExpression());
}
const Token& NextToken()
{
if (fCurrentToken.type == TOKEN_END_OF_LINE)
return fCurrentToken;
if (fReuseToken) {
fReuseToken = false;
//printf("next token (recycled): '%s'\n", fCurrentToken.string.String());
return fCurrentToken;
}
while (*fCurrentChar != 0 && isspace(*fCurrentChar))
fCurrentChar++;
if (*fCurrentChar == 0)
return fCurrentToken = Token("", 0, _CurrentPos(), TOKEN_END_OF_LINE);
bool decimal = *fCurrentChar == '.' || *fCurrentChar == ',';
if (decimal || isdigit(*fCurrentChar)) {
if (fHexSupport && *fCurrentChar == '0' && fCurrentChar[1] == 'x')
return _ParseHexNumber();
BString temp;
const char* begin = fCurrentChar;
// optional digits before the comma
while (isdigit(*fCurrentChar)) {
temp << *fCurrentChar;
fCurrentChar++;
}
// optional post comma part
// (required if there are no digits before the comma)
if (*fCurrentChar == '.' || *fCurrentChar == ',') {
temp << '.';
fCurrentChar++;
// optional post comma digits
while (isdigit(*fCurrentChar)) {
temp << *fCurrentChar;
fCurrentChar++;
}
}
// optional exponent part
if (*fCurrentChar == 'e' || *fCurrentChar == 'E') {
temp << *fCurrentChar;
fCurrentChar++;
// optional exponent sign
if (*fCurrentChar == '+' || *fCurrentChar == '-') {
temp << *fCurrentChar;
fCurrentChar++;
}
// required exponent digits
if (!isdigit(*fCurrentChar)) {
throw ParseException("missing exponent in constant",
fCurrentChar - begin);
}
while (isdigit(*fCurrentChar)) {
temp << *fCurrentChar;
fCurrentChar++;
}
}
int32 length = fCurrentChar - begin;
BString test = temp;
test << "&_";
double value;
char t[2];
int32 matches = sscanf(test.String(), "%lf&%s", &value, t);
if (matches != 2) {
throw ParseException("error in constant",
_CurrentPos() - length);
}
fCurrentToken = Token(begin, length, _CurrentPos() - length,
TOKEN_CONSTANT);
fCurrentToken.value = temp.String();
} else if (isalpha(*fCurrentChar) && *fCurrentChar != 'x') {
const char* begin = fCurrentChar;
while (*fCurrentChar != 0 && (isalpha(*fCurrentChar)
|| isdigit(*fCurrentChar))) {
fCurrentChar++;
}
int32 length = fCurrentChar - begin;
fCurrentToken = Token(begin, length, _CurrentPos() - length,
TOKEN_IDENTIFIER);
} else {
int32 type = TOKEN_NONE;
switch (*fCurrentChar) {
case '+':
type = TOKEN_PLUS;
break;
case '-':
type = TOKEN_MINUS;
break;
case '*':
type = TOKEN_STAR;
break;
case '/':
case '\\':
case ':':
type = TOKEN_SLASH;
break;
case '%':
type = TOKEN_MODULO;
break;
case '^':
type = TOKEN_POWER;
break;
case '(':
type = TOKEN_OPENING_BRACKET;
break;
case ')':
type = TOKEN_CLOSING_BRACKET;
break;
case '&':
type = TOKEN_AND;
break;
case '|':
type = TOKEN_OR;
break;
case '~':
type = TOKEN_NOT;
break;
case '\n':
type = TOKEN_END_OF_LINE;
break;
case 'x':
if (!fHexSupport) {
type = TOKEN_STAR;
break;
}
// fall through
default:
throw ParseException("unexpected character", _CurrentPos());
}
fCurrentToken = Token(fCurrentChar, 1, _CurrentPos(), type);
fCurrentChar++;
}
//printf("next token: '%s'\n", fCurrentToken.string.String());
return fCurrentToken;
}
const Token& _NextTokenExpression()
{
if (isdigit(*fCurrentChar)) {
// number
const char* begin = fCurrentChar++;
if (*fCurrentChar == 'x') {
// hex number
fCurrentChar++;
while (*fCurrentChar != 0
&& (isdigit(*fCurrentChar)
|| strchr("abcdefABCDEF", *fCurrentChar))) {
fCurrentChar++;
}
if (fCurrentChar - begin == 2)
parse_exception("invalid hex number", begin - fString);
} else {
// decimal number
while (*fCurrentChar != 0 && isdigit(*fCurrentChar))
fCurrentChar++;
}
int32 length = fCurrentChar - begin;
fCurrentToken.SetTo(begin, length, _CurrentPos() - length,
TOKEN_CONSTANT);
fCurrentToken.value = strtoull(fCurrentToken.string, NULL, 0);
} else if (isalpha(*fCurrentChar) || *fCurrentChar == '_'
|| *fCurrentChar == '$' || *fCurrentChar == '@') {
// identifier
const char* begin = fCurrentChar;
fCurrentChar++;
while (*fCurrentChar != 0
&& (isalpha(*fCurrentChar) || *fCurrentChar == '_'
|| isdigit(*fCurrentChar))) {
fCurrentChar++;
}
int32 length = fCurrentChar - begin;
fCurrentToken.SetTo(begin, length, _CurrentPos() - length,
TOKEN_IDENTIFIER);
} else {
const char* begin = fCurrentChar;
char c = *fCurrentChar;
fCurrentChar++;
int32 flags = 0;
switch (c) {
case '=':
fCurrentChar--;
case '+':
case '-':
case '*':
case '/':
case '%':
if (*fCurrentChar == '=') {
fCurrentChar++;
flags = TOKEN_ASSIGN_FLAG;
}
case '(':
case ')':
case '[':
case ']':
case '{':
case '}':
case ';':
{
int32 length = fCurrentChar - begin;
fCurrentToken.SetTo(begin, length, _CurrentPos() - length,
c | flags);
break;
}
case '"':
{
fCurrentChar--;
_QuotedString();
break;
}
default:
{
fCurrentChar--;
_UnquotedString();
break;
}
}
}
return fCurrentToken;
}