/*
* C, C++ and doc style comments. Return token or zero for no token.
*/
static int getComment(EcInput *input, EcToken *tp, int c)
{
EcStream *stream;
int form, startLine;
startLine = tp->stream->lineNumber;
stream = input->stream;
form = c;
// TODO - would be great to warn about comment nested in a comment.
for (form = c; c > 0;) {
c = getNextChar(stream);
if (c <= 0) {
/*
* Unterminated Comment
*/
addFormattedStringToToken(tp, "Unterminated comment starting on line %d", startLine);
makeToken(tp, 0, form == '/' ? T_EOF: T_ERR, 0);
return 1;
}
if (form == '/') {
if (c == '\n') {
break;
}
} else {
if (c == '*') {
c = getNextChar(stream);
if (c == '/') {
break;
}
addCharToToken(tp, '*');
putBackChar(stream, c);
} else if (c == '/') {
c = getNextChar(stream);
if (c == '*') {
/*
* Nested comment
*/
if (input->compiler->warnLevel > 0) {
ecReportError(input->compiler, "warning", stream->name, stream->lineNumber, 0,
stream->column, "Possible nested comment");
}
}
addCharToToken(tp, '/');
}
}
addCharToToken(tp, c);
}
return 0;
}
/*
C, C++ and doc style comments. Return token or zero for no token.
*/
static int getComment(EcCompiler *cp, EcToken *tp, int c)
{
EcStream *stream;
int form, startLine;
startLine = cp->stream->loc.lineNumber;
stream = cp->stream;
form = c;
for (form = c; c > 0;) {
c = getNextChar(stream);
if (c <= 0) {
/*
Unterminated Comment
*/
addFormattedStringToToken(tp, "Unterminated comment starting on line %d", startLine);
makeToken(tp, 0, form == '/' ? T_EOF: T_ERR, 0);
return 1;
}
if (form == '/') {
if (c == '\n' || c == '\r') {
break;
}
} else {
if (c == '*') {
c = getNextChar(stream);
if (c == '/') {
break;
}
addCharToToken(tp, '*');
putBackChar(stream, c);
} else if (c == '/') {
c = getNextChar(stream);
if (c == '*') {
/*
Nested comment
*/
if (cp->warnLevel > 0) {
ecError(cp, "Warning", &stream->loc, "Possible nested comment");
}
}
addCharToToken(tp, '/');
}
}
addCharToToken(tp, c);
}
return 0;
}
static int makeAlphaToken(EcCompiler *cp, EcToken *tp, int c)
{
ReservedWord *rp;
EcStream *stream;
/*
We know that c is an alpha already
*/
stream = cp->stream;
while (isalnum((uchar) c) || c == '_' || c == '$' || c == '\\') {
if (c == '\\') {
c = getNextChar(stream);
if (c == '\n' || c == '\r') {
break;
} else if (c == 'u') {
c = decodeNumber(cp, 16, 4);
// TODO - for now, mask back to 8 bits.
c = c & 0xff;
}
}
addCharToToken(tp, c);
c = getNextChar(stream);
}
if (c) {
putBackChar(stream, c);
}
rp = (ReservedWord*) mprLookupKey(cp->keywords, tp->text);
if (rp) {
setTokenID(tp, rp->tokenId, rp->subId, rp->groupMask);
} else {
setTokenID(tp, T_ID, -1, 0);
}
return finalizeToken(tp);
}
static int makeSubToken(EcToken *tp, int c, int tokenId, int subId, int groupMask)
{
if (addCharToToken(tp, c) < 0) {
return T_ERR;
}
return finishToken(tp, tokenId, subId, groupMask);
}
static int makeToken(EcToken *tp, int c, int tokenId, int groupMask)
{
if (c && addCharToToken(tp, c) < 0) {
return T_ERR;
}
return finishToken(tp, tokenId, -1, groupMask);
}
static int makeSubToken(EcToken *tp, int c, int tokenId, int subId, int groupMask)
{
if (addCharToToken(tp, c) < 0) {
return T_ERR;
}
setTokenID(tp, tokenId, subId, groupMask);
return finalizeToken(tp);
}
static int makeQuotedToken(EcCompiler *cp, EcToken *tp, int c)
{
EcStream *stream;
int quoteType;
stream = cp->stream;
quoteType = c;
for (c = getNextChar(stream); c && c != quoteType; c = getNextChar(stream)) {
if (c == 0) {
return makeToken(tp, 0, T_ERR, 0);
}
if (c == '\\') {
c = getNextChar(stream);
switch (c) {
// TBD -- others
case '\\':
break;
case '\'':
case '\"':
break;
case 'b':
c = '\b';
break;
case 'f':
c = '\f';
break;
case 'n':
c = '\n';
break;
case 'r':
c = '\r';
break;
case 't':
c = '\t';
break;
case 'u':
c = decodeNumber(cp, 16, 4);
break;
case 'x':
c = decodeNumber(cp, 16, 2);
break;
case 'v':
c = '\v';
break;
case '0':
c = decodeNumber(cp, 8, 3);
break;
default:
break;
}
}
addCharToToken(tp, c);
}
assert(tp->text);
setTokenID(tp, T_STRING, -1, 0);
return finalizeToken(tp);
}
static int addStringToToken(EcToken *tp, char *str)
{
char *cp;
for (cp = str; *cp; cp++) {
if (addCharToToken(tp, *cp) < 0) {
return MPR_ERR_MEMORY;
}
}
return 0;
}
PUBLIC int ecGetRegExpToken(EcCompiler *cp, wchar *prefix)
{
EcToken *token, *tp;
EcStream *stream;
wchar *pp;
int c;
stream = cp->stream;
tp = token = cp->token;
assert(tp != 0);
initializeToken(tp, stream);
for (pp = prefix; pp && *pp; pp++) {
addCharToToken(tp, *pp);
}
while (1) {
c = getNextChar(stream);
switch (c) {
case -1:
return makeToken(tp, 0, T_ERR, 0);
case 0:
if (stream->flags & EC_STREAM_EOL) {
return makeToken(tp, 0, T_NOP, 0);
}
return makeToken(tp, 0, T_EOF, 0);
case '/':
addCharToToken(tp, '/');
while (1) {
c = getNextChar(stream);
if (c != 'g' && c != 'i' && c != 'm' && c != 'y' && c != 'x' && c != 'X' && c != 'U' && c != 's') {
putBackChar(stream, c);
break;
}
addCharToToken(tp, c);
}
return makeToken(tp, 0, T_REGEXP, 0);
case '\\':
c = getNextChar(stream);
if (c == '\r' || c == '\n' || c == 0) {
ecError(cp, "Warning", &stream->loc, "Illegal newline in regular expression");
return makeToken(tp, 0, T_ERR, 0);
}
addCharToToken(tp, '\\');
addCharToToken(tp, c);
break;
case '\r':
case '\n':
ecError(cp, "Warning", &stream->loc, "Illegal newline in regular expression");
return makeToken(tp, 0, T_ERR, 0);
default:
addCharToToken(tp, c);
}
}
}
// TODO - handle triple quoting
static int getQuotedToken(EcInput *input, EcToken *tp, int c)
{
EcStream *stream;
int quoteType;
stream = input->stream;
quoteType = c;
for (c = getNextChar(stream); c && c != quoteType; c = getNextChar(stream)) {
if (c == 0) {
return makeToken(tp, 0, T_ERR, 0);
}
if (c == '\\') {
c = getNextChar(stream);
switch (c) {
// TBD -- others
case '\\':
break;
case '\'':
case '\"':
break;
case 'b':
c = '\b';
break;
case 'f':
c = '\f';
break;
case 'n':
c = '\n';
break;
case 'r':
c = '\r';
break;
case 't':
c = '\t';
break;
case 'u':
case 'x':
c = decodeNumber(input, 16, 4);
break;
case '0':
c = decodeNumber(input, 8, 3);
break;
default:
break;
}
}
addCharToToken(tp, c);
}
return finishToken(tp, T_STRING, -1, 0);
}
/*
* TODO rationalize with ecParser T_NUMBER code. This could be a lot faster.
*/
static int getNumberToken(EcInput *input, EcToken *tp, int c)
{
EcStream *stream;
int lowc, isHex, isFloat;
isHex = isFloat = 0;
stream = input->stream;
if (c == '0') {
addCharToToken(tp, c);
c = getNextChar(stream);
if (tolower(c) == 'x') {
do {
addCharToToken(tp, c);
c = getNextChar(stream);
} while (isxdigit(c));
putBackChar(stream, c);
return finishToken(tp, T_NUMBER, -1, 0);
}
}
lowc = tolower(c);
while (isdigit(lowc) || lowc == '.' || lowc == 'e' || lowc == 'f') {
if (lowc == '.' || lowc == 'e' || lowc == 'f') {
isFloat++;
}
addCharToToken(tp, c);
c = getNextChar(stream);
lowc = tolower(c);
}
putBackChar(stream, c);
return finishToken(tp, T_NUMBER, -1, 0);
}
static int getAlphaToken(EcInput *input, EcToken *tp, int c)
{
ReservedWord *rp;
EcStream *stream;
/*
* We know that c is an alpha already
*/
// TBD -- does ES4 allow $
stream = input->stream;
while (isalnum(c) || c == '_' || c == '$' || c == '\\') {
if (c == '\\') {
c = getNextChar(stream);
if (c == '\n' || c == '\r') {
break;
} else if (c == 'u') {
c = decodeNumber(input, 16, 4);
}
}
addCharToToken(tp, c);
c = getNextChar(stream);
}
if (c) {
putBackChar(stream, c);
}
// TODO - need to take into account contextually reserved and
// full reserved words.
rp = (ReservedWord*) mprLookupHash(input->lexer->keywords, (char*) tp->text);
if (rp) {
return finishToken(tp, rp->tokenId, rp->subId, rp->groupMask);
} else {
return finishToken(tp, T_ID, -1, 0);
}
}
/*
Hex: 0(x|X)[DIGITS]
Octal: 0[DIGITS]
Float: [DIGITS].[DIGITS][(e|E)[+|-]DIGITS]
*/
static int makeNumberToken(EcCompiler *cp, EcToken *tp, int c)
{
EcStream *stream;
stream = cp->stream;
if (c == '0') {
c = getNextChar(stream);
if (tolower((uchar) c) == 'x') {
/* Hex */
addCharToToken(tp, '0');
do {
addCharToToken(tp, c);
c = getNextChar(stream);
} while (isxdigit(c));
putBackChar(stream, c);
setTokenID(tp, T_NUMBER, -1, 0);
return finalizeToken(tp);
} else if ('0' <= c && c <= '7') {
/* Octal */
addCharToToken(tp, '0');
do {
addCharToToken(tp, c);
c = getNextChar(stream);
} while ('0' <= c && c <= '7');
putBackChar(stream, c);
setTokenID(tp, T_NUMBER, -1, 0);
return finalizeToken(tp);
} else {
putBackChar(stream, c);
c = '0';
}
}
/*
Float
*/
while (isdigit((uchar) c)) {
addCharToToken(tp, c);
c = getNextChar(stream);
}
if (c == '.') {
addCharToToken(tp, c);
c = getNextChar(stream);
}
while (isdigit((uchar) c)) {
addCharToToken(tp, c);
c = getNextChar(stream);
}
if (tolower((uchar) c) == 'e') {
addCharToToken(tp, c);
c = getNextChar(stream);
if (c == '+' || c == '-') {
addCharToToken(tp, c);
c = getNextChar(stream);
}
while (isdigit((uchar) c)) {
addCharToToken(tp, c);
c = getNextChar(stream);
}
}
putBackChar(stream, c);
setTokenID(tp, T_NUMBER, -1, 0);
return finalizeToken(tp);
}
PUBLIC int ecGetToken(EcCompiler *cp)
{
EcToken *tp;
EcStream *stream;
int c;
if ((tp = getLexToken(cp)) == NULL) {
return T_ERR;
}
if (tp->tokenId) {
return tp->tokenId;
}
stream = cp->stream;
while (1) {
c = getNextChar(stream);
/*
Overloadable operators
+ - ~ * / % < > <= >= == << >> >>> & | === != !==
TODO FUTURE, we could allow also: ".", "[", "(" and unary !, ^
*/
switch (c) {
default:
if (isdigit((uchar) c)) {
return makeNumberToken(cp, tp, c);
} else if (c == '\\') {
c = getNextChar(stream);
if (c == '\n') {
break;
}
putBackChar(stream, c);
c = '\n';
}
if (isalpha((uchar) c) || c == '_' || c == '\\' || c == '$') {
return makeAlphaToken(cp, tp, c);
}
return makeToken(tp, 0, T_ERR, 0);
case -1:
return makeToken(tp, 0, T_ERR, 0);
case 0:
if (stream->flags & EC_STREAM_EOL) {
return makeToken(tp, 0, T_NOP, 0);
}
return makeToken(tp, 0, T_EOF, 0);
case ' ':
case '\f':
case '\t':
case '\v':
case 0xA0: /* No break space */
break;
case '\r':
case '\n':
break;
case '"':
case '\'':
return makeQuotedToken(cp, tp, c);
case '#':
return makeToken(tp, c, T_HASH, 0);
case '[':
// EJS extension to consider this an operator
return makeToken(tp, c, T_LBRACKET, G_OPERATOR);
case ']':
return makeToken(tp, c, T_RBRACKET, 0);
case '(':
// EJS extension to consider this an operator
return makeToken(tp, c, T_LPAREN, G_OPERATOR);
case ')':
return makeToken(tp, c, T_RPAREN, 0);
case '{':
return makeToken(tp, c, T_LBRACE, 0);
case '}':
return makeToken(tp, c, T_RBRACE, 0);
case '@':
return makeToken(tp, c, T_AT, 0);
case ';':
return makeToken(tp, c, T_SEMICOLON, 0);
case ',':
return makeToken(tp, c, T_COMMA, 0);
case '?':
return makeToken(tp, c, T_QUERY, 0);
case '~':
return makeToken(tp, c, T_TILDE, G_OPERATOR);
case '+':
c = getNextChar(stream);
if (c == '+') {
addCharToToken(tp, '+');
return makeToken(tp, c, T_PLUS_PLUS, G_OPERATOR);
} else if (c == '=') {
addCharToToken(tp, '+');
return makeSubToken(tp, c, T_ASSIGN, T_PLUS_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
}
putBackChar(stream, c);
return makeToken(tp, '+', T_PLUS, G_OPERATOR);
case '-':
c = getNextChar(stream);
if (isdigit((uchar) c)) {
putBackChar(stream, c);
return makeToken(tp, '-', T_MINUS, G_OPERATOR);
} else if (c == '-') {
addCharToToken(tp, '-');
return makeToken(tp, c, T_MINUS_MINUS, G_OPERATOR);
} else if (c == '=') {
addCharToToken(tp, '-');
return makeSubToken(tp, c, T_ASSIGN, T_MINUS_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
}
putBackChar(stream, c);
return makeToken(tp, '-', T_MINUS, G_OPERATOR);
case '*':
c = getNextChar(stream);
if (c == '=') {
addCharToToken(tp, '*');
return makeSubToken(tp, c, T_ASSIGN, T_MUL_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
}
putBackChar(stream, c);
return makeToken(tp, '*', T_MUL, G_OPERATOR);
case '/':
c = getNextChar(stream);
if (c == '=') {
addCharToToken(tp, '/');
return makeSubToken(tp, c, T_ASSIGN, T_DIV_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
} else if (c == '>') {
addCharToToken(tp, '/');
return makeToken(tp, c, T_SLASH_GT, G_OPERATOR);
} else if (c == '*' || c == '/') {
/*
C and C++ comments
*/
if (getComment(cp, tp, c) < 0) {
return tp->tokenId;
}
/*
Doc comments are: [slash]**. The second "*' becomes the first char of the comment.
Don't regard: [slash]*** (three stars) as a comment.
*/
if (cp->doc) {
if (tp->text && tp->text[0] == '*' && tp->text[1] != '*') {
cp->docToken = mprMemdup(tp->text, tp->length * sizeof(wchar));
}
}
initializeToken(tp, stream);
break;
}
putBackChar(stream, c);
return makeToken(tp, '/', T_DIV, G_OPERATOR);
case '%':
c = getNextChar(stream);
if (c == '=') {
addCharToToken(tp, '%');
return makeSubToken(tp, c, T_ASSIGN, T_MOD_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
}
putBackChar(stream, c);
return makeToken(tp, '%', T_MOD, G_OPERATOR);
case '.':
c = getNextChar(stream);
if (c == '.') {
c = getNextChar(stream);
if (c == '.') {
addStringToToken(tp, "..");
return makeToken(tp, c, T_ELIPSIS, 0);
}
putBackChar(stream, c);
addCharToToken(tp, '.');
return makeToken(tp, '.', T_DOT_DOT, 0);
#if FUTURE
} else if (c == '<') {
addCharToToken(tp, '.');
return makeToken(tp, c, T_DOT_LESS, 0);
#endif
} else if (isdigit((uchar) c)) {
putBackChar(stream, c);
return makeNumberToken(cp, tp, '.');
}
putBackChar(stream, c);
// EJS extension to consider this an operator
return makeToken(tp, '.', T_DOT, G_OPERATOR);
case ':':
c = getNextChar(stream);
if (c == ':') {
addCharToToken(tp, ':');
return makeToken(tp, c, T_COLON_COLON, 0);
}
putBackChar(stream, c);
return makeToken(tp, ':', T_COLON, 0);
case '!':
c = getNextChar(stream);
if (c == '=') {
c = getNextChar(stream);
if (c == '=') {
addStringToToken(tp, "!=");
return makeToken(tp, c, T_STRICT_NE, G_OPERATOR);
}
putBackChar(stream, c);
addCharToToken(tp, '!');
return makeToken(tp, '=', T_NE, G_OPERATOR);
}
putBackChar(stream, c);
return makeToken(tp, '!', T_LOGICAL_NOT, G_OPERATOR);
case '&':
c = getNextChar(stream);
if (c == '&') {
addCharToToken(tp, '&');
c = getNextChar(stream);
if (c == '=') {
addCharToToken(tp, '&');
return makeSubToken(tp, '=', T_ASSIGN, T_LOGICAL_AND_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
}
putBackChar(stream, c);
return makeToken(tp, '&', T_LOGICAL_AND, G_OPERATOR);
} else if (c == '=') {
addCharToToken(tp, '&');
return makeSubToken(tp, c, T_ASSIGN, T_BIT_AND_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
}
putBackChar(stream, c);
return makeToken(tp, '&', T_BIT_AND, G_OPERATOR);
case '<':
c = getNextChar(stream);
if (c == '=') {
addCharToToken(tp, '<');
return makeToken(tp, c, T_LE, G_OPERATOR);
} else if (c == '<') {
c = getNextChar(stream);
if (c == '=') {
addStringToToken(tp, "<<");
return makeSubToken(tp, c, T_ASSIGN, T_LSH_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
}
putBackChar(stream, c);
addCharToToken(tp, '<');
return makeToken(tp, c, T_LSH, G_OPERATOR);
} else if (c == '/') {
addCharToToken(tp, '<');
return makeToken(tp, c, T_LT_SLASH, 0);
}
putBackChar(stream, c);
return makeToken(tp, '<', T_LT, G_OPERATOR);
case '=':
c = getNextChar(stream);
if (c == '=') {
c = getNextChar(stream);
if (c == '=') {
addStringToToken(tp, "==");
return makeToken(tp, c, T_STRICT_EQ, G_OPERATOR);
}
putBackChar(stream, c);
addCharToToken(tp, '=');
return makeToken(tp, c, T_EQ, G_OPERATOR);
}
putBackChar(stream, c);
return makeToken(tp, '=', T_ASSIGN, G_OPERATOR);
case '>':
c = getNextChar(stream);
if (c == '=') {
addCharToToken(tp, '<');
return makeToken(tp, c, T_GE, G_OPERATOR);
} else if (c == '>') {
c = getNextChar(stream);
if (c == '=') {
addStringToToken(tp, ">>");
return makeSubToken(tp, c, T_ASSIGN, T_RSH_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
} else if (c == '>') {
c = getNextChar(stream);
if (c == '=') {
addStringToToken(tp, ">>>");
return makeSubToken(tp, c, T_ASSIGN, T_RSH_ZERO_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
}
putBackChar(stream, c);
addStringToToken(tp, ">>");
return makeToken(tp, '>', T_RSH_ZERO, G_OPERATOR);
}
putBackChar(stream, c);
addCharToToken(tp, '>');
return makeToken(tp, '>', T_RSH, G_OPERATOR);
}
putBackChar(stream, c);
return makeToken(tp, '>', T_GT, G_OPERATOR);
case '^':
c = getNextChar(stream);
if (c == '^') {
addCharToToken(tp, '^');
c = getNextChar(stream);
if (c == '=') {
addCharToToken(tp, '^');
return makeSubToken(tp, '=', T_ASSIGN, T_LOGICAL_XOR_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
}
putBackChar(stream, c);
return makeToken(tp, '^', T_LOGICAL_XOR, G_OPERATOR);
} else if (c == '=') {
addCharToToken(tp, '^');
return makeSubToken(tp, '=', T_ASSIGN, T_BIT_XOR_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
}
putBackChar(stream, c);
return makeToken(tp, '^', T_BIT_XOR, G_OPERATOR);
case '|':
c = getNextChar(stream);
if (c == '|') {
addCharToToken(tp, '|');
c = getNextChar(stream);
if (c == '=') {
addCharToToken(tp, '|');
return makeSubToken(tp, '=', T_ASSIGN, T_LOGICAL_OR_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
}
putBackChar(stream, c);
return makeToken(tp, '|', T_LOGICAL_OR, G_OPERATOR);
} else if (c == '=') {
addCharToToken(tp, '|');
return makeSubToken(tp, '=', T_ASSIGN, T_BIT_OR_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
}
putBackChar(stream, c);
return makeToken(tp, '|', T_BIT_OR, G_OPERATOR);
}
}
}
int ecGetRegExpToken(EcInput *input)
{
EcToken *token, *tp;
EcStream *stream;
int c;
stream = input->stream;
tp = token = input->token;
mprAssert(tp != 0);
initializeToken(tp, stream);
addCharToToken(tp, '/');
while (1) {
c = getNextChar(stream);
switch (c) {
case -1:
return makeToken(tp, 0, T_ERR, 0);
case 0:
if (stream->flags & EC_STREAM_EOL) {
return makeToken(tp, 0, T_NOP, 0);
}
return makeToken(tp, 0, T_EOF, 0);
case '/':
addCharToToken(tp, '/');
while (1) {
c = getNextChar(stream);
if (c != 'g' && c != 'i' && c != 'm' && c != 'y') {
putBackChar(stream, c);
break;
}
addCharToToken(tp, c);
}
return makeToken(tp, 0, T_REGEXP, 0);
case '\\':
c = getNextChar(stream);
if (c == '\r' || c == '\n' || c == 0) {
ecReportError(input->compiler, "warning", stream->name, stream->lineNumber, 0, stream->column,
"Illegal newline in regular expression");
return makeToken(tp, 0, T_ERR, 0);
}
addCharToToken(tp, '\\');
addCharToToken(tp, c);
break;
case '\r':
case '\n':
ecReportError(input->compiler, "warning", stream->name, stream->lineNumber, 0, stream->column,
"Illegal newline in regular expression");
return makeToken(tp, 0, T_ERR, 0);
default:
addCharToToken(tp, c);
}
}
}
int ecGetToken(EcInput *input)
{
EcToken *token, *tp;
EcStream *stream;
int c;
// TODO - functionalize this section
token = input->token;
if ((tp = input->putBack) != 0) {
input->putBack = tp->next;
input->token = tp;
/*
* Move any old token to free list
*/
if (token) {
token->next = input->freeTokens;
input->freeTokens = token;
}
return tp->tokenId;
}
if (token == 0) {
// TBD -- need an API for this
input->token = mprAllocObjZeroed(input, EcToken);
if (input->token == 0) {
// TBD -- err code
return -1;
}
input->token->lineNumber = 1;
}
stream = input->stream;
tp = input->token;
mprAssert(tp);
initializeToken(tp, stream);
while (1) {
c = getNextChar(stream);
/*
* Overloadable operators
*
* + - ~ * / % < > <= >= == << >> >>> & | === != !==
*
* TODO FUTURE, we could allow also: ".", "[", "("
*
* TODO: what about unary !, ^
*/
switch (c) {
default:
number:
if (isdigit(c)) {
return getNumberToken(input, tp, c);
} else if (c == '\\') {
c = getNextChar(stream);
if (c == '\n') {
break;
}
putBackChar(stream, c);
c = '\n';
}
if (isalpha(c) || c == '_' || c == '\\' || c == '$') {
return getAlphaToken(input, tp, c);
}
return makeToken(tp, 0, T_ERR, 0);
case -1:
return makeToken(tp, 0, T_ERR, 0);
case 0:
if (stream->flags & EC_STREAM_EOL) {
return makeToken(tp, 0, T_NOP, 0);
} else {
return makeToken(tp, 0, T_EOF, 0);
}
case ' ':
case '\t':
break;
case '\r':
case '\n':
if (tp->textLen == 0 && tp->lineNumber != stream->lineNumber) {
tp->currentLine = 0;
}
break;
case '"':
case '\'':
return getQuotedToken(input, tp, c);
case '#':
return makeToken(tp, c, T_HASH, 0);
case '[':
// EJS extension to consider this an operator
return makeToken(tp, c, T_LBRACKET, G_OPERATOR);
case ']':
return makeToken(tp, c, T_RBRACKET, 0);
case '(':
// EJS extension to consider this an operator
return makeToken(tp, c, T_LPAREN, G_OPERATOR);
case ')':
return makeToken(tp, c, T_RPAREN, 0);
case '{':
return makeToken(tp, c, T_LBRACE, 0);
case '}':
return makeToken(tp, c, T_RBRACE, 0);
case '@':
return makeToken(tp, c, T_AT, 0);
case ';':
return makeToken(tp, c, T_SEMICOLON, 0);
case ',':
return makeToken(tp, c, T_COMMA, 0);
case '?':
return makeToken(tp, c, T_QUERY, 0);
case '~':
return makeToken(tp, c, T_TILDE, G_OPERATOR);
case '+':
c = getNextChar(stream);
if (c == '+') {
addCharToToken(tp, '+');
return makeToken(tp, c, T_PLUS_PLUS, G_OPERATOR);
} else if (c == '=') {
addCharToToken(tp, '+');
return makeSubToken(tp, c, T_ASSIGN, T_PLUS_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
}
putBackChar(stream, c);
return makeToken(tp, '+', T_PLUS, G_OPERATOR);
case '-':
c = getNextChar(stream);
if (isdigit(c)) {
putBackChar(stream, c);
return makeToken(tp, '-', T_MINUS, G_OPERATOR);
} else if (c == '-') {
addCharToToken(tp, '-');
return makeToken(tp, c, T_MINUS_MINUS, G_OPERATOR);
} else if (c == '=') {
addCharToToken(tp, '-');
return makeSubToken(tp, c, T_ASSIGN, T_MINUS_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
}
putBackChar(stream, c);
return makeToken(tp, '-', T_MINUS, G_OPERATOR);
case '*':
c = getNextChar(stream);
if (c == '=') {
addCharToToken(tp, '*');
return makeSubToken(tp, c, T_ASSIGN, T_MUL_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
}
putBackChar(stream, c);
return makeToken(tp, '*', T_MUL, G_OPERATOR);
case '/':
c = getNextChar(stream);
if (c == '=') {
addCharToToken(tp, '/');
return makeSubToken(tp, c, T_ASSIGN, T_DIV_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
} else if (c == '>') {
addCharToToken(tp, '/');
return makeToken(tp, c, T_SLASH_GT, G_OPERATOR);
} else if (c == '*' || c == '/') {
/*
* C and C++ comments
*/
if (getComment(input, tp, c) < 0) {
return tp->tokenId;
}
#if BLD_FEATURE_EJS_DOC
if (tp->text && tp->text[0] == '*') {
mprFree(input->doc);
input->doc = mprStrdup(input, (char*) tp->text);
}
#endif
initializeToken(tp, stream);
break;
}
putBackChar(stream, c);
return makeToken(tp, '/', T_DIV, G_OPERATOR);
case '%':
c = getNextChar(stream);
if (c == '=') {
addCharToToken(tp, '%');
return makeSubToken(tp, c, T_ASSIGN, T_MOD_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
}
putBackChar(stream, c);
return makeToken(tp, '%', T_MOD, G_OPERATOR);
case '.':
c = getNextChar(stream);
if (c == '.') {
c = getNextChar(stream);
if (c == '.') {
addStringToToken(tp, "..");
return makeToken(tp, c, T_ELIPSIS, 0);
}
putBackChar(stream, c);
addCharToToken(tp, '.');
return makeToken(tp, '.', T_DOT_DOT, 0);
} else if (c == '<') {
addCharToToken(tp, '.');
return makeToken(tp, c, T_DOT_LESS, 0);
} else if (isdigit(c)) {
putBackChar(stream, c);
goto number;
}
putBackChar(stream, c);
// EJS extension to consider this an operator
return makeToken(tp, '.', T_DOT, G_OPERATOR);
case ':':
c = getNextChar(stream);
if (c == ':') {
addCharToToken(tp, ':');
return makeToken(tp, c, T_COLON_COLON, 0);
}
putBackChar(stream, c);
return makeToken(tp, ':', T_COLON, 0);
case '!':
c = getNextChar(stream);
if (c == '=') {
c = getNextChar(stream);
if (c == '=') {
addStringToToken(tp, "!=");
return makeToken(tp, c, T_STRICT_NE, G_OPERATOR);
}
putBackChar(stream, c);
addCharToToken(tp, '!');
return makeToken(tp, '=', T_NE, G_OPERATOR);
}
putBackChar(stream, c);
return makeToken(tp, '!', T_LOGICAL_NOT, G_OPERATOR);
#if UNUSED
case '~':
c = getNextChar(stream);
if (c == '=') {
addStringToToken(tp, "~=");
return makeSubToken(tp, c, T_ASSIGN, T_BIT_NEG_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
}
putBackChar(stream, c);
return makeToken(tp, '~', T_BIT_NEG, G_OPERATOR);
#endif
case '&':
c = getNextChar(stream);
if (c == '&') {
addCharToToken(tp, '&');
c = getNextChar(stream);
if (c == '=') {
addCharToToken(tp, '&');
return makeSubToken(tp, '=', T_ASSIGN, T_LOGICAL_AND_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
}
putBackChar(stream, c);
return makeToken(tp, '&', T_LOGICAL_AND, G_OPERATOR);
} else if (c == '=') {
addCharToToken(tp, '&');
return makeSubToken(tp, c, T_ASSIGN, T_BIT_AND_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
}
putBackChar(stream, c);
return makeToken(tp, '&', T_BIT_AND, G_OPERATOR);
case '<':
c = getNextChar(stream);
if (c == '=') {
addCharToToken(tp, '<');
return makeToken(tp, c, T_LE, G_OPERATOR);
} else if (c == '<') {
c = getNextChar(stream);
if (c == '=') {
addStringToToken(tp, "<<");
return makeSubToken(tp, c, T_ASSIGN, T_LSH_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
}
putBackChar(stream, c);
addCharToToken(tp, '<');
return makeToken(tp, c, T_LSH, G_OPERATOR);
} else if (c == '/') {
addCharToToken(tp, '<');
return makeToken(tp, c, T_LT_SLASH, 0);
}
putBackChar(stream, c);
return makeToken(tp, '<', T_LT, G_OPERATOR);
case '=':
c = getNextChar(stream);
if (c == '=') {
c = getNextChar(stream);
if (c == '=') {
addStringToToken(tp, "==");
return makeToken(tp, c, T_STRICT_EQ, G_OPERATOR);
}
putBackChar(stream, c);
addCharToToken(tp, '=');
return makeToken(tp, c, T_EQ, G_OPERATOR);
}
putBackChar(stream, c);
return makeToken(tp, '=', T_ASSIGN, G_OPERATOR);
case '>':
c = getNextChar(stream);
if (c == '=') {
addCharToToken(tp, '<');
return makeToken(tp, c, T_GE, G_OPERATOR);
} else if (c == '>') {
c = getNextChar(stream);
if (c == '=') {
addStringToToken(tp, ">>");
return makeSubToken(tp, c, T_ASSIGN, T_RSH_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
} else if (c == '>') {
c = getNextChar(stream);
if (c == '=') {
addStringToToken(tp, ">>>");
return makeSubToken(tp, c, T_ASSIGN, T_RSH_ZERO_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
}
putBackChar(stream, c);
addStringToToken(tp, ">>");
return makeToken(tp, '>', T_RSH_ZERO, G_OPERATOR);
}
putBackChar(stream, c);
addCharToToken(tp, '>');
return makeToken(tp, '>', T_RSH, G_OPERATOR);
}
putBackChar(stream, c);
return makeToken(tp, '>', T_GT, G_OPERATOR);
case '^':
c = getNextChar(stream);
if (c == '^') {
addCharToToken(tp, '^');
c = getNextChar(stream);
if (c == '=') {
addCharToToken(tp, '^');
return makeSubToken(tp, '=', T_ASSIGN, T_LOGICAL_XOR_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
}
putBackChar(stream, c);
return makeToken(tp, '^', T_LOGICAL_XOR, G_OPERATOR);
} else if (c == '=') {
addCharToToken(tp, '^');
return makeSubToken(tp, '=', T_ASSIGN, T_BIT_XOR_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
}
putBackChar(stream, c);
return makeToken(tp, '^', T_BIT_XOR, G_OPERATOR);
case '|':
c = getNextChar(stream);
if (c == '|') {
addCharToToken(tp, '|');
c = getNextChar(stream);
if (c == '=') {
addCharToToken(tp, '|');
return makeSubToken(tp, '=', T_ASSIGN, T_LOGICAL_OR_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
}
putBackChar(stream, c);
return makeToken(tp, '|', T_LOGICAL_OR, G_OPERATOR);
} else if (c == '=') {
addCharToToken(tp, '|');
return makeSubToken(tp, '=', T_ASSIGN, T_BIT_OR_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
}
putBackChar(stream, c);
return makeToken(tp, '|', T_BIT_OR, G_OPERATOR);
}
}
}