/* * C, C++ and doc style comments. Return token or zero for no token. */ static int getComment(EcInput *input, EcToken *tp, int c) { EcStream *stream; int form, startLine; startLine = tp->stream->lineNumber; stream = input->stream; form = c; // TODO - would be great to warn about comment nested in a comment. for (form = c; c > 0;) { c = getNextChar(stream); if (c <= 0) { /* * Unterminated Comment */ addFormattedStringToToken(tp, "Unterminated comment starting on line %d", startLine); makeToken(tp, 0, form == '/' ? T_EOF: T_ERR, 0); return 1; } if (form == '/') { if (c == '\n') { break; } } else { if (c == '*') { c = getNextChar(stream); if (c == '/') { break; } addCharToToken(tp, '*'); putBackChar(stream, c); } else if (c == '/') { c = getNextChar(stream); if (c == '*') { /* * Nested comment */ if (input->compiler->warnLevel > 0) { ecReportError(input->compiler, "warning", stream->name, stream->lineNumber, 0, stream->column, "Possible nested comment"); } } addCharToToken(tp, '/'); } } addCharToToken(tp, c); } return 0; }
/* C, C++ and doc style comments. Return token or zero for no token. */ static int getComment(EcCompiler *cp, EcToken *tp, int c) { EcStream *stream; int form, startLine; startLine = cp->stream->loc.lineNumber; stream = cp->stream; form = c; for (form = c; c > 0;) { c = getNextChar(stream); if (c <= 0) { /* Unterminated Comment */ addFormattedStringToToken(tp, "Unterminated comment starting on line %d", startLine); makeToken(tp, 0, form == '/' ? T_EOF: T_ERR, 0); return 1; } if (form == '/') { if (c == '\n' || c == '\r') { break; } } else { if (c == '*') { c = getNextChar(stream); if (c == '/') { break; } addCharToToken(tp, '*'); putBackChar(stream, c); } else if (c == '/') { c = getNextChar(stream); if (c == '*') { /* Nested comment */ if (cp->warnLevel > 0) { ecError(cp, "Warning", &stream->loc, "Possible nested comment"); } } addCharToToken(tp, '/'); } } addCharToToken(tp, c); } return 0; }
static int makeAlphaToken(EcCompiler *cp, EcToken *tp, int c) { ReservedWord *rp; EcStream *stream; /* We know that c is an alpha already */ stream = cp->stream; while (isalnum((uchar) c) || c == '_' || c == '$' || c == '\\') { if (c == '\\') { c = getNextChar(stream); if (c == '\n' || c == '\r') { break; } else if (c == 'u') { c = decodeNumber(cp, 16, 4); // TODO - for now, mask back to 8 bits. c = c & 0xff; } } addCharToToken(tp, c); c = getNextChar(stream); } if (c) { putBackChar(stream, c); } rp = (ReservedWord*) mprLookupKey(cp->keywords, tp->text); if (rp) { setTokenID(tp, rp->tokenId, rp->subId, rp->groupMask); } else { setTokenID(tp, T_ID, -1, 0); } return finalizeToken(tp); }
static int makeSubToken(EcToken *tp, int c, int tokenId, int subId, int groupMask) { if (addCharToToken(tp, c) < 0) { return T_ERR; } return finishToken(tp, tokenId, subId, groupMask); }
static int makeToken(EcToken *tp, int c, int tokenId, int groupMask) { if (c && addCharToToken(tp, c) < 0) { return T_ERR; } return finishToken(tp, tokenId, -1, groupMask); }
static int makeSubToken(EcToken *tp, int c, int tokenId, int subId, int groupMask) { if (addCharToToken(tp, c) < 0) { return T_ERR; } setTokenID(tp, tokenId, subId, groupMask); return finalizeToken(tp); }
static int makeQuotedToken(EcCompiler *cp, EcToken *tp, int c) { EcStream *stream; int quoteType; stream = cp->stream; quoteType = c; for (c = getNextChar(stream); c && c != quoteType; c = getNextChar(stream)) { if (c == 0) { return makeToken(tp, 0, T_ERR, 0); } if (c == '\\') { c = getNextChar(stream); switch (c) { // TBD -- others case '\\': break; case '\'': case '\"': break; case 'b': c = '\b'; break; case 'f': c = '\f'; break; case 'n': c = '\n'; break; case 'r': c = '\r'; break; case 't': c = '\t'; break; case 'u': c = decodeNumber(cp, 16, 4); break; case 'x': c = decodeNumber(cp, 16, 2); break; case 'v': c = '\v'; break; case '0': c = decodeNumber(cp, 8, 3); break; default: break; } } addCharToToken(tp, c); } assert(tp->text); setTokenID(tp, T_STRING, -1, 0); return finalizeToken(tp); }
static int addStringToToken(EcToken *tp, char *str) { char *cp; for (cp = str; *cp; cp++) { if (addCharToToken(tp, *cp) < 0) { return MPR_ERR_MEMORY; } } return 0; }
PUBLIC int ecGetRegExpToken(EcCompiler *cp, wchar *prefix) { EcToken *token, *tp; EcStream *stream; wchar *pp; int c; stream = cp->stream; tp = token = cp->token; assert(tp != 0); initializeToken(tp, stream); for (pp = prefix; pp && *pp; pp++) { addCharToToken(tp, *pp); } while (1) { c = getNextChar(stream); switch (c) { case -1: return makeToken(tp, 0, T_ERR, 0); case 0: if (stream->flags & EC_STREAM_EOL) { return makeToken(tp, 0, T_NOP, 0); } return makeToken(tp, 0, T_EOF, 0); case '/': addCharToToken(tp, '/'); while (1) { c = getNextChar(stream); if (c != 'g' && c != 'i' && c != 'm' && c != 'y' && c != 'x' && c != 'X' && c != 'U' && c != 's') { putBackChar(stream, c); break; } addCharToToken(tp, c); } return makeToken(tp, 0, T_REGEXP, 0); case '\\': c = getNextChar(stream); if (c == '\r' || c == '\n' || c == 0) { ecError(cp, "Warning", &stream->loc, "Illegal newline in regular expression"); return makeToken(tp, 0, T_ERR, 0); } addCharToToken(tp, '\\'); addCharToToken(tp, c); break; case '\r': case '\n': ecError(cp, "Warning", &stream->loc, "Illegal newline in regular expression"); return makeToken(tp, 0, T_ERR, 0); default: addCharToToken(tp, c); } } }
// TODO - handle triple quoting static int getQuotedToken(EcInput *input, EcToken *tp, int c) { EcStream *stream; int quoteType; stream = input->stream; quoteType = c; for (c = getNextChar(stream); c && c != quoteType; c = getNextChar(stream)) { if (c == 0) { return makeToken(tp, 0, T_ERR, 0); } if (c == '\\') { c = getNextChar(stream); switch (c) { // TBD -- others case '\\': break; case '\'': case '\"': break; case 'b': c = '\b'; break; case 'f': c = '\f'; break; case 'n': c = '\n'; break; case 'r': c = '\r'; break; case 't': c = '\t'; break; case 'u': case 'x': c = decodeNumber(input, 16, 4); break; case '0': c = decodeNumber(input, 8, 3); break; default: break; } } addCharToToken(tp, c); } return finishToken(tp, T_STRING, -1, 0); }
/* * TODO rationalize with ecParser T_NUMBER code. This could be a lot faster. */ static int getNumberToken(EcInput *input, EcToken *tp, int c) { EcStream *stream; int lowc, isHex, isFloat; isHex = isFloat = 0; stream = input->stream; if (c == '0') { addCharToToken(tp, c); c = getNextChar(stream); if (tolower(c) == 'x') { do { addCharToToken(tp, c); c = getNextChar(stream); } while (isxdigit(c)); putBackChar(stream, c); return finishToken(tp, T_NUMBER, -1, 0); } } lowc = tolower(c); while (isdigit(lowc) || lowc == '.' || lowc == 'e' || lowc == 'f') { if (lowc == '.' || lowc == 'e' || lowc == 'f') { isFloat++; } addCharToToken(tp, c); c = getNextChar(stream); lowc = tolower(c); } putBackChar(stream, c); return finishToken(tp, T_NUMBER, -1, 0); }
static int getAlphaToken(EcInput *input, EcToken *tp, int c) { ReservedWord *rp; EcStream *stream; /* * We know that c is an alpha already */ // TBD -- does ES4 allow $ stream = input->stream; while (isalnum(c) || c == '_' || c == '$' || c == '\\') { if (c == '\\') { c = getNextChar(stream); if (c == '\n' || c == '\r') { break; } else if (c == 'u') { c = decodeNumber(input, 16, 4); } } addCharToToken(tp, c); c = getNextChar(stream); } if (c) { putBackChar(stream, c); } // TODO - need to take into account contextually reserved and // full reserved words. rp = (ReservedWord*) mprLookupHash(input->lexer->keywords, (char*) tp->text); if (rp) { return finishToken(tp, rp->tokenId, rp->subId, rp->groupMask); } else { return finishToken(tp, T_ID, -1, 0); } }
/* Hex: 0(x|X)[DIGITS] Octal: 0[DIGITS] Float: [DIGITS].[DIGITS][(e|E)[+|-]DIGITS] */ static int makeNumberToken(EcCompiler *cp, EcToken *tp, int c) { EcStream *stream; stream = cp->stream; if (c == '0') { c = getNextChar(stream); if (tolower((uchar) c) == 'x') { /* Hex */ addCharToToken(tp, '0'); do { addCharToToken(tp, c); c = getNextChar(stream); } while (isxdigit(c)); putBackChar(stream, c); setTokenID(tp, T_NUMBER, -1, 0); return finalizeToken(tp); } else if ('0' <= c && c <= '7') { /* Octal */ addCharToToken(tp, '0'); do { addCharToToken(tp, c); c = getNextChar(stream); } while ('0' <= c && c <= '7'); putBackChar(stream, c); setTokenID(tp, T_NUMBER, -1, 0); return finalizeToken(tp); } else { putBackChar(stream, c); c = '0'; } } /* Float */ while (isdigit((uchar) c)) { addCharToToken(tp, c); c = getNextChar(stream); } if (c == '.') { addCharToToken(tp, c); c = getNextChar(stream); } while (isdigit((uchar) c)) { addCharToToken(tp, c); c = getNextChar(stream); } if (tolower((uchar) c) == 'e') { addCharToToken(tp, c); c = getNextChar(stream); if (c == '+' || c == '-') { addCharToToken(tp, c); c = getNextChar(stream); } while (isdigit((uchar) c)) { addCharToToken(tp, c); c = getNextChar(stream); } } putBackChar(stream, c); setTokenID(tp, T_NUMBER, -1, 0); return finalizeToken(tp); }
PUBLIC int ecGetToken(EcCompiler *cp) { EcToken *tp; EcStream *stream; int c; if ((tp = getLexToken(cp)) == NULL) { return T_ERR; } if (tp->tokenId) { return tp->tokenId; } stream = cp->stream; while (1) { c = getNextChar(stream); /* Overloadable operators + - ~ * / % < > <= >= == << >> >>> & | === != !== TODO FUTURE, we could allow also: ".", "[", "(" and unary !, ^ */ switch (c) { default: if (isdigit((uchar) c)) { return makeNumberToken(cp, tp, c); } else if (c == '\\') { c = getNextChar(stream); if (c == '\n') { break; } putBackChar(stream, c); c = '\n'; } if (isalpha((uchar) c) || c == '_' || c == '\\' || c == '$') { return makeAlphaToken(cp, tp, c); } return makeToken(tp, 0, T_ERR, 0); case -1: return makeToken(tp, 0, T_ERR, 0); case 0: if (stream->flags & EC_STREAM_EOL) { return makeToken(tp, 0, T_NOP, 0); } return makeToken(tp, 0, T_EOF, 0); case ' ': case '\f': case '\t': case '\v': case 0xA0: /* No break space */ break; case '\r': case '\n': break; case '"': case '\'': return makeQuotedToken(cp, tp, c); case '#': return makeToken(tp, c, T_HASH, 0); case '[': // EJS extension to consider this an operator return makeToken(tp, c, T_LBRACKET, G_OPERATOR); case ']': return makeToken(tp, c, T_RBRACKET, 0); case '(': // EJS extension to consider this an operator return makeToken(tp, c, T_LPAREN, G_OPERATOR); case ')': return makeToken(tp, c, T_RPAREN, 0); case '{': return makeToken(tp, c, T_LBRACE, 0); case '}': return makeToken(tp, c, T_RBRACE, 0); case '@': return makeToken(tp, c, T_AT, 0); case ';': return makeToken(tp, c, T_SEMICOLON, 0); case ',': return makeToken(tp, c, T_COMMA, 0); case '?': return makeToken(tp, c, T_QUERY, 0); case '~': return makeToken(tp, c, T_TILDE, G_OPERATOR); case '+': c = getNextChar(stream); if (c == '+') { addCharToToken(tp, '+'); return makeToken(tp, c, T_PLUS_PLUS, G_OPERATOR); } else if (c == '=') { addCharToToken(tp, '+'); return makeSubToken(tp, c, T_ASSIGN, T_PLUS_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN); } putBackChar(stream, c); return makeToken(tp, '+', T_PLUS, G_OPERATOR); case '-': c = getNextChar(stream); if (isdigit((uchar) c)) { putBackChar(stream, c); return makeToken(tp, '-', T_MINUS, G_OPERATOR); } else if (c == '-') { addCharToToken(tp, '-'); return makeToken(tp, c, T_MINUS_MINUS, G_OPERATOR); } else if (c == '=') { addCharToToken(tp, '-'); return makeSubToken(tp, c, T_ASSIGN, T_MINUS_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN); } putBackChar(stream, c); return makeToken(tp, '-', T_MINUS, G_OPERATOR); case '*': c = getNextChar(stream); if (c == '=') { addCharToToken(tp, '*'); return makeSubToken(tp, c, T_ASSIGN, T_MUL_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN); } putBackChar(stream, c); return makeToken(tp, '*', T_MUL, G_OPERATOR); case '/': c = getNextChar(stream); if (c == '=') { addCharToToken(tp, '/'); return makeSubToken(tp, c, T_ASSIGN, T_DIV_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN); } else if (c == '>') { addCharToToken(tp, '/'); return makeToken(tp, c, T_SLASH_GT, G_OPERATOR); } else if (c == '*' || c == '/') { /* C and C++ comments */ if (getComment(cp, tp, c) < 0) { return tp->tokenId; } /* Doc comments are: [slash]**. The second "*' becomes the first char of the comment. Don't regard: [slash]*** (three stars) as a comment. */ if (cp->doc) { if (tp->text && tp->text[0] == '*' && tp->text[1] != '*') { cp->docToken = mprMemdup(tp->text, tp->length * sizeof(wchar)); } } initializeToken(tp, stream); break; } putBackChar(stream, c); return makeToken(tp, '/', T_DIV, G_OPERATOR); case '%': c = getNextChar(stream); if (c == '=') { addCharToToken(tp, '%'); return makeSubToken(tp, c, T_ASSIGN, T_MOD_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN); } putBackChar(stream, c); return makeToken(tp, '%', T_MOD, G_OPERATOR); case '.': c = getNextChar(stream); if (c == '.') { c = getNextChar(stream); if (c == '.') { addStringToToken(tp, ".."); return makeToken(tp, c, T_ELIPSIS, 0); } putBackChar(stream, c); addCharToToken(tp, '.'); return makeToken(tp, '.', T_DOT_DOT, 0); #if FUTURE } else if (c == '<') { addCharToToken(tp, '.'); return makeToken(tp, c, T_DOT_LESS, 0); #endif } else if (isdigit((uchar) c)) { putBackChar(stream, c); return makeNumberToken(cp, tp, '.'); } putBackChar(stream, c); // EJS extension to consider this an operator return makeToken(tp, '.', T_DOT, G_OPERATOR); case ':': c = getNextChar(stream); if (c == ':') { addCharToToken(tp, ':'); return makeToken(tp, c, T_COLON_COLON, 0); } putBackChar(stream, c); return makeToken(tp, ':', T_COLON, 0); case '!': c = getNextChar(stream); if (c == '=') { c = getNextChar(stream); if (c == '=') { addStringToToken(tp, "!="); return makeToken(tp, c, T_STRICT_NE, G_OPERATOR); } putBackChar(stream, c); addCharToToken(tp, '!'); return makeToken(tp, '=', T_NE, G_OPERATOR); } putBackChar(stream, c); return makeToken(tp, '!', T_LOGICAL_NOT, G_OPERATOR); case '&': c = getNextChar(stream); if (c == '&') { addCharToToken(tp, '&'); c = getNextChar(stream); if (c == '=') { addCharToToken(tp, '&'); return makeSubToken(tp, '=', T_ASSIGN, T_LOGICAL_AND_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN); } putBackChar(stream, c); return makeToken(tp, '&', T_LOGICAL_AND, G_OPERATOR); } else if (c == '=') { addCharToToken(tp, '&'); return makeSubToken(tp, c, T_ASSIGN, T_BIT_AND_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN); } putBackChar(stream, c); return makeToken(tp, '&', T_BIT_AND, G_OPERATOR); case '<': c = getNextChar(stream); if (c == '=') { addCharToToken(tp, '<'); return makeToken(tp, c, T_LE, G_OPERATOR); } else if (c == '<') { c = getNextChar(stream); if (c == '=') { addStringToToken(tp, "<<"); return makeSubToken(tp, c, T_ASSIGN, T_LSH_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN); } putBackChar(stream, c); addCharToToken(tp, '<'); return makeToken(tp, c, T_LSH, G_OPERATOR); } else if (c == '/') { addCharToToken(tp, '<'); return makeToken(tp, c, T_LT_SLASH, 0); } putBackChar(stream, c); return makeToken(tp, '<', T_LT, G_OPERATOR); case '=': c = getNextChar(stream); if (c == '=') { c = getNextChar(stream); if (c == '=') { addStringToToken(tp, "=="); return makeToken(tp, c, T_STRICT_EQ, G_OPERATOR); } putBackChar(stream, c); addCharToToken(tp, '='); return makeToken(tp, c, T_EQ, G_OPERATOR); } putBackChar(stream, c); return makeToken(tp, '=', T_ASSIGN, G_OPERATOR); case '>': c = getNextChar(stream); if (c == '=') { addCharToToken(tp, '<'); return makeToken(tp, c, T_GE, G_OPERATOR); } else if (c == '>') { c = getNextChar(stream); if (c == '=') { addStringToToken(tp, ">>"); return makeSubToken(tp, c, T_ASSIGN, T_RSH_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN); } else if (c == '>') { c = getNextChar(stream); if (c == '=') { addStringToToken(tp, ">>>"); return makeSubToken(tp, c, T_ASSIGN, T_RSH_ZERO_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN); } putBackChar(stream, c); addStringToToken(tp, ">>"); return makeToken(tp, '>', T_RSH_ZERO, G_OPERATOR); } putBackChar(stream, c); addCharToToken(tp, '>'); return makeToken(tp, '>', T_RSH, G_OPERATOR); } putBackChar(stream, c); return makeToken(tp, '>', T_GT, G_OPERATOR); case '^': c = getNextChar(stream); if (c == '^') { addCharToToken(tp, '^'); c = getNextChar(stream); if (c == '=') { addCharToToken(tp, '^'); return makeSubToken(tp, '=', T_ASSIGN, T_LOGICAL_XOR_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN); } putBackChar(stream, c); return makeToken(tp, '^', T_LOGICAL_XOR, G_OPERATOR); } else if (c == '=') { addCharToToken(tp, '^'); return makeSubToken(tp, '=', T_ASSIGN, T_BIT_XOR_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN); } putBackChar(stream, c); return makeToken(tp, '^', T_BIT_XOR, G_OPERATOR); case '|': c = getNextChar(stream); if (c == '|') { addCharToToken(tp, '|'); c = getNextChar(stream); if (c == '=') { addCharToToken(tp, '|'); return makeSubToken(tp, '=', T_ASSIGN, T_LOGICAL_OR_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN); } putBackChar(stream, c); return makeToken(tp, '|', T_LOGICAL_OR, G_OPERATOR); } else if (c == '=') { addCharToToken(tp, '|'); return makeSubToken(tp, '=', T_ASSIGN, T_BIT_OR_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN); } putBackChar(stream, c); return makeToken(tp, '|', T_BIT_OR, G_OPERATOR); } } }
int ecGetRegExpToken(EcInput *input) { EcToken *token, *tp; EcStream *stream; int c; stream = input->stream; tp = token = input->token; mprAssert(tp != 0); initializeToken(tp, stream); addCharToToken(tp, '/'); while (1) { c = getNextChar(stream); switch (c) { case -1: return makeToken(tp, 0, T_ERR, 0); case 0: if (stream->flags & EC_STREAM_EOL) { return makeToken(tp, 0, T_NOP, 0); } return makeToken(tp, 0, T_EOF, 0); case '/': addCharToToken(tp, '/'); while (1) { c = getNextChar(stream); if (c != 'g' && c != 'i' && c != 'm' && c != 'y') { putBackChar(stream, c); break; } addCharToToken(tp, c); } return makeToken(tp, 0, T_REGEXP, 0); case '\\': c = getNextChar(stream); if (c == '\r' || c == '\n' || c == 0) { ecReportError(input->compiler, "warning", stream->name, stream->lineNumber, 0, stream->column, "Illegal newline in regular expression"); return makeToken(tp, 0, T_ERR, 0); } addCharToToken(tp, '\\'); addCharToToken(tp, c); break; case '\r': case '\n': ecReportError(input->compiler, "warning", stream->name, stream->lineNumber, 0, stream->column, "Illegal newline in regular expression"); return makeToken(tp, 0, T_ERR, 0); default: addCharToToken(tp, c); } } }
int ecGetToken(EcInput *input) { EcToken *token, *tp; EcStream *stream; int c; // TODO - functionalize this section token = input->token; if ((tp = input->putBack) != 0) { input->putBack = tp->next; input->token = tp; /* * Move any old token to free list */ if (token) { token->next = input->freeTokens; input->freeTokens = token; } return tp->tokenId; } if (token == 0) { // TBD -- need an API for this input->token = mprAllocObjZeroed(input, EcToken); if (input->token == 0) { // TBD -- err code return -1; } input->token->lineNumber = 1; } stream = input->stream; tp = input->token; mprAssert(tp); initializeToken(tp, stream); while (1) { c = getNextChar(stream); /* * Overloadable operators * * + - ~ * / % < > <= >= == << >> >>> & | === != !== * * TODO FUTURE, we could allow also: ".", "[", "(" * * TODO: what about unary !, ^ */ switch (c) { default: number: if (isdigit(c)) { return getNumberToken(input, tp, c); } else if (c == '\\') { c = getNextChar(stream); if (c == '\n') { break; } putBackChar(stream, c); c = '\n'; } if (isalpha(c) || c == '_' || c == '\\' || c == '$') { return getAlphaToken(input, tp, c); } return makeToken(tp, 0, T_ERR, 0); case -1: return makeToken(tp, 0, T_ERR, 0); case 0: if (stream->flags & EC_STREAM_EOL) { return makeToken(tp, 0, T_NOP, 0); } else { return makeToken(tp, 0, T_EOF, 0); } case ' ': case '\t': break; case '\r': case '\n': if (tp->textLen == 0 && tp->lineNumber != stream->lineNumber) { tp->currentLine = 0; } break; case '"': case '\'': return getQuotedToken(input, tp, c); case '#': return makeToken(tp, c, T_HASH, 0); case '[': // EJS extension to consider this an operator return makeToken(tp, c, T_LBRACKET, G_OPERATOR); case ']': return makeToken(tp, c, T_RBRACKET, 0); case '(': // EJS extension to consider this an operator return makeToken(tp, c, T_LPAREN, G_OPERATOR); case ')': return makeToken(tp, c, T_RPAREN, 0); case '{': return makeToken(tp, c, T_LBRACE, 0); case '}': return makeToken(tp, c, T_RBRACE, 0); case '@': return makeToken(tp, c, T_AT, 0); case ';': return makeToken(tp, c, T_SEMICOLON, 0); case ',': return makeToken(tp, c, T_COMMA, 0); case '?': return makeToken(tp, c, T_QUERY, 0); case '~': return makeToken(tp, c, T_TILDE, G_OPERATOR); case '+': c = getNextChar(stream); if (c == '+') { addCharToToken(tp, '+'); return makeToken(tp, c, T_PLUS_PLUS, G_OPERATOR); } else if (c == '=') { addCharToToken(tp, '+'); return makeSubToken(tp, c, T_ASSIGN, T_PLUS_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN); } putBackChar(stream, c); return makeToken(tp, '+', T_PLUS, G_OPERATOR); case '-': c = getNextChar(stream); if (isdigit(c)) { putBackChar(stream, c); return makeToken(tp, '-', T_MINUS, G_OPERATOR); } else if (c == '-') { addCharToToken(tp, '-'); return makeToken(tp, c, T_MINUS_MINUS, G_OPERATOR); } else if (c == '=') { addCharToToken(tp, '-'); return makeSubToken(tp, c, T_ASSIGN, T_MINUS_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN); } putBackChar(stream, c); return makeToken(tp, '-', T_MINUS, G_OPERATOR); case '*': c = getNextChar(stream); if (c == '=') { addCharToToken(tp, '*'); return makeSubToken(tp, c, T_ASSIGN, T_MUL_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN); } putBackChar(stream, c); return makeToken(tp, '*', T_MUL, G_OPERATOR); case '/': c = getNextChar(stream); if (c == '=') { addCharToToken(tp, '/'); return makeSubToken(tp, c, T_ASSIGN, T_DIV_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN); } else if (c == '>') { addCharToToken(tp, '/'); return makeToken(tp, c, T_SLASH_GT, G_OPERATOR); } else if (c == '*' || c == '/') { /* * C and C++ comments */ if (getComment(input, tp, c) < 0) { return tp->tokenId; } #if BLD_FEATURE_EJS_DOC if (tp->text && tp->text[0] == '*') { mprFree(input->doc); input->doc = mprStrdup(input, (char*) tp->text); } #endif initializeToken(tp, stream); break; } putBackChar(stream, c); return makeToken(tp, '/', T_DIV, G_OPERATOR); case '%': c = getNextChar(stream); if (c == '=') { addCharToToken(tp, '%'); return makeSubToken(tp, c, T_ASSIGN, T_MOD_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN); } putBackChar(stream, c); return makeToken(tp, '%', T_MOD, G_OPERATOR); case '.': c = getNextChar(stream); if (c == '.') { c = getNextChar(stream); if (c == '.') { addStringToToken(tp, ".."); return makeToken(tp, c, T_ELIPSIS, 0); } putBackChar(stream, c); addCharToToken(tp, '.'); return makeToken(tp, '.', T_DOT_DOT, 0); } else if (c == '<') { addCharToToken(tp, '.'); return makeToken(tp, c, T_DOT_LESS, 0); } else if (isdigit(c)) { putBackChar(stream, c); goto number; } putBackChar(stream, c); // EJS extension to consider this an operator return makeToken(tp, '.', T_DOT, G_OPERATOR); case ':': c = getNextChar(stream); if (c == ':') { addCharToToken(tp, ':'); return makeToken(tp, c, T_COLON_COLON, 0); } putBackChar(stream, c); return makeToken(tp, ':', T_COLON, 0); case '!': c = getNextChar(stream); if (c == '=') { c = getNextChar(stream); if (c == '=') { addStringToToken(tp, "!="); return makeToken(tp, c, T_STRICT_NE, G_OPERATOR); } putBackChar(stream, c); addCharToToken(tp, '!'); return makeToken(tp, '=', T_NE, G_OPERATOR); } putBackChar(stream, c); return makeToken(tp, '!', T_LOGICAL_NOT, G_OPERATOR); #if UNUSED case '~': c = getNextChar(stream); if (c == '=') { addStringToToken(tp, "~="); return makeSubToken(tp, c, T_ASSIGN, T_BIT_NEG_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN); } putBackChar(stream, c); return makeToken(tp, '~', T_BIT_NEG, G_OPERATOR); #endif case '&': c = getNextChar(stream); if (c == '&') { addCharToToken(tp, '&'); c = getNextChar(stream); if (c == '=') { addCharToToken(tp, '&'); return makeSubToken(tp, '=', T_ASSIGN, T_LOGICAL_AND_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN); } putBackChar(stream, c); return makeToken(tp, '&', T_LOGICAL_AND, G_OPERATOR); } else if (c == '=') { addCharToToken(tp, '&'); return makeSubToken(tp, c, T_ASSIGN, T_BIT_AND_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN); } putBackChar(stream, c); return makeToken(tp, '&', T_BIT_AND, G_OPERATOR); case '<': c = getNextChar(stream); if (c == '=') { addCharToToken(tp, '<'); return makeToken(tp, c, T_LE, G_OPERATOR); } else if (c == '<') { c = getNextChar(stream); if (c == '=') { addStringToToken(tp, "<<"); return makeSubToken(tp, c, T_ASSIGN, T_LSH_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN); } putBackChar(stream, c); addCharToToken(tp, '<'); return makeToken(tp, c, T_LSH, G_OPERATOR); } else if (c == '/') { addCharToToken(tp, '<'); return makeToken(tp, c, T_LT_SLASH, 0); } putBackChar(stream, c); return makeToken(tp, '<', T_LT, G_OPERATOR); case '=': c = getNextChar(stream); if (c == '=') { c = getNextChar(stream); if (c == '=') { addStringToToken(tp, "=="); return makeToken(tp, c, T_STRICT_EQ, G_OPERATOR); } putBackChar(stream, c); addCharToToken(tp, '='); return makeToken(tp, c, T_EQ, G_OPERATOR); } putBackChar(stream, c); return makeToken(tp, '=', T_ASSIGN, G_OPERATOR); case '>': c = getNextChar(stream); if (c == '=') { addCharToToken(tp, '<'); return makeToken(tp, c, T_GE, G_OPERATOR); } else if (c == '>') { c = getNextChar(stream); if (c == '=') { addStringToToken(tp, ">>"); return makeSubToken(tp, c, T_ASSIGN, T_RSH_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN); } else if (c == '>') { c = getNextChar(stream); if (c == '=') { addStringToToken(tp, ">>>"); return makeSubToken(tp, c, T_ASSIGN, T_RSH_ZERO_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN); } putBackChar(stream, c); addStringToToken(tp, ">>"); return makeToken(tp, '>', T_RSH_ZERO, G_OPERATOR); } putBackChar(stream, c); addCharToToken(tp, '>'); return makeToken(tp, '>', T_RSH, G_OPERATOR); } putBackChar(stream, c); return makeToken(tp, '>', T_GT, G_OPERATOR); case '^': c = getNextChar(stream); if (c == '^') { addCharToToken(tp, '^'); c = getNextChar(stream); if (c == '=') { addCharToToken(tp, '^'); return makeSubToken(tp, '=', T_ASSIGN, T_LOGICAL_XOR_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN); } putBackChar(stream, c); return makeToken(tp, '^', T_LOGICAL_XOR, G_OPERATOR); } else if (c == '=') { addCharToToken(tp, '^'); return makeSubToken(tp, '=', T_ASSIGN, T_BIT_XOR_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN); } putBackChar(stream, c); return makeToken(tp, '^', T_BIT_XOR, G_OPERATOR); case '|': c = getNextChar(stream); if (c == '|') { addCharToToken(tp, '|'); c = getNextChar(stream); if (c == '=') { addCharToToken(tp, '|'); return makeSubToken(tp, '=', T_ASSIGN, T_LOGICAL_OR_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN); } putBackChar(stream, c); return makeToken(tp, '|', T_LOGICAL_OR, G_OPERATOR); } else if (c == '=') { addCharToToken(tp, '|'); return makeSubToken(tp, '=', T_ASSIGN, T_BIT_OR_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN); } putBackChar(stream, c); return makeToken(tp, '|', T_BIT_OR, G_OPERATOR); } } }