Beispiel #1
0
/*
 *  C, C++ and doc style comments. Return token or zero for no token.
 */
static int getComment(EcInput *input, EcToken *tp, int c)
{
    EcStream    *stream;
    int         form, startLine;

    startLine = tp->stream->lineNumber;

    stream = input->stream;
    form = c;

    //  TODO - would be great to warn about comment nested in a comment.

    for (form = c; c > 0;) {
        c = getNextChar(stream);
        if (c <= 0) {
            /*
             *  Unterminated Comment
             */
            addFormattedStringToToken(tp, "Unterminated comment starting on line %d", startLine);
            makeToken(tp, 0, form == '/' ? T_EOF: T_ERR, 0);
            return 1;
        }

        if (form == '/') {
            if (c == '\n') {
                break;
            }

        } else {
            if (c == '*') {
                c = getNextChar(stream);
                if (c == '/') {
                    break;
                }
                addCharToToken(tp, '*');
                putBackChar(stream, c);

            } else if (c == '/') {
                c = getNextChar(stream);
                if (c == '*') {
                    /*
                     *  Nested comment
                     */
                    if (input->compiler->warnLevel > 0) {
                        ecReportError(input->compiler, "warning", stream->name, stream->lineNumber, 0,
                                stream->column, "Possible nested comment");
                    }
                }
                addCharToToken(tp, '/');
            }
        }
        addCharToToken(tp, c);
    }

    return 0;
}
Beispiel #2
0
/*
    C, C++ and doc style comments. Return token or zero for no token.
 */
static int getComment(EcCompiler *cp, EcToken *tp, int c)
{
    EcStream    *stream;
    int         form, startLine;

    startLine = cp->stream->loc.lineNumber;
    stream = cp->stream;
    form = c;

    for (form = c; c > 0;) {
        c = getNextChar(stream);
        if (c <= 0) {
            /*
                Unterminated Comment
             */
            addFormattedStringToToken(tp, "Unterminated comment starting on line %d", startLine);
            makeToken(tp, 0, form == '/' ? T_EOF: T_ERR, 0);
            return 1;
        }
        if (form == '/') {
            if (c == '\n' || c == '\r') {
                break;
            }
        } else {
            if (c == '*') {
                c = getNextChar(stream);
                if (c == '/') {
                    break;
                }
                addCharToToken(tp, '*');
                putBackChar(stream, c);

            } else if (c == '/') {
                c = getNextChar(stream);
                if (c == '*') {
                    /*
                        Nested comment
                     */
                    if (cp->warnLevel > 0) {
                        ecError(cp, "Warning", &stream->loc, "Possible nested comment");
                    }
                }
                addCharToToken(tp, '/');
            }
        }
        addCharToToken(tp, c);
    }
    return 0;
}
Beispiel #3
0
static int makeAlphaToken(EcCompiler *cp, EcToken *tp, int c)
{
    ReservedWord    *rp;
    EcStream        *stream;

    /*
        We know that c is an alpha already
     */
    stream = cp->stream;

    while (isalnum((uchar) c) || c == '_' || c == '$' || c == '\\') {
        if (c == '\\') {
            c = getNextChar(stream);
            if (c == '\n' || c == '\r') {
                break;
            } else if (c == 'u') {
                c = decodeNumber(cp, 16, 4);
                //  TODO - for now, mask back to 8 bits.
                c = c & 0xff;
            }
        }
        addCharToToken(tp, c);
        c = getNextChar(stream);
    }
    if (c) {
        putBackChar(stream, c);
    }
    rp = (ReservedWord*) mprLookupKey(cp->keywords, tp->text);
    if (rp) {
        setTokenID(tp, rp->tokenId, rp->subId, rp->groupMask);
    } else {
        setTokenID(tp, T_ID, -1, 0);
    }
    return finalizeToken(tp);
}
Beispiel #4
0
static int makeSubToken(EcToken *tp, int c, int tokenId, int subId, int groupMask)
{
    if (addCharToToken(tp, c) < 0) {
        return T_ERR;
    }
    return finishToken(tp, tokenId, subId, groupMask);
}
Beispiel #5
0
static int makeToken(EcToken *tp, int c, int tokenId, int groupMask)
{
    if (c && addCharToToken(tp, c) < 0) {
        return T_ERR;
    }
    return finishToken(tp, tokenId, -1, groupMask);
}
Beispiel #6
0
static int makeSubToken(EcToken *tp, int c, int tokenId, int subId, int groupMask)
{
    if (addCharToToken(tp, c) < 0) {
        return T_ERR;
    }
    setTokenID(tp, tokenId, subId, groupMask);
    return finalizeToken(tp);
}
Beispiel #7
0
static int makeQuotedToken(EcCompiler *cp, EcToken *tp, int c)
{
    EcStream    *stream;
    int         quoteType;

    stream = cp->stream;
    quoteType = c;

    for (c = getNextChar(stream); c && c != quoteType; c = getNextChar(stream)) {
        if (c == 0) {
            return makeToken(tp, 0, T_ERR, 0);
        }
        if (c == '\\') {
            c = getNextChar(stream);
            switch (c) {
            //  TBD -- others
            case '\\':
                break;
            case '\'':
            case '\"':
                break;
            case 'b':
                c = '\b';
                break;
            case 'f':
                c = '\f';
                break;
            case 'n':
                c = '\n';
                break;
            case 'r':
                c = '\r';
                break;
            case 't':
                c = '\t';
                break;
            case 'u':
                c = decodeNumber(cp, 16, 4);
                break;
            case 'x':
                c = decodeNumber(cp, 16, 2);
                break;
            case 'v':
                c = '\v';
                break;
            case '0':
                c = decodeNumber(cp, 8, 3);
                break;
            default:
                break;
            }
        }
        addCharToToken(tp, c);
    }
    assert(tp->text);
    setTokenID(tp, T_STRING, -1, 0);
    return finalizeToken(tp);
}
Beispiel #8
0
static int addStringToToken(EcToken *tp, char *str)
{
    char    *cp;

    for (cp = str; *cp; cp++) {
        if (addCharToToken(tp, *cp) < 0) {
            return MPR_ERR_MEMORY;
        }
    }
    return 0;
}
Beispiel #9
0
PUBLIC int ecGetRegExpToken(EcCompiler *cp, wchar *prefix)
{
    EcToken     *token, *tp;
    EcStream    *stream;
    wchar       *pp;
    int         c;

    stream = cp->stream;
    tp = token = cp->token;
    assert(tp != 0);

    initializeToken(tp, stream);

    for (pp = prefix; pp && *pp; pp++) {
        addCharToToken(tp, *pp);
    }
    while (1) {
        c = getNextChar(stream);
        switch (c) {
        case -1:
            return makeToken(tp, 0, T_ERR, 0);

        case 0:
            if (stream->flags & EC_STREAM_EOL) {
                return makeToken(tp, 0, T_NOP, 0);
            }
            return makeToken(tp, 0, T_EOF, 0);

        case '/':
            addCharToToken(tp, '/');
            while (1) {
                c = getNextChar(stream);
                if (c != 'g' && c != 'i' && c != 'm' && c != 'y' && c != 'x' && c != 'X' && c != 'U' && c != 's') {
                    putBackChar(stream, c);
                    break;
                }
                addCharToToken(tp, c);
            }
            return makeToken(tp, 0, T_REGEXP, 0);

        case '\\':
            c = getNextChar(stream);
            if (c == '\r' || c == '\n' || c == 0) {
                ecError(cp, "Warning", &stream->loc, "Illegal newline in regular expression");
                return makeToken(tp, 0, T_ERR, 0);
            }
            addCharToToken(tp, '\\');
            addCharToToken(tp, c);
            break;

        case '\r':
        case '\n':
            ecError(cp, "Warning", &stream->loc, "Illegal newline in regular expression");
            return makeToken(tp, 0, T_ERR, 0);

        default:
            addCharToToken(tp, c);
        }
    }
}
Beispiel #10
0
//  TODO - handle triple quoting
static int getQuotedToken(EcInput *input, EcToken *tp, int c)
{
    EcStream    *stream;
    int         quoteType;

    stream = input->stream;
    quoteType = c;

    for (c = getNextChar(stream); c && c != quoteType; c = getNextChar(stream)) {
        if (c == 0) {
            return makeToken(tp, 0, T_ERR, 0);
        }
        if (c == '\\') {
            c = getNextChar(stream);
            switch (c) {
            //  TBD -- others
            case '\\':
                break;
            case '\'':
            case '\"':
                break;
            case 'b':
                c = '\b';
                break;
            case 'f':
                c = '\f';
                break;
            case 'n':
                c = '\n';
                break;
            case 'r':
                c = '\r';
                break;
            case 't':
                c = '\t';
                break;
            case 'u':
            case 'x':
                c = decodeNumber(input, 16, 4);
                break;
            case '0':
                c = decodeNumber(input, 8, 3);
                break;
            default:
                break;
            }
        }
        addCharToToken(tp, c);
    }
    return finishToken(tp, T_STRING, -1, 0);
}
Beispiel #11
0
/*
 *  TODO rationalize with ecParser T_NUMBER code. This could be a lot faster.
 */
static int getNumberToken(EcInput *input, EcToken *tp, int c)
{
    EcStream    *stream;
    int         lowc, isHex, isFloat;

    isHex = isFloat = 0;

    stream = input->stream;
    if (c == '0') {
        addCharToToken(tp, c);
        c = getNextChar(stream);
        if (tolower(c) == 'x') {
            do {
                addCharToToken(tp, c);
                c = getNextChar(stream);
            } while (isxdigit(c));

            putBackChar(stream, c);
            return finishToken(tp, T_NUMBER, -1, 0);
        }
    }

    lowc = tolower(c);
    while (isdigit(lowc) || lowc == '.' || lowc == 'e' || lowc == 'f') {
        if (lowc == '.' || lowc == 'e' || lowc == 'f') {
            isFloat++;
        }
        addCharToToken(tp, c);
        c = getNextChar(stream);
        lowc = tolower(c);
    }

    putBackChar(stream, c);

    return finishToken(tp, T_NUMBER, -1, 0);
}
Beispiel #12
0
static int getAlphaToken(EcInput *input, EcToken *tp, int c)
{
    ReservedWord    *rp;
    EcStream        *stream;

    /*
     *  We know that c is an alpha already
     */
    //  TBD -- does ES4 allow $

    stream = input->stream;

    while (isalnum(c) || c == '_' || c == '$' || c == '\\') {
        if (c == '\\') {
            c = getNextChar(stream);
            if (c == '\n' || c == '\r') {
                break;
            } else if (c == 'u') {
                c = decodeNumber(input, 16, 4);
            }
        }
        addCharToToken(tp, c);
        c = getNextChar(stream);
    }
    if (c) {
        putBackChar(stream, c);
    }

    //  TODO - need to take into account contextually reserved and
    //  full reserved words.
    rp = (ReservedWord*) mprLookupHash(input->lexer->keywords, (char*) tp->text);
    if (rp) {
        return finishToken(tp, rp->tokenId, rp->subId, rp->groupMask);
    } else {
        return finishToken(tp, T_ID, -1, 0);
    }
}
Beispiel #13
0
/*
    Hex:        0(x|X)[DIGITS]
    Octal:      0[DIGITS]
    Float:      [DIGITS].[DIGITS][(e|E)[+|-]DIGITS]
 */
static int makeNumberToken(EcCompiler *cp, EcToken *tp, int c)
{
    EcStream    *stream;

    stream = cp->stream;
    if (c == '0') {
        c = getNextChar(stream);
        if (tolower((uchar) c) == 'x') {
            /* Hex */
            addCharToToken(tp, '0');
            do {
                addCharToToken(tp, c);
                c = getNextChar(stream);
            } while (isxdigit(c));
            putBackChar(stream, c);
            setTokenID(tp, T_NUMBER, -1, 0);
            return finalizeToken(tp);

        } else if ('0' <= c && c <= '7') {
            /* Octal */
            addCharToToken(tp, '0');
            do {
                addCharToToken(tp, c);
                c = getNextChar(stream);
            } while ('0' <= c && c <= '7');
            putBackChar(stream, c);
            setTokenID(tp, T_NUMBER, -1, 0);
            return finalizeToken(tp);

        } else {
            putBackChar(stream, c);
            c = '0';
        }
    }

    /*
        Float
     */
    while (isdigit((uchar) c)) {
        addCharToToken(tp, c);
        c = getNextChar(stream);
    }
    if (c == '.') {
        addCharToToken(tp, c);
        c = getNextChar(stream);
    }
    while (isdigit((uchar) c)) {
        addCharToToken(tp, c);
        c = getNextChar(stream);
    }
    if (tolower((uchar) c) == 'e') {
        addCharToToken(tp, c);
        c = getNextChar(stream);
        if (c == '+' || c == '-') {
            addCharToToken(tp, c);
            c = getNextChar(stream);
        }
        while (isdigit((uchar) c)) {
            addCharToToken(tp, c);
            c = getNextChar(stream);
        }
    }
    putBackChar(stream, c);
    setTokenID(tp, T_NUMBER, -1, 0);
    return finalizeToken(tp);
}
Beispiel #14
0
PUBLIC int ecGetToken(EcCompiler *cp)
{
    EcToken     *tp;
    EcStream    *stream;
    int         c;

    if ((tp = getLexToken(cp)) == NULL) {
        return T_ERR;
    }
    if (tp->tokenId) {
        return tp->tokenId;
    }
    stream = cp->stream;

    while (1) {
        c = getNextChar(stream);
        /*
            Overloadable operators
            + - ~ * / % < > <= >= == << >> >>> & | === != !==
            TODO FUTURE, we could allow also:  ".", "[", "(" and unary !, ^
         */
        switch (c) {
        default:
            if (isdigit((uchar) c)) {
                return makeNumberToken(cp, tp, c);

            } else if (c == '\\') {
                c = getNextChar(stream);
                if (c == '\n') {
                    break;
                }
                putBackChar(stream, c);
                c = '\n';
            }
            if (isalpha((uchar) c) || c == '_' || c == '\\' || c == '$') {
                return makeAlphaToken(cp, tp, c);
            }
            return makeToken(tp, 0, T_ERR, 0);

        case -1:
            return makeToken(tp, 0, T_ERR, 0);

        case 0:
            if (stream->flags & EC_STREAM_EOL) {
                return makeToken(tp, 0, T_NOP, 0);
            }
            return makeToken(tp, 0, T_EOF, 0);

        case ' ':
        case '\f':
        case '\t':
        case '\v':
        case 0xA0:      /* No break space */
            break;

        case '\r':
        case '\n':
            break;

        case '"':
        case '\'':
            return makeQuotedToken(cp, tp, c);

        case '#':
            return makeToken(tp, c, T_HASH, 0);

        case '[':
            //  EJS extension to consider this an operator
            return makeToken(tp, c, T_LBRACKET, G_OPERATOR);

        case ']':
            return makeToken(tp, c, T_RBRACKET, 0);

        case '(':
            //  EJS extension to consider this an operator
            return makeToken(tp, c, T_LPAREN, G_OPERATOR);

        case ')':
            return makeToken(tp, c, T_RPAREN, 0);

        case '{':
            return makeToken(tp, c, T_LBRACE, 0);

        case '}':
            return makeToken(tp, c, T_RBRACE, 0);

        case '@':
            return makeToken(tp, c, T_AT, 0);

        case ';':
            return makeToken(tp, c, T_SEMICOLON, 0);

        case ',':
            return makeToken(tp, c, T_COMMA, 0);

        case '?':
            return makeToken(tp, c, T_QUERY, 0);

        case '~':
            return makeToken(tp, c, T_TILDE, G_OPERATOR);

        case '+':
            c = getNextChar(stream);
            if (c == '+') {
                addCharToToken(tp, '+');
                return makeToken(tp, c, T_PLUS_PLUS, G_OPERATOR);
            } else if (c == '=') {
                addCharToToken(tp, '+');
                return makeSubToken(tp, c, T_ASSIGN, T_PLUS_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
            }
            putBackChar(stream, c);
            return makeToken(tp, '+', T_PLUS, G_OPERATOR);

        case '-':
            c = getNextChar(stream);
            if (isdigit((uchar) c)) {
                putBackChar(stream, c);
                return makeToken(tp, '-', T_MINUS, G_OPERATOR);

            } else if (c == '-') {
                addCharToToken(tp, '-');
                return makeToken(tp, c, T_MINUS_MINUS, G_OPERATOR);

            } else if (c == '=') {
                addCharToToken(tp, '-');
                return makeSubToken(tp, c, T_ASSIGN, T_MINUS_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
            }
            putBackChar(stream, c);
            return makeToken(tp, '-', T_MINUS, G_OPERATOR);

        case '*':
            c = getNextChar(stream);
            if (c == '=') {
                addCharToToken(tp, '*');
                return makeSubToken(tp, c, T_ASSIGN, T_MUL_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
            }
            putBackChar(stream, c);
            return makeToken(tp, '*', T_MUL, G_OPERATOR);

        case '/':
            c = getNextChar(stream);
            if (c == '=') {
                addCharToToken(tp, '/');
                return makeSubToken(tp, c, T_ASSIGN, T_DIV_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);

            } else if (c == '>') {
                addCharToToken(tp, '/');
                return makeToken(tp, c, T_SLASH_GT, G_OPERATOR);

            } else if (c == '*' || c == '/') {
                /*
                    C and C++ comments
                 */
                if (getComment(cp, tp, c) < 0) {
                    return tp->tokenId;
                }
                /*
                    Doc comments are: [slash]**. The second "*' becomes the first char of the comment.
                    Don't regard: [slash]*** (three stars) as a comment.
                 */
                if (cp->doc) {
                    if (tp->text && tp->text[0] == '*' && tp->text[1] != '*') {
                        cp->docToken = mprMemdup(tp->text, tp->length * sizeof(wchar));
                    }
                }
                initializeToken(tp, stream);
                break;
            }
            putBackChar(stream, c);
            return makeToken(tp, '/', T_DIV, G_OPERATOR);

        case '%':
            c = getNextChar(stream);
            if (c == '=') {
                addCharToToken(tp, '%');
                return makeSubToken(tp, c, T_ASSIGN, T_MOD_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
            }
            putBackChar(stream, c);
            return makeToken(tp, '%', T_MOD, G_OPERATOR);

        case '.':
            c = getNextChar(stream);
            if (c == '.') {
                c = getNextChar(stream);
                if (c == '.') {
                    addStringToToken(tp, "..");
                    return makeToken(tp, c, T_ELIPSIS, 0);
                }
                putBackChar(stream, c);
                addCharToToken(tp, '.');
                return makeToken(tp, '.', T_DOT_DOT, 0);
#if FUTURE
            } else if (c == '<') {
                addCharToToken(tp, '.');
                return makeToken(tp, c, T_DOT_LESS, 0);
#endif
            } else if (isdigit((uchar) c)) {
                putBackChar(stream, c);
                return makeNumberToken(cp, tp, '.');
            }
            putBackChar(stream, c);
            //  EJS extension to consider this an operator
            return makeToken(tp, '.', T_DOT, G_OPERATOR);

        case ':':
            c = getNextChar(stream);
            if (c == ':') {
                addCharToToken(tp, ':');
                return makeToken(tp, c, T_COLON_COLON, 0);
            }
            putBackChar(stream, c);
            return makeToken(tp, ':', T_COLON, 0);

        case '!':
            c = getNextChar(stream);
            if (c == '=') {
                c = getNextChar(stream);
                if (c == '=') {
                    addStringToToken(tp, "!=");
                    return makeToken(tp, c, T_STRICT_NE, G_OPERATOR);
                }
                putBackChar(stream, c);
                addCharToToken(tp, '!');
                return makeToken(tp, '=', T_NE, G_OPERATOR);
            }
            putBackChar(stream, c);
            return makeToken(tp, '!', T_LOGICAL_NOT, G_OPERATOR);

        case '&':
            c = getNextChar(stream);
            if (c == '&') {
                addCharToToken(tp, '&');
                c = getNextChar(stream);
                if (c == '=') {
                    addCharToToken(tp, '&');
                    return makeSubToken(tp, '=', T_ASSIGN, T_LOGICAL_AND_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
                }
                putBackChar(stream, c);
                return makeToken(tp, '&', T_LOGICAL_AND, G_OPERATOR);

            } else if (c == '=') {
                addCharToToken(tp, '&');
                return makeSubToken(tp, c, T_ASSIGN, T_BIT_AND_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
            }
            putBackChar(stream, c);
            return makeToken(tp, '&', T_BIT_AND, G_OPERATOR);

        case '<':
            c = getNextChar(stream);
            if (c == '=') {
                addCharToToken(tp, '<');
                return makeToken(tp, c, T_LE, G_OPERATOR);
            } else if (c == '<') {
                c = getNextChar(stream);
                if (c == '=') {
                    addStringToToken(tp, "<<");
                    return makeSubToken(tp, c, T_ASSIGN, T_LSH_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
                }
                putBackChar(stream, c);
                addCharToToken(tp, '<');
                return makeToken(tp, c, T_LSH, G_OPERATOR);

            } else if (c == '/') {
                addCharToToken(tp, '<');
                return makeToken(tp, c, T_LT_SLASH, 0);
            }
            putBackChar(stream, c);
            return makeToken(tp, '<', T_LT, G_OPERATOR);

        case '=':
            c = getNextChar(stream);
            if (c == '=') {
                c = getNextChar(stream);
                if (c == '=') {
                    addStringToToken(tp, "==");
                    return makeToken(tp, c, T_STRICT_EQ, G_OPERATOR);
                }
                putBackChar(stream, c);
                addCharToToken(tp, '=');
                return makeToken(tp, c, T_EQ, G_OPERATOR);
            }
            putBackChar(stream, c);
            return makeToken(tp, '=', T_ASSIGN, G_OPERATOR);

        case '>':
            c = getNextChar(stream);
            if (c == '=') {
                addCharToToken(tp, '<');
                return makeToken(tp, c, T_GE, G_OPERATOR);
            } else if (c == '>') {
                c = getNextChar(stream);
                if (c == '=') {
                    addStringToToken(tp, ">>");
                    return makeSubToken(tp, c, T_ASSIGN, T_RSH_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
                } else if (c == '>') {
                    c = getNextChar(stream);
                    if (c == '=') {
                        addStringToToken(tp, ">>>");
                        return makeSubToken(tp, c, T_ASSIGN, T_RSH_ZERO_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
                    }
                    putBackChar(stream, c);
                    addStringToToken(tp, ">>");
                    return makeToken(tp, '>', T_RSH_ZERO, G_OPERATOR);
                }
                putBackChar(stream, c);
                addCharToToken(tp, '>');
                return makeToken(tp, '>', T_RSH, G_OPERATOR);
            }
            putBackChar(stream, c);
            return makeToken(tp, '>', T_GT, G_OPERATOR);

        case '^':
            c = getNextChar(stream);
            if (c == '^') {
                addCharToToken(tp, '^');
                c = getNextChar(stream);
                if (c == '=') {
                    addCharToToken(tp, '^');
                    return makeSubToken(tp, '=', T_ASSIGN, T_LOGICAL_XOR_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
                }
                putBackChar(stream, c);
                return makeToken(tp, '^', T_LOGICAL_XOR, G_OPERATOR);

            } else if (c == '=') {
                addCharToToken(tp, '^');
                return makeSubToken(tp, '=', T_ASSIGN, T_BIT_XOR_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
            }
            putBackChar(stream, c);
            return makeToken(tp, '^', T_BIT_XOR, G_OPERATOR);

        case '|':
            c = getNextChar(stream);
            if (c == '|') {
                addCharToToken(tp, '|');
                c = getNextChar(stream);
                if (c == '=') {
                    addCharToToken(tp, '|');
                    return makeSubToken(tp, '=', T_ASSIGN, T_LOGICAL_OR_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
                }
                putBackChar(stream, c);
                return makeToken(tp, '|', T_LOGICAL_OR, G_OPERATOR);

            } else if (c == '=') {
                addCharToToken(tp, '|');
                return makeSubToken(tp, '=', T_ASSIGN, T_BIT_OR_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);

            }
            putBackChar(stream, c);
            return makeToken(tp, '|', T_BIT_OR, G_OPERATOR);
        }
    }
}
Beispiel #15
0
int ecGetRegExpToken(EcInput *input)
{
    EcToken     *token, *tp;
    EcStream    *stream;
    int         c;

    stream = input->stream;
    tp = token = input->token;
    mprAssert(tp != 0);

    initializeToken(tp, stream);
    addCharToToken(tp, '/');

    while (1) {

        c = getNextChar(stream);

        switch (c) {
        case -1:
            return makeToken(tp, 0, T_ERR, 0);

        case 0:
            if (stream->flags & EC_STREAM_EOL) {
                return makeToken(tp, 0, T_NOP, 0);
            }
            return makeToken(tp, 0, T_EOF, 0);

        case '/':
            addCharToToken(tp, '/');
            while (1) {
                c = getNextChar(stream);
                if (c != 'g' && c != 'i' && c != 'm' && c != 'y') {
                    putBackChar(stream, c);
                    break;
                }
                addCharToToken(tp, c);
            }
            return makeToken(tp, 0, T_REGEXP, 0);

        case '\\':
            c = getNextChar(stream);
            if (c == '\r' || c == '\n' || c == 0) {
                ecReportError(input->compiler, "warning", stream->name, stream->lineNumber, 0, stream->column,
                    "Illegal newline in regular expression");
                return makeToken(tp, 0, T_ERR, 0);
            }
            addCharToToken(tp, '\\');
            addCharToToken(tp, c);
            break;

        case '\r':
        case '\n':
            ecReportError(input->compiler, "warning", stream->name, stream->lineNumber, 0, stream->column,
                "Illegal newline in regular expression");
            return makeToken(tp, 0, T_ERR, 0);

        default:
            addCharToToken(tp, c);
        }
    }
}
Beispiel #16
0
int ecGetToken(EcInput *input)
{
    EcToken     *token, *tp;
    EcStream    *stream;
    int         c;

    //  TODO - functionalize this section

    token = input->token;

    if ((tp = input->putBack) != 0) {
        input->putBack = tp->next;
        input->token = tp;

        /*
         *  Move any old token to free list
         */
        if (token) {
            token->next = input->freeTokens;
            input->freeTokens = token;
        }
        return tp->tokenId;
    }

    if (token == 0) {
        //  TBD -- need an API for this
        input->token = mprAllocObjZeroed(input, EcToken);
        if (input->token == 0) {
            //  TBD -- err code
            return -1;
        }
        input->token->lineNumber = 1;
    }

    stream = input->stream;
    tp = input->token;
    mprAssert(tp);

    initializeToken(tp, stream);

    while (1) {

        c = getNextChar(stream);

        /*
         *  Overloadable operators
         *
         *  + - ~ * / % < > <= >= == << >> >>> & | === != !==
         *
         *  TODO FUTURE, we could allow also:  ".", "[", "("
         *
         *  TODO: what about unary !, ^
         */
        switch (c) {
        default:
        number:
            if (isdigit(c)) {
                return getNumberToken(input, tp, c);

            } else if (c == '\\') {
                c = getNextChar(stream);
                if (c == '\n') {
                    break;
                }
                putBackChar(stream, c);
                c = '\n';
            }
            if (isalpha(c) || c == '_' || c == '\\' || c == '$') {
                return getAlphaToken(input, tp, c);
            }
            return makeToken(tp, 0, T_ERR, 0);

        case -1:
            return makeToken(tp, 0, T_ERR, 0);

        case 0:
            if (stream->flags & EC_STREAM_EOL) {
                return makeToken(tp, 0, T_NOP, 0);
            } else {
                return makeToken(tp, 0, T_EOF, 0);
            }

        case ' ':
        case '\t':
            break;

        case '\r':
        case '\n':
            if (tp->textLen == 0 && tp->lineNumber != stream->lineNumber) {
                tp->currentLine = 0;
            }
            break;

        case '"':
        case '\'':
            return getQuotedToken(input, tp, c);

        case '#':
            return makeToken(tp, c, T_HASH, 0);

        case '[':
            //  EJS extension to consider this an operator
            return makeToken(tp, c, T_LBRACKET, G_OPERATOR);

        case ']':
            return makeToken(tp, c, T_RBRACKET, 0);

        case '(':
            //  EJS extension to consider this an operator
            return makeToken(tp, c, T_LPAREN, G_OPERATOR);

        case ')':
            return makeToken(tp, c, T_RPAREN, 0);

        case '{':
            return makeToken(tp, c, T_LBRACE, 0);

        case '}':
            return makeToken(tp, c, T_RBRACE, 0);

        case '@':
            return makeToken(tp, c, T_AT, 0);

        case ';':
            return makeToken(tp, c, T_SEMICOLON, 0);

        case ',':
            return makeToken(tp, c, T_COMMA, 0);

        case '?':
            return makeToken(tp, c, T_QUERY, 0);

        case '~':
            return makeToken(tp, c, T_TILDE, G_OPERATOR);

        case '+':
            c = getNextChar(stream);
            if (c == '+') {
                addCharToToken(tp, '+');
                return makeToken(tp, c, T_PLUS_PLUS, G_OPERATOR);
            } else if (c == '=') {
                addCharToToken(tp, '+');
                return makeSubToken(tp, c, T_ASSIGN, T_PLUS_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
            }
            putBackChar(stream, c);
            return makeToken(tp, '+', T_PLUS, G_OPERATOR);

        case '-':
            c = getNextChar(stream);
            if (isdigit(c)) {
                putBackChar(stream, c);
                return makeToken(tp, '-', T_MINUS, G_OPERATOR);

            } else if (c == '-') {
                addCharToToken(tp, '-');
                return makeToken(tp, c, T_MINUS_MINUS, G_OPERATOR);

            } else if (c == '=') {
                addCharToToken(tp, '-');
                return makeSubToken(tp, c, T_ASSIGN, T_MINUS_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
            }
            putBackChar(stream, c);
            return makeToken(tp, '-', T_MINUS, G_OPERATOR);

        case '*':
            c = getNextChar(stream);
            if (c == '=') {
                addCharToToken(tp, '*');
                return makeSubToken(tp, c, T_ASSIGN, T_MUL_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
            }
            putBackChar(stream, c);
            return makeToken(tp, '*', T_MUL, G_OPERATOR);

        case '/':
            c = getNextChar(stream);
            if (c == '=') {
                addCharToToken(tp, '/');
                return makeSubToken(tp, c, T_ASSIGN, T_DIV_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);

            } else if (c == '>') {
                addCharToToken(tp, '/');
                return makeToken(tp, c, T_SLASH_GT, G_OPERATOR);

            } else if (c == '*' || c == '/') {
                /*
                 *  C and C++ comments
                 */
                if (getComment(input, tp, c) < 0) {
                    return tp->tokenId;
                }
#if BLD_FEATURE_EJS_DOC
                if (tp->text && tp->text[0] == '*') {
                    mprFree(input->doc);
                    input->doc = mprStrdup(input, (char*) tp->text);
                }
#endif
                initializeToken(tp, stream);
                break;
            }
            putBackChar(stream, c);
            return makeToken(tp, '/', T_DIV, G_OPERATOR);

        case '%':
            c = getNextChar(stream);
            if (c == '=') {
                addCharToToken(tp, '%');
                return makeSubToken(tp, c, T_ASSIGN, T_MOD_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
            }
            putBackChar(stream, c);
            return makeToken(tp, '%', T_MOD, G_OPERATOR);

        case '.':
            c = getNextChar(stream);
            if (c == '.') {
                c = getNextChar(stream);
                if (c == '.') {
                    addStringToToken(tp, "..");
                    return makeToken(tp, c, T_ELIPSIS, 0);
                }
                putBackChar(stream, c);
                addCharToToken(tp, '.');
                return makeToken(tp, '.', T_DOT_DOT, 0);

            } else if (c == '<') {
                addCharToToken(tp, '.');
                return makeToken(tp, c, T_DOT_LESS, 0);

            } else if (isdigit(c)) {
                putBackChar(stream, c);
                goto number;
            }
            putBackChar(stream, c);
            //  EJS extension to consider this an operator
            return makeToken(tp, '.', T_DOT, G_OPERATOR);

        case ':':
            c = getNextChar(stream);
            if (c == ':') {
                addCharToToken(tp, ':');
                return makeToken(tp, c, T_COLON_COLON, 0);
            }
            putBackChar(stream, c);
            return makeToken(tp, ':', T_COLON, 0);

        case '!':
            c = getNextChar(stream);
            if (c == '=') {
                c = getNextChar(stream);
                if (c == '=') {
                    addStringToToken(tp, "!=");
                    return makeToken(tp, c, T_STRICT_NE, G_OPERATOR);
                }
                putBackChar(stream, c);
                addCharToToken(tp, '!');
                return makeToken(tp, '=', T_NE, G_OPERATOR);
            }
            putBackChar(stream, c);
            return makeToken(tp, '!', T_LOGICAL_NOT, G_OPERATOR);

#if UNUSED
        case '~':
            c = getNextChar(stream);
            if (c == '=') {
                addStringToToken(tp, "~=");
                return makeSubToken(tp, c, T_ASSIGN, T_BIT_NEG_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
            }
            putBackChar(stream, c);
            return makeToken(tp, '~', T_BIT_NEG, G_OPERATOR);
#endif

        case '&':
            c = getNextChar(stream);
            if (c == '&') {
                addCharToToken(tp, '&');
                c = getNextChar(stream);
                if (c == '=') {
                    addCharToToken(tp, '&');
                    return makeSubToken(tp, '=', T_ASSIGN, T_LOGICAL_AND_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
                }
                putBackChar(stream, c);
                return makeToken(tp, '&', T_LOGICAL_AND, G_OPERATOR);

            } else if (c == '=') {
                addCharToToken(tp, '&');
                return makeSubToken(tp, c, T_ASSIGN, T_BIT_AND_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
            }
            putBackChar(stream, c);
            return makeToken(tp, '&', T_BIT_AND, G_OPERATOR);

        case '<':
            c = getNextChar(stream);
            if (c == '=') {
                addCharToToken(tp, '<');
                return makeToken(tp, c, T_LE, G_OPERATOR);
            } else if (c == '<') {
                c = getNextChar(stream);
                if (c == '=') {
                    addStringToToken(tp, "<<");
                    return makeSubToken(tp, c, T_ASSIGN, T_LSH_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
                }
                putBackChar(stream, c);
                addCharToToken(tp, '<');
                return makeToken(tp, c, T_LSH, G_OPERATOR);

            } else if (c == '/') {
                addCharToToken(tp, '<');
                return makeToken(tp, c, T_LT_SLASH, 0);
            }
            putBackChar(stream, c);
            return makeToken(tp, '<', T_LT, G_OPERATOR);

        case '=':
            c = getNextChar(stream);
            if (c == '=') {
                c = getNextChar(stream);
                if (c == '=') {
                    addStringToToken(tp, "==");
                    return makeToken(tp, c, T_STRICT_EQ, G_OPERATOR);
                }
                putBackChar(stream, c);
                addCharToToken(tp, '=');
                return makeToken(tp, c, T_EQ, G_OPERATOR);
            }
            putBackChar(stream, c);
            return makeToken(tp, '=', T_ASSIGN, G_OPERATOR);

        case '>':
            c = getNextChar(stream);
            if (c == '=') {
                addCharToToken(tp, '<');
                return makeToken(tp, c, T_GE, G_OPERATOR);
            } else if (c == '>') {
                c = getNextChar(stream);
                if (c == '=') {
                    addStringToToken(tp, ">>");
                    return makeSubToken(tp, c, T_ASSIGN, T_RSH_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
                } else if (c == '>') {
                    c = getNextChar(stream);
                    if (c == '=') {
                        addStringToToken(tp, ">>>");
                        return makeSubToken(tp, c, T_ASSIGN, T_RSH_ZERO_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
                    }
                    putBackChar(stream, c);
                    addStringToToken(tp, ">>");
                    return makeToken(tp, '>', T_RSH_ZERO, G_OPERATOR);
                }
                putBackChar(stream, c);
                addCharToToken(tp, '>');
                return makeToken(tp, '>', T_RSH, G_OPERATOR);
            }
            putBackChar(stream, c);
            return makeToken(tp, '>', T_GT, G_OPERATOR);

        case '^':
            c = getNextChar(stream);
            if (c == '^') {
                addCharToToken(tp, '^');
                c = getNextChar(stream);
                if (c == '=') {
                    addCharToToken(tp, '^');
                    return makeSubToken(tp, '=', T_ASSIGN, T_LOGICAL_XOR_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
                }
                putBackChar(stream, c);
                return makeToken(tp, '^', T_LOGICAL_XOR, G_OPERATOR);

            } else if (c == '=') {
                addCharToToken(tp, '^');
                return makeSubToken(tp, '=', T_ASSIGN, T_BIT_XOR_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
            }
            putBackChar(stream, c);
            return makeToken(tp, '^', T_BIT_XOR, G_OPERATOR);

        case '|':
            c = getNextChar(stream);
            if (c == '|') {
                addCharToToken(tp, '|');
                c = getNextChar(stream);
                if (c == '=') {
                    addCharToToken(tp, '|');
                    return makeSubToken(tp, '=', T_ASSIGN, T_LOGICAL_OR_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);
                }
                putBackChar(stream, c);
                return makeToken(tp, '|', T_LOGICAL_OR, G_OPERATOR);

            } else if (c == '=') {
                addCharToToken(tp, '|');
                return makeSubToken(tp, '=', T_ASSIGN, T_BIT_OR_ASSIGN, G_OPERATOR | G_COMPOUND_ASSIGN);

            }
            putBackChar(stream, c);
            return makeToken(tp, '|', T_BIT_OR, G_OPERATOR);
        }
    }
}