Пример #1
0
/* NextToken - read the next token */
static int NextToken(ParseContext *c)
{
    int ch, tkn;

    /* skip leading blanks */
    ch = SkipSpaces(c);

    /* remember the start of the current token */
    c->tokenOffset = (int)(c->sys->linePtr - c->sys->lineBuf);

    /* check the next character */
    switch (ch) {
    case EOF:
        tkn = T_EOL;
        break;
    case '"':
        tkn = StringToken(c);
        break;
    case '\'':
        tkn = CharToken(c);
        break;
    case '<':
        if ((ch = GetChar(c)) == '=')
            tkn = T_LE;
        else if (ch == '>')
            tkn = T_NE;
        else if (ch == '<')
            tkn = T_SHL;
        else {
            UngetC(c);
            tkn = '<';
        }
        break;
    case '>':
        if ((ch = GetChar(c)) == '=')
            tkn = T_GE;
        else if (ch == '>')
            tkn = T_SHR;
        else {
            UngetC(c);
            tkn = '>';
        }
        break;
    case '0':
        switch (GetChar(c)) {
        case 'x':
        case 'X':
            tkn = HexNumberToken(c);
            break;
        case 'b':
        case 'B':
            tkn = BinaryNumberToken(c);
            break;
        default:
            UngetC(c);
            tkn = NumberToken(c, '0');
            break;
        }
        break;
    default:
        if (isdigit(ch))
            tkn = NumberToken(c, ch);
        else if (IdentifierCharP(ch)) {
            char *savePtr;
            switch (tkn = IdentifierToken(c,ch)) {
            case T_ELSE:
                savePtr = c->sys->linePtr;
                if ((ch = SkipSpaces(c)) != EOF && IdentifierCharP(ch)) {
                    switch (IdentifierToken(c, ch)) {
                    case T_IF:
                        tkn = T_ELSE_IF;
                        break;
                    default:
                        c->sys->linePtr = savePtr;
                        break;
                    }
                }
                else
                    c->sys->linePtr = savePtr;
                break;
            case T_END:
                savePtr = c->sys->linePtr;
                if ((ch = SkipSpaces(c)) != EOF && IdentifierCharP(ch)) {
                    switch (IdentifierToken(c, ch)) {
                    case T_DEF:
                        tkn = T_END_DEF;
                        break;
                    case T_IF:
                        tkn = T_END_IF;
                        break;
                    default:
                        c->sys->linePtr = savePtr;
                        break;
                    }
                }
                else
                    c->sys->linePtr = savePtr;
                break;
            case T_DO:
                savePtr = c->sys->linePtr;
                if ((ch = SkipSpaces(c)) != EOF && IdentifierCharP(ch)) {
                    switch (IdentifierToken(c, ch)) {
                    case T_WHILE:
                        tkn = T_DO_WHILE;
                        break;
                    case T_UNTIL:
                        tkn = T_DO_UNTIL;
                        break;
                    default:
                        c->sys->linePtr = savePtr;
                        break;
                    }
                }
                else
                    c->sys->linePtr = savePtr;
                break;
            case T_LOOP:
                savePtr = c->sys->linePtr;
                if ((ch = SkipSpaces(c)) != EOF && IdentifierCharP(ch)) {
                    switch (IdentifierToken(c, ch)) {
                    case T_WHILE:
                        tkn = T_LOOP_WHILE;
                        break;
                    case T_UNTIL:
                        tkn = T_LOOP_UNTIL;
                        break;
                    default:
                        c->sys->linePtr = savePtr;
                        break;
                    }
                }
                else
                    c->sys->linePtr = savePtr;
                break;
            }
        }
        else
            tkn = ch;
    }

    /* return the token */
    return tkn;
}
Пример #2
0
//-----------------------------------------------------------------------------
TOKEN GetNextToken(READFILE Stream) {

    int CurrentChar,PreviousChar;
    int Index;
//----static so it doesn't have to get allocated everytime (very often!)
    static SuperString LocalValue;

//DEBUG printf("char was ==%c==\n",CurrentCharacter(Stream));
    if (Stream->Overshot) {
//DEBUG printf("overshot\n");
        CurrentChar = CurrentCharacter(Stream);
    } else {
//DEBUG printf("get next\n");
        CurrentChar = NextCharacter(Stream);
    }
    Stream->Overshot = 0;

//----Skip whitespace
    while (isspace(CurrentChar)) {
        PreviousChar = CurrentChar;
        CurrentChar = NextCharacter(Stream);
//----Check for a blank line, if required
        if (Stream->NeedNonLogicTokens && PreviousChar == '\n' && 
CurrentChar == '\n') {
            return(BuildToken(blank_line_token,""));
        }
    }

//DEBUG printf("char is ==%c==\n",CurrentChar);
    switch (CurrentChar) {
        case '/':
            Index = 0;
            LocalValue[Index++] = CurrentChar;
            PreviousChar = CurrentChar;
            CurrentChar = NextCharacter(Stream);
            if (CurrentChar == '*') {
                LocalValue[Index] = CurrentChar;
                while (CurrentChar != EOF && (CurrentChar != '/' || 
PreviousChar != '*')) {
                    PreviousChar = CurrentChar;
                    CurrentChar = NextCharacter(Stream);
                    IncrementTokenIndex(Stream,&Index);
                    LocalValue[Index] = CurrentChar;
                }
                if (CurrentChar == '/') {
//----Add eoln if it's there
                    CurrentChar = NextCharacter(Stream);
                    if (CurrentChar == '\n') {
                        IncrementTokenIndex(Stream,&Index);
                        LocalValue[Index] = CurrentChar;
                    } else {
                        Stream->Overshot = 1;
                    }
                    IncrementTokenIndex(Stream,&Index);
                    LocalValue[Index] = '\0';
                    if (Stream->NeedNonLogicTokens) {
                        return(BuildToken(comment_token,LocalValue));
                    } else {
                        return(GetNextToken(Stream));
                    }
                } else {
                    CharacterError(Stream);
                }
            } else {
                CharacterError(Stream);
            }
            break;
        case '%':
        case '#':
            if (Stream->NeedNonLogicTokens) {
                Index = 0;
                do {
                    LocalValue[Index] = CurrentChar;
                    IncrementTokenIndex(Stream,&Index);
                    CurrentChar = NextCharacter(Stream);
                } while (CurrentChar != '\n' && CurrentChar != EOF);
                LocalValue[Index] = '\0';
                Stream->Overshot = 1;
                return(BuildToken(comment_token,LocalValue));
            } else {
//----Discard sequences of comments (recursive approach gave stack overflow)
                do {
                    while (CurrentChar != '\n' && CurrentChar != EOF) {
                        CurrentChar = NextCharacter(Stream);
                    }
                    CurrentChar = NextCharacter(Stream);
                } while (CurrentChar == '%' || CurrentChar == '#');
                Stream->Overshot = 1;
                return(GetNextToken(Stream));
            }
            break;
        case '\'':
            return(QuotedToken(Stream,'\'',lower_word));
            break;
        case '(':
            return(BuildToken(punctuation,"("));
            break;
        case ')':
            return(BuildToken(punctuation,")"));
            break;
        case '[':
            return(BuildToken(punctuation,"["));
            break;
        case ']':
            return(BuildToken(punctuation,"]"));
            break;
        case '!':
            CurrentChar = NextCharacter(Stream);
            if (CurrentChar == '=') {
                return(BuildToken(lower_word,"!="));
            } else if (CurrentChar == '>') {
                return(BuildToken(quantifier,"!>"));
            } else if (CurrentChar == '!') {
                return(BuildToken(unary_connective,"!!"));
            } else {
                Stream->Overshot = 1;
                return(BuildToken(quantifier,"!"));
            }
            break;
        case '?':
            CurrentChar = NextCharacter(Stream);
            if (CurrentChar == '*') {
                return(BuildToken(quantifier,"?*"));
            } else if (CurrentChar == '?') {
                return(BuildToken(unary_connective,"??"));
            } else {
                Stream->Overshot = 1;
                return(BuildToken(quantifier,"?"));
            }
            break;
        case '^':
            return(BuildToken(quantifier,"^"));
            break;
        case '.':
            return(BuildToken(punctuation,"."));
            break;
        case ':':
            return(BuildToken(punctuation,":"));
            break;
        case ',':
            return(BuildToken(punctuation,","));
            break;
        case '<':
            CurrentChar = NextCharacter(Stream);
            if (CurrentChar == '='){
                CurrentChar = NextCharacter(Stream);
                if (CurrentChar == '>') {
                    return(BuildToken(binary_connective,"<=>"));
                } else {
                    Stream->Overshot = 1;
                    return(BuildToken(binary_connective,"<="));
                }
            } else if (CurrentChar == '~') {
                CurrentChar = NextCharacter(Stream);
                if (CurrentChar == '>') {
                    return(BuildToken(binary_connective,"<~>"));
                } else {
                    CharacterError(Stream);
                }
            } else if (CurrentChar == '<') {
                return(BuildToken(punctuation,"<<"));
            } else {
                CharacterError(Stream);
            }
            break;
        case '=':
            CurrentChar = NextCharacter(Stream);
            if (CurrentChar == '>') {
                return(BuildToken(binary_connective,"=>"));
            } else {
                Stream->Overshot = 1;
                return(BuildToken(lower_word,"="));
            }
            break;
        case '~':
            CurrentChar = NextCharacter(Stream);
            if (CurrentChar == '|') {
                return(BuildToken(binary_connective,"~|"));
            } else if (CurrentChar == '&') {
                return(BuildToken(binary_connective,"~&"));
            } else {
                Stream->Overshot = 1;
                return(BuildToken(unary_connective,"~"));
            }
            break;
        case '+':
            CurrentChar = NextCharacter(Stream);
            if (CurrentChar == '+') {
                return(BuildToken(unary_connective,"++"));
            } else if (NumberToken(Stream,'+',CurrentChar,LocalValue)) {
                return(BuildToken(number,LocalValue));
            } else {
                Stream->Overshot = 1;
                return(BuildToken(binary_connective,"+"));
            }
            break;
        case '-':
            CurrentChar = NextCharacter(Stream);
            if (CurrentChar == '-') {
                CurrentChar = NextCharacter(Stream);
                if (CurrentChar == '>') {
                    return(BuildToken(binary_connective,"-->"));
                } else {
                    Stream->Overshot = 1;
                    return(BuildToken(unary_connective,"--"));
                }
//----Code copied from below for numbers
            } else if (NumberToken(Stream,'-',CurrentChar,LocalValue)) {
                return(BuildToken(number,LocalValue));
            } else {
                Stream->Overshot = 1;
                return(BuildToken(punctuation,"-"));
            }
            break;
        case '"':
            return(QuotedToken(Stream,'"',distinct_object));
            break;
        case '|':
            return(BuildToken(binary_connective,"|"));
            break;
        case '&':
            return(BuildToken(binary_connective,"&"));
            break;
        case '@':
            CurrentChar = NextCharacter(Stream);
            if (CurrentChar == '+') {
                return(BuildToken(quantifier,"@+"));
            } else if (CurrentChar == '-') {
                return(BuildToken(quantifier,"@-"));
            } else {
                Stream->Overshot = 1;
                return(BuildToken(binary_connective,"@"));
            }
            break;
        case '>':
            return(BuildToken(binary_connective,">"));
            break;
        case '*':
            return(BuildToken(binary_connective,"*"));
            break;
        case EOF:
            return(BuildToken(endeof,""));
            break;
        default:
            Index = 0;
            if (CurrentChar == '$' || islower(CurrentChar)) {
                do {
                    LocalValue[Index] = CurrentChar;
                    CurrentChar = NextCharacter(Stream);
                    IncrementTokenIndex(Stream,&Index);
                } while (isalnum(CurrentChar) || CurrentChar=='_' ||
//----Allow second $ for system predicates and functions
(Index == 1 && CurrentChar == '$' && LocalValue[0] == '$'));
                LocalValue[Index] = '\0';
                Stream->Overshot = 1;
//----Ensure $ words have some length
                Index = 0;
                while (LocalValue[Index] == '$') {
                    Index++;
                }
                if (Index > 0 && !islower(LocalValue[Index])) {
                    CharacterError(Stream);
                }
//----Replace equal by = for now (can remove in future)
//----At some point I did comment this out, but it's still needed for
//----reformatting old, e.g., EP proofs.
//                if (!strcmp(LocalValue,"equal")) {
//                    strcpy(LocalValue,"=");
//                }
                return(BuildToken(lower_word,LocalValue));
            } else if (isupper(CurrentChar)) {
                do {
                    LocalValue[Index] = CurrentChar;
                    CurrentChar = NextCharacter(Stream);
                    IncrementTokenIndex(Stream,&Index);
                } while (isalnum(CurrentChar) || (CurrentChar=='_'));
                LocalValue[Index] = '\0';
                Stream->Overshot = 1;
//----Nasty hack to allow end of file to be specified by user on input stream
                if (!strcmp(LocalValue,"EOF__")) {
                    return(BuildToken(endeof,""));
                } else {
                    return(BuildToken(upper_word,LocalValue));
                }
//----Numbers
            } else if (NumberToken(Stream,'\0',CurrentChar,LocalValue)) {
                return(BuildToken(number,LocalValue));
            } else {
                CharacterError(Stream);
            }
            break;
    }
//----Need a default return for the error cases which compiler doesn't get
    return(NULL);
}