/* La función getToken el siguiente token * en el archivo fuente */ TokenType getToken(void) { /* indice para guardar en tokenString */ int tokenStringIndex = 0; /* guarda el token actual para ser retornado */ TokenType currentToken; /* estado actual - siempre comienza con START */ StateType state = START; /* flag para indicar el guardado en tokenString */ int save; /* flags para indicar el estado de un NUM */ int real = 0; int negativo = 0; int cientifico = 0; while (state != DONE) { int c = getNextChar(); save = TRUE; switch (state) { case START: if (isdigit(c) || (c == '-')) state = INNUM; else if (isalpha(c)) state = INID; else if (c == '=') state = INASSIGN; else if ((c == ' ') || (c == '\t') || (c == '\n')) save = FALSE; else if (c == '/') { c = getNextChar(); if (c == '/') { save = FALSE; state = INCOMMENT; } else { ungetNextChar(); state = START; } } else { state = DONE; switch (c) { case EOF: save = FALSE; currentToken = ENDFILE; break; case '<': currentToken = LT; break; case '+': currentToken = PLUS; break; case '-': currentToken = MINUS; break; case '*': currentToken = TIMES; break; case '/': currentToken = OVER; break; case '(': currentToken = LPAREN; break; case ')': currentToken = RPAREN; break; case ';': currentToken = SEMI; break; default: currentToken = ERROR; break; } } break; case INCOMMENT: save = FALSE; if (c == EOF) { state = DONE; currentToken = ENDFILE; } else if (c == '\n') state = START; break; case INASSIGN: state = DONE; if (c == '=') currentToken = EQ; else { ungetNextChar(); currentToken =ASSIGN; } break; case INNUM: if (!isdigit(c)) { if(c=='-' && negativo == 0){ negativo = 1; }else if (c=='.' && real == 0){ real = 1; }else if (c=='e' && cientifico == 0){ cientifico = 1; c = getNextChar(); real = 0; if (c=='-'){ negativo = 0; }else{ ungetNextChar(); } }else{ /* backup en el input */ ungetNextChar(); save = FALSE; state = DONE; currentToken = NUM; } } break; case INID: if (!isalpha(c)) { /* backup en el input */ ungetNextChar(); save = FALSE; state = DONE; currentToken = ID; } break; case DONE: default: /* nunca deberia suceder */ fprintf(listing,"Scanner Bug: state= %d\n",state); state = DONE; currentToken = ERROR; break; } if ((save) && (tokenStringIndex <= MAXTOKENLEN)) tokenString[tokenStringIndex++] = (char) c; if (state == DONE) { tokenString[tokenStringIndex] = '\0'; if (currentToken == ID) currentToken = reservedLookup(tokenString); } } if (TraceScan) { fprintf(listing,"\t%d: ",lineno); printToken(currentToken,tokenString); } return currentToken; } /* fin del getToken */
/* function getToken returns the * next token in source file */ TokenType getToken(void) { /* index for storing into tokenString */ int tokenStringIndex = 0; /* holds current token to be returned */ TokenType currentToken; /* current state - always begins at START */ StateType state = START; /* flag to indicate save to tokenString */ int save; while (state != DONE) { int c = getNextChar(); save = TRUE; switch (state) { case START: if (isdigit(c)) state = INNUM; else if (isalpha(c)) state = INID; else if (c == '=') state = INASSIGN; else if (c == '<') state = INLESS; else if (c == '>') state = INGREATER; else if (c == '!') state = INNOTEQ; else if ((c == ' ') || (c == '\t') || (c == '\n')) save = FALSE; else if (c == '/') { save = FALSE; state = INCOMMENT1; } else { state = DONE; switch (c) { case EOF: save = FALSE; currentToken = ENDFILE; break; case '+': currentToken = PLUS; break; case '-': currentToken = MINUS; break; case '*': currentToken = TIMES; break; case ';': currentToken = SEMI; break; case ',': currentToken = COMMA; break; case '(': currentToken = LPAREN; break; case ')': currentToken = RPAREN; break; case '[': currentToken = SLBRACKET; break; case ']': currentToken = SRBRACKET; break; case '{': currentToken = LBRACKET; break; case '}': currentToken = RBRACKET; break; default: currentToken = ERROR; break; } } break; case INCOMMENT1: save = FALSE; if (c == '*') { state = INCOMMENT2; } else { state = DONE; ungetNextChar(); currentToken = OVER; } break; case INCOMMENT2: save = FALSE; if(c == EOF) { state = DONE; currentToken = ENDFILE; } else if (c == '*') { state = INCOMMENT3; } break; case INCOMMENT3: save = FALSE; if (c == '/') { state = START; } else { state = INCOMMENT2; } break; case INASSIGN: state = DONE; if (c == '=') currentToken = EQ; else { /* backup in the input */ ungetNextChar(); save = FALSE; currentToken = ASSIGN; } break; case INLESS: state = DONE; if (c == '=') currentToken = LESSEQ; else { /* backup in the input */ ungetNextChar(); save = FALSE; currentToken = LESS; } break; case INGREATER: state = DONE; if (c == '=') currentToken = GREATEREQ; else { /* backup in the input */ ungetNextChar(); save = FALSE; currentToken = GREATER; } break; case INNOTEQ: state = DONE; if (c == '=') currentToken = NOTEQ; else { /* backup in the input */ ungetNextChar(); save = FALSE; currentToken = ERROR; } break; case INNUM: if (!isdigit(c)) { /* backup in the input */ ungetNextChar(); save = FALSE; state = DONE; currentToken = NUM; } break; case INID: if (!isalpha(c)) { /* backup in the input */ ungetNextChar(); save = FALSE; state = DONE; currentToken = ID; } break; case DONE: default: /* should never happen */ fprintf(listing,"Scanner Bug: state= %d\n",state); state = DONE; currentToken = ERROR; break; } if ((save) && (tokenStringIndex <= MAXTOKENLEN)) tokenString[tokenStringIndex++] = (char) c; if (state == DONE) { tokenString[tokenStringIndex] = '\0'; if (currentToken == ID) currentToken = reservedLookup(tokenString); } } if (TraceScan) { fprintf(listing,"\t%d: ",lineno); printToken(currentToken,tokenString); } return currentToken; } /* end getToken */
/* function getToken returns the * next token in source file */ AsmTokenType TAsmAnalyze::getToken(void) { int tokenStrIdx = 0; /* index for storing into tokenString */ tokenString = ""; AsmTokenType curToken; /* holds current token to be returned */ StateType state = START; /* current state - always begins at START */ int save; /* flag to indicate save to tokenString */ //linepos=0; //bufsize=0; //cout<<linepos<<endl; while (state != DONE) { char c = getNextChar(); save = TRUE; switch (state) { case START: if (IsName(c)) // Is Name state = INID; else if (isdigit(c)) // Is Hex Number state = INHEXNUM; else if (c == '#') state = INSHARP; else if (c == ',') state = INXCHAR; else if ((c == ' ') || (c == '\t')) save = FALSE; else if (c == '[') { save = FALSE; state = INSTRING; } else if (c == '{') { save = FALSE; state = INCOMMENT; } else if (c == ';') { save = FALSE; state = INCOMMENT_SINGLE; } else { state = DONE; switch (c) { case EOF: save = FALSE; curToken = ENDFILE; break; case ':': curToken = COLON; break; case '=': curToken = EQUAL; break; case '@': curToken = ATCHAR; break; case '\n': curToken = ENDLINE; break; default: curToken = TOKENERROR; } } break; case INCOMMENT: save = FALSE; if (c == EOF) state = DONE; else if (c == '}') state = START; break; case INCOMMENT_SINGLE: save = FALSE; if (c == '\n') { state = DONE; curToken = ENDLINE; } break; case INXCHAR: state = DONE; if (c == 'X') curToken = XCHAR; else { ungetNextChar(); /* backup in the input */ save = FALSE; curToken = TOKENERROR; } break; case INSHARP: if (!isxdigit(c)) { ungetNextChar(); /* backup in the input */ save = FALSE; state = DONE; if (tokenStrIdx > 1) curToken = SHARP; else curToken = TOKENERROR; } break; case INSTRING: save = TRUE; if (c == ']') { ungetNextChar(); /* backup in the input */ save = FALSE; state = DONE; curToken = STRING; } break; case INID: if (!IsName(c) && !isdigit(c)) { ungetNextChar(); /* backup in the input */ save = FALSE; state = DONE; curToken = ID; } break; case INHEXNUM: if (!isxdigit(c)) { ungetNextChar(); /* backup in the input */ save = FALSE; state = DONE; curToken = HEXNUM; } break; case DONE: default: //cout<<"Scanner Bug: state= "<<state<<endl; state = DONE; curToken = TOKENERROR; break; } if ((save) && (tokenStrIdx <= MAXTOKENLEN)) { //tokenString[tokenStrIdx++] = c; tokenString+=c; tokenStrIdx++; //cout<<tokenStrIdx<<" "<<c<<"\n"; } if (state == DONE) { //tokenString[tokenStrIdx++] = '\0'; if (curToken == ID) curToken = reservedLookup(tokenString); /*if (curToken == ID) { if (IsHex(tokenString)) curToken = HEXNUM; else curToken = ID; }*/ } } TokenPos++; //cout<<"\t"<<lineno<<": "; //cout<<tokenString<<" "<<TokenPos<<endl; //printToken(curToken, tokenString); CurrentToken = curToken; return curToken; }
/* function getToken returns the * next token in source file */ TokenType getToken(void) { /* index for storing into tokenString */ int tokenStringIndex = 0; /* holds current token to be returned */ TokenType currentToken; /* current state - always begins at START */ StateType state = START; /* flag to indicate save to tokenString */ int save; while (state != DONE) { int c = getNextChar(); save = true; //........... switch (state) { case START: if (isdigit(c)) state = INNUM; else if (isalpha(c)) state = INID; //else if (c == ':') //............... //state = INASSIGN; else if (c == '<') state = INLE; else if (c == '>') state = INGE; else if (c == '=') state = INEQ; else if (c == '!') state = INNE; else if ((c == ' ') || (c == '\t') || (c == '\n')) save = false; else if (c == '/') { //save = FALSE; state = ENTERING_COMMENT; } else { state = DONE; switch (c) { case EOF: save = false; currentToken = ENDFILE; break; case '+': currentToken = PLUS; break; case '-': currentToken = MINUS; break; case '*': currentToken = TIMES; break; case '/': currentToken = ENTERING_COMMENT; break; case '(': currentToken = LPAREN; break; case ')': currentToken = RPAREN; break; case ';': currentToken = SEMI; break; case '[': currentToken = LBRACKET; break; case ']': currentToken = RBRACKET; break; case '{': currentToken = LBRACE; break; case '}': currentToken = RBRACE; break; default: currentToken = ERROR; break; } } break; /*case ENTERING_COMMENT: save = false; if (c == EOF) //???????????????? { state = DONE; currentToken = ENDFILE; } else if (c == '*') state = INCOMMENT; else { currentToken = TIMES; save = true; state = DONE; } break; */ case ENTERING_COMMENT: save = false; if(c == '*') state = INCOMMENT; else { ungetNextChar(); currentToken = OVER; state = DONE; save = true; } break; case INCOMMENT: save = false; if(c == '*') state = EXITING_COMMENT; break; case EXITING_COMMENT: save = false; if(c == '/') state = START; else if (c == '*') state = EXITING_COMMENT; else state = INCOMMENT; break; case INLE: state = DONE; if (c == '=') currentToken = LE; else { /* backup in the input */ ungetNextChar(); //save = FALSE; currentToken = LT; } break; case INGE: state = DONE; if (c == '=') currentToken = GE; else { /* backup in the input */ ungetNextChar(); //save = FALSE; currentToken = GT; } break; case INEQ: state = DONE; if (c == '=') currentToken = EQ; else { /* backup in the input */ ungetNextChar(); //save = FALSE; currentToken = ASSIGN; } break; case INNE: //不等号。。。。。。。。。。。。 state = DONE; if (c == '=') currentToken = NE; else { /* backup in the input */ ungetNextChar(); save = false; currentToken = ERROR; } break; case INNUM: if (!isdigit(c)) { /* backup in the input */ ungetNextChar(); save = false; state = DONE; currentToken = NUM; } break; case INID: if (!isalpha(c)) { /* backup in the input */ ungetNextChar(); save = false; state = DONE; currentToken = ID; } break; case DONE: default: /* should never happen */ //fprintf(listing,"Scanner Bug: state= %d\n",state); state = DONE; currentToken = ERROR; break; } if ((save) && (tokenStringIndex <= MAXTOKENLEN)) { tokenString[tokenStringIndex++] = (char) c; //tokenIndex[tokenStringIndex] = currentToken; } if (state == DONE) { tokenString[tokenStringIndex] = '\0'; if (currentToken == ID) { currentToken = reservedLookup(tokenString); } } } /* if (TraceScan) { fprintf(listing,"\t%d: ",lineno); printToken(currentToken,tokenString); }*/ //printf("YES\n"); //fputs(ansToken[currentToken],target); printf("%s\n",ansToken[currentToken]); return currentToken; } /* end getToken */
// function getToken returns the // next token in source file TokenType getToken(void) { // index for storing into tokenString int tokenStringIndex = 0; // holds current token to be returned TokenType currentToken; // current state - always begins at START StateType state = START; // flag to indicate save to tokenString int save; while (state != DONE) { char c = getNextChar(); save = TRUE; switch (state) { case START: if (isdigit(c)) state = INNUM; else if (isalpha(c)) state = INID; else if (c == ':') state = INASSIGN; else if ((c == ' ') || (c == '\t') || (c == '\n')) save = FALSE; else if (c == '{') { save = FALSE; state = INCOMMENT; } else { state = DONE; switch (c) { case EOF: save = FALSE; currentToken = ENDFILE; break; case '=': currentToken = EQ; break; case '<': currentToken = LT; break; case '+': currentToken = PLUS; break; case '-': currentToken = MINUS; break; case '*': currentToken = TIMES; break; case '/': currentToken = OVER; break; case '(': currentToken = LPAREN; case ')': currentToken = RPAREN; case ';': currentToken = SEMI; break; default: currentToken = ERROR; break; } } break; case INCOMMENT: save = FALSE; if (c == '}') state = START; break; case INASSIGN: state = DONE; if (c == '=') currentToken = ASSIGN; else { // backup in the input ungetNextChar(); save = FALSE; currentToken = ERROR; } break; case INNUM: if (!isdigit(c)) { // backup in the input ungetNextChar(); save = FALSE; state = DONE; currentToken = NUM; } break; case INID: if (!isalpha(c)) { // backup in the input ungetNextChar(); save = FALSE; state = DONE; currentToken = ID; } break; case DONE: default: // should never happen fprintf(listing, "Scanner Bug: state= %d\n", state); state = DONE; currentToken = ERROR; break; } if ((save) && (tokenStringIndex <= MAXTOKENLEN)) tokenString [ tokenStringIndex ++ ] = c; if (state == DONE) { tokenString [ tokenStringIndex ] = '\0'; if (currentToken == ID) currentToken = reservedLookup(tokenString); } } if (TraceScan) { fprintf(listing, "\t%d: ", lineno); printToken(currentToken, tokenString); } return currentToken; } // end getToken
//returns the token type of next token and corresponding string is stored in tokenString TokenType getToken(){ TokenType currentToken = OTHER; char ch; int saveCharInToken = 1; StateType state = START; int tokenStringIndex = 0; bzero(tokenString, MAX_STR_LEN); while(state != END){ ch = getNextChar(); switch(state){ case START : if(isdigit(ch)){ if(ch == '0') state = INT; else state = INT_DEC; } else if(ch == '\'') state = CHAR; else if(ch == '\"') state = STRING; else if(ch == '_' || isalpha(ch)) state = ID; else if(ch == '+' || ch == '-' || ch == '*' || ch == '/' || ch == '%') state = OP; else if(ch == '<' || ch == '>' || ch == '=' || ch == '!') state = OP; else if(ch == '&' || ch == '|') state = COND_OP; else if(ch == '[' || ch == ']' || ch == '{' || ch == '}' || ch == '(' || ch == ')' || ch == ',' || ch == ';') state = DELIMIT; else if(isWhiteSpace(ch)) continue; else{ handleTokenError(ch, 0, 0); currentToken = OTHER; state = END; saveCharInToken = 0; } break; case INT : if(ch == 'x') state = INT_EXPECT_HEX; else if(isdigit(ch)) state = INT_DEC; else { state = END; currentToken = INTLITERAL; saveCharInToken = 0; ungetNextChar(); } break; case INT_DEC : if(!isdigit(ch)) { state = END; currentToken = INTLITERAL; saveCharInToken = 0; ungetNextChar(); } break; case INT_EXPECT_HEX : if(isxdigit(ch)) state = INT_HEX; else { handleTokenError(ch, 0, 0); currentToken = ERROR; } break; case INT_HEX : if(!isxdigit(ch)) { state = END; currentToken = INTLITERAL; saveCharInToken = 0; ungetNextChar(); } break; case CHAR : if(ch >= 32 && ch <= 126 && ch != '\'' && ch != '\"' && ch != '\\') state = CHAR_EXPECT_QUOTE; else if(ch == '\\') state = ESCAPE_CHAR; else { handleTokenError(ch, 0, 0); state = CHAR_EXPECT_QUOTE; currentToken = ERROR; } break; case CHAR_EXPECT_QUOTE : if(ch == '\'') { state = END; if(currentToken != ERROR) currentToken = CHARLITERAL; saveCharInToken = 1; } else { handleTokenError(ch, 1, '\''); currentToken = ERROR; } break; case ESCAPE_CHAR : if(ch == 'n' || ch == 't' || ch == '\'' || ch == '\"' || ch == '\\') state = CHAR_EXPECT_QUOTE; else { handleTokenError(ch, 0, 0); state = CHAR_EXPECT_QUOTE; currentToken = ERROR; } break; case STRING : if(ch == '\\') state = STRING_ESCAPE_CHAR; else if(ch == '\"'){ state = END; if(currentToken != ERROR) currentToken = STRINGLITERAL; saveCharInToken = 1; } else if(!(ch >= 32 && ch <= 126 && ch != '\'' && ch != '\\')) { handleTokenError(ch, 1, '\"'); currentToken = ERROR; state = END; } break; case STRING_ESCAPE_CHAR : if(ch == 'n' || ch == 't' || ch == '\'' || ch == '\"' || ch == '\\') state = STRING; else { handleTokenError(ch, 0, 0); state = STRING; currentToken = ERROR; } break; case ID : if(!isalnum(ch) && ch != '_') { state = END; currentToken = IDENTIFIER; saveCharInToken = 0; ungetNextChar(); } break; case OP : if(tokenString[0] == '/' && ch == '/') { state = COMMENT; currentToken = COMMENT_TOK; } else if(ch == '=') { state = END; currentToken = OPERATOR; saveCharInToken = 1; } else{ state = END; currentToken = OPERATOR; saveCharInToken = 0; ungetNextChar(); } break; case COND_OP : if(ch == tokenString[0]) { state = END; currentToken = OPERATOR; saveCharInToken = 1; } else { handleTokenError(ch, 1, tokenString[0]); currentToken = ERROR; } break; case COMMENT : if(ch == '\n') { state = END; saveCharInToken = 0; ungetNextChar(); } break; case DELIMIT : state = END; currentToken = DELIMITER; saveCharInToken = 0; ungetNextChar(); } if(ch == EOF) break; if(saveCharInToken && currentToken != ERROR) tokenString[tokenStringIndex++] = ch; //printf("ch = %c\tsave = %d\tstate = %s\ttoken = %s\ttokenString = %s\n", ch, saveCharInToken, stateTypeStr[state], tokenTypeStr[currentToken], tokenString); } tokenString[tokenStringIndex] = '\0'; if(currentToken == COMMENT_TOK) return getToken(); if(currentToken == IDENTIFIER && isBoolean(tokenString)) currentToken = BOOLEANLITERAL; else if(currentToken == IDENTIFIER && isKeyword(tokenString)) currentToken = KEYWORD; return currentToken; }