Пример #1
0
Token* readIdentKeyword(void) {
  Token *token = makeToken(TK_NONE, lineNo, colNo);
  int count = 1;

  token->string[0] = (char)currentChar;
  readChar();

  while ((currentChar != EOF) &&
	 ((charCodes[currentChar] == CHAR_LETTER) || (charCodes[currentChar] == CHAR_DIGIT))) {
    if (count <= MAX_IDENT_LEN) token->string[count++] = (char)currentChar;
    readChar();
  }

  if (count > MAX_IDENT_LEN) {
    error(ERR_IDENTTOOLONG, token->lineNo, token->colNo);
    return token;
  }

  token->string[count] = '\0';
  token->tokenType = checkKeyword(token->string);

  if (token->tokenType == TK_NONE)
    token->tokenType = TK_IDENT;

  return token;
}
Пример #2
0
Token* readIdentKeyword(void) {

  // CuongDD: 28/08/2014
  Token *token = makeToken(TK_IDENT, lineNo, colNo);
  int index = 0;
  // If the next char is alphnumeric, keep reading
  while(charCodes[currentChar] == CHAR_LETTER || charCodes[currentChar] == CHAR_DIGIT) {
    token->string[index++] = currentChar;
    readChar(); 
  }

  token->string[index] = '\0';

  // Check if the ident is too long
  if (index > MAX_IDENT_LEN) {
    error(ERR_IDENTTOOLONG, lineNo, colNo);
  } else {
    // Check keyword
    TokenType tkType = checkKeyword(token->string);

    if (tkType != TK_NONE) {
      token->tokenType = tkType;
    }
  }

  return token;
}
Пример #3
0
Token* readIdentKeyword(void) 
{
	Token *token = makeToken(TK_IDENT, colNo, lineNo);
	char string[MAX_IDENT_LEN + 1];
	int i = 0;
	while (currentChar != EOF 
		&& (charCodes[currentChar] == CHAR_DIGIT 
		|| charCodes[currentChar] == CHAR_LETTER)) 
	{
		if (i >= MAX_IDENT_LEN) 
		{
			if (!CATCH_TOOLONG_ERR)
				error(ERR_IDENTTOOLONG, lineNo, colNo);
			readChar();
			continue;
		}
		string[i++] = currentChar;
		readChar();
	}
	string[i] = 0;
	
	// make token, because checkKeyword returns TK_NONE if kw is not type of Keyword, 
	// so we make token type is TK_IDENT instead of TK_NONE
	TokenType tokenType = checkKeyword(string);
	if (tokenType == TK_NONE)
		token->tokenType = TK_IDENT;
	else
		token->tokenType = tokenType;
	strcpy(token->string, string);
	return token;
}
Пример #4
0
Token* readIdentKeyword(void) {
  int count = 0;
  Token* token = makeToken(TK_IDENT, lineNo, colNo);

  while (charCodes[currentChar] == CHAR_LETTER || charCodes[currentChar] == CHAR_DIGIT) {
    // Add current character to identifier
    token->string[count] = currentChar;

    // Increase identifier length
    count++;

    // Get next character
    readChar();
  }
  // End string
  token->string[count] = '\0';

  // Limit identifier length
  if (count > MAX_IDENT_LEN) {
    // Announce error
    error(ERR_IDENTTOOLONG, lineNo, colNo - count);
  } else {
    // If this identifier is a keyword
    TokenType type = checkKeyword(token->string);

    // Otherwise
    if (type != TK_NONE) {
      token->tokenType = type;
    }
  }

  return token;
}
Пример #5
0
Token scanner( FILE *source )
{
    char c;
    Token token;

    if(!tokenBufferEmpty) {
        tokenBufferEmpty = 1;
        return tokenBuffer;
    }

    while( !feof(source) ){
        c = fgetc(source);
        while( isspace(c) )
            c = fgetc(source);

        if( isdigit(c) )
            return getNumericToken(source, c);
        else if( isalpha(c) ){
            token = getWordToken(source, c);
            return checkKeyword(token);
        }
        else
            return getOperatorToken(source, c);
    }
    token.tok[0] = '\0';
    token.type = EOFsymbol;
    return token;
}
Пример #6
0
Token* readString(void) {
  // TO DO doc khi van con la chu hoa so, luu vao token
  Token *token = makeToken(TK_NONE, lineNo, colNo);
  int count = 0;

  readChar();
  while ((currentChar != EOF)&&(charCodes[currentChar]!= CHAR_NHAYKEP)) {
    if ((count <= MAX_IDENT_LEN)&& (currentChar!='\\')&& (currentChar!= '\n')){
      token->string[count++] = (char)currentChar;}
    else if (currentChar == '\n'){
      // bao loi o day
      return token;
    } else if (currentChar =='\\') {

	switch(currentChar){
	case '\\':
	  token->string[count++] = '\\'; break;
	case 'n':
	  token->string[count++] = '\n'; break;
	case 't':
	  token->string[count++] = '\t'; break;
	case 'a':
	  token->string[count++] = '\a'; break;
	case '\'':
	  token->string[count++] = '\''; break;
	case '\"':
	  token->string[count++] = '\"'; break;
	default:
	  // bao loi o day
	  return token;
	}
    }
    readChar();
  }

  if (count > MAX_IDENT_LEN) {
    error(ERR_IDENTTOOLONG, token->lineNo, token->colNo);
    return token;
  }

  token->string[count] = '\0';
  token->tokenType = checkKeyword(token->string);
  if (token->tokenType == TK_NONE)
    token->tokenType = TK_STRING;

  return token;
}
Пример #7
0
int main(int argc, string argv[])
{
    if (argc != 2)
    {
        printf("Not enough or too much arguments!\n");
        return 1;
    }
    
    string key = argv[1];
    if (false == checkKeyword(key))
    {
        printf("Keyword contains non-alphabetical characters\n");
        return 1;
    }
    
    
    string message = GetString();
    encryptVigenere(message, key);
    
    return 0;
}
Пример #8
0
Token* readIdentKeyword(void) {
	int i = 0;
    Token *token = makeToken(TK_IDENT, lineNo, colNo);
    char *str = token->string;
    while (charCodes[currentChar] == CHAR_LETTER || charCodes[currentChar] == CHAR_DIGIT) {
        str[i++] = currentChar;
        readChar();
    }
    str[i] = '\0';
 
    TokenType type = checkKeyword(str);
    if (type != TK_NONE) {
        token->tokenType = type;
    }
 
    while (charCodes[currentChar] == CHAR_LETTER || charCodes[currentChar] == CHAR_DIGIT)
        readChar();
 
    return token;
  //TODO
}
Пример #9
0
Token* readString(void) {
    Token *token = makeToken(TK_NONE, lineNo, colNo);
    int count = 0;

    readChar();
    while((currentChar !=EOF) && (charCodes[currentChar] != CHAR_DOUBLEQUOTE)) {
        if (currentChar == '\n') {
            error(ERR_INVALIDSTR, token->lineNo, token->colNo);
            return token;
        } else {
            if (charCodes[currentChar] != CHAR_SPLASH) {
                if (count <= MAX_IDENT_LEN) token->string[count++] = (char)currentChar;
            } else {
                if (currentChar != EOF) {
                    readChar();
                    char c ;
                    int rescan = 0;
                    switch(currentChar) {
                    case 'n':
                        c = '\n';
                        break;
                    case 'a':
                        c = '\a';
                        break;
                    case 'b':
                        c = '\b';
                        break;
                    case 't':
                        c = '\t';
                        break;
                    case 'f':
                        c = '\f';
                        break;
                    case 'r':
                        c = '\r';
                        break;
                    case 'v':
                        c = '\v';
                        break;
                    case 'x':
                        readChar();
                        char hex[3];
                        int i = 0;
                        int h = 0;
                        while ((currentChar >='0' && currentChar <='9') || (currentChar >='A' && currentChar <='F')) {
                            if ( i < 2)
                                hex[i++] = (char) currentChar;
                            readChar();
                        }
                        hex[i] = '\0';
                        rescan = 1;
                        sscanf(hex,"%2x",&h);
                        if (h < 0 || i == 0) {
                            error(ERR_INVALIDCONSTANT, token->lineNo, token->colNo);
                            return token;
                        }
                        c = (char) h;
                        break;
                    case '\'':
                        c='\'';
                        break;
                    case '\"':
                        c = '\"';
                        break;
                    case '0':
                        readChar();
                        if (charCodes[currentChar] == CHAR_SPACE)
                            c = '\0';
                        else {
                            char oct[4];
                            int i = 0;
                            int o = 0;
                            while(currentChar <= '7' && currentChar >='0') {
                                if (i < 3)
                                    oct[i++] = (char)currentChar;
                                readChar();
                            }
                            oct[i] = '\0';
                            rescan = 1;
                            sscanf(oct,"%3o",&o);
                            if (o < 0 || i == 0) {
                                error(ERR_INVALIDCONSTANT, token->lineNo, token->colNo);
                                return token;
                            }
                            c = (char) o;
                        }
                        break;
                    default:
                        c = currentChar;
                        break;
                    }
                    if (c) {
                        if (count <= MAX_IDENT_LEN) token->string[count++] = c;
                    }
                    if (rescan) continue;
                }
            }
            readChar();
        }
    }

    if (count > MAX_IDENT_LEN) {
        error(ERR_IDENTTOOLONG, token->lineNo, token->colNo);
        return token;
    }
    readChar();

    token->string[count] = '\0';
    token->tokenType = checkKeyword(token->string);

    if (token->tokenType == TK_NONE)
        token->tokenType = TK_STRING;

    return token;
}
Пример #10
0
tokenInfo getNextToken(FILE* fp, fileState* fstate)
{
    //size_t size;
    
    if(fstate->buffer==NULL)
    {
        fstate->buffer = (char*)malloc(sizeof(char)*MAX_LENGTH);
        fstate->size = getStream(fp,fstate->buffer,MAX_LENGTH);
        fstate->lookahead = 0;
        fstate->lineno = 1;
        fstate->colmno = 1;
    }
    
    Buffer temp = (Buffer)malloc(sizeof(char)*MAX_LENGTH_TOKEN);
    
    
    
    state mystate;
    mystate = start;
    
    int bufferIndex;
    
    int startline;
    int startcolm;

    token_type_enum type = NONE;

    tokenInfo token;
    
    int returnVal;
    int idcount;
    
    while(1)
    {
        switch(mystate)
        {
            case start:
            startlabel:
                startline = fstate->lineno;
                startcolm = fstate->colmno;
                idcount=0;
                bufferIndex = 0;
                if(fstate->buffer[fstate->lookahead]=='=')
                    mystate = oneequal;
                else if(fstate->buffer[fstate->lookahead]=='>')
                    mystate = greater;
                else if(fstate->buffer[fstate->lookahead]=='<')
                    mystate = less;
                else if(fstate->buffer[fstate->lookahead]=='@')
                {
                    token = makeToken(startline, startcolm, SIZE, "@");
                    updateFileState(fp,fstate);
                    debugPrint("SIZE\n");
                    return token;
                }
                else if(fstate->buffer[fstate->lookahead]=='+')
                {
                    token = makeToken(startline, startcolm, PLUS, "+");
                    updateFileState(fp,fstate);
                    debugPrint("PLUS\n");
                    return token;
                }
                else if(fstate->buffer[fstate->lookahead]=='-')
                {
                    token = makeToken(startline, startcolm, MINUS, "-");
                    updateFileState(fp,fstate);
                    debugPrint("MINUS\n");
                    return token;
                }
                else if(fstate->buffer[fstate->lookahead]=='*')
                {
                    token = makeToken(startline, startcolm, MUL, "*");
                    updateFileState(fp,fstate);
                    debugPrint("MUL\n");
                    return token;
                }
                else if(fstate->buffer[fstate->lookahead]=='/')
                {
                    token = makeToken(startline, startcolm, DIV, "/");
                    updateFileState(fp,fstate);
                    debugPrint("DIV\n");
                    return token;
                }
                else if(fstate->buffer[fstate->lookahead]=='[')
                {
                    token = makeToken(startline, startcolm, SQO, "[");
                    updateFileState(fp,fstate);
                    debugPrint("SQO\n");
                    return token;
                }
                else if(fstate->buffer[fstate->lookahead]=='(')
                {
                    token = makeToken(startline, startcolm, OP,"(");
                    updateFileState(fp,fstate);
                    debugPrint("OP\n");
                    return token;
                }
                else if(fstate->buffer[fstate->lookahead]==']')
                {
                    token = makeToken(startline, startcolm, SQC, "]");
                    updateFileState(fp,fstate);
                    debugPrint("SQC\n");
                    return token;
                }
                else if(fstate->buffer[fstate->lookahead]==')')
                {
                    token = makeToken(startline, startcolm, CL, ")");
                    updateFileState(fp,fstate);
                    debugPrint("CL\n");
                    return token;
                }
                else if(fstate->buffer[fstate->lookahead]==';')
                {
                    token = makeToken(startline, startcolm, SEMICOLON, ";");
                    updateFileState(fp,fstate);
                    debugPrint("SEMICOLON\n");
                    return token;
                }
                else if(fstate->buffer[fstate->lookahead]==',')
                {
                    token = makeToken(startline, startcolm, COMMA, ",");
                    updateFileState(fp,fstate);
                    debugPrint("COMMA\n");
                    return token;
                }
                else if(fstate->buffer[fstate->lookahead]=='#')
                {
                    mystate = dumb;
                }
                else if(fstate->buffer[fstate->lookahead]=='_')
                {
                    temp[bufferIndex] = fstate->buffer[fstate->lookahead];
                    bufferIndex++;
                    mystate = funid;
                }
                else if(isInside(fstate->buffer[fstate->lookahead],numbers,0,9))
                {
                    temp[bufferIndex] = fstate->buffer[fstate->lookahead];
                    bufferIndex++;
                    mystate = num;
                }
                else if(isInside(fstate->buffer[fstate->lookahead],alphabets,0,51))
                {
                    temp[bufferIndex] = fstate->buffer[fstate->lookahead];
                    bufferIndex++;
                    mystate = id;
                    idcount++;
                }
                else if(fstate->buffer[fstate->lookahead]=='"')
                {
                    temp[bufferIndex] = fstate->buffer[fstate->lookahead];
                    bufferIndex++;
                    mystate = stringbegin;
                }
                else if(fstate->buffer[fstate->lookahead]=='.')
                {
                    temp[bufferIndex] = fstate->buffer[fstate->lookahead];
                    bufferIndex++;
                    mystate = logicalbegin;
                }
                
                else if(isspace(fstate->buffer[fstate->lookahead]))
                {
                    mystate = start;
                }
                else if(fstate->buffer[fstate->lookahead] == '$')
                {
                    return makeToken(startline,startcolm,NONE,"$");
                }
                else
                {
                    temp[bufferIndex] = fstate->buffer[fstate->lookahead];
                    temp[bufferIndex+1] = '\0';
                    mystate = error;
                }
                break;
            
            case error:
                printf("%s:%d:%d:Unknown lexeme %s\n",fstate->filename,startline,startcolm,temp);
                
                if(isspace(fstate->buffer[fstate->lookahead]))
                	goto errorhandle;
                
                mystate = gotowhite;
                bufferIndex = 0;
                break;
                
            case errorid:
                printf("%s:Identifier at %d:%d is longer than 20 characters.\n",fstate->filename,startline,startcolm);
                
                if(isspace(fstate->buffer[fstate->lookahead]))
                	goto errorhandle;
                
                mystate = gotowhite;
                bufferIndex = 0;
                break;
            
            case gotowhite:
            errorhandle:
                if(isspace(fstate->buffer[fstate->lookahead]))
                {
                    //mystate = start;
                    token = makeToken(startline, startcolm, ERROR, temp);
                    updateFileState(fp,fstate);
                    debugPrint("ERROR\n");
                    return token;
                }
                break;
            
            case oneequal:
                if(fstate->buffer[fstate->lookahead]=='=')
                {
                    mystate = start;
                    token = makeToken(startline, startcolm, EQ, "==");
                    updateFileState(fp,fstate);
                    return token;
                    debugPrint("EQ\n");
                }
                    
                else if(fstate->buffer[fstate->lookahead]=='/')
                    mystate = equalnot;
                else
                {
                    token = makeToken(startline, startcolm, ASSIGNOP, "=");
                    
                    debugPrint("ASSIGNOP\n");
                    return token;
                    mystate = start;
                    goto startlabel;
                }
                break;
            
            case greater:
                if(fstate->buffer[fstate->lookahead]=='=')
                {
                    token = makeToken(startline, startcolm, GE, ">=");
                    
                    debugPrint("GE\n");
                    updateFileState(fp,fstate);
                    return token;
                    mystate = start;
                }
                else
                {
                    token = makeToken(startline, startcolm, GT, ">");
                    
                    debugPrint("GT\n");
                    return token;
                    mystate = start;
                    goto startlabel;
                }
                break;
            
            case less:
                if(fstate->buffer[fstate->lookahead]=='=')
                {
                    token = makeToken(startline, startcolm, LE, "<=");
                    
                    debugPrint("LE\n");
                    updateFileState(fp,fstate);
                    return token;
                    mystate = start;
                }
                else
                {
                    token = makeToken(startline, startcolm, LT, "<");
                    
                    debugPrint("LT\n");
                    return token;
                    mystate = start;
                    goto startlabel;
                }
                break;
            
            case equalnot:
                if(fstate->buffer[fstate->lookahead]=='=')
                {
                    token = makeToken(startline, startcolm, NE, "=/=");
                    
                    debugPrint("NE\n");
                    updateFileState(fp,fstate);
                    return token;
                    mystate = start;
                }
                else
                {
                    mystate = error;
                }
                break;
            
            case funid:
                if(isInside(fstate->buffer[fstate->lookahead],alphabets,0,51))
                {
                    temp[bufferIndex] = fstate->buffer[fstate->lookahead];
                    bufferIndex++;
                    mystate = funidalpha;
                }
                else
                {
                    mystate = error;
                }
                break;
                
                    
            case funidalpha:
                if(isInside(fstate->buffer[fstate->lookahead],alphabets,0,51)||
                    isInside(fstate->buffer[fstate->lookahead],numbers,0,9))
                {
                    temp[bufferIndex] = fstate->buffer[fstate->lookahead];
                    bufferIndex++;
                    mystate = funidalpha;
                }
                else
                {
                    temp[bufferIndex] = '\0';
                    bufferIndex = 0;
                    
                    if(strcmp(temp,"_main")==0)
                        type = MAIN;
                    else
                        type = FUNID;
                    
                    token = makeToken(startline, startcolm, type, temp);
                    
                                        
                    debugPrint("FUNID %s\n",temp);
                    return token;
                    /*
                    mystate = start;
                    goto startlabel;
                    */
                }
                break;
                
            case id:
                if(isInside(fstate->buffer[fstate->lookahead],alphabets,0,51))
                {
                    temp[bufferIndex] = fstate->buffer[fstate->lookahead];
                    bufferIndex++;
                    mystate = id;
                    idcount++;
                    if(idcount>20)
                    {
                        mystate=errorid;
                        break;
                    }
                }
                else if(isInside(fstate->buffer[fstate->lookahead],numbers,0,9))
                {
                	temp[bufferIndex] = fstate->buffer[fstate->lookahead];
                    bufferIndex++;
                    mystate = idnum;
                    idcount++;
                    if(idcount>20)
                    {
                        mystate=errorid;
                        break;
                    }
                    
                	
                }
                else
                {
                    temp[bufferIndex] = '\0';
                    bufferIndex = 0;
                    
                    idcount++;
                    
                    
                    type = checkKeyword(temp);
                    
                    token = makeToken(startline, startcolm, type, temp);
                    
                    
                    debugPrint("ID %s\n",temp);
                    return token;
                    mystate = start;
                    goto startlabel;
                }
                break;
            
            case idnum:
            	if(isInside(fstate->buffer[fstate->lookahead],numbers,0,9))
            	{
            		temp[bufferIndex] = fstate->buffer[fstate->lookahead];
            		temp[bufferIndex+1] = '\0';
                    bufferIndex = 0;
                    
            		mystate=error;
            		break;
        		}
        		else
        		{
                    temp[bufferIndex] = '\0';
                    bufferIndex = 0;
                    idcount++;
                    
                    token = makeToken(startline, startcolm, ID, temp);
                    debugPrint("ID %s\n",temp);
                    //updateFileState(fp,fstate);
                    return token;
                    
                    
                    mystate = start;
        		}
        		break;
        		
        			
            
            case dumb:
                if(fstate->buffer[fstate->lookahead]=='\n')
                mystate = start;
                break;
                
            case stringbegin:
                if(isInside(fstate->buffer[fstate->lookahead],alphabets,0,51))
                {
                    temp[bufferIndex] = fstate->buffer[fstate->lookahead];
                    bufferIndex++;
                    mystate = stringbeginalpha;
                }
                else if(isspace(fstate->buffer[fstate->lookahead]))
                {
                    temp[bufferIndex] = fstate->buffer[fstate->lookahead];
                    bufferIndex++;
                    mystate = stringbeginalpha;
                }
                else
                {
                    mystate = error;
                }
                
                break;
                
            case stringbeginalpha:
                if(isInside(fstate->buffer[fstate->lookahead],alphabets,0,51))
                {
                    temp[bufferIndex] = fstate->buffer[fstate->lookahead];
                    bufferIndex++;
                    mystate = stringbeginalpha;
                }
                else if(fstate->buffer[fstate->lookahead]=='"')
                {
                    temp[bufferIndex] = fstate->buffer[fstate->lookahead];
                    temp[bufferIndex+1] = '\0';
                    bufferIndex = 0;
                    
                    token = makeToken(startline, startcolm, STR, temp);
                    
                    
                    debugPrint("STR %s\n",temp);
                    updateFileState(fp,fstate);
                    return token;
                    mystate = start;
                }
                else if(isspace(fstate->buffer[fstate->lookahead]))
                {
                    temp[bufferIndex] = fstate->buffer[fstate->lookahead];
                    bufferIndex++;
                    mystate = stringbeginalpha;
                }
                else
                {
                    mystate = error;
                }
                break;
                
            case logicalbegin:
                if(isInside(fstate->buffer[fstate->lookahead],alphabets,0,51))
                {
                    temp[bufferIndex] = fstate->buffer[fstate->lookahead];
                    bufferIndex++;
                    mystate = logicalalpha;
                }
                else
                {
                    mystate = error;
                }   
                
                break;
            
            case logicalalpha:
                if(isInside(fstate->buffer[fstate->lookahead],alphabets,0,51))
                {
                    temp[bufferIndex] = fstate->buffer[fstate->lookahead];
                    bufferIndex++;
                    mystate = logicalalpha;
                }
                
                else if(fstate->buffer[fstate->lookahead]=='.')
                {
                    temp[bufferIndex] = fstate->buffer[fstate->lookahead];
                    temp[bufferIndex+1] = '\0';
                    bufferIndex = 0;
                    
                    if(strcmp(temp,".and.")==0)
                    {
                        type = AND;
                    }
                    else if(strcmp(temp,".or.")==0)
                    {
                        type = OR;
                    }
                    else if(strcmp(temp,".not.")==0)
                    {
                        type = NOT;
                    }
                    else
                        type = NONE;
                    
                    
                    if(type==NONE)
                    {
                        mystate = error;
                    }
                    else
                    {
                        debugPrint("LogicalOp %s\n",temp);
                        token = makeToken(startline, startcolm, type, temp);
                        updateFileState(fp,fstate);
                        return token;
                        mystate = start;
                    }
                    
                }
                else
                {
                    mystate = error;
                }
                break;
            
            case num:
                if(isInside(fstate->buffer[fstate->lookahead],numbers,0,9))
                {
                    temp[bufferIndex] = fstate->buffer[fstate->lookahead];
                    bufferIndex++;
                    mystate = num;
                }
                else if(fstate->buffer[fstate->lookahead]=='.')
                {
                    temp[bufferIndex] = fstate->buffer[fstate->lookahead];
                    bufferIndex++;
                    mystate = numdot;
                }
                else
                {
                    temp[bufferIndex] = '\0';
                    bufferIndex=0;
                    
                    token = makeToken(startline, startcolm, NUM, temp);
                    
                    
                    debugPrint("NUM %s\n",temp);
                    return token;
                    mystate = start;
                    goto startlabel;
                }
                break;
            
            case numdot:
                if(isInside(fstate->buffer[fstate->lookahead],numbers,0,9))
                {
                    temp[bufferIndex] = fstate->buffer[fstate->lookahead];
                    bufferIndex++;
                    mystate = numdot1;
                }
                else if(isInside(fstate->buffer[fstate->lookahead],alphabets,0,51))
                {
                    temp[--bufferIndex] = '\0';
                    bufferIndex = 0;
                    
                    token = makeToken(startline, startcolm, NUM, temp);
                    debugPrint("NUM %s\n",temp);
                    
                    fstate->lookahead--;
                    return token;
                }
                else
                {
                    mystate = error;
                }
                break;
                
            case numdot1:
                if(isInside(fstate->buffer[fstate->lookahead],numbers,0,9))
                {
                    temp[bufferIndex] = fstate->buffer[fstate->lookahead];
                    temp[bufferIndex+1] = '\0';
                    bufferIndex = 0;
                    
                    token = makeToken(startline, startcolm, RNUM, temp);
                    debugPrint("RNUM %s\n",temp);
                    updateFileState(fp,fstate);
                    return token;
                    mystate = start;
                }
                else
                {
                    mystate = error;
                }
                break;
            
            
        }
        
        updateFileState(fp,fstate);
    }
    
    
}
Пример #11
0
LEXTOKEN Lexan::nextToken(void)
{
	int c;
	LEXSTATE state = ST_DEFAULT;

	float float_dms = 0.1;	// Multiplier of float number (after point)
	int exp = 0;		// Exponent
	int exp_sign = 1;	// 1 or -1

	stringstream ss;


	while(true)
	{
		if(m_source.empty())
			return LEX_EOF;
		else
			c = m_source.top()->get();

		switch(state)
		{
		case ST_DEFAULT:
			if(isspace(c))
			{
				if(c == '\n')
					m_source.top()->incLine();
				break;
			}
			if(isalpha(c) || c == '_')
			{
				state = ST_NAME;
				m_string = c;
				break;
			}
			if(c == '0')
			{
				state = ST_INT_ZERO;
				m_int = 0;
				break;
			}
			if(c >= '1' && c <= '9')// != 0, this is not OCT number!
			{
				state = ST_INT;
				m_int = c - '0';
				break;
			}

			if(c == ',')
				return LEX_COMMA;
			if(c == ';')
				return LEX_SEMICOLON;
			if(c == '(')
				return LEX_LPA;
			if(c == ')')
				return LEX_RPA;
			if(c == '{')
				return LEX_LVA;
			if(c == '}')
				return LEX_RVA;
			if(c == '[')
				return LEX_LSA;
			if(c == ']')
				return LEX_RSA;
			if(c == '?')
				return LEX_QUESTIONMARK;
			if(c == ':')
				return LEX_COLON;

			if(c == '=')
			{
				state = ST_OP_ASSIGN;
				break;
			}
			if(c == '+')
			{
				state = ST_OP_PLUS;
				break;
			}
			if(c == '-')
			{
				state = ST_OP_MINUS;
				break;
			}
			if(c == '*')
			{
				state = ST_OP_MULT;
				break;
			}
			if(c == '/')
			{
				state = ST_OP_DIV;
				break;
			}
			if(c == '%')
			{
				state = ST_OP_MOD;
				break;
			}
			if(c == '.')
			{
				state = ST_DOT;
				break;
			}
			if(c == '<')
			{
				state = ST_OP_LESS;
				break;
			}
			if(c == '>')
			{
				state = ST_OP_GREATER;
				break;
			}
			if(c == '!')
			{
				state = ST_OP_NOT;
				break;
			}
			if(c == '&')
			{
				state = ST_OP_AND;
				break;
			}
			if(c == '|')
			{
				state = ST_OP_OR;
				break;
			}

			if(c == '"')
			{
				state = ST_STRING;
				m_string.clear();
				break;
			}

			if(c == EOF)		// End of file
			{
				delete m_source.top();
				m_source.pop();
				// state = ST_DEFAULT;// We are here
				break;
			}

			ss.clear();
			ss << _("Unexpected character: '")
				<< (char)c << "' (ascii " << c << ")";
			ERR_PP(getPosition(), ss.str());

			return LEX_ERROR;

		case ST_CPP_COMMENT:		// //
			if(c == '\n')
			{
				state = ST_DEFAULT;
				m_source.top()->incLine();
				break;
			}
			if(c == EOF)		// End of file
			{
				delete m_source.top();
				m_source.pop();
				state = ST_DEFAULT;
				break;
			}

			// Stay in this state
			break;

		case ST_C_COMMENT:		// /*
			if(c == '*')
			{
				state = ST_C_COMMENT_END;
				break;
			}
			if(c == '\n')
			{
				m_source.top()->incLine();
				break;
			}
			if(c == EOF)
			{
				ERR_PP(getPosition(), _("Unexpected end of source: unterminated /* c-style */ comment"));

				delete m_source.top();
				m_source.pop();
				state = ST_DEFAULT;

				return LEX_ERROR;
			}

			// Stay in this state
			break;

		case ST_C_COMMENT_END:		// */
			if(c == '/')
			{
				state = ST_DEFAULT;
				break;
			}
			if(c == '*')// /****something*like*this*****/
			{
				// Stay in this state
				break;
			}
			if(c == '\n')
			{
				// Stay in this state
				m_source.top()->incLine();
				break;
			}
			if(c == EOF)
			{
				ERR_PP(getPosition(), _("Unexpected end of source: unterminated /* c-style */ comment"));

				delete m_source.top();
				m_source.pop();
				state = ST_DEFAULT;

				return LEX_ERROR;
			}

			// Everything else
			state = ST_C_COMMENT;
			break;

		case ST_NAME:			// identifier
			if(isalnum(c) || c == '_')
			{
				// Stay in this state
				m_string += c;
				break;
			}

			unget();
			return checkKeyword();

		case ST_STRING:			// "
			if(c == '\\')// Escape character
			{
				state = ST_STRING_ESC;
				break;
			}
			if(c == '"')// End of string
			{
				return LEX_STRING;
			}
			if(c == '\n')
			{
				m_source.top()->incLine();
				ERR_PP(getPosition(), _("Unexpected end of line: unterminated \"string\" constant"));
				return LEX_ERROR;
			}
			if(c == EOF)
			{
				ERR_PP(getPosition(), _("Unexpected end of source: unterminated \"string\" constant"));

				delete m_source.top();
				m_source.pop();
				state = ST_DEFAULT;

				return LEX_ERROR;
			}

			// Stay in this state
			m_string += c;
			break;

		case ST_STRING_ESC:		// \escape
			if(c == 'a')// \a - Alert (Bell)
			{
				m_string += '\a';
				state = ST_STRING;
				break;
			}
			if(c == 'b')// \b - Backspace
			{
				m_string += '\b';
				state = ST_STRING;
				break;
			}
			if(c == 'f')// \f - Formfeed
			{
				m_string += '\f';
				state = ST_STRING;
				break;
			}
			if(c == 'n')// \n - Newline
			{
				m_string += '\n';
				state = ST_STRING;
				break;
			}
			if(c == 'r')// \r - Carriage return
			{
				m_string += '\r';
				state = ST_STRING;
				break;
			}
			if(c == 't')// \t - Horizontal tab
			{
				m_string += '\t';
				state = ST_STRING;
				break;
			}
			if(c == 'v')// \v - Vertical tab
			{
				m_string += '\v';
				state = ST_STRING;
				break;
			}
			if(c == 'x')// \xHH - HEX
			{
				m_int = 0;
				state = ST_STRING_ESC_HEX;
				break;
			}
			if(c >= '0' && c <= '7')// \000 - OCT
			{
				m_int = c - '0';
				state = ST_STRING_ESC_OCT;
				break;
			}
			if(c == '\\')// \\ Backslash
			{
				m_string += '\\';
				state = ST_STRING;
				break;
			}
			if(c == '\'')// \' Single quote
			{
				m_string += '\'';
				state = ST_STRING;
				break;
			}
			if(c == '\"')// \" Double quote
			{
				m_string += '\"';
				state = ST_STRING;
				break;
			}
			if(c == '\r')// Trash MS Windows \r
			{
				WARN_PP(getPosition(), _("Detected \\r character (CR, 13 in ascii) in string escape sequence after backslash. This would be without problem if you want to use multiline string here and your script uses CR-LF style of line ending (common for text editors under MS Windows)"));

				// Stay in this state and hope \n will continue
				break;
			}
			if(c == '\n')// Multiline string
			{
				state = ST_STRING;
				m_source.top()->incLine();
				break;
			}
			if(c == EOF)
			{
				ERR_PP(getPosition(), _("Unexpected end of source: unterminated \"string\" constant"));

				delete m_source.top();
				m_source.pop();
				state = ST_DEFAULT;

				return LEX_ERROR;
			}

			ss.clear();
			ss << _("Unrecognized escape sequence in \"string\" constant: '")
				<< (char)c << "' (ascii " << c << ")";
			WARN_PP(getPosition(), ss.str());

			m_string += '\\';
			m_string += c;
			state = ST_STRING;
			break;

		case ST_STRING_ESC_HEX:		// \xA
			if(isdigit(c))
			{
				m_int = c - '0';
				state = ST_STRING_ESC_HEX_1;
				break;
			}
			c = tolower(c);
			if(c >= 'a' && c <= 'f')
			{
				m_int = 10 + c - 'a';
				state = ST_STRING_ESC_HEX_1;
				break;
			}
			if(c == EOF)
			{
				ERR_PP(getPosition(), _("Unexpected end of source: unterminated \"string\" constant"));

				delete m_source.top();
				m_source.pop();
				state = ST_DEFAULT;

				return LEX_ERROR;
			}

			ss.clear();
			ss << _("Character is not valid in \"string\" HEX escape sequence context: '")
				<< (char)c << "' (ascii " << c << ")";
			WARN_PP(getPosition(), ss.str());

			m_string += "\\x";
			unget();
			state = ST_STRING;
			break;

		case ST_STRING_ESC_HEX_1:	// \xAB
			if(isdigit(c))
			{
				m_int <<= 4;
				m_int += c - '0';
				m_string += m_int;
				state = ST_STRING;
				break;
			}
			c = tolower(c);
			if(c >= 'a' && c <= 'f')
			{
				m_int <<= 4;
				m_int += 10 + c - 'a';
				m_string += m_int;
				state = ST_STRING;
				break;
			}
			if(c == EOF)
			{
				ERR_PP(getPosition(), _("Unexpected end of source: unterminated \"string\" constant"));

				delete m_source.top();
				m_source.pop();
				state = ST_DEFAULT;

				return LEX_ERROR;
			}

			ss.clear();
			ss << _("Character is not valid in \"string\" HEX escape sequence context: '")
				<< (char)c << "' (ascii " << c << ")";
			WARN_PP(getPosition(), ss.str());

			m_string += "\\x";
			m_string += (m_int < 10) ? '0' + m_int : 'a' + m_int - 10;
			unget();
			state = ST_STRING;
			break;

		case ST_STRING_ESC_OCT:		// \12
			if(c >= '0' && c <= '7')
			{
				m_int <<= 3;
				m_int += c - '0';
				state = ST_STRING_ESC_OCT_1;
				break;
			}
			if(c == EOF)
			{
				ERR_PP(getPosition(), _("Unexpected end of source: unterminated \"string\" constant"));

				delete m_source.top();
				m_source.pop();
				state = ST_DEFAULT;

				return LEX_ERROR;
			}

			ss.clear();
			ss << _("Character is not valid in \"string\" OCT escape sequence context: '")
				<< (char)c << "' (ascii " << c << ")";
			WARN_PP(getPosition(), ss.str());

			m_string += '\\';
			m_string += '0' + m_int;
			unget();
			state = ST_STRING;
			break;

		case ST_STRING_ESC_OCT_1:	// \123
			if(c >= '0' && c <= '7')
			{
				m_int <<= 3;
				m_int += c - '0';

				if(m_int > 255)
				{
					WARN_PP(getPosition(), _("The value of \"string\" OCT escape sequence is too big ( > 0377)"));

					m_string += '\\';
					m_string += '0' + (m_int>>6);
					m_string += '0' + (m_int>>3)%8;
					m_string += '0' + m_int%8;
				}
				else
				{
					m_string += m_int;
				}

				state = ST_STRING;
				break;
			}