示例#1
0
文件: TlpLexer.cpp 项目: lncosie/ogdf
bool Lexer::tokenizeLine()
{
	while(m_begin != m_end && isspace(*m_begin)) {
		++m_begin;
	}

	// We got an end of a line or a comment.
	if(m_begin == m_end || *m_begin == ';') {
		return true;
	}

	if(*m_begin == '(') {
		m_tokens.push_back(Token(Token::tok_leftParen, line(), column()));
		++m_begin;
		return tokenizeLine();
	}

	if(*m_begin == ')') {
		m_tokens.push_back(Token(Token::tok_rightParen, line(), column()));
		++m_begin;
		return tokenizeLine();
	}

	if(*m_begin == '"') {
		return tokenizeString() && tokenizeLine();
	}

	if(isIdentifier(*m_begin)) {
		return tokenizeIdentifier() && tokenizeLine();
	}

	OGDF_ERROR("Unexpected character \"" << *m_begin << "\" at (" << line() << ", " << column() << ").");
	return false;
}
示例#2
0
//
// Fill in token information for the next token, except for the token class.
// Returns the enum value of the token class of the next token found.
// Return 0 (EndOfTokens) on end of input.
//
EHlslTokenClass HlslScanContext::tokenizeClass(HlslToken& token)
{
    do {
        parserToken = &token;
        TPpToken ppToken;
        int token = ppContext.tokenize(ppToken);
        if (token == EndOfInput)
            return EHTokNone;

        tokenText = ppToken.name;
        loc = ppToken.loc;
        parserToken->loc = loc;
        switch (token) {
        case ';':                       return EHTokSemicolon;
        case ',':                       return EHTokComma;
        case ':':                       return EHTokColon;
        case '=':                       return EHTokAssign;
        case '(':                       return EHTokLeftParen;
        case ')':                       return EHTokRightParen;
        case '.':                       return EHTokDot;
        case '!':                       return EHTokBang;
        case '-':                       return EHTokDash;
        case '~':                       return EHTokTilde;
        case '+':                       return EHTokPlus;
        case '*':                       return EHTokStar;
        case '/':                       return EHTokSlash;
        case '%':                       return EHTokPercent;
        case '<':                       return EHTokLeftAngle;
        case '>':                       return EHTokRightAngle;
        case '|':                       return EHTokVerticalBar;
        case '^':                       return EHTokCaret;
        case '&':                       return EHTokAmpersand;
        case '?':                       return EHTokQuestion;
        case '[':                       return EHTokLeftBracket;
        case ']':                       return EHTokRightBracket;
        case '{':                       return EHTokLeftBrace;
        case '}':                       return EHTokRightBrace;
        case '\\':
            parseContext.error(loc, "illegal use of escape character", "\\", "");
            break;

        case PPAtomAddAssign:          return EHTokAddAssign;
        case PPAtomSubAssign:          return EHTokSubAssign;
        case PPAtomMulAssign:          return EHTokMulAssign;
        case PPAtomDivAssign:          return EHTokDivAssign;
        case PPAtomModAssign:          return EHTokModAssign;

        case PpAtomRight:              return EHTokRightOp;
        case PpAtomLeft:               return EHTokLeftOp;

        case PpAtomRightAssign:        return EHTokRightAssign;
        case PpAtomLeftAssign:         return EHTokLeftAssign;
        case PpAtomAndAssign:          return EHTokAndAssign;
        case PpAtomOrAssign:           return EHTokOrAssign;
        case PpAtomXorAssign:          return EHTokXorAssign;

        case PpAtomAnd:                return EHTokAndOp;
        case PpAtomOr:                 return EHTokOrOp;
        case PpAtomXor:                return EHTokXorOp;

        case PpAtomEQ:                 return EHTokEqOp;
        case PpAtomGE:                 return EHTokGeOp;
        case PpAtomNE:                 return EHTokNeOp;
        case PpAtomLE:                 return EHTokLeOp;

        case PpAtomDecrement:          return EHTokDecOp;
        case PpAtomIncrement:          return EHTokIncOp;

        case PpAtomColonColon:         return EHTokColonColon;

        case PpAtomConstInt:           parserToken->i = ppToken.ival;       return EHTokIntConstant;
        case PpAtomConstUint:          parserToken->i = ppToken.ival;       return EHTokUintConstant;
        case PpAtomConstFloat:         parserToken->d = ppToken.dval;       return EHTokFloatConstant;
        case PpAtomConstDouble:        parserToken->d = ppToken.dval;       return EHTokDoubleConstant;
        case PpAtomIdentifier:
        {
            EHlslTokenClass token = tokenizeIdentifier();
            return token;
        }

        case PpAtomConstString: {
            parserToken->string = NewPoolTString(tokenText);
            return EHTokStringConstant;
        }

        case EndOfInput:               return EHTokNone;

        default:
            char buf[2];
            buf[0] = (char)token;
            buf[1] = 0;
            parseContext.error(loc, "unexpected token", buf, "");
            break;
        }
    } while (true);
}
示例#3
0
int TScanContext::tokenize(TPpContext* pp, TParserToken& token)
{
    do {
        parserToken = &token;
        TPpToken ppToken;
        tokenText = pp->tokenize(&ppToken);
        if (tokenText == nullptr)
            return 0;

        loc = ppToken.loc;
        parserToken->sType.lex.loc = loc;
        switch (ppToken.token) {
        case ';':  afterType = false;   return SEMICOLON;
        case ',':  afterType = false;   return COMMA;
        case ':':                       return COLON;
        case '=':  afterType = false;   return EQUAL;
        case '(':  afterType = false;   return LEFT_PAREN;
        case ')':  afterType = false;   return RIGHT_PAREN;
        case '.':  field = true;        return DOT;
        case '!':                       return BANG;
        case '-':                       return DASH;
        case '~':                       return TILDE;
        case '+':                       return PLUS;
        case '*':                       return STAR;
        case '/':                       return SLASH;
        case '%':                       return PERCENT;
        case '<':                       return LEFT_ANGLE;
        case '>':                       return RIGHT_ANGLE;
        case '|':                       return VERTICAL_BAR;
        case '^':                       return CARET;
        case '&':                       return AMPERSAND;
        case '?':                       return QUESTION;
        case '[':                       return LEFT_BRACKET;
        case ']':                       return RIGHT_BRACKET;
        case '{':                       return LEFT_BRACE;
        case '}':                       return RIGHT_BRACE;
        case '\\':
            parseContext.error(loc, "illegal use of escape character", "\\", "");
            break;

        case PpAtomAdd:                return ADD_ASSIGN;
        case PpAtomSub:                return SUB_ASSIGN;
        case PpAtomMul:                return MUL_ASSIGN;
        case PpAtomDiv:                return DIV_ASSIGN;
        case PpAtomMod:                return MOD_ASSIGN;

        case PpAtomRight:              return RIGHT_OP;
        case PpAtomLeft:               return LEFT_OP;

        case PpAtomRightAssign:        return RIGHT_ASSIGN;
        case PpAtomLeftAssign:         return LEFT_ASSIGN;
        case PpAtomAndAssign:          return AND_ASSIGN;
        case PpAtomOrAssign:           return OR_ASSIGN;
        case PpAtomXorAssign:          return XOR_ASSIGN;

        case PpAtomAnd:                return AND_OP;
        case PpAtomOr:                 return OR_OP;
        case PpAtomXor:                return XOR_OP;

        case PpAtomEQ:                 return EQ_OP;
        case PpAtomGE:                 return GE_OP;
        case PpAtomNE:                 return NE_OP;
        case PpAtomLE:                 return LE_OP;

        case PpAtomDecrement:          return DEC_OP;
        case PpAtomIncrement:          return INC_OP;
                                   
        case PpAtomConstInt:           parserToken->sType.lex.i = ppToken.ival;       return INTCONSTANT;
        case PpAtomConstUint:          parserToken->sType.lex.i = ppToken.ival;       return UINTCONSTANT;
        case PpAtomConstFloat:         parserToken->sType.lex.d = ppToken.dval;       return FLOATCONSTANT;
        case PpAtomConstDouble:        parserToken->sType.lex.d = ppToken.dval;       return DOUBLECONSTANT;
        case PpAtomIdentifier:         return tokenizeIdentifier();

        case EndOfInput:               return 0;

        default:
            char buf[2];
            buf[0] = (char)ppToken.token;
            buf[1] = 0;
            parseContext.error(loc, "unexpected token", buf, "");
            break;
        }
    } while (true);
}
示例#4
0
	bool XmlLexer::next()
	{

		int nextChar = file.peek();
		bool nextValid = true;
		
		if( !file.good() )
		{
		
			nextChar = Token::END_OF_FILE;
		
		}
		
		switch( nextChar )
		{
		
			case Token::CARET_OPEN:
			{
			
				_token.type = Token::CARET_OPEN;
				_token.string = Token::CARET_OPEN;
				++_token.column;
				file.get();
				break;
				
			}
			
			case Token::CARET_CLOSE:
			{
			
				_token.type = Token::CARET_CLOSE;
				_token.string = Token::CARET_CLOSE;
				++_token.column;
				file.get();
				break;
					
			}
			
			case Token::BACKSLASH:
			{
			
				_token.type = Token::BACKSLASH;
				_token.string = Token::BACKSLASH;
				++_token.column;
				file.get();
				break;
			
			}
			
			case Token::END_OF_FILE:
			{
			
				_token.type = Token::END_OF_FILE;
				_token.string = Token::END_OF_FILE;
				++_token.column;
				nextValid = false;				
				break;
		
			}
			
			default:
			{
			
				tokenizeIdentifier();
				break;
				
			}
		
		}	
		
		if( nextValid )
		{
		
			while( devourComment() );
	
		}
		
		report( "Got token: " << _token.string );
		
		return nextValid;
	
	}
示例#5
0
int TScanContext::tokenize(TPpContext* pp, TParserToken& token)
{
    do {
        parserToken = &token;
        TPpToken ppToken;
        tokenText = pp->tokenize(&ppToken);
        if (tokenText == 0)
            return 0;

        loc = ppToken.loc;
        parserToken->sType.lex.loc = loc;
        switch (ppToken.token) {
        case ';':  afterType = false;   return SEMICOLON;
        case ',':  afterType = false;   return COMMA;
        case ':':                       return COLON;
        case '=':  afterType = false;   return EQUAL;
        case '(':  afterType = false;   return LEFT_PAREN;
        case ')':  afterType = false;   return RIGHT_PAREN;
        case '.':  field = true;        return DOT;
        case '!':                       return BANG;
        case '-':                       return DASH;
        case '~':                       return TILDE;
        case '+':                       return PLUS;
        case '*':                       return STAR;
        case '/':                       return SLASH;
        case '%':                       return PERCENT;
        case '<':                       return LEFT_ANGLE;
        case '>':                       return RIGHT_ANGLE;
        case '|':                       return VERTICAL_BAR;
        case '^':                       return CARET;
        case '&':                       return AMPERSAND;
        case '?':                       return QUESTION;
        case '[':                       return LEFT_BRACKET;
        case ']':                       return RIGHT_BRACKET;
        case '{':                       return LEFT_BRACE;
        case '}':                       return RIGHT_BRACE;
        case '\\':
            parseContext.error(loc, "illegal use of escape character", "\\", "");
            break;

        case CPP_AND_OP:                return AND_OP;
        case CPP_SUB_ASSIGN:            return SUB_ASSIGN;
        case CPP_MOD_ASSIGN:            return MOD_ASSIGN;
        case CPP_ADD_ASSIGN:            return ADD_ASSIGN;
        case CPP_DIV_ASSIGN:            return DIV_ASSIGN;
        case CPP_MUL_ASSIGN:            return MUL_ASSIGN;
        case CPP_EQ_OP:                 return EQ_OP;
        case CPP_XOR_OP:                return XOR_OP;
        case CPP_GE_OP:                 return GE_OP;
        case CPP_RIGHT_OP:              return RIGHT_OP;
        case CPP_LE_OP:                 return LE_OP;
        case CPP_LEFT_OP:               return LEFT_OP;
        case CPP_DEC_OP:                return DEC_OP;
        case CPP_NE_OP:                 return NE_OP;
        case CPP_OR_OP:                 return OR_OP;
        case CPP_INC_OP:                return INC_OP;
        case CPP_RIGHT_ASSIGN:          return RIGHT_ASSIGN;
        case CPP_LEFT_ASSIGN:           return LEFT_ASSIGN;
        case CPP_AND_ASSIGN:            return AND_ASSIGN;
        case CPP_OR_ASSIGN:             return OR_ASSIGN;
        case CPP_XOR_ASSIGN:            return XOR_ASSIGN;
                                   
        case CPP_INTCONSTANT:           parserToken->sType.lex.i = ppToken.ival;       return INTCONSTANT;
        case CPP_UINTCONSTANT:          parserToken->sType.lex.i = ppToken.ival;       return UINTCONSTANT;
        case CPP_FLOATCONSTANT:         parserToken->sType.lex.d = ppToken.dval;       return FLOATCONSTANT;
        case CPP_DOUBLECONSTANT:        parserToken->sType.lex.d = ppToken.dval;       return DOUBLECONSTANT;
        case CPP_IDENTIFIER:            return tokenizeIdentifier();

        case EOF:                       return 0;
                                   
        default:
            char buf[2];
            buf[0] = (char)ppToken.token;
            buf[1] = 0;
            parseContext.error(loc, "unexpected token", buf, "");
            break;
        }
    } while (true);
}