bool Lexer::tokenizeLine()
{
while(m_begin != m_end && isspace(*m_begin)) {
++m_begin;
}
// We got an end of a line or a comment.
if(m_begin == m_end || *m_begin == ';') {
return true;
}
if(*m_begin == '(') {
m_tokens.push_back(Token(Token::tok_leftParen, line(), column()));
++m_begin;
return tokenizeLine();
}
if(*m_begin == ')') {
m_tokens.push_back(Token(Token::tok_rightParen, line(), column()));
++m_begin;
return tokenizeLine();
}
if(*m_begin == '"') {
return tokenizeString() && tokenizeLine();
}
if(isIdentifier(*m_begin)) {
return tokenizeIdentifier() && tokenizeLine();
}
OGDF_ERROR("Unexpected character \"" << *m_begin << "\" at (" << line() << ", " << column() << ").");
return false;
}
//
// Fill in token information for the next token, except for the token class.
// Returns the enum value of the token class of the next token found.
// Return 0 (EndOfTokens) on end of input.
//
EHlslTokenClass HlslScanContext::tokenizeClass(HlslToken& token)
{
do {
parserToken = &token;
TPpToken ppToken;
int token = ppContext.tokenize(ppToken);
if (token == EndOfInput)
return EHTokNone;
tokenText = ppToken.name;
loc = ppToken.loc;
parserToken->loc = loc;
switch (token) {
case ';': return EHTokSemicolon;
case ',': return EHTokComma;
case ':': return EHTokColon;
case '=': return EHTokAssign;
case '(': return EHTokLeftParen;
case ')': return EHTokRightParen;
case '.': return EHTokDot;
case '!': return EHTokBang;
case '-': return EHTokDash;
case '~': return EHTokTilde;
case '+': return EHTokPlus;
case '*': return EHTokStar;
case '/': return EHTokSlash;
case '%': return EHTokPercent;
case '<': return EHTokLeftAngle;
case '>': return EHTokRightAngle;
case '|': return EHTokVerticalBar;
case '^': return EHTokCaret;
case '&': return EHTokAmpersand;
case '?': return EHTokQuestion;
case '[': return EHTokLeftBracket;
case ']': return EHTokRightBracket;
case '{': return EHTokLeftBrace;
case '}': return EHTokRightBrace;
case '\\':
parseContext.error(loc, "illegal use of escape character", "\\", "");
break;
case PPAtomAddAssign: return EHTokAddAssign;
case PPAtomSubAssign: return EHTokSubAssign;
case PPAtomMulAssign: return EHTokMulAssign;
case PPAtomDivAssign: return EHTokDivAssign;
case PPAtomModAssign: return EHTokModAssign;
case PpAtomRight: return EHTokRightOp;
case PpAtomLeft: return EHTokLeftOp;
case PpAtomRightAssign: return EHTokRightAssign;
case PpAtomLeftAssign: return EHTokLeftAssign;
case PpAtomAndAssign: return EHTokAndAssign;
case PpAtomOrAssign: return EHTokOrAssign;
case PpAtomXorAssign: return EHTokXorAssign;
case PpAtomAnd: return EHTokAndOp;
case PpAtomOr: return EHTokOrOp;
case PpAtomXor: return EHTokXorOp;
case PpAtomEQ: return EHTokEqOp;
case PpAtomGE: return EHTokGeOp;
case PpAtomNE: return EHTokNeOp;
case PpAtomLE: return EHTokLeOp;
case PpAtomDecrement: return EHTokDecOp;
case PpAtomIncrement: return EHTokIncOp;
case PpAtomColonColon: return EHTokColonColon;
case PpAtomConstInt: parserToken->i = ppToken.ival; return EHTokIntConstant;
case PpAtomConstUint: parserToken->i = ppToken.ival; return EHTokUintConstant;
case PpAtomConstFloat: parserToken->d = ppToken.dval; return EHTokFloatConstant;
case PpAtomConstDouble: parserToken->d = ppToken.dval; return EHTokDoubleConstant;
case PpAtomIdentifier:
{
EHlslTokenClass token = tokenizeIdentifier();
return token;
}
case PpAtomConstString: {
parserToken->string = NewPoolTString(tokenText);
return EHTokStringConstant;
}
case EndOfInput: return EHTokNone;
default:
char buf[2];
buf[0] = (char)token;
buf[1] = 0;
parseContext.error(loc, "unexpected token", buf, "");
break;
}
} while (true);
}
int TScanContext::tokenize(TPpContext* pp, TParserToken& token)
{
do {
parserToken = &token;
TPpToken ppToken;
tokenText = pp->tokenize(&ppToken);
if (tokenText == nullptr)
return 0;
loc = ppToken.loc;
parserToken->sType.lex.loc = loc;
switch (ppToken.token) {
case ';': afterType = false; return SEMICOLON;
case ',': afterType = false; return COMMA;
case ':': return COLON;
case '=': afterType = false; return EQUAL;
case '(': afterType = false; return LEFT_PAREN;
case ')': afterType = false; return RIGHT_PAREN;
case '.': field = true; return DOT;
case '!': return BANG;
case '-': return DASH;
case '~': return TILDE;
case '+': return PLUS;
case '*': return STAR;
case '/': return SLASH;
case '%': return PERCENT;
case '<': return LEFT_ANGLE;
case '>': return RIGHT_ANGLE;
case '|': return VERTICAL_BAR;
case '^': return CARET;
case '&': return AMPERSAND;
case '?': return QUESTION;
case '[': return LEFT_BRACKET;
case ']': return RIGHT_BRACKET;
case '{': return LEFT_BRACE;
case '}': return RIGHT_BRACE;
case '\\':
parseContext.error(loc, "illegal use of escape character", "\\", "");
break;
case PpAtomAdd: return ADD_ASSIGN;
case PpAtomSub: return SUB_ASSIGN;
case PpAtomMul: return MUL_ASSIGN;
case PpAtomDiv: return DIV_ASSIGN;
case PpAtomMod: return MOD_ASSIGN;
case PpAtomRight: return RIGHT_OP;
case PpAtomLeft: return LEFT_OP;
case PpAtomRightAssign: return RIGHT_ASSIGN;
case PpAtomLeftAssign: return LEFT_ASSIGN;
case PpAtomAndAssign: return AND_ASSIGN;
case PpAtomOrAssign: return OR_ASSIGN;
case PpAtomXorAssign: return XOR_ASSIGN;
case PpAtomAnd: return AND_OP;
case PpAtomOr: return OR_OP;
case PpAtomXor: return XOR_OP;
case PpAtomEQ: return EQ_OP;
case PpAtomGE: return GE_OP;
case PpAtomNE: return NE_OP;
case PpAtomLE: return LE_OP;
case PpAtomDecrement: return DEC_OP;
case PpAtomIncrement: return INC_OP;
case PpAtomConstInt: parserToken->sType.lex.i = ppToken.ival; return INTCONSTANT;
case PpAtomConstUint: parserToken->sType.lex.i = ppToken.ival; return UINTCONSTANT;
case PpAtomConstFloat: parserToken->sType.lex.d = ppToken.dval; return FLOATCONSTANT;
case PpAtomConstDouble: parserToken->sType.lex.d = ppToken.dval; return DOUBLECONSTANT;
case PpAtomIdentifier: return tokenizeIdentifier();
case EndOfInput: return 0;
default:
char buf[2];
buf[0] = (char)ppToken.token;
buf[1] = 0;
parseContext.error(loc, "unexpected token", buf, "");
break;
}
} while (true);
}
bool XmlLexer::next()
{
int nextChar = file.peek();
bool nextValid = true;
if( !file.good() )
{
nextChar = Token::END_OF_FILE;
}
switch( nextChar )
{
case Token::CARET_OPEN:
{
_token.type = Token::CARET_OPEN;
_token.string = Token::CARET_OPEN;
++_token.column;
file.get();
break;
}
case Token::CARET_CLOSE:
{
_token.type = Token::CARET_CLOSE;
_token.string = Token::CARET_CLOSE;
++_token.column;
file.get();
break;
}
case Token::BACKSLASH:
{
_token.type = Token::BACKSLASH;
_token.string = Token::BACKSLASH;
++_token.column;
file.get();
break;
}
case Token::END_OF_FILE:
{
_token.type = Token::END_OF_FILE;
_token.string = Token::END_OF_FILE;
++_token.column;
nextValid = false;
break;
}
default:
{
tokenizeIdentifier();
break;
}
}
if( nextValid )
{
while( devourComment() );
}
report( "Got token: " << _token.string );
return nextValid;
}
int TScanContext::tokenize(TPpContext* pp, TParserToken& token)
{
do {
parserToken = &token;
TPpToken ppToken;
tokenText = pp->tokenize(&ppToken);
if (tokenText == 0)
return 0;
loc = ppToken.loc;
parserToken->sType.lex.loc = loc;
switch (ppToken.token) {
case ';': afterType = false; return SEMICOLON;
case ',': afterType = false; return COMMA;
case ':': return COLON;
case '=': afterType = false; return EQUAL;
case '(': afterType = false; return LEFT_PAREN;
case ')': afterType = false; return RIGHT_PAREN;
case '.': field = true; return DOT;
case '!': return BANG;
case '-': return DASH;
case '~': return TILDE;
case '+': return PLUS;
case '*': return STAR;
case '/': return SLASH;
case '%': return PERCENT;
case '<': return LEFT_ANGLE;
case '>': return RIGHT_ANGLE;
case '|': return VERTICAL_BAR;
case '^': return CARET;
case '&': return AMPERSAND;
case '?': return QUESTION;
case '[': return LEFT_BRACKET;
case ']': return RIGHT_BRACKET;
case '{': return LEFT_BRACE;
case '}': return RIGHT_BRACE;
case '\\':
parseContext.error(loc, "illegal use of escape character", "\\", "");
break;
case CPP_AND_OP: return AND_OP;
case CPP_SUB_ASSIGN: return SUB_ASSIGN;
case CPP_MOD_ASSIGN: return MOD_ASSIGN;
case CPP_ADD_ASSIGN: return ADD_ASSIGN;
case CPP_DIV_ASSIGN: return DIV_ASSIGN;
case CPP_MUL_ASSIGN: return MUL_ASSIGN;
case CPP_EQ_OP: return EQ_OP;
case CPP_XOR_OP: return XOR_OP;
case CPP_GE_OP: return GE_OP;
case CPP_RIGHT_OP: return RIGHT_OP;
case CPP_LE_OP: return LE_OP;
case CPP_LEFT_OP: return LEFT_OP;
case CPP_DEC_OP: return DEC_OP;
case CPP_NE_OP: return NE_OP;
case CPP_OR_OP: return OR_OP;
case CPP_INC_OP: return INC_OP;
case CPP_RIGHT_ASSIGN: return RIGHT_ASSIGN;
case CPP_LEFT_ASSIGN: return LEFT_ASSIGN;
case CPP_AND_ASSIGN: return AND_ASSIGN;
case CPP_OR_ASSIGN: return OR_ASSIGN;
case CPP_XOR_ASSIGN: return XOR_ASSIGN;
case CPP_INTCONSTANT: parserToken->sType.lex.i = ppToken.ival; return INTCONSTANT;
case CPP_UINTCONSTANT: parserToken->sType.lex.i = ppToken.ival; return UINTCONSTANT;
case CPP_FLOATCONSTANT: parserToken->sType.lex.d = ppToken.dval; return FLOATCONSTANT;
case CPP_DOUBLECONSTANT: parserToken->sType.lex.d = ppToken.dval; return DOUBLECONSTANT;
case CPP_IDENTIFIER: return tokenizeIdentifier();
case EOF: return 0;
default:
char buf[2];
buf[0] = (char)ppToken.token;
buf[1] = 0;
parseContext.error(loc, "unexpected token", buf, "");
break;
}
} while (true);
}