static int
multi_symbol_token(void)
{
static struct {
char c1, c2;
int token;
} *tab, table[] = {
{ '+', '=', PLUS_EQ },
{ '+', '+', PLUS_PLUS },
{ '-', '=', MINUS_EQ },
{ '-', '-', MINUS_MINUS },
{ '/', '/', SLASH_SLASH },
{ '=', '=', EQ_EQ },
{ '!', '=', NOT_EQ },
{ '~', '>', TILDE_GT },
{ '>', '=', GT_EQ },
{ '<', '=', LT_EQ },
{ '&', '&', LAZY_AND },
{ '|', '|', LAZY_OR },
/*{ ':', ':', CONTEXT },*/
{ 0, 0, 0 }
};
for (tab = table; tab->c1; tab++) {
if (*yytext == tab->c1 && nextc == tab->c2) {
input();
RETURN_TOKEN(tab->token);
}
}
RETURN_TOKEN(*yytext);
}
SQInteger SQLexer::Lex()
{
_lasttokenline = _currentline;
while(CUR_CHAR != SQUIRREL_EOB) {
switch(CUR_CHAR){
case _SC('\t'): case _SC('\r'): case _SC(' '): NEXT(); continue;
case _SC('\n'):
_currentline++;
_prevtoken=_curtoken;
_curtoken=_SC('\n');
NEXT();
_currentcolumn=1;
continue;
case _SC('#'): LexLineComment(); continue;
case _SC('/'):
NEXT();
switch(CUR_CHAR){
case _SC('*'):
NEXT();
LexBlockComment();
continue;
case _SC('/'):
LexLineComment();
continue;
case _SC('='):
NEXT();
RETURN_TOKEN(TK_DIVEQ);
continue;
case _SC('>'):
NEXT();
RETURN_TOKEN(TK_ATTR_CLOSE);
continue;
default:
RETURN_TOKEN('/');
}
case _SC('='):
NEXT();
if (CUR_CHAR != _SC('=')){ RETURN_TOKEN('=') }
else { NEXT(); RETURN_TOKEN(TK_EQ); }
case _SC('<'):
NEXT();
switch(CUR_CHAR) {
case _SC('='):
NEXT();
if(CUR_CHAR == _SC('>')) {
NEXT();
RETURN_TOKEN(TK_3WAYSCMP);
}
RETURN_TOKEN(TK_LE)
break;
case _SC('-'): NEXT(); RETURN_TOKEN(TK_NEWSLOT); break;
case _SC('<'): NEXT(); RETURN_TOKEN(TK_SHIFTL); break;
case _SC('/'): NEXT(); RETURN_TOKEN(TK_ATTR_OPEN); break;
}
RETURN_TOKEN('<');
case _SC('>'):
NEXT();
if (CUR_CHAR == _SC('=')){ NEXT(); RETURN_TOKEN(TK_GE);}
else if(CUR_CHAR == _SC('>')){
NEXT();
if(CUR_CHAR == _SC('>')){
NEXT();
RETURN_TOKEN(TK_USHIFTR);
}
RETURN_TOKEN(TK_SHIFTR);
}
else { RETURN_TOKEN('>') }
case _SC('!'):
NEXT();
if (CUR_CHAR != _SC('=')){ RETURN_TOKEN('!')}
else { NEXT(); RETURN_TOKEN(TK_NE); }
case _SC('@'): {
SQInteger stype;
NEXT();
if(CUR_CHAR != _SC('"')) {
RETURN_TOKEN('@');
}
if((stype=ReadString('"',true))!=-1) {
RETURN_TOKEN(stype);
}
Error(_SC("error parsing the string"));
}
case _SC('"'):
case _SC('\''): {
SQInteger stype;
if((stype=ReadString(CUR_CHAR,false))!=-1){
RETURN_TOKEN(stype);
}
Error(_SC("error parsing the string"));
}
case _SC('{'): case _SC('}'): case _SC('('): case _SC(')'): case _SC('['): case _SC(']'):
case _SC(';'): case _SC(','): case _SC('?'): case _SC('^'): case _SC('~'):
{SQInteger ret = CUR_CHAR;
NEXT(); RETURN_TOKEN(ret); }
case _SC('.'):
NEXT();
if (CUR_CHAR != _SC('.')){ RETURN_TOKEN('.') }
NEXT();
if (CUR_CHAR != _SC('.')){ Error(_SC("invalid token '..'")); }
NEXT();
RETURN_TOKEN(TK_VARPARAMS);
case _SC('&'):
NEXT();
if (CUR_CHAR != _SC('&')){ RETURN_TOKEN('&') }
else { NEXT(); RETURN_TOKEN(TK_AND); }
case _SC('|'):
NEXT();
if (CUR_CHAR != _SC('|')){ RETURN_TOKEN('|') }
else { NEXT(); RETURN_TOKEN(TK_OR); }
case _SC(':'):
NEXT();
if (CUR_CHAR != _SC(':')){ RETURN_TOKEN(':') }
else { NEXT(); RETURN_TOKEN(TK_DOUBLE_COLON); }
case _SC('*'):
NEXT();
if (CUR_CHAR == _SC('=')){ NEXT(); RETURN_TOKEN(TK_MULEQ);}
else RETURN_TOKEN('*');
case _SC('%'):
NEXT();
if (CUR_CHAR == _SC('=')){ NEXT(); RETURN_TOKEN(TK_MODEQ);}
else RETURN_TOKEN('%');
case _SC('-'):
NEXT();
if (CUR_CHAR == _SC('=')){ NEXT(); RETURN_TOKEN(TK_MINUSEQ);}
else if (CUR_CHAR == _SC('-')){ NEXT(); RETURN_TOKEN(TK_MINUSMINUS);}
else RETURN_TOKEN('-');
case _SC('+'):
NEXT();
if (CUR_CHAR == _SC('=')){ NEXT(); RETURN_TOKEN(TK_PLUSEQ);}
else if (CUR_CHAR == _SC('+')){ NEXT(); RETURN_TOKEN(TK_PLUSPLUS);}
else RETURN_TOKEN('+');
case SQUIRREL_EOB:
return 0;
default:{
if (scisdigit(CUR_CHAR)) {
SQInteger ret = ReadNumber();
RETURN_TOKEN(ret);
}
else if (scisalpha(CUR_CHAR) || CUR_CHAR == _SC('_')) {
SQInteger t = ReadID();
RETURN_TOKEN(t);
}
else {
SQInteger c = CUR_CHAR;
if (sciscntrl((int)c)) Error(_SC("unexpected character(control)"));
NEXT();
RETURN_TOKEN(c);
}
RETURN_TOKEN(0);
}
}
}
return 0;
}
int
yylex(void)
{
char *s;
uDatum u;
#ifdef DISTRIB
extern Int *higherPriority;
#endif /* DISTRIB */
/* initial for a token */
clear_text();
if (bof) {
bof = 0;
if (nextc == '%') { /* comment line */
skip_percent_comment();/* return next token */
end_text();
clear_text();
}
/* printf("bof \n"); */
if (nextc == '>' || nextc == '<') getAgentName();
}
restart:
while (isspace(nextc) && (nextc != '\n') && (nextc != '\r')) {
/* skip blanks */
input();
}
if ((nextc == '\n') || (nextc == '\r')) {
input();
if (nextc == '%') { /* comment line */
skip_percent_comment();/* return next token */
end_text();
clear_text();
}
/* printf("restart\n"); */
if (nextc == '>' || nextc == '<') getAgentName();
goto restart;
}
end_text();
clear_text();
if (nextc == 0) { /* EOF */
#ifdef DISTRIB
if (setHighPrior) { *higherPriority=0; setHighPrior = 0; }
#endif
return 0; /* don't do input() so next token is EOF */
}
#ifdef TTYEDEN
prompt = promptsemi;
#endif
if (isdigit(nextc) || nextc == '.') /* a number or a dot */
return number_token();
if (isalpha(nextc) || nextc == '_') /* a keyword or an identifier */
{
append_agentName = 1;
return id_token();
}
/*if (nextc == '^') {
input();
append_agentName = 1;
return context_token();
}*/
if (nextc == '\'') {
while (input(), nextc != '\'') {
if (nextc == '\\')
input();
else if (nextc == 0)
error("unexpected end-of-file in character constant");
}
input(); /* read the quote */
/* mustn't alter the following two lines */
end_text(); /* accept the text */
last_char = '\0'; /* delete the last quote */
backslash(yytext + 1); /* resolve the backslashs */
if (yytext[2]) /* more than one char */
error("single char expected");
u.i = yytext[1];
yylval.dp = makedatum(MYCHAR, u);
return CONSTANT;
}
if (nextc == '\"') {
#ifdef TTYEDEN
prompt = promptchar;
#endif
while (input(), nextc != '\"') {
if (nextc == '\\')
input();
else if (nextc == 0)
error("unexpected end-of-file in string constant");
}
input(); /* read the quote */
/* mustn't alter the following two lines */
end_text(); /* accept the text */
last_char = '\0'; /* delete the last quote */
backslash(yytext + 1); /* resolve the backslashs */
s = (char *) emalloc(strlen(yytext));
strcpy(s, yytext + 1);
u.s = s;
yylval.dp = makedatum(STRING, u);
return CONSTANT;
}
input(); /* accept the char, must be the 1st char */
switch (*yytext) { /* the 1st char */
case '@':
yylval.dp = makedatum(UNDEF, u); /* bug fix Ash + Carters */
RETURN_TOKEN(CONSTANT);
break;
case '~': /* for agency --sun */
if (isalpha(nextc) || nextc == '_') {
clear_text();
append_agentName = 0;
return id_token();
}
break;
case '$':
while (isdigit(nextc))
input();
if (yyleng == 1)
RETURN_TOKEN('$');
yylval.narg = atoi(yytext + 1);
RETURN_TOKEN(ARG);
case '/':
#ifdef TTYEDEN
prompt = promptcomment;
#endif
if (nextc == '*') { /* nested comment */
skip_comment(); /* return next token */
end_text();
clear_text();
#ifdef TTYEDEN
prompt = prompt1;
#endif
goto restart;
}
break;
case '#':
if (nextc == '#') {
skip_one_line_comment();
end_text();
clear_text();
#ifdef TTYEDEN
prompt = prompt1;
#endif
goto restart;
}
break;
}
return (multi_symbol_token());
}
ConfigMapLexer::Token ConfigMapLexer::nextToken()
{
if(tkn)
{
Token t = *tkn;
delete tkn;
tkn = 0;
return t;
}
std::stringstream value(std::stringstream::in | std::stringstream::out);
int tknLine = line;
int tknColumn = column + 1;
State_T state = NONE;
char l = EOF;
while(true)
{
++column;
char c = static_cast<char>(input->get());
switch(c)
{
case EOF:
RETURN_TOKEN(MK_TOKEN(T::CMT_EOF, std::string(1, c)), T::CMT_VALUE);
case ' ':
if(state == ESCAPE)
{
return MK_TOKEN(T::CMT_ERROR, std::string(1, c));
}
else if(state == NONE);
else
{
value << c;
}
break;
case '\r':
break;
case '\n':
++line;
column = 1;
switch(state)
{
case ESCAPE:
return MK_TOKEN(T::CMT_ERROR, std::string(1, c));
case EOL_C: // leave end of line comment
{
state = NONE;
std::string s = value.str();
value.str("");
return MK_TOKEN(T::CMT_COMMENT, std::string(s));
}
default:
value << c;
case NONE:
;
}
break;
case ',':
if(state == STRING || state == EOL_C || state == BLOCK_C)
{
value << c;
break;
}
else
RETURN_TOKEN(MK_TOKEN(T::CMT_COMMA, std::string(1, c)), T::CMT_VALUE);
case '=':
if(state == STRING || state == EOL_C || state == BLOCK_C)
{
value << c;
break;
}
else
RETURN_TOKEN(MK_TOKEN(T::CMT_EQ, std::string(1, c)), T::CMT_KEY);
case ';':
if(state == STRING || state == EOL_C || state == BLOCK_C)
{
value << c;
break;
}
else
RETURN_TOKEN(MK_TOKEN(T::CMT_SEMICOLON, std::string(1, c)), T::CMT_VALUE);
case '[':
if(state == STRING || state == EOL_C || state == BLOCK_C)
{
value << c;
break;
}
else
return MK_TOKEN(T::CMT_ABL, std::string(1, c));
case ']':
if(state == STRING || state == EOL_C || state == BLOCK_C)
{
value << c;
break;
}
else
RETURN_TOKEN(MK_TOKEN(T::CMT_ABR, std::string(1, c)), T::CMT_VALUE);
case '{':
if(state == STRING || state == EOL_C || state == BLOCK_C)
{
value << c;
break;
}
else
return MK_TOKEN(T::CMT_CBL, std::string(1, c));
case '}':
if(state == STRING || state == EOL_C || state == BLOCK_C)
{
value << c;
break;
}
else
return MK_TOKEN(T::CMT_CBR, std::string(1, c));
case '*':
if(l == '/' && !state) // enter block comment
{
value.str("");
state = BLOCK_C;
}
else
{
value << c;
}
break;
case '/':
if(l == '*' && (state == BLOCK_C)) // leave block comment
{
std::string s = value.str();
value.str("");
state = NONE;
return MK_TOKEN(T::CMT_COMMENT, std::string(s, 0, s.length() - 1));
}
else if(l == '/' && state == NONE)
{
value.str("");
state = EOL_C;
}
else
{
value << c;
}
break;
case '"':
if(state == STRING) // leave quoted string
{
std::string s = value.str();
value.str("");
state = NONE;
return MK_TOKEN(T::CMT_VALUE, std::string(s));
}
else if(state == NONE) // enter quoted string
{
tknLine = line;
tknColumn = column;
state = STRING;
}
else if(state == ESCAPE)
{
value << c;
state = STRING;
}
else
{
return MK_TOKEN(T::CMT_ERROR, std::string(1, c));
}
break;
case '\\':
if(state == STRING) // escape next character
{
state = ESCAPE;
}
else if(state == ESCAPE)
{
value << c;
state = STRING;
}
else
{
return MK_TOKEN(T::CMT_ERROR, std::string(1, c));
}
break;
default:
if(state == STRING || state == EOL_C || state == BLOCK_C)
{
value << c;
}
else if(state == NONE)
{
if(isalnum(c) || c == '_' || c == '+' || c == '-' || c == '.')
{
value << c;
}
else
{
return MK_TOKEN(T::CMT_ERROR, std::string(1, c));
}
}
else
{
return MK_TOKEN(T::CMT_ERROR, std::string(1, c));
}
}
l = c;
}
}