/******************************************************************
scan()
See prlex.h for return values other than characters (i.e. > 256)
*******************************************************************/
int scan()
{
ini_scan();
getachar();
if(Ch == EOF)
return(EOF);
switch(Ctype[Ch])
{
case DI:
MY_ASSERT(isdigit(Ch)); /* double check */
case SI:
MY_ASSERT(isdigit(Ch) || Ch == '-' || Ch == '+');
return(scan_number(Ch));
case QU:
MY_ASSERT(Ch == '"');
scan_string();
return(TOKEN_STRING);
case BR:
MY_ASSERT(Ch == ')' || Ch == '(');
return(Ch);
#ifdef CLIPS_SYNTAX
case QE:
scan_identifier(?);
return(TOKEN_VAR);
case AL:
case AU:
case OT:
scan_identifier(Ch);
return(TOKEN_ATOM);
#else
case AL:
MY_ASSERT(islower(Ch));
scan_identifier(Ch);
return(TOKEN_ATOM);
case US:
MY_ASSERT(Ch == '_');
case AU:
scan_identifier(Ch);
return(TOKEN_VAR);
#endif
case CC:
return(SCAN_ERR);
#ifdef CHARACTER
case AP:
return(scan_character());
#endif
default:
return(Ch);
}
}
int yylex(void) {
int status;
#ifdef DEBUG
printf("yylex is called\n"
" first_line: %d\n"
" first_column: %d\n"
" last_line: %d\n"
" last_column: %d\n",
yylloc.first_line,
yylloc.first_column,
yylloc.last_line,
yylloc.last_column);
#endif
skip_white_space:
while(*src == ' ' || *src == '\t') {
++yylloc.last_column;
++src;
}
/* skip comments */
if(*src == ';') {
while(*src != '\n' && *src != 0) {
++yylloc.last_column;
++src;
}
}
/* when a newline appears, update yylloc accordingly and start to skip
whitespace again */
if(*src == '\n') {
++yylloc.last_line;
yylloc.last_column = 1;
++src;
goto skip_white_space;
}
yylloc.first_line = yylloc.last_line;
yylloc.first_column = yylloc.last_column;
status = scan_multibyte_operator();
if(status != 0)
return status;
status = scan_int();
if(status != 0)
return status;
status = scan_string();
if(status != 0)
return status;
status = scan_special_identifier();
if(status != 0)
return status;
status = scan_identifier();
if(status != 0)
return status;
/* Return a single char or, when end of string, 0 */
++yylloc.last_column;
#ifdef DEBUG
if(*src == 0)
printf("yylex returns EOF\n");
else
printf("yylex returns '%c'\n", *src);
#endif
return *src++;
}
bool tokz_get_token(Tokenizer *tokz, Token *tok)
{
int c, c2, e;
if (!(tokz->flags&TOKZ_READ_FROM_BUFFER))
assert(tokz->file!=NULL);
tok_free(tok);
if(!TOK_IS_INVALID(&(tokz->ungettok))){
*tok=tokz->ungettok;
tokz->ungettok.type=TOK_INVALID;
return TRUE;
}
while(1){
e=0;
do{
c=GETCH();
}while(c!='\n' && c!=EOF && isspace(c));
tok->line=tokz->line;
switch(c){
case EOF:
TOK_SET_OP(tok, OP_EOF);
return TRUE;
case '\n':
INC_LINE();
if(tokz->flags&TOKZ_IGNORE_NEXTLINE)
continue;
TOK_SET_OP(tok, OP_NEXTLINE);
return TRUE;
case '\\':
do{
c=GETCH();
if(c==EOF){
TOK_SET_OP(tok, OP_EOF);
return FALSE;
}
if(!isspace(c) && e==0){
e=E_TOKZ_EOL_EXPECTED;
tokz_warn_error(tokz, tokz->line, e);
if(!(tokz->flags&TOKZ_ERROR_TOLERANT))
return FALSE;
}
}while(c!='\n');
INC_LINE();
continue;
case '#':
if(tokz->flags&TOKZ_READ_COMMENTS){
e=scan_line_comment(tok, tokz);
break;
}else if((e=skip_line_comment(tokz))){
break;
}
continue;
case '/':
c2=GETCH();
if(c2=='='){
TOK_SET_OP(tok, OP_AS_DIV);
return TRUE;
}
if(c2!='*'){
UNGETCH(c2);
TOK_SET_OP(tok, OP_DIV);
return TRUE;
}
if(tokz->flags&TOKZ_READ_COMMENTS){
e=scan_c_comment(tok, tokz);
break;
}else if((e=skip_c_comment(tokz))){
break;
}
continue;
case '\"':
e=scan_string(tok, tokz, TRUE);
break;
case '\'':
e=scan_char(tok, tokz);
break;
default:
if(('0'<=c && c<='9') || c=='-' || c=='+'){
e=scan_number(tok, tokz, c);
break;
}
if(START_IDENT(c))
e=scan_identifier(tok, tokz, c);
else
e=scan_op(tok, tokz, c);
}
if(!e)
return TRUE;
tokz_warn_error(tokz, tokz->line, e);
return FALSE;
}
}
