/****************************************************************** scan() See prlex.h for return values other than characters (i.e. > 256) *******************************************************************/ int scan() { ini_scan(); getachar(); if(Ch == EOF) return(EOF); switch(Ctype[Ch]) { case DI: MY_ASSERT(isdigit(Ch)); /* double check */ case SI: MY_ASSERT(isdigit(Ch) || Ch == '-' || Ch == '+'); return(scan_number(Ch)); case QU: MY_ASSERT(Ch == '"'); scan_string(); return(TOKEN_STRING); case BR: MY_ASSERT(Ch == ')' || Ch == '('); return(Ch); #ifdef CLIPS_SYNTAX case QE: scan_identifier(?); return(TOKEN_VAR); case AL: case AU: case OT: scan_identifier(Ch); return(TOKEN_ATOM); #else case AL: MY_ASSERT(islower(Ch)); scan_identifier(Ch); return(TOKEN_ATOM); case US: MY_ASSERT(Ch == '_'); case AU: scan_identifier(Ch); return(TOKEN_VAR); #endif case CC: return(SCAN_ERR); #ifdef CHARACTER case AP: return(scan_character()); #endif default: return(Ch); } }
int yylex(void) { int status; #ifdef DEBUG printf("yylex is called\n" " first_line: %d\n" " first_column: %d\n" " last_line: %d\n" " last_column: %d\n", yylloc.first_line, yylloc.first_column, yylloc.last_line, yylloc.last_column); #endif skip_white_space: while(*src == ' ' || *src == '\t') { ++yylloc.last_column; ++src; } /* skip comments */ if(*src == ';') { while(*src != '\n' && *src != 0) { ++yylloc.last_column; ++src; } } /* when a newline appears, update yylloc accordingly and start to skip whitespace again */ if(*src == '\n') { ++yylloc.last_line; yylloc.last_column = 1; ++src; goto skip_white_space; } yylloc.first_line = yylloc.last_line; yylloc.first_column = yylloc.last_column; status = scan_multibyte_operator(); if(status != 0) return status; status = scan_int(); if(status != 0) return status; status = scan_string(); if(status != 0) return status; status = scan_special_identifier(); if(status != 0) return status; status = scan_identifier(); if(status != 0) return status; /* Return a single char or, when end of string, 0 */ ++yylloc.last_column; #ifdef DEBUG if(*src == 0) printf("yylex returns EOF\n"); else printf("yylex returns '%c'\n", *src); #endif return *src++; }
bool tokz_get_token(Tokenizer *tokz, Token *tok) { int c, c2, e; if (!(tokz->flags&TOKZ_READ_FROM_BUFFER)) assert(tokz->file!=NULL); tok_free(tok); if(!TOK_IS_INVALID(&(tokz->ungettok))){ *tok=tokz->ungettok; tokz->ungettok.type=TOK_INVALID; return TRUE; } while(1){ e=0; do{ c=GETCH(); }while(c!='\n' && c!=EOF && isspace(c)); tok->line=tokz->line; switch(c){ case EOF: TOK_SET_OP(tok, OP_EOF); return TRUE; case '\n': INC_LINE(); if(tokz->flags&TOKZ_IGNORE_NEXTLINE) continue; TOK_SET_OP(tok, OP_NEXTLINE); return TRUE; case '\\': do{ c=GETCH(); if(c==EOF){ TOK_SET_OP(tok, OP_EOF); return FALSE; } if(!isspace(c) && e==0){ e=E_TOKZ_EOL_EXPECTED; tokz_warn_error(tokz, tokz->line, e); if(!(tokz->flags&TOKZ_ERROR_TOLERANT)) return FALSE; } }while(c!='\n'); INC_LINE(); continue; case '#': if(tokz->flags&TOKZ_READ_COMMENTS){ e=scan_line_comment(tok, tokz); break; }else if((e=skip_line_comment(tokz))){ break; } continue; case '/': c2=GETCH(); if(c2=='='){ TOK_SET_OP(tok, OP_AS_DIV); return TRUE; } if(c2!='*'){ UNGETCH(c2); TOK_SET_OP(tok, OP_DIV); return TRUE; } if(tokz->flags&TOKZ_READ_COMMENTS){ e=scan_c_comment(tok, tokz); break; }else if((e=skip_c_comment(tokz))){ break; } continue; case '\"': e=scan_string(tok, tokz, TRUE); break; case '\'': e=scan_char(tok, tokz); break; default: if(('0'<=c && c<='9') || c=='-' || c=='+'){ e=scan_number(tok, tokz, c); break; } if(START_IDENT(c)) e=scan_identifier(tok, tokz, c); else e=scan_op(tok, tokz, c); } if(!e) return TRUE; tokz_warn_error(tokz, tokz->line, e); return FALSE; } }