static void pushtoken(rconlexer* lexer) { lexer->pushedback = 1; if(rcondebug > 1) dumptoken(lexer,1); }
int dcelex(YYSTYPE* lvalp, DCEparsestate* state) { DCElexstate* lexstate = state->lexstate; int token; int c; int len; char* p=NULL; token = 0; ncbytesclear(lexstate->yytext); ncbytesnull(lexstate->yytext); p=lexstate->next; while(token == 0 && (c=*p)) { if(c <= ' ' || c >= '\177') {p++; continue;} if(c == '"') { int more = 1; ceaddyytext(lexstate,c); /* We have a SCAN_STRINGCONST */ while(more && (c=*(++p))) { switch (c) { case '"': p++; more=0; break; case '\\': c=*(++p); switch (c) { case 'r': c = '\r'; break; case 'n': c = '\n'; break; case 'f': c = '\f'; break; case 't': c = '\t'; break; case 'x': { int d1,d2; c = '?'; ++p; d1 = tohex(*p++); if(d1 < 0) { dceerror(state,"Illegal \\xDD in SCAN_STRING"); } else { d2 = tohex(*p++); if(d2 < 0) { dceerror(state,"Illegal \\xDD in SCAN_STRING"); } else { c=(((unsigned int)d1)<<4) | (unsigned int)d2; } } } break; default: break; } break; default: break; } ceaddyytext(lexstate,c); } token=SCAN_STRINGCONST; } else if(strchr(numchars1,c) != NULL) { /* we might have a SCAN_NUMBERCONST */ int isnumber = 0; char* yytext; char* endpoint; ceaddyytext(lexstate,c); for(p++;(c=*p);p++) { if(strchr(numcharsn,c) == NULL) break; ceaddyytext(lexstate,c); } /* See if this is a number */ ncbytesnull(lexstate->yytext); yytext = ncbytescontents(lexstate->yytext); (void)strtoll(yytext,&endpoint,10); if(*yytext != '\0' && *endpoint == '\0') isnumber = 1; else { (void)strtod(yytext,&endpoint); if(*yytext != '\0' && *endpoint == '\0') isnumber = 1; /* maybe */ } /* A number followed by an id char is assumed to just be a funny id */ if(isnumber && (*p == '\0' || strchr(wordcharsn,*p) == NULL)) { token = SCAN_NUMBERCONST; } else { /* Now, if the funny word has a "." in it, we have to back up to that dot */ char* dotpoint = strchr(yytext,'.'); if(dotpoint != NULL) { p = dotpoint; *dotpoint = '\0'; } token = SCAN_WORD; } } else if(strchr(wordchars1,c) != NULL) { /* we have a SCAN_WORD */ ceaddyytext(lexstate,c); for(p++;(c=*p);p++) { if(strchr(wordcharsn,c) == NULL) break; ceaddyytext(lexstate,c); } token=SCAN_WORD; } else { /* we have a single char token */ token = c; ceaddyytext(lexstate,c); p++; } } lexstate->next = p; len = ncbyteslength(lexstate->yytext); if(len > MAX_TOKEN_LENGTH) len = MAX_TOKEN_LENGTH; strncpy(lexstate->lasttokentext,ncbytescontents(lexstate->yytext),len); lexstate->lasttokentext[len] = '\0'; lexstate->lasttoken = token; if(dcedebug) dumptoken(lexstate); /*Put return value onto Bison stack*/ if(ncbyteslength(lexstate->yytext) == 0) *lvalp = NULL; else { *lvalp = ncbytesdup(lexstate->yytext); nclistpush(lexstate->reclaim,(void*)*lvalp); } return token; }
int daplex(YYSTYPE* lvalp, DAPparsestate* state) { DAPlexstate* lexstate = state->lexstate; int token; int c; unsigned int i; char* p; char* tmp; YYSTYPE lval = NULL; token = 0; ocbytesclear(lexstate->yytext); /* invariant: p always points to current char */ for(p=lexstate->next; token==0&&(c=*p); p++) { if(c == '\n') { lexstate->lineno++; } else if(c <= ' ' || c == '\177') { /* whitespace: ignore */ } else if(c == '#') { /* single line comment */ while((c=*(++p))) { if(c == '\n') break; } } else if(strchr(lexstate->worddelims,c) != NULL) { /* don't put in lexstate->yytext to avoid memory leak */ token = c; } else if(c == '"') { int more = 1; /* We have a string token; will be reported as WORD_STRING */ while(more && (c=*(++p))) { if(c == '"') { more = 0; continue; } #ifdef DAP2ENCODE if(c == '\\') { /* Resolve spec ambiguity about handling of \c: 1. !KEEPSLASH: convert \c to c for any character c 2. KEEPSLASH: convert \c to \c for any character c; that is, keep the backslash. It is clear that the problem being addressed was \". But it is unclear what to to do about \n: convert to Ascii LF or leave as \n. This code will leave as \n and assume higher levels of code will address the issue. */ #ifdef KEEPSLASH dapaddyytext(lexstate,c); #endif c=*(++p); if(c == '\0') more = 0; } #else /*Non-standard*/ switch (c) { case '\\': c=*(++p); switch (c) { case 'r': c = '\r'; break; case 'n': c = '\n'; break; case 'f': c = '\f'; break; case 't': c = '\t'; break; case 'x': { int d1,d2; c = '?'; ++p; d1 = tohex(*p++); if(d1 < 0) { daperror(state,"Illegal \\xDD in TOKEN_STRING"); } else { d2 = tohex(*p++); if(d2 < 0) { daperror(state,"Illegal \\xDD in TOKEN_STRING"); } else { c=(((unsigned int)d1)<<4) | (unsigned int)d2; } } } break; default: break; } break; default: break; } #endif /*!DAP2ENCODE*/ if(more) dapaddyytext(lexstate,c); } token=WORD_STRING; } else if(strchr(lexstate->wordchars1,c) != NULL) { int isdatamark = 0; /* we have a WORD_WORD */ dapaddyytext(lexstate,c); while((c=*(++p))) { #ifdef URLCVT if(c == '%' && p[1] != 0 && p[2] != 0 && strchr(hexdigits,p[1]) != NULL && strchr(hexdigits,p[2]) != NULL) { int d1,d2; d1 = tohex(p[1]); d2 = tohex(p[2]); if(d1 >= 0 || d2 >= 0) { c=(((unsigned int)d1)<<4) | (unsigned int)d2; p+=2; } } else { if(strchr(lexstate->wordcharsn,c) == NULL) { p--; break; } } dapaddyytext(lexstate,c); #else if(strchr(lexstate->wordcharsn,c) == NULL) { p--; break; } dapaddyytext(lexstate,c); #endif } /* Special check for Data: */ tmp = ocbytescontents(lexstate->yytext); if(strcmp(tmp,"Data")==0 && *p == ':') { dapaddyytext(lexstate,*p); p++; if(p[0] == '\n') { token = SCAN_DATA; isdatamark = 1; p++; } else if(p[0] == '\r' && p[1] == '\n') { token = SCAN_DATA; isdatamark = 1; p+=2; } } if(!isdatamark) { /* check for keyword */ token=WORD_WORD; /* assume */ for(i=0;; i++) { if(keywords[i] == NULL) break; if(strcasecmp(keywords[i],tmp)==0) { token=keytokens[i]; break; } } } } else { /* illegal */ } } lexstate->next = p; strncpy(lexstate->lasttokentext,ocbytescontents(lexstate->yytext),MAX_TOKEN_LENGTH); lexstate->lasttoken = token; if(ocdebug >= 2) dumptoken(lexstate); /*Put return value onto Bison stack*/ if(ocbyteslength(lexstate->yytext) == 0) lval = NULL; else { lval = ocbytesdup(lexstate->yytext); oclistpush(lexstate->reclaim,(void*)lval); } if(lvalp) *lvalp = lval; return token; /* Return the type of the token. */ }
static int nexttoken(rconlexer* lexer) { int token; int c; if(lexer->pushedback) {token = lexer->token; lexer->pushedback = 0; goto done;} token = 0; textclear(&lexer->text); while(token==0) { c=readc(lexer); lexer->charno++; if(c == EOF) { token = EOF; lexer->charno--; break; } else if(c == '\n') { lexer->lineno++; lexer->charno = 1; } else if(c == '/') { c = readc(lexer); if(c == '/') {/* single line comment */ while((c=readc(lexer)) != EOF) {if(c == '\n') break;} } else { pushback(lexer,c); c = '/'; } } if(c <= ' ' || c == '\177') { /* ignore */ } else if(strchr(delims,c) != NULL) { textadd(&lexer->text,c); token = c; } else if(c == '"') { int more = 1; while(more) { c = readc(lexer); switch (c) { case EOF: goto fail; case '"': more=0; break; case '\\': textadd(&lexer->text,c); c=readc(lexer); textadd(&lexer->text,c); break; default: textadd(&lexer->text,c); } } if(!removeescapes(lexer)) goto fail; token=_STRING; } else { /* Treat like a string without quotes */ textadd(&lexer->text,c); while((c=readc(lexer))) { if(c == '/' || c <= ' ' || c == '\177') {pushback(lexer,c); break;} else if(strchr(delims,c) != NULL) {pushback(lexer,c); break;} textadd(&lexer->text,c); } if(!removeescapes(lexer)) goto fail; /* check for keyword */ if(strcmp(lexer->text.text,"true") == 0) { token = _TRUE; } else if(strcmp(lexer->text.text,"false") == 0) { token = _FALSE; } else if(strcmp(lexer->text.text,"null") == 0) { token = _NULL; } else { /* See if this looks like a number */ double d; if(sscanf(lexer->text.text,"%lg",&d) == 1) token = _NUMBER; else token = _STRING; } } } done: lexer->token = token; if(rcondebug > 1) dumptoken(lexer,0); return token; fail: return EOF; }
int daplex(YYSTYPE* lvalp, DAPparsestate* state) { DAPlexstate* lexstate = state->lexstate; int token; int c; unsigned int i; char* p=lexstate->next; char* tmp; token = 0; ocbytesclear(lexstate->yytext); /* invariant: p always points to current char */ for(p=lexstate->next;token==0&&(c=*p);p++) { if(c == '\n') { lexstate->lineno++; } else if(c <= ' ' || c == '\177') { /* whitespace: ignore */ } else if(c == '#') { /* single line comment */ while((c=*(++p))) {if(c == '\n') break;} } else if(strchr(lexstate->worddelims,c) != NULL) { /* don't put in lexstate->yytext to avoid memory leak */ token = c; } else if(c == '"') { int more = 1; /* We have a string token; will be reported as SCAN_WORD */ while(more && (c=*(++p))) { #ifdef NONSTDCVT switch (c) { case '"': more=0; break; case '\\': c=*(++p); switch (c) { case 'r': c = '\r'; break; case 'n': c = '\n'; break; case 'f': c = '\f'; break; case 't': c = '\t'; break; case 'x': { int d1,d2; c = '?'; ++p; d1 = tohex(*p++); if(d1 < 0) { daperror(state,"Illegal \\xDD in TOKEN_STRING"); } else { d2 = tohex(*p++); if(d2 < 0) { daperror(state,"Illegal \\xDD in TOKEN_STRING"); } else { c=(((unsigned int)d1)<<4) | (unsigned int)d2; } } } break; default: break; } break; default: break; } #else /*!NONSTDCVT*/ if(c == '"') more = 0; else if(c == '\\') { c=*(++p); if(c == '\0') more = false; if(c != '"') {c = '\\'; --p;} } #endif /*!NONSTDCVT*/ if(more) dapaddyytext(lexstate,c); } token=SCAN_WORD; } else if(strchr(lexstate->wordchars1,c) != NULL) { /* we have a SCAN_WORD */ dapaddyytext(lexstate,c); while((c=*(++p))) { #ifdef URLCVT if(c == '%' && p[1] != 0 && p[2] != 0 && strchr(hexdigits,p[1]) != NULL && strchr(hexdigits,p[2]) != NULL) { #ifdef WRONG /* Should not unescape %xx occurrences */ int d1,d2; d1 = tohex(p[1]); d2 = tohex(p[2]); if(d1 >= 0 || d2 >= 0) { c=(((unsigned int)d1)<<4) | (unsigned int)d2; p+=2; } #endif } else { if(strchr(lexstate->wordcharsn,c) == NULL) {p--; break;} } dapaddyytext(lexstate,c); #else if(strchr(lexstate->wordcharsn,c) == NULL) {p--; break;} dapaddyytext(lexstate,c); #endif } /* Special check for Data: */ tmp = ocbytescontents(lexstate->yytext); if(strcmp(tmp,"Data")==0 && *p == ':') { dapaddyytext(lexstate,*p); p++; token = SCAN_DATA; } else { /* check for keyword */ token=SCAN_WORD; /* assume */ for(i=0;;i++) { if(keywords[i] == NULL) break; if(strcasecmp(keywords[i],tmp)==0) { token=keytokens[i]; break; } } } } else { /* illegal */ } } lexstate->next = p; strncpy(lexstate->lasttokentext,ocbytescontents(lexstate->yytext),MAX_TOKEN_LENGTH); lexstate->lasttoken = token; if(ocdebug >= 2) dumptoken(lexstate); /*Put return value onto Bison stack*/ if(ocbyteslength(lexstate->yytext) == 0) *lvalp = NULL; else { *lvalp = ocbytesdup(lexstate->yytext); oclistpush(lexstate->reclaim,(ocelem)*lvalp); } return token; /* Return the type of the token. */ }