static int llex (LexState *ls, SemInfo *seminfo) { luaZ_resetbuffer(ls->buff); for (;;) { switch (ls->current) { case '\n': case '\r': { /* line breaks */ inclinenumber(ls); break; } case ' ': case '\f': case '\t': case '\v': { /* spaces */ next(ls); break; } case '-': { /* '-' or '--' (comment) */ next(ls); if (ls->current != '-') return '-'; /* else is a comment */ next(ls); if (ls->current == '[') { /* long comment? */ int sep = skip_sep(ls); luaZ_resetbuffer(ls->buff); /* 'skip_sep' may dirty the buffer */ if (sep >= 0) { read_long_string(ls, NULL, sep); /* skip long comment */ luaZ_resetbuffer(ls->buff); /* previous call may dirty the buff. */ break; } } /* else short comment */ while (!currIsNewline(ls) && ls->current != EOZ) next(ls); /* skip until end of line (or end of file) */ break; } case '[': { /* long string or simply '[' */ int sep = skip_sep(ls); if (sep >= 0) { read_long_string(ls, seminfo, sep); return TK_STRING; } else if (sep != -1) /* '[=...' missing second bracket */ lexerror(ls, "invalid long string delimiter", TK_STRING); return '['; } case '=': { next(ls); if (check_next1(ls, '=')) return TK_EQ; else return '='; } case '<': { next(ls); if (check_next1(ls, '=')) return TK_LE; else if (check_next1(ls, '<')) return TK_SHL; else return '<'; } case '>': { next(ls); if (check_next1(ls, '=')) return TK_GE; else if (check_next1(ls, '>')) return TK_SHR; else return '>'; } case '/': { next(ls); if (check_next1(ls, '/')) return TK_IDIV; else return '/'; } case '~': { next(ls); if (check_next1(ls, '=')) return TK_NE; else return '~'; } case ':': { next(ls); if (check_next1(ls, ':')) return TK_DBCOLON; else return ':'; } case '"': case '\'': { /* short literal strings */ read_string(ls, ls->current, seminfo); return TK_STRING; } case '`': { /* relative paths */ read_string(ls, ls->current, seminfo); return TK_PATH; } case '.': { /* '.', '..', '...', or number */ save_and_next(ls); if (check_next1(ls, '.')) { if (check_next1(ls, '.')) return TK_DOTS; /* '...' */ else return TK_CONCAT; /* '..' */ } else if (!lisdigit(ls->current)) return '.'; else return read_numeral(ls, seminfo); } case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': { return read_numeral(ls, seminfo); } case EOZ: { return TK_EOS; } default: { if (lislalpha(ls->current)) { /* identifier or reserved word? */ TString *ts; do { save_and_next(ls); } while (lislalnum(ls->current)); ts = luaX_newstring(ls, luaZ_buffer(ls->buff), luaZ_bufflen(ls->buff)); seminfo->ts = ts; if (isreserved(ts)) /* reserved word? */ return ts->extra - 1 + FIRST_RESERVED; else { return TK_NAME; } } else { /* single-char tokens (+ - / ...) */ int c = ls->current; next(ls); return c; } } } } }
static int llex(LexState *ls, TValue *tv) { lj_str_resetbuf(&ls->sb); for (;;) { if (lj_ctype_isident(ls->current)) { GCstr *s; if (lj_ctype_isdigit(ls->current)) { /* Numeric literal. */ read_numeral(ls, tv); return TK_number; } /* Identifier or reserved word. */ do { save_and_next(ls); } while (lj_ctype_isident(ls->current)); s = lj_parse_keepstr(ls, ls->sb.buf, ls->sb.n); if (s->reserved > 0) /* Reserved word? */ return TK_OFS + s->reserved; setstrV(ls->L, tv, s); return TK_name; } switch (ls->current) { case '\n': case '\r': inclinenumber(ls); continue; case ' ': case '\t': case '\v': case '\f': next(ls); continue; case '-': next(ls); if (ls->current != '-') return '-'; /* else is a comment */ next(ls); if (ls->current == '[') { int sep = skip_sep(ls); lj_str_resetbuf(&ls->sb); /* `skip_sep' may dirty the buffer */ if (sep >= 0) { read_long_string(ls, NULL, sep); /* long comment */ lj_str_resetbuf(&ls->sb); continue; } } /* else short comment */ while (!currIsNewline(ls) && ls->current != END_OF_STREAM) next(ls); continue; case '[': { int sep = skip_sep(ls); if (sep >= 0) { read_long_string(ls, tv, sep); return TK_string; } else if (sep == -1) { return '['; } else { lj_lex_error(ls, TK_string, LJ_ERR_XLDELIM); continue; } } case '=': next(ls); if (ls->current != '=') return '='; else { next(ls); return TK_eq; } case '<': next(ls); if (ls->current != '=') return '<'; else { next(ls); return TK_le; } case '>': next(ls); if (ls->current != '=') return '>'; else { next(ls); return TK_ge; } case '~': next(ls); if (ls->current != '=') return '~'; else { next(ls); return TK_ne; } case '"': case '\'': read_string(ls, ls->current, tv); return TK_string; case '.': save_and_next(ls); if (check_next(ls, ".")) { if (check_next(ls, ".")) return TK_dots; /* ... */ else return TK_concat; /* .. */ } else if (!lj_ctype_isdigit(ls->current)) { return '.'; } else { read_numeral(ls, tv); return TK_number; } case END_OF_STREAM: return TK_eof; default: { int c = ls->current; next(ls); return c; /* Single-char tokens (+ - / ...). */ } } } }
int luaX_lex (LexState *LS, SemInfo *seminfo) { for (;;) { switch (LS->current) { case '\n': { inclinenumber(LS); continue; } case '-': { next(LS); if (LS->current != '-') return '-'; /* else is a comment */ next(LS); if (LS->current == '[' && (next(LS), LS->current == '[')) read_long_string(LS, NULL); /* long comment */ else /* short comment */ while (LS->current != '\n' && LS->current != EOZ) next(LS); continue; } case '[': { next(LS); if (LS->current != '[') return '['; else { read_long_string(LS, seminfo); return TK_STRING; } } case '=': { next(LS); if (LS->current != '=') return '='; else { next(LS); return TK_EQ; } } case '<': { next(LS); if (LS->current == '<') { next(LS); return TK_SHL; } else if (LS->current != '=') return '<'; else { next(LS); return TK_LE; } } case '>': { next(LS); if (LS->current == '>') { next(LS); return TK_SHR; } else if (LS->current != '=') return '>'; else { next(LS); return TK_GE; } } case '~': { next(LS); if (LS->current != '=') return '~'; else { next(LS); return TK_NE; } } case '"': case '\'': { read_string(LS, LS->current, seminfo); return TK_STRING; } case '.': { next(LS); if (LS->current == '.') { next(LS); if (LS->current == '.') { next(LS); return TK_DOTS; /* ... */ } else return TK_CONCAT; /* .. */ } else if (!lex_isdigit(LS->current)) return '.'; else { return read_numeral(LS, 1, seminfo); } } case EOZ: { return TK_EOS; } default: { if (isspace(LS->current)) { next(LS); continue; } else if (lex_isdigit(LS->current)) { return (read_numeral(LS, 0, seminfo)); } else if (lex_isalpha(LS->current) || LS->current == '_') { char saveCh = 0; size_t l; TString *ts; if (LS->current == 'L') { next(LS); if (LS->current == '"') { read_wstring(LS, LS->current, seminfo); return TK_WSTRING; } saveCh = 'L'; } /* identifier or reserved word */ l = readname(LS, saveCh); ts = luaS_newlstr(LS->L, luaZ_buffer(LS->buff), l); if (ts->tsv.reserved > 0) /* reserved word? */ return ts->tsv.reserved - 1 + FIRST_RESERVED; seminfo->ts = ts; return TK_NAME; } else { int c = LS->current; if (iscntrl(c)) luaX_error(LS, "invalid control char", luaO_pushfstring(LS->L, "char(%d)", c)); next(LS); return c; /* single-char tokens (+ - / ...) */ } } } } }
static int llex (tt_Lexer *L) { for (;;) { switch (ttL_current(L)) { case '\n': case '\r': { /* line breaks */ ttL_inclinenumber(L); break; } case ' ': case '\f': case '\t': case '\v': { /* spaces */ ttL_next(L); break; } case '-': { /* '-' or '--' (comment) */ ttL_next(L); if (ttL_current(L) != '-') return '-'; /* else is a comment */ ttL_next(L); if (ttL_current(L) == '[') { /* long comment? */ int sep = skip_sep(L); ttL_resetbuffer(L); /* `skip_sep' may dirty the buffer */ if (sep >= 0) { read_long_string(L, 0, sep); /* skip long comment */ ttL_resetbuffer(L); /* previous call may dirty the buff. */ break; } } /* else short comment */ while (!ttL_isnewline(L) && ttL_current(L) != TTL_EOZ) ttL_next(L); /* skip until end of line (or end of file) */ break; } case '[': { /* long string or simply '[' */ int sep = skip_sep(L); if (sep >= 0) { read_long_string(L, 1, sep); return TTK_STRING; } else if (sep == -1) return '['; else ttL_lexerror(L, "invalid long string delimiter", TTK_STRING); } case '=': { ttL_next(L); if (ttL_current(L) != '=') return '='; else { ttL_next(L); return TTK_EQ; } } case '<': { ttL_next(L); if (ttL_current(L) != '=') return '<'; else { ttL_next(L); return TTK_LE; } } case '>': { ttL_next(L); if (ttL_current(L) != '=') return '>'; else { ttL_next(L); return TTK_GE; } } case '~': { ttL_next(L); if (ttL_current(L) != '=') return '~'; else { ttL_next(L); return TTK_NE; } } case ':': { ttL_next(L); if (ttL_current(L) != ':') return ':'; else { ttL_next(L); return TTK_DBCOLON; } } case '"': case '\'': { /* short literal strings */ read_string(L, ttL_current(L)); return TTK_STRING; } case '.': { /* '.', '..', '...', or number */ ttL_save_and_next(L); if (ttL_check_next(L, ".")) { if (ttL_check_next(L, ".")) return TTK_DOTS; /* '...' */ else return TTK_CONCAT; /* '..' */ } else if (!isdigit(ttL_current(L))) return '.'; /* else go through */ } case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': { read_numeral(L); return TTK_NUMBER; } case TTL_EOZ: { return TTK_EOS; } default: { if (isalpha(ttL_current(L)) || ttL_current(L) == '_') { /* identifier or reserved word? */ int reversed; do { ttL_save_and_next(L); } while (isalnum(ttL_current(L)) || ttL_current(L) == '_'); if ((reversed = ttL_iskeyword(L, ltokens, TTK_LUA_NUM_RESERVED)) != 0) return reversed + TTK_FIRST_REVERSED; return TTK_NAME; } else { /* single-char tokens (+ - / ...) */ int c = ttL_current(L); ttL_next(L); return c; } } } } }