/* ** this function is quite liberal in what it accepts, as 'luaO_str2num' ** will reject ill-formed numerals. */ static int read_numeral (LexState *ls, SemInfo *seminfo) { TValue obj; const char *expo = "Ee"; int first = ls->current; lua_assert(lisdigit(ls->current)); save_and_next(ls); if (first == '0' && check_next2(ls, "xX")) /* hexadecimal? */ expo = "Pp"; for (;;) { if (check_next2(ls, expo)) /* exponent part? */ check_next2(ls, "-+"); /* optional exponent sign */ if (lisxdigit(ls->current)) save_and_next(ls); else if (ls->current == '.') save_and_next(ls); else break; } save(ls, '\0'); if (luaO_str2num(luaZ_buffer(ls->buff), &obj) == 0) /* format error? */ lexerror(ls, "malformed number", TK_FLT); if (ttisinteger(&obj)) { seminfo->i = ivalue(&obj); return TK_INT; } else { lua_assert(ttisfloat(&obj)); seminfo->r = fltvalue(&obj); return TK_FLT; } }
/* LUA_NUMBER */ static void read_numeral (LexState *LS, int comma, SemInfo *seminfo) { size_t l = 0; checkbuffer(LS, l); if (comma) save(LS, '.', l); while (isdigit(LS->current)) { checkbuffer(LS, l); save_and_next(LS, l); } if (LS->current == '.') { save_and_next(LS, l); if (LS->current == '.') { save_and_next(LS, l); save(LS, '\0', l); luaX_lexerror(LS, "ambiguous syntax (decimal point x string concatenation)", TK_NUMBER); } } while (isdigit(LS->current)) { checkbuffer(LS, l); save_and_next(LS, l); } if (LS->current == 'e' || LS->current == 'E') { save_and_next(LS, l); /* read `E' */ if (LS->current == '+' || LS->current == '-') save_and_next(LS, l); /* optional exponent sign */ while (isdigit(LS->current)) { checkbuffer(LS, l); save_and_next(LS, l); } } save(LS, '\0', l); if (!luaO_str2d(luaZ_buffer(LS->buff), &seminfo->r)) luaX_lexerror(LS, "malformed number", TK_NUMBER); }
/* ** change all characters 'from' in buffer to 'to' */ static void buffreplace (LexState *ls, char from, char to) { if (from != to) { size_t n = luaZ_bufflen(ls->buff); char *p = luaZ_buffer(ls->buff); while (n--) if (p[n] == from) p[n] = to; } }
static const char *txtToken (LexState *ls, int token) { switch (token) { case TK_NAME: case TK_STRING: case TK_FLT: case TK_INT: save(ls, '\0'); return luaO_pushfstring(ls->L, "'%s'", luaZ_buffer(ls->buff)); default: return luaX_token2str(ls, token); } }
static void read_string (LexState *LS, int del, SemInfo *seminfo) { size_t l = 0; checkbuffer(LS, l); save_and_next(LS, l); while (LS->current != del) { checkbuffer(LS, l); switch (LS->current) { case EOZ: save(LS, '\0', l); luaX_lexerror(LS, "unfinished string", TK_EOS); break; /* to avoid warnings */ case '\n': save(LS, '\0', l); luaX_lexerror(LS, "unfinished string", TK_STRING); break; /* to avoid warnings */ case '\\': next(LS); /* do not save the `\' */ switch (LS->current) { case 'a': save(LS, '\a', l); next(LS); break; case 'b': save(LS, '\b', l); next(LS); break; case 'f': save(LS, '\f', l); next(LS); break; case 'n': save(LS, '\n', l); next(LS); break; case 'r': save(LS, '\r', l); next(LS); break; case 't': save(LS, '\t', l); next(LS); break; case 'v': save(LS, '\v', l); next(LS); break; case '\n': save(LS, '\n', l); inclinenumber(LS); break; case EOZ: break; /* will raise an error next loop */ default: { if (!isdigit(LS->current)) save_and_next(LS, l); /* handles \\, \", \', and \? */ else { /* \xxx */ int c = 0; int i = 0; do { c = 10*c + (LS->current-'0'); next(LS); } while (++i<3 && isdigit(LS->current)); if (c > UCHAR_MAX) { save(LS, '\0', l); luaX_lexerror(LS, "escape sequence too large", TK_STRING); } save(LS, c, l); } } } break; default: save_and_next(LS, l); } } save_and_next(LS, l); /* skip delimiter */ save(LS, '\0', l); seminfo->ts = luaS_newlstr(LS->L, luaZ_buffer(LS->buff) + 1, l - 3); }
void luaX_syntaxerror (LexState *ls, const char *msg) { const char *lasttoken; switch (ls->t.token) { case TK_NAME: lasttoken = getstr(ls->t.seminfo.ts); break; case TK_STRING: case TK_NUMBER: lasttoken = luaZ_buffer(ls->buff); break; default: lasttoken = luaX_token2str(ls, ls->t.token); break; } luaX_error(ls, msg, lasttoken); }
static void read_long_string (LexState *LS, SemInfo *seminfo) { int cont = 0; size_t l = 0; checkbuffer(LS, l); save(LS, '[', l); /* save first `[' */ save_and_next(LS, l); /* pass the second `[' */ if (LS->current == '\n') /* string starts with a newline? */ inclinenumber(LS); /* skip it */ for (;;) { checkbuffer(LS, l); switch (LS->current) { case EOZ: save(LS, '\0', l); luaX_lexerror(LS, (seminfo) ? "unfinished long string" : "unfinished long comment", TK_EOS); break; /* to avoid warnings */ case '[': save_and_next(LS, l); if (LS->current == '[') { cont++; save_and_next(LS, l); } continue; case ']': save_and_next(LS, l); if (LS->current == ']') { if (cont == 0) goto endloop; cont--; save_and_next(LS, l); } continue; case '\n': save(LS, '\n', l); inclinenumber(LS); if (!seminfo) l = 0; /* reset buffer to avoid wasting space */ continue; default: save_and_next(LS, l); } } endloop: save_and_next(LS, l); /* skip the second `]' */ save(LS, '\0', l); if (seminfo) seminfo->ts = luaS_newlstr(LS->L, luaZ_buffer(LS->buff) + 2, l - 5); }
static void read_long_string (LexState *ls, SemInfo *seminfo, int sep) { int line = ls->linenumber; /* initial line (for error message) */ save_and_next(ls); /* skip 2nd '[' */ if (currIsNewline(ls)) /* string starts with a newline? */ inclinenumber(ls); /* skip it */ for (;;) { switch (ls->current) { case EOZ: { /* error */ const char *what = (seminfo ? "string" : "comment"); const char *msg = luaO_pushfstring(ls->L, "unfinished long %s (starting at line %d)", what, line); lexerror(ls, msg, TK_EOS); break; /* to avoid warnings */ } case ']': { if (skip_sep(ls) == sep) { save_and_next(ls); /* skip 2nd ']' */ goto endloop; } break; } case '\n': case '\r': { save(ls, '\n'); inclinenumber(ls); if (!seminfo) luaZ_resetbuffer(ls->buff); /* avoid wasting space */ break; } default: { if (seminfo) save_and_next(ls); else next(ls); } } } endloop: if (seminfo) seminfo->ts = luaX_newstring(ls, luaZ_buffer(ls->buff) + (2 + sep), luaZ_bufflen(ls->buff) - 2*(2 + sep)); }
static int llex (LexState *ls, SemInfo *seminfo) { luaZ_resetbuffer(ls->buff); for (;;) { switch (ls->current) { case '\n': case '\r': { /* line breaks */ inclinenumber(ls); break; } case ' ': case '\f': case '\t': case '\v': { /* spaces */ next(ls); break; } case '-': { /* '-' or '--' (comment) */ next(ls); if (ls->current != '-') return '-'; /* else is a comment */ next(ls); if (ls->current == '[') { /* long comment? */ int sep = skip_sep(ls); luaZ_resetbuffer(ls->buff); /* 'skip_sep' may dirty the buffer */ if (sep >= 0) { read_long_string(ls, NULL, sep); /* skip long comment */ luaZ_resetbuffer(ls->buff); /* previous call may dirty the buff. */ break; } } /* else short comment */ while (!currIsNewline(ls) && ls->current != EOZ) next(ls); /* skip until end of line (or end of file) */ break; } case '[': { /* long string or simply '[' */ int sep = skip_sep(ls); if (sep >= 0) { read_long_string(ls, seminfo, sep); return TK_STRING; } else if (sep != -1) /* '[=...' missing second bracket */ lexerror(ls, "invalid long string delimiter", TK_STRING); return '['; } case '=': { next(ls); if (check_next1(ls, '=')) return TK_EQ; else return '='; } case '<': { next(ls); if (check_next1(ls, '=')) return TK_LE; else if (check_next1(ls, '<')) return TK_SHL; else return '<'; } case '>': { next(ls); if (check_next1(ls, '=')) return TK_GE; else if (check_next1(ls, '>')) return TK_SHR; else return '>'; } case '/': { next(ls); if (check_next1(ls, '/')) return TK_IDIV; else return '/'; } case '~': { next(ls); if (check_next1(ls, '=')) return TK_NE; else return '~'; } case ':': { next(ls); if (check_next1(ls, ':')) return TK_DBCOLON; else return ':'; } case '"': case '\'': { /* short literal strings */ read_string(ls, ls->current, seminfo); return TK_STRING; } case '`': { /* relative paths */ read_string(ls, ls->current, seminfo); return TK_PATH; } case '.': { /* '.', '..', '...', or number */ save_and_next(ls); if (check_next1(ls, '.')) { if (check_next1(ls, '.')) return TK_DOTS; /* '...' */ else return TK_CONCAT; /* '..' */ } else if (!lisdigit(ls->current)) return '.'; else return read_numeral(ls, seminfo); } case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': { return read_numeral(ls, seminfo); } case EOZ: { return TK_EOS; } default: { if (lislalpha(ls->current)) { /* identifier or reserved word? */ TString *ts; do { save_and_next(ls); } while (lislalnum(ls->current)); ts = luaX_newstring(ls, luaZ_buffer(ls->buff), luaZ_bufflen(ls->buff)); seminfo->ts = ts; if (isreserved(ts)) /* reserved word? */ return ts->extra - 1 + FIRST_RESERVED; else { return TK_NAME; } } else { /* single-char tokens (+ - / ...) */ int c = ls->current; next(ls); return c; } } } } }
static void read_string (LexState *ls, int del, SemInfo *seminfo) { save_and_next(ls); /* keep delimiter (for error messages) */ while (ls->current != del) { switch (ls->current) { case EOZ: lexerror(ls, "unfinished string", TK_EOS); break; /* to avoid warnings */ case '\n': case '\r': lexerror(ls, "unfinished string", TK_STRING); break; /* to avoid warnings */ case '\\': { /* escape sequences */ int c; /* final character to be saved */ save_and_next(ls); /* keep '\\' for error messages */ switch (ls->current) { case 'a': c = '\a'; goto read_save; case 'b': c = '\b'; goto read_save; case 'f': c = '\f'; goto read_save; case 'n': c = '\n'; goto read_save; case 'r': c = '\r'; goto read_save; case 't': c = '\t'; goto read_save; case 'v': c = '\v'; goto read_save; case 'x': c = readhexaesc(ls); goto read_save; case 'u': utf8esc(ls); goto no_save; case '\n': case '\r': inclinenumber(ls); c = '\n'; goto only_save; case '\\': case '\"': case '\'': c = ls->current; goto read_save; case EOZ: goto no_save; /* will raise an error next loop */ case 'z': { /* zap following span of spaces */ luaZ_buffremove(ls->buff, 1); /* remove '\\' */ next(ls); /* skip the 'z' */ while (lisspace(ls->current)) { if (currIsNewline(ls)) inclinenumber(ls); else next(ls); } goto no_save; } default: { esccheck(ls, lisdigit(ls->current), "invalid escape sequence"); c = readdecesc(ls); /* digital escape '\ddd' */ goto only_save; } } read_save: next(ls); /* go through */ only_save: luaZ_buffremove(ls->buff, 1); /* remove '\\' */ save(ls, c); /* go through */ no_save: break; } default: save_and_next(ls); } } save_and_next(ls); /* skip delimiter */ seminfo->ts = luaX_newstring(ls, luaZ_buffer(ls->buff) + 1, luaZ_bufflen(ls->buff) - 2); }
int luaX_lex (LexState *LS, SemInfo *seminfo) { for (;;) { switch (LS->current) { case '\n': { inclinenumber(LS); continue; } case '-': { next(LS); if (LS->current != '-') return '-'; /* else is a comment */ next(LS); if (LS->current == '[' && (next(LS), LS->current == '[')) read_long_string(LS, NULL); /* long comment */ else /* short comment */ while (LS->current != '\n' && LS->current != EOZ) next(LS); continue; } case '[': { next(LS); if (LS->current != '[') return '['; else { read_long_string(LS, seminfo); return TK_STRING; } } case '=': { next(LS); if (LS->current != '=') return '='; else { next(LS); return TK_EQ; } } case '<': { next(LS); if (LS->current == '<') { next(LS); return TK_SHL; } else if (LS->current != '=') return '<'; else { next(LS); return TK_LE; } } case '>': { next(LS); if (LS->current == '>') { next(LS); return TK_SHR; } else if (LS->current != '=') return '>'; else { next(LS); return TK_GE; } } case '~': { next(LS); if (LS->current != '=') return '~'; else { next(LS); return TK_NE; } } case '"': case '\'': { read_string(LS, LS->current, seminfo); return TK_STRING; } case '.': { next(LS); if (LS->current == '.') { next(LS); if (LS->current == '.') { next(LS); return TK_DOTS; /* ... */ } else return TK_CONCAT; /* .. */ } else if (!lex_isdigit(LS->current)) return '.'; else { return read_numeral(LS, 1, seminfo); } } case EOZ: { return TK_EOS; } default: { if (isspace(LS->current)) { next(LS); continue; } else if (lex_isdigit(LS->current)) { return (read_numeral(LS, 0, seminfo)); } else if (lex_isalpha(LS->current) || LS->current == '_') { char saveCh = 0; size_t l; TString *ts; if (LS->current == 'L') { next(LS); if (LS->current == '"') { read_wstring(LS, LS->current, seminfo); return TK_WSTRING; } saveCh = 'L'; } /* identifier or reserved word */ l = readname(LS, saveCh); ts = luaS_newlstr(LS->L, luaZ_buffer(LS->buff), l); if (ts->tsv.reserved > 0) /* reserved word? */ return ts->tsv.reserved - 1 + FIRST_RESERVED; seminfo->ts = ts; return TK_NAME; } else { int c = LS->current; if (iscntrl(c)) luaX_error(LS, "invalid control char", luaO_pushfstring(LS->L, "char(%d)", c)); next(LS); return c; /* single-char tokens (+ - / ...) */ } } } } }
static void read_wstring (LexState *LS, int del, SemInfo *seminfo) { size_t l = 0; checkbuffer(LS, l * 2); wsave_and_next(LS, l); while (LS->current != del) { checkbuffer(LS, l * 2); switch (LS->current) { case EOZ: wsave(LS, '\0', l); luaX_lexerror(LS, "unfinished string", TK_EOS); break; /* to avoid warnings */ case '\n': wsave(LS, '\0', l); luaX_lexerror(LS, "unfinished string", TK_STRING); break; /* to avoid warnings */ case '\\': next(LS); /* do not save the `\' */ switch (LS->current) { case 'a': wsave(LS, '\a', l); next(LS); break; case 'b': wsave(LS, '\b', l); next(LS); break; case 'f': wsave(LS, '\f', l); next(LS); break; case 'n': wsave(LS, '\n', l); next(LS); break; case 'r': wsave(LS, '\r', l); next(LS); break; case 't': wsave(LS, '\t', l); next(LS); break; case 'v': wsave(LS, '\v', l); next(LS); break; case '\n': wsave(LS, '\n', l); inclinenumber(LS); break; case EOZ: break; /* will raise an error next loop */ case 'x': { int ch; next(LS); ch = tolower(LS->current); if (!lex_isdigit(ch) && !(ch >= 'a' && ch <= 'f') ) save(LS, 'x', l); /* handles \\, \", \', and \? */ else { /* \xxx */ int c = 0; int i = 0; int numDigits = 4; do { ch = tolower(LS->current); if (lex_isdigit(ch)) c = 16*c + (ch-'0'); else if (ch >= 'a' && ch <= 'f') c = 16*c + (ch-'a') + 10; next(LS); ch = tolower(LS->current); } while (++i<numDigits && (lex_isdigit(ch) || (ch >= 'a' && ch <= 'f'))); wsave(LS, c, l); } break; } default: { if (!lex_isdigit(LS->current)) wsave_and_next(LS, l); /* handles \\, \", \', and \? */ else { /* \xxx */ int c = 0; int i = 0; do { c = 10*c + (LS->current-'0'); next(LS); } while (++i<3 && lex_isdigit(LS->current)); if (c > UCHAR_MAX) { wsave(LS, '\0', l); luaX_lexerror(LS, "escape sequence too large", TK_STRING); } wsave(LS, c, l); } } } break; default: wsave_and_next(LS, l); } } wsave_and_next(LS, l); /* skip delimiter */ wsave(LS, '\0', l); seminfo->ts = luaS_newlwstr(LS->L, (const lua_WChar*)(luaZ_buffer(LS->buff) + 1 * 2), (l - 3 * 2) / 2); }
/* LUA_NUMBER */ static int read_numeral (LexState *LS, int period, SemInfo *seminfo) { int isReal = 0; int startsWithZero = LS->current == '0'; size_t l = 0; checkbuffer(LS, l); if (period) { save(LS, '.', l); isReal = 1; } if (startsWithZero) { next(LS); if (LS->current == 'x') { /* Process a hex number */ int ch = 0; int c = 0; int i = 0; int numDigits = 8; next(LS); do { ch = tolower(LS->current); if (lex_isdigit(ch)) c = 16*c + (ch-'0'); else if (ch >= 'a' && ch <= 'f') c = 16*c + (ch-'a') + 10; next(LS); ch = tolower(LS->current); } while (++i<numDigits && (lex_isdigit(ch) || (ch >= 'a' && ch <= 'f'))); seminfo->r = c; return TK_NUMBER; } else { checkbuffer(LS, 1); save(LS, '0', l); } } while (lex_isdigit(LS->current)) { checkbuffer(LS, l); save_and_next(LS, l); } if (LS->current == '.') { isReal = 1; save_and_next(LS, l); if (LS->current == '.') { save_and_next(LS, l); save(LS, '\0', l); luaX_lexerror(LS, "ambiguous syntax (decimal point x string concatenation)", TK_NUMBER); } } while (lex_isdigit(LS->current)) { checkbuffer(LS, l); save_and_next(LS, l); } if (LS->current == 'e' || LS->current == 'E') { isReal = 1; save_and_next(LS, l); /* read `E' */ if (LS->current == '+' || LS->current == '-') save_and_next(LS, l); /* optional exponent sign */ while (lex_isdigit(LS->current)) { checkbuffer(LS, l); save_and_next(LS, l); } } save(LS, '\0', l); if (isReal) { if (!luaO_str2d(luaZ_buffer(LS->buff), &seminfo->r)) luaX_lexerror(LS, "malformed number", TK_NUMBER); return TK_NUMBER; } else { if (!luaO_str2d(luaZ_buffer(LS->buff), &seminfo->r)) luaX_lexerror(LS, "malformed integer", TK_NUMBER); return TK_NUMBER; } }
static void luaX_lexerror (LexState *ls, const char *s, int token) { if (token == TK_EOS) luaX_error(ls, s, luaX_token2str(ls, token)); else luaX_error(ls, s, luaZ_buffer(ls->buff)); }