static void expand_token (struct obstack *obs, token_type t, token_data *td, int line) { symbol *sym; switch (t) { /* TOKSW */ case TOKEN_EOF: case TOKEN_MACDEF: break; case TOKEN_OPEN: case TOKEN_COMMA: case TOKEN_CLOSE: case TOKEN_SIMPLE: case TOKEN_STRING: shipout_text (obs, TOKEN_DATA_TEXT (td), strlen (TOKEN_DATA_TEXT (td)), line); break; case TOKEN_WORD: sym = lookup_symbol (TOKEN_DATA_TEXT (td), SYMBOL_LOOKUP); if (sym == NULL || SYMBOL_TYPE (sym) == TOKEN_VOID || (SYMBOL_TYPE (sym) == TOKEN_FUNC && SYMBOL_BLIND_NO_ARGS (sym) && peek_token () != TOKEN_OPEN)) { #ifdef ENABLE_CHANGEWORD shipout_text (obs, TOKEN_DATA_ORIG_TEXT (td), strlen (TOKEN_DATA_ORIG_TEXT (td)), line); #else shipout_text (obs, TOKEN_DATA_TEXT (td), strlen (TOKEN_DATA_TEXT (td)), line); #endif } else expand_macro (sym); break; default: M4ERROR ((warning_status, 0, "INTERNAL ERROR: bad token type in expand_token ()")); abort (); } }
token_type next_token (token_data *td) { int ch; int quote_level; token_type type; #ifdef ENABLE_CHANGEWORD int startpos; char *orig_text = 0; #endif obstack_free (&token_stack, token_bottom); obstack_1grow (&token_stack, '\0'); token_bottom = obstack_finish (&token_stack); ch = peek_input (); if (ch == CHAR_EOF) { return TOKEN_EOF; #ifdef DEBUG_INPUT fprintf (stderr, "next_token -> EOF\n"); #endif } if (ch == CHAR_MACRO) { init_macro_token (td); (void) next_char (); return TOKEN_MACDEF; } (void) next_char (); if (MATCH (ch, bcomm.string)) { obstack_grow (&token_stack, bcomm.string, bcomm.length); while ((ch = next_char ()) != CHAR_EOF && !MATCH (ch, ecomm.string)) obstack_1grow (&token_stack, ch); if (ch != CHAR_EOF) obstack_grow (&token_stack, ecomm.string, ecomm.length); type = TOKEN_STRING; } #ifdef ENABLE_CHANGEWORD else if (default_word_regexp && (isalpha (ch) || ch == '_')) #else else if (isalpha (ch) || ch == '_') #endif { obstack_1grow (&token_stack, ch); while ((ch = peek_input ()) != CHAR_EOF && (isalnum (ch) || ch == '_')) { obstack_1grow (&token_stack, ch); (void) next_char (); } type = TOKEN_WORD; } #ifdef ENABLE_CHANGEWORD else if (!default_word_regexp && strchr (word_start, ch)) { obstack_1grow (&token_stack, ch); while (1) { ch = peek_input (); if (ch == CHAR_EOF) break; obstack_1grow (&token_stack, ch); startpos = re_search (&word_regexp, obstack_base (&token_stack), obstack_object_size (&token_stack), 0, 0, ®s); if (startpos != 0 || regs.end [0] != obstack_object_size (&token_stack)) { *(((char *) obstack_base (&token_stack) + obstack_object_size (&token_stack)) - 1) = '\0'; break; } next_char (); } obstack_1grow (&token_stack, '\0'); orig_text = obstack_finish (&token_stack); if (regs.start[1] != -1) obstack_grow (&token_stack,orig_text + regs.start[1], regs.end[1] - regs.start[1]); else obstack_grow (&token_stack, orig_text,regs.end[0]); type = TOKEN_WORD; } #endif /* ENABLE_CHANGEWORD */ else if (!MATCH (ch, lquote.string)) { type = TOKEN_SIMPLE; obstack_1grow (&token_stack, ch); } else { quote_level = 1; while (1) { ch = next_char (); if (ch == CHAR_EOF) M4ERROR ((EXIT_FAILURE, 0, "ERROR: EOF in string")); if (MATCH (ch, rquote.string)) { if (--quote_level == 0) break; obstack_grow (&token_stack, rquote.string, rquote.length); } else if (MATCH (ch, lquote.string)) { quote_level++; obstack_grow (&token_stack, lquote.string, lquote.length); } else obstack_1grow (&token_stack, ch); } type = TOKEN_STRING; } obstack_1grow (&token_stack, '\0'); TOKEN_DATA_TYPE (td) = TOKEN_TEXT; TOKEN_DATA_TEXT (td) = obstack_finish (&token_stack); #ifdef ENABLE_CHANGEWORD if (orig_text == NULL) orig_text = TOKEN_DATA_TEXT (td); TOKEN_DATA_ORIG_TEXT (td) = orig_text; #endif #ifdef DEBUG_INPUT fprintf (stderr, "next_token -> %d (%s)\n", type, TOKEN_DATA_TEXT (td)); #endif return type; }
token_type next_token (token_data *td, int *line) { int ch; int quote_level; token_type type; #ifdef ENABLE_CHANGEWORD int startpos; char *orig_text = NULL; #endif const char *file; int dummy; obstack_free (&token_stack, token_bottom); if (!line) line = &dummy; /* Can't consume character until after CHAR_MACRO is handled. */ ch = peek_input (); if (ch == CHAR_EOF) { #ifdef DEBUG_INPUT xfprintf (stderr, "next_token -> EOF\n"); #endif next_char (); return TOKEN_EOF; } if (ch == CHAR_MACRO) { init_macro_token (td); next_char (); #ifdef DEBUG_INPUT xfprintf (stderr, "next_token -> MACDEF (%s)\n", find_builtin_by_addr (TOKEN_DATA_FUNC (td))->name); #endif return TOKEN_MACDEF; } next_char (); /* Consume character we already peeked at. */ file = current_file; *line = current_line; if (MATCH (ch, bcomm.string, true)) { obstack_grow (&token_stack, bcomm.string, bcomm.length); while ((ch = next_char ()) != CHAR_EOF && !MATCH (ch, ecomm.string, true)) obstack_1grow (&token_stack, ch); if (ch != CHAR_EOF) obstack_grow (&token_stack, ecomm.string, ecomm.length); else /* current_file changed to "" if we see CHAR_EOF, use the previous value we stored earlier. */ M4ERROR_AT_LINE ((EXIT_FAILURE, 0, file, *line, "ERROR: end of file in comment")); type = TOKEN_STRING; } else if (default_word_regexp && (isalpha (ch) || ch == '_')) { obstack_1grow (&token_stack, ch); while ((ch = peek_input ()) != CHAR_EOF && (isalnum (ch) || ch == '_')) { obstack_1grow (&token_stack, ch); next_char (); } type = TOKEN_WORD; } #ifdef ENABLE_CHANGEWORD else if (!default_word_regexp && word_regexp.fastmap[ch]) { obstack_1grow (&token_stack, ch); while (1) { ch = peek_input (); if (ch == CHAR_EOF) break; obstack_1grow (&token_stack, ch); startpos = re_search (&word_regexp, (char *) obstack_base (&token_stack), obstack_object_size (&token_stack), 0, 0, ®s); if (startpos || regs.end [0] != (regoff_t) obstack_object_size (&token_stack)) { *(((char *) obstack_base (&token_stack) + obstack_object_size (&token_stack)) - 1) = '\0'; break; } next_char (); } obstack_1grow (&token_stack, '\0'); orig_text = (char *) obstack_finish (&token_stack); if (regs.start[1] != -1) obstack_grow (&token_stack,orig_text + regs.start[1], regs.end[1] - regs.start[1]); else obstack_grow (&token_stack, orig_text,regs.end[0]); type = TOKEN_WORD; } #endif /* ENABLE_CHANGEWORD */ else if (!MATCH (ch, lquote.string, true)) { switch (ch) { case '(': type = TOKEN_OPEN; break; case ',': type = TOKEN_COMMA; break; case ')': type = TOKEN_CLOSE; break; default: type = TOKEN_SIMPLE; break; } obstack_1grow (&token_stack, ch); } else { bool fast = lquote.length == 1 && rquote.length == 1; quote_level = 1; while (1) { /* Try scanning a buffer first. */ const char *buffer = (isp && isp->type == INPUT_STRING ? isp->u.u_s.string : NULL); if (buffer && *buffer) { size_t len = isp->u.u_s.end - buffer; const char *p = buffer; do { p = (char *) memchr2 (p, *lquote.string, *rquote.string, buffer + len - p); } while (p && fast && (*p++ == *rquote.string ? --quote_level : ++quote_level)); if (p) { if (fast) { assert (!quote_level); obstack_grow (&token_stack, buffer, p - buffer - 1); isp->u.u_s.string += p - buffer; break; } obstack_grow (&token_stack, buffer, p - buffer); ch = to_uchar (*p); isp->u.u_s.string += p - buffer + 1; } else { obstack_grow (&token_stack, buffer, len); isp->u.u_s.string += len; continue; } } /* Fall back to a byte. */ else ch = next_char (); if (ch == CHAR_EOF) /* current_file changed to "" if we see CHAR_EOF, use the previous value we stored earlier. */ M4ERROR_AT_LINE ((EXIT_FAILURE, 0, file, *line, "ERROR: end of file in string")); if (MATCH (ch, rquote.string, true)) { if (--quote_level == 0) break; obstack_grow (&token_stack, rquote.string, rquote.length); } else if (MATCH (ch, lquote.string, true)) { quote_level++; obstack_grow (&token_stack, lquote.string, lquote.length); } else obstack_1grow (&token_stack, ch); } type = TOKEN_STRING; } obstack_1grow (&token_stack, '\0'); TOKEN_DATA_TYPE (td) = TOKEN_TEXT; TOKEN_DATA_TEXT (td) = (char *) obstack_finish (&token_stack); #ifdef ENABLE_CHANGEWORD if (orig_text == NULL) orig_text = TOKEN_DATA_TEXT (td); TOKEN_DATA_ORIG_TEXT (td) = orig_text; #endif #ifdef DEBUG_INPUT xfprintf (stderr, "next_token -> %s (%s)\n", token_type_string (type), TOKEN_DATA_TEXT (td)); #endif return type; }