/* 8b. Drop whitespace. */ static void phase8b_get (token_ty *tp) { for (;;) { phase8a_get (tp); if (tp->type == token_type_white_space) continue; if (tp->type == token_type_eoln) { /* We have to track the last occurrence of a string. One mode of xgettext allows to group an extracted message with a comment for documentation. The rule which states which comment is assumed to be grouped with the message says it should immediately precede it. Our interpretation: between the last line of the comment and the line in which the keyword is found must be no line with non-white space tokens. */ ++newline_count; if (last_non_comment_line > last_comment_line) savable_comment_reset (); continue; } break; } }
static void phase5_get (token_ty *tp) { static char *buffer; static int bufmax; int bufpos; int c; if (phase5_pushback_length) { *tp = phase5_pushback[--phase5_pushback_length]; return; } for (;;) { tp->line_number = line_number; c = phase2_getc (); switch (c) { case EOF: tp->type = token_type_eof; return; case '\n': if (last_non_comment_line > last_comment_line) savable_comment_reset (); /* FALLTHROUGH */ case '\r': case '\t': case ' ': /* Ignore whitespace and comments. */ continue; } last_non_comment_line = tp->line_number; switch (c) { case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': case 'Y': case 'Z': case '_': case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': case 's': case 't': case 'u': case 'v': case 'w': case 'x': case 'y': case 'z': case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': /* Symbol, or part of a number. */ bufpos = 0; for (;;) { if (bufpos >= bufmax) { bufmax = 2 * bufmax + 10; buffer = xrealloc (buffer, bufmax); } buffer[bufpos++] = c; c = phase2_getc (); switch (c) { case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': case 'Y': case 'Z': case '_': case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': case 's': case 't': case 'u': case 'v': case 'w': case 'x': case 'y': case 'z': case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': continue; default: if (bufpos == 1 && buffer[0] == '_' && c == '(') { tp->type = token_type_i18n; return; } phase2_ungetc (c); break; } break; } if (bufpos >= bufmax) { bufmax = 2 * bufmax + 10; buffer = xrealloc (buffer, bufmax); } buffer[bufpos] = '\0'; tp->string = xstrdup (buffer); tp->type = token_type_symbol; return; case '"': bufpos = 0; for (;;) { c = phase7_getc (); if (c == EOF || c == P7_QUOTES) break; if (bufpos >= bufmax) { bufmax = 2 * bufmax + 10; buffer = xrealloc (buffer, bufmax); } buffer[bufpos++] = c; } if (bufpos >= bufmax) { bufmax = 2 * bufmax + 10; buffer = xrealloc (buffer, bufmax); } buffer[bufpos] = '\0'; tp->string = xstrdup (buffer); tp->type = token_type_string_literal; tp->comment = add_reference (savable_comment); return; case '(': tp->type = token_type_lparen; return; case ')': tp->type = token_type_rparen; return; case ',': tp->type = token_type_comma; return; default: /* We could carefully recognize each of the 2 and 3 character operators, but it is not necessary, as we only need to recognize gettext invocations. Don't bother. */ tp->type = token_type_other; return; } } }
static void x_awk_lex (token_ty *tp) { static char *buffer; static int bufmax; int bufpos; int c; for (;;) { tp->line_number = line_number; c = phase2_getc (); switch (c) { case EOF: tp->type = token_type_eof; return; case '\n': if (last_non_comment_line > last_comment_line) savable_comment_reset (); /* Newline is not allowed inside expressions. It usually introduces a fresh statement. FIXME: Newlines after any of ',' '{' '?' ':' '||' '&&' 'do' 'else' does *not* introduce a fresh statement. */ prefer_division_over_regexp = false; /* FALLTHROUGH */ case '\t': case ' ': /* Ignore whitespace and comments. */ continue; case '\\': /* Backslash ought to be immediately followed by a newline. */ continue; } last_non_comment_line = tp->line_number; switch (c) { case '.': { int c2 = phase2_getc (); phase2_ungetc (c2); if (!(c2 >= '0' && c2 <= '9')) { tp->type = token_type_other; prefer_division_over_regexp = false; return; } } /* FALLTHROUGH */ case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': case 'Y': case 'Z': case '_': case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': case 's': case 't': case 'u': case 'v': case 'w': case 'x': case 'y': case 'z': case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': /* Symbol, or part of a number. */ bufpos = 0; for (;;) { if (bufpos >= bufmax) { bufmax = 2 * bufmax + 10; buffer = xrealloc (buffer, bufmax); } buffer[bufpos++] = c; c = phase2_getc (); switch (c) { case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': case 'Y': case 'Z': case '_': case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': case 's': case 't': case 'u': case 'v': case 'w': case 'x': case 'y': case 'z': case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': continue; default: if (bufpos == 1 && buffer[0] == '_' && c == '"') { tp->type = token_type_i18nstring; goto case_string; } phase2_ungetc (c); break; } break; } if (bufpos >= bufmax) { bufmax = 2 * bufmax + 10; buffer = xrealloc (buffer, bufmax); } buffer[bufpos] = '\0'; tp->string = xstrdup (buffer); tp->type = token_type_symbol; /* Most identifiers can be variable names; after them we must interpret '/' as division operator. But for awk's builtin keywords we have three cases: (a) Must interpret '/' as division operator. "length". (b) Must interpret '/' as start of a regular expression. "do", "exit", "print", "printf", "return". (c) '/' after this keyword in invalid anyway. All others. I used the following script for the distinction. for k in $awk_keywords; do echo; echo $k; awk "function foo () { $k / 10 }" < /dev/null done */ if (strcmp (buffer, "do") == 0 || strcmp (buffer, "exit") == 0 || strcmp (buffer, "print") == 0 || strcmp (buffer, "printf") == 0 || strcmp (buffer, "return") == 0) prefer_division_over_regexp = false; else prefer_division_over_regexp = true; return; case '"': tp->type = token_type_string; case_string: bufpos = 0; for (;;) { c = phase7_getc (); if (c == EOF || c == P7_QUOTES) break; if (bufpos >= bufmax) { bufmax = 2 * bufmax + 10; buffer = xrealloc (buffer, bufmax); } buffer[bufpos++] = c; } if (bufpos >= bufmax) { bufmax = 2 * bufmax + 10; buffer = xrealloc (buffer, bufmax); } buffer[bufpos] = '\0'; tp->string = xstrdup (buffer); prefer_division_over_regexp = true; return; case '(': tp->type = token_type_lparen; prefer_division_over_regexp = false; return; case ')': tp->type = token_type_rparen; prefer_division_over_regexp = true; return; case ',': tp->type = token_type_comma; prefer_division_over_regexp = false; return; case ';': tp->type = token_type_semicolon; prefer_division_over_regexp = false; return; case ']': tp->type = token_type_other; prefer_division_over_regexp = true; return; case '/': if (!prefer_division_over_regexp) { /* Regular expression. Counting brackets is non-trivial. [[] is balanced, and so is [\]]. Also, /[/]/ is balanced and ends at the third slash. Do not count [ or ] if either one is preceded by a \. A '[' should be counted if a) it is the first one so far (brackets == 0), or b) it is the '[' in '[:'. A ']' should be counted if not preceded by a \. According to POSIX, []] is how you put a ] into a set. Try to handle that too. */ int brackets = 0; bool pos0 = true; /* true at start of regexp */ bool pos1_open = false; /* true after [ at start of regexp */ bool pos2_open_not = false; /* true after [^ at start of regexp */ for (;;) { c = phase1_getc (); if (c == EOF || c == '\n') { phase1_ungetc (c); error_with_progname = false; error (0, 0, _("%s:%d: warning: unterminated regular expression"), logical_file_name, line_number); error_with_progname = true; break; } else if (c == '[') { if (brackets == 0) brackets++; else { c = phase1_getc (); if (c == ':') brackets++; phase1_ungetc (c); } if (pos0) { pos0 = false; pos1_open = true; continue; } } else if (c == ']') { if (!(pos1_open || pos2_open_not)) brackets--; } else if (c == '^') { if (pos1_open) { pos1_open = false; pos2_open_not = true; continue; } } else if (c == '\\') { c = phase1_getc (); /* Backslash-newline is valid and ignored. */ } else if (c == '/') { if (brackets <= 0) break; } pos0 = false; pos1_open = false; pos2_open_not = false; } tp->type = token_type_other; prefer_division_over_regexp = false; return; } /* FALLTHROUGH */ default: /* We could carefully recognize each of the 2 and 3 character operators, but it is not necessary, as we only need to recognize gettext invocations. Don't bother. */ tp->type = token_type_other; prefer_division_over_regexp = false; return; } } }
static void phase6_get (token_ty *tp) { static token_ty *buf; static int bufmax; int bufpos; int j; if (phase6_pushback_length) { *tp = phase6_pushback[--phase6_pushback_length]; return; } for (;;) { /* Get the next token. If it is not a '#' at the beginning of a line (ignoring whitespace), return immediately. */ phaseX_get (tp); if (tp->type != token_type_hash) return; /* Accumulate the rest of the directive in a buffer, until the "define" keyword is seen or until end of line. */ bufpos = 0; for (;;) { phaseX_get (tp); if (tp->type == token_type_eoln || tp->type == token_type_eof) break; /* Before the "define" keyword and inside other directives white space is irrelevant. So just throw it away. */ if (tp->type != token_type_white_space) { /* If it is a #define directive, return immediately, thus treating the body of the #define directive like normal input. */ if (bufpos == 0 && tp->type == token_type_name && strcmp (tp->string, "define") == 0) return; /* Accumulate. */ if (bufpos >= bufmax) { bufmax = 2 * bufmax + 10; buf = xrealloc (buf, bufmax * sizeof (buf[0])); } buf[bufpos++] = *tp; } } /* If it is a #line directive, with no macros to expand, act on it. Ignore all other directives. */ if (bufpos >= 3 && buf[0].type == token_type_name && strcmp (buf[0].string, "line") == 0 && buf[1].type == token_type_number && buf[2].type == token_type_string_literal) { logical_file_name = xstrdup (buf[2].string); line_number = buf[1].number; } if (bufpos >= 2 && buf[0].type == token_type_number && buf[1].type == token_type_string_literal) { logical_file_name = xstrdup (buf[1].string); line_number = buf[0].number; } /* Release the storage held by the directive. */ for (j = 0; j < bufpos; ++j) free_token (&buf[j]); /* We must reset the selected comments. */ savable_comment_reset (); } }
static void phase4_get (token_ty *tp) { static char *buffer; static int bufmax; int bufpos; int c; if (phase4_pushback_length) { *tp = phase4_pushback[--phase4_pushback_length]; return; } tp->string = NULL; for (;;) { tp->line_number = line_number; c = phase3_getc (); switch (c) { case EOF: tp->type = token_type_eof; return; case '\n': if (last_non_comment_line > last_comment_line) savable_comment_reset (); /* FALLTHROUGH */ case ' ': case '\t': case '\r': /* Ignore whitespace. */ continue; } last_non_comment_line = tp->line_number; switch (c) { case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': case 'Y': case 'Z': case '_': case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': case 's': case 't': case 'u': case 'v': case 'w': case 'x': case 'y': case 'z': case 127: case 128: case 129: case 130: case 131: case 132: case 133: case 134: case 135: case 136: case 137: case 138: case 139: case 140: case 141: case 142: case 143: case 144: case 145: case 146: case 147: case 148: case 149: case 150: case 151: case 152: case 153: case 154: case 155: case 156: case 157: case 158: case 159: case 160: case 161: case 162: case 163: case 164: case 165: case 166: case 167: case 168: case 169: case 170: case 171: case 172: case 173: case 174: case 175: case 176: case 177: case 178: case 179: case 180: case 181: case 182: case 183: case 184: case 185: case 186: case 187: case 188: case 189: case 190: case 191: case 192: case 193: case 194: case 195: case 196: case 197: case 198: case 199: case 200: case 201: case 202: case 203: case 204: case 205: case 206: case 207: case 208: case 209: case 210: case 211: case 212: case 213: case 214: case 215: case 216: case 217: case 218: case 219: case 220: case 221: case 222: case 223: case 224: case 225: case 226: case 227: case 228: case 229: case 230: case 231: case 232: case 233: case 234: case 235: case 236: case 237: case 238: case 239: case 240: case 241: case 242: case 243: case 244: case 245: case 246: case 247: case 248: case 249: case 250: case 251: case 252: case 253: case 254: case 255: bufpos = 0; for (;;) { if (bufpos >= bufmax) { bufmax = 2 * bufmax + 10; buffer = xrealloc (buffer, bufmax); } buffer[bufpos++] = c; c = phase1_getc (); switch (c) { case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': case 'Y': case 'Z': case '_': case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': case 's': case 't': case 'u': case 'v': case 'w': case 'x': case 'y': case 'z': case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': case 127: case 128: case 129: case 130: case 131: case 132: case 133: case 134: case 135: case 136: case 137: case 138: case 139: case 140: case 141: case 142: case 143: case 144: case 145: case 146: case 147: case 148: case 149: case 150: case 151: case 152: case 153: case 154: case 155: case 156: case 157: case 158: case 159: case 160: case 161: case 162: case 163: case 164: case 165: case 166: case 167: case 168: case 169: case 170: case 171: case 172: case 173: case 174: case 175: case 176: case 177: case 178: case 179: case 180: case 181: case 182: case 183: case 184: case 185: case 186: case 187: case 188: case 189: case 190: case 191: case 192: case 193: case 194: case 195: case 196: case 197: case 198: case 199: case 200: case 201: case 202: case 203: case 204: case 205: case 206: case 207: case 208: case 209: case 210: case 211: case 212: case 213: case 214: case 215: case 216: case 217: case 218: case 219: case 220: case 221: case 222: case 223: case 224: case 225: case 226: case 227: case 228: case 229: case 230: case 231: case 232: case 233: case 234: case 235: case 236: case 237: case 238: case 239: case 240: case 241: case 242: case 243: case 244: case 245: case 246: case 247: case 248: case 249: case 250: case 251: case 252: case 253: case 254: case 255: continue; default: phase1_ungetc (c); break; } break; } if (bufpos >= bufmax) { bufmax = 2 * bufmax + 10; buffer = xrealloc (buffer, bufmax); } buffer[bufpos] = 0; tp->string = xstrdup (buffer); tp->type = token_type_symbol; return; case '\'': /* Single-quoted string literal. */ bufpos = 0; for (;;) { c = phase1_getc (); if (c == EOF || c == '\'') break; if (c == '\\') { c = phase1_getc (); if (c != '\\' && c != '\'') { phase1_ungetc (c); c = '\\'; } } if (bufpos >= bufmax) { bufmax = 2 * bufmax + 10; buffer = xrealloc (buffer, bufmax); } buffer[bufpos++] = c; } if (bufpos >= bufmax) { bufmax = 2 * bufmax + 10; buffer = xrealloc (buffer, bufmax); } buffer[bufpos] = 0; tp->type = token_type_string_literal; tp->string = xstrdup (buffer); tp->comment = add_reference (savable_comment); return; case '"': /* Double-quoted string literal. */ tp->type = token_type_string_literal; bufpos = 0; for (;;) { c = phase1_getc (); if (c == EOF || c == '"') break; if (c == '$') { c = phase1_getc (); if ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_' || c == '{' || c >= 0x7f) { /* String with variables. */ tp->type = token_type_other; continue; } phase1_ungetc (c); c = '$'; } if (c == '{') { c = phase1_getc (); if (c == '$') { /* String with expressions. */ tp->type = token_type_other; continue; } phase1_ungetc (c); c = '{'; } if (c == '\\') { int n, j; c = phase1_getc (); switch (c) { case '"': case '\\': case '$': break; case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': n = 0; for (j = 0; j < 3; ++j) { n = n * 8 + c - '0'; c = phase1_getc (); switch (c) { default: break; case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': continue; } break; } phase1_ungetc (c); c = n; break; case 'x': n = 0; for (j = 0; j < 2; ++j) { c = phase1_getc (); switch (c) { case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': n = n * 16 + c - '0'; break; case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': n = n * 16 + 10 + c - 'A'; break; case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': n = n * 16 + 10 + c - 'a'; break; default: phase1_ungetc (c); c = 0; break; } if (c == 0) break; } if (j == 0) { phase1_ungetc ('x'); c = '\\'; } else c = n; break; case 'n': c = '\n'; break; case 't': c = '\t'; break; case 'r': c = '\r'; break; default: phase1_ungetc (c); c = '\\'; break; } } if (bufpos >= bufmax) { bufmax = 2 * bufmax + 10; buffer = xrealloc (buffer, bufmax); } buffer[bufpos++] = c; } if (bufpos >= bufmax) { bufmax = 2 * bufmax + 10; buffer = xrealloc (buffer, bufmax); } buffer[bufpos] = 0; if (tp->type == token_type_string_literal) { tp->string = xstrdup (buffer); tp->comment = add_reference (savable_comment); } return; case '?': case '%': { int c2 = phase1_getc (); if (c2 == '>') { /* ?> and %> terminate PHP mode and switch back to HTML mode. */ skip_html (); tp->type = token_type_other; } else { phase1_ungetc (c2); tp->type = (c == '%' ? token_type_operator1 : token_type_other); } return; } case '(': tp->type = token_type_lparen; return; case ')': tp->type = token_type_rparen; return; case ',': tp->type = token_type_comma; return; case '[': tp->type = token_type_lbracket; return; case ']': tp->type = token_type_rbracket; return; case '.': tp->type = token_type_dot; return; case '*': case '/': tp->type = token_type_operator1; return; case '+': case '-': { int c2 = phase1_getc (); if (c2 == c) /* ++ or -- */ tp->type = token_type_operator1; else /* + or - */ { phase1_ungetc (c2); tp->type = token_type_operator2; } return; } case '!': case '~': case '@': tp->type = token_type_operator2; return; case '<': { int c2 = phase1_getc (); if (c2 == '<') { int c3 = phase1_getc (); if (c3 == '<') { /* Start of here document. Parse whitespace, then label, then newline. */ do c = phase3_getc (); while (c == ' ' || c == '\t' || c == '\n' || c == '\r'); bufpos = 0; do { if (bufpos >= bufmax) { bufmax = 2 * bufmax + 10; buffer = xrealloc (buffer, bufmax); } buffer[bufpos++] = c; c = phase3_getc (); } while (c != EOF && c != '\n' && c != '\r'); /* buffer[0..bufpos-1] now contains the label. */ /* Now skip the here document. */ for (;;) { c = phase1_getc (); if (c == EOF) break; if (c == '\n' || c == '\r') { int bufidx = 0; while (bufidx < bufpos) { c = phase1_getc (); if (c == EOF) break; if (c != buffer[bufidx]) { phase1_ungetc (c); break; } bufidx++; } if (bufidx == bufpos) { c = phase1_getc (); if (c != ';') phase1_ungetc (c); c = phase1_getc (); if (c == '\n' || c == '\r') break; } } } /* FIXME: Ideally we should turn the here document into a string literal if it didn't contain $ substitution. And we should also respect backslash escape sequences like in double-quoted strings. */ tp->type = token_type_other; return; } phase1_ungetc (c3); } /* < / script > terminates PHP mode and switches back to HTML mode. */ while (c2 == ' ' || c2 == '\t' || c2 == '\n' || c2 == '\r') c2 = phase1_getc (); if (c2 == '/') { do c2 = phase1_getc (); while (c2 == ' ' || c2 == '\t' || c2 == '\n' || c2 == '\r'); if (c2 == 's' || c2 == 'S') { c2 = phase1_getc (); if (c2 == 'c' || c2 == 'C') { c2 = phase1_getc (); if (c2 == 'r' || c2 == 'R') { c2 = phase1_getc (); if (c2 == 'i' || c2 == 'I') { c2 = phase1_getc (); if (c2 == 'p' || c2 == 'P') { c2 = phase1_getc (); if (c2 == 't' || c2 == 'T') { do c2 = phase1_getc (); while (c2 == ' ' || c2 == '\t' || c2 == '\n' || c2 == '\r'); if (c2 == '>') { skip_html (); } else phase1_ungetc (c2); } else phase1_ungetc (c2); } else phase1_ungetc (c2); } else phase1_ungetc (c2); } else phase1_ungetc (c2); } else phase1_ungetc (c2); } else phase1_ungetc (c2); } else phase1_ungetc (c2); tp->type = token_type_other; return; } case '`': /* Execution operator. */ default: /* We could carefully recognize each of the 2 and 3 character operators, but it is not necessary, as we only need to recognize gettext invocations. Don't bother. */ tp->type = token_type_other; return; } } }
static void phase2_get (token_ty *tp) { static char *buffer; static int bufmax; int bufpos; int c; if (phase2_pushback_length) { *tp = phase2_pushback[--phase2_pushback_length]; return; } tp->string = NULL; for (;;) { tp->line_number = line_number; c = phase1_getc (); switch (c) { case EOF: tp->type = token_type_eof; return; case '"': { /* Comment. */ int lineno; comment_start (); lineno = line_number; for (;;) { c = phase1_getc (); if (c == '"' || c == EOF) break; if (c == '\n') { comment_line_end (); comment_start (); } else { /* We skip all leading white space, but not EOLs. */ if (!(buflen == 0 && (c == ' ' || c == '\t'))) comment_add (c); } } comment_line_end (); last_comment_line = lineno; continue; } case '\n': if (last_non_comment_line > last_comment_line) savable_comment_reset (); /* FALLTHROUGH */ case ' ': case '\t': case '\r': /* Ignore whitespace. */ continue; } last_non_comment_line = tp->line_number; switch (c) { case '\'': /* String literal. */ bufpos = 0; for (;;) { c = phase1_getc (); if (c == EOF) break; if (c == '\'') { c = phase1_getc (); if (c != '\'') { phase1_ungetc (c); break; } } if (bufpos >= bufmax) { bufmax = 2 * bufmax + 10; buffer = xrealloc (buffer, bufmax); } buffer[bufpos++] = c; } if (bufpos >= bufmax) { bufmax = 2 * bufmax + 10; buffer = xrealloc (buffer, bufmax); } buffer[bufpos] = 0; tp->type = token_type_string_literal; tp->string = xstrdup (buffer); return; case '+': case '-': case '*': case '/': case '~': case '|': case ',': case '<': case '>': case '=': case '&': case '@': case '?': case '%': case '\\': { char *name; int c2 = phase1_getc (); switch (c2) { case '+': case '-': case '*': case '/': case '~': case '|': case ',': case '<': case '>': case '=': case '&': case '@': case '?': case '%': name = xmalloc (3); name[0] = c; name[1] = c2; name[2] = '\0'; tp->type = token_type_symbol; tp->string = name; return; default: phase1_ungetc (c2); break; } name = xmalloc (2); name[0] = c; name[1] = '\0'; tp->type = token_type_symbol; tp->string = name; return; } case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': case 'Y': case 'Z': case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': case 's': case 't': case 'u': case 'v': case 'w': case 'x': case 'y': case 'z': /* Recognize id or id":"[id":"]* or id":"[id":"]*id. */ bufpos = 0; for (;;) { if (bufpos >= bufmax) { bufmax = 2 * bufmax + 10; buffer = xrealloc (buffer, bufmax); } buffer[bufpos++] = c; c = phase1_getc (); switch (c) { case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': case 'Y': case 'Z': case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': case 's': case 't': case 'u': case 'v': case 'w': case 'x': case 'y': case 'z': case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': continue; case ':': if (bufpos >= bufmax) { bufmax = 2 * bufmax + 10; buffer = xrealloc (buffer, bufmax); } buffer[bufpos++] = c; c = phase1_getc (); switch (c) { case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': case 'Y': case 'Z': case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': case 's': case 't': case 'u': case 'v': case 'w': case 'x': case 'y': case 'z': continue; default: phase1_ungetc (c); break; } break; default: phase1_ungetc (c); break; } break; } if (bufpos >= bufmax) { bufmax = 2 * bufmax + 10; buffer = xrealloc (buffer, bufmax); } buffer[bufpos] = '\0'; tp->string = xstrdup (buffer); tp->type = token_type_symbol; return; case '#': /* Uniquification operator. */ tp->type = token_type_uniq; return; case '$': c = phase1_getc (); tp->type = token_type_other; return; default: tp->type = token_type_other; return; } } }
static void phase3_get (token_ty *tp) { int c; int c2; int c_start; if (phase3_pushback_length) { *tp = phase3_pushback[--phase3_pushback_length]; return; } tp->string = NULL; for (;;) { tp->line_number = line_number; c = phase2_getc (); switch (c) { case EOF: tp->type = token_type_eof; return; case '\n': if (last_non_comment_line > last_comment_line) savable_comment_reset (); /* Intentionally not breaking. */ case ' ': case '\t': case '\f': continue; case '+': case '-': case '*': case '/': case '^': case '%': case '#': tp->type = token_type_operator1; return; case '<': case '>': case '=': c2 = phase1_getc (); if (c2 != '=') phase1_ungetc (c2); tp->type = token_type_operator2; return; case '~': c2 = phase1_getc (); if (c2 == '=') { tp->type = token_type_operator2; return; } else phase1_ungetc (c2); continue; case '(': tp->type = token_type_lparen; return; case ')': tp->type = token_type_rparen; return; case ',': tp->type = token_type_comma; return; case ';': tp->type = token_type_other; return; /* There are three operators beginning with a dot. '.', '..' and '...'. The most useful for us is the string concatenation operator ('..'). */ case '.': c = phase1_getc (); if (c == '.') { c = phase1_getc (); if (c == '.') { tp->type = token_type_other; return; } else { phase1_ungetc (c); tp->type = token_type_doubledot; return; } } else if (c >= '0' && c <= '9') { /* It's a number. We aren't interested in the actual numeric value, so ignore the dot and let next iteration eat the number. */ phase1_ungetc (c); continue; } else { phase1_ungetc (c); tp->type = token_type_dot; return; } case '"': case '\'': c_start = c; string_start (); for (;;) { /* We need unprocessed characters from phase 1. */ c = phase1_getc (); /* We got '\', this is probably an escape sequence. */ if (c == '\\') { c = phase1_getc (); switch (c) { case 'a': string_add ('\a'); break; case 'b': string_add ('\b'); break; case 'f': string_add ('\f'); break; case 'n': string_add ('\n'); break; case 'r': string_add ('\r'); break; case 't': string_add ('\t'); break; case 'v': string_add ('\v'); break; case 'x': { int num = 0; int i = 0; for (i = 0; i < 2; i++) { c = phase1_getc (); if (c >= '0' && c <= '9') num += c - '0'; else if (c >= 'a' && c <= 'f') num += c - 'a' + 10; else if (c >= 'A' && c <= 'F') num += c - 'A' + 10; else { phase1_ungetc (c); break; } if (i == 0) num *= 16; } if (i == 2) string_add (num); } break; case 'z': /* Ignore the following whitespace. */ do { c = phase1_getc (); } while (c == ' ' || c == '\n' || c == '\t' || c == '\r' || c == '\f' || c == '\v'); phase1_ungetc (c); break; default: /* Check if it's a '\ddd' sequence. */ if (c >= '0' && c <= '9') { int num = 0; int i = 0; while (c >= '0' && c <= '9' && i < 3) { num *= 10; num += (c - '0'); c = phase1_getc (); i++; } /* The last read character is either a non-number or another number after our '\ddd' sequence. We need to ungetc it. */ phase1_ungetc (c); /* The sequence number is too big, this causes a lexical error. Ignore it. */ if (num < 256) string_add (num); } else string_add (c); } } else if (c == c_start || c == EOF || c == '\n') { /* End of string. */ string_end (); tp->string = xstrdup (string_buf); tp->comment = add_reference (savable_comment); tp->type = token_type_string; return; } else string_add (c); } break; case '[': c = phase1_getc (); /* Count the number of equal signs. */ int esigns = 0; while (c == '=') { esigns++; c = phase1_getc (); } if (c != '[') { /* We did not find what we were looking for, ungetc it. */ phase1_ungetc (c); if (esigns == 0) { /* Our current character isn't '[' and we got 0 equal signs, so the first '[' must have been a left bracket. */ tp->type = token_type_lbracket; return; } else /* Lexical error, ignore it. */ continue; } string_start (); for (;;) { c = phase1_getc (); if (c == ']') { c = phase1_getc (); /* Count the number of equal signs. */ int esigns2 = 0; while (c == '=') { esigns2++; c = phase1_getc (); } if (c == ']' && esigns == esigns2) { /* We got ']==...==]', where the number of equal signs matches the number of equal signs in the opening bracket. */ string_end (); tp->string = xstrdup (string_buf); tp->comment = add_reference (savable_comment); tp->type = token_type_string; return; } else { /* Otherwise we got either ']==' garbage or ']==...==]' with a different number of equal signs. Add ']' and equal signs to the string, and ungetc the current character, because the second ']' might be a part of another closing long bracket, e.g. '==]===]'. */ phase1_ungetc (c); string_add (']'); while (esigns2--) string_add ('='); } } else { if (c == EOF) { string_end (); tp->string = xstrdup (string_buf); tp->comment = add_reference (savable_comment); tp->type = token_type_string; return; } else string_add (c); } } break; case ']': tp->type = token_type_rbracket; return; default: if (c >= '0' && c <= '9') { while (c >= '0' && c <= '9') c = phase1_getc (); if (c == '.') { c = phase1_getc (); while (c >= '0' && c <= '9') c = phase1_getc (); } if (c == 'e' || c == 'E') { if (c == '+' || c == '-') c = phase1_getc (); while (c >= '0' && c <= '9') c = phase1_getc (); } phase1_ungetc (c); tp->type = token_type_number; return; } else if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_') { string_start (); while ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9')) { string_add (c); c = phase1_getc (); } string_end (); phase1_ungetc (c); if (strcmp (string_buf, "not") == 0) tp->type = token_type_operator1; else if (strcmp (string_buf, "and") == 0) tp->type = token_type_operator2; else if (strcmp (string_buf, "or") == 0) tp->type = token_type_operator2; else { tp->string = xstrdup (string_buf); tp->type = token_type_symbol; } return; } else tp->type = token_type_other; } } }