/// Read the next token as a string. void tokenizer_t::read_string() { long len; int do_loop = 1; size_t paran_count = 0; // Up to 96 open parens, before we give up on good error reporting. const size_t paran_offsets_max = 96; size_t paran_offsets[paran_offsets_max]; // Where the open bracket is. size_t offset_of_bracket = 0; const wchar_t *const start = this->buff; bool is_first = true; enum tok_mode_t { mode_regular_text = 0, // regular text mode_subshell = 1, // inside of subshell mode_array_brackets = 2, // inside of array brackets mode_array_brackets_and_subshell = 3 // inside of array brackets and subshell, like in '$foo[(ech' } mode = mode_regular_text; while (1) { if (!myal(*this->buff)) { if (*this->buff == L'\\') { const wchar_t *error_location = this->buff; this->buff++; if (*this->buff == L'\0') { if ((!this->accept_unfinished)) { TOK_CALL_ERROR(this, TOK_UNTERMINATED_ESCAPE, UNTERMINATED_ESCAPE_ERROR, error_location); return; } // Since we are about to increment tok->buff, decrement it first so the // increment doesn't go past the end of the buffer. See issue #389. this->buff--; do_loop = 0; } this->buff++; continue; } switch (mode) { case mode_regular_text: { switch (*this->buff) { case L'(': { paran_count = 1; paran_offsets[0] = this->buff - this->orig_buff; mode = mode_subshell; break; } case L'[': { if (this->buff != start) { mode = mode_array_brackets; offset_of_bracket = this->buff - this->orig_buff; } break; } case L'\'': case L'"': { const wchar_t *end = quote_end(this->buff); if (end) { this->buff = end; } else { const wchar_t *error_loc = this->buff; this->buff += wcslen(this->buff); if (!this->accept_unfinished) { TOK_CALL_ERROR(this, TOK_UNTERMINATED_QUOTE, QUOTE_ERROR, error_loc); return; } do_loop = 0; } break; } default: { if (!tok_is_string_character(*(this->buff), is_first)) { do_loop = 0; } } } break; } case mode_array_brackets_and_subshell: case mode_subshell: { switch (*this->buff) { case L'\'': case L'\"': { const wchar_t *end = quote_end(this->buff); if (end) { this->buff = end; } else { const wchar_t *error_loc = this->buff; this->buff += wcslen(this->buff); if ((!this->accept_unfinished)) { TOK_CALL_ERROR(this, TOK_UNTERMINATED_QUOTE, QUOTE_ERROR, error_loc); return; } do_loop = 0; } break; } case L'(': { if (paran_count < paran_offsets_max) { paran_offsets[paran_count] = this->buff - this->orig_buff; } paran_count++; break; } case L')': { assert(paran_count > 0); paran_count--; if (paran_count == 0) { mode = (mode == mode_array_brackets_and_subshell ? mode_array_brackets : mode_regular_text); } break; } case L'\0': { do_loop = 0; break; } } break; } case mode_array_brackets: { switch (*this->buff) { case L'(': { paran_count = 1; paran_offsets[0] = this->buff - this->orig_buff; mode = mode_array_brackets_and_subshell; break; } case L']': { mode = mode_regular_text; break; } case L'\0': { do_loop = 0; break; } } break; } } } if (!do_loop) break; this->buff++; is_first = false; } if ((!this->accept_unfinished) && (mode != mode_regular_text)) { switch (mode) { case mode_subshell: { // Determine the innermost opening paran offset by interrogating paran_offsets. assert(paran_count > 0); size_t offset_of_open_paran = 0; if (paran_count <= paran_offsets_max) { offset_of_open_paran = paran_offsets[paran_count - 1]; } TOK_CALL_ERROR(this, TOK_UNTERMINATED_SUBSHELL, PARAN_ERROR, this->orig_buff + offset_of_open_paran); break; } case mode_array_brackets: case mode_array_brackets_and_subshell: { TOK_CALL_ERROR(this, TOK_UNTERMINATED_SLICE, SQUARE_BRACKET_ERROR, this->orig_buff + offset_of_bracket); break; } default: { assert(0 && "Unexpected mode in read_string"); break; } } return; } len = this->buff - start; this->last_token.assign(start, len); this->last_type = TOK_STRING; }
void tokenizer_t::tok_next() { if (this->last_type == TOK_ERROR) { this->has_next = false; return; } if (!this->has_next) { // wprintf( L"EOL\n" ); this->last_type = TOK_END; return; } while (1) { if (this->buff[0] == L'\\' && this->buff[1] == L'\n') { this->buff += 2; this->continue_line_after_comment = true; } else if (my_iswspace(this->buff[0])) { this->buff++; } else { break; } } while (*this->buff == L'#') { if (this->show_comments) { this->last_pos = this->buff - this->orig_buff; this->read_comment(); if (this->buff[0] == L'\n' && this->continue_line_after_comment) this->buff++; return; } while (*(this->buff) != L'\n' && *(this->buff) != L'\0') this->buff++; if (this->buff[0] == L'\n' && this->continue_line_after_comment) this->buff++; while (my_iswspace(*(this->buff))) this->buff++; } this->continue_line_after_comment = false; this->last_pos = this->buff - this->orig_buff; switch (*this->buff) { case L'\0': { this->last_type = TOK_END; // fwprintf( stderr, L"End of string\n" ); this->has_next = false; break; } case L'\r': // carriage-return case L'\n': // newline case L';': { this->last_type = TOK_END; this->buff++; // Hack: when we get a newline, swallow as many as we can. This compresses multiple // subsequent newlines into a single one. if (!this->show_blank_lines) { while (*this->buff == L'\n' || *this->buff == 13 /* CR */ || *this->buff == ' ' || *this->buff == '\t') { this->buff++; } } this->last_token.clear(); break; } case L'&': { this->last_type = TOK_BACKGROUND; this->buff++; break; } case L'|': { this->last_token = L"1"; this->last_type = TOK_PIPE; this->buff++; break; } case L'>': case L'<': case L'^': { // There's some duplication with the code in the default case below. The key difference // here is that we must never parse these as a string; a failed redirection is an error! enum token_type mode = TOK_NONE; int fd = -1; size_t consumed = read_redirection_or_fd_pipe(this->buff, &mode, &fd); if (consumed == 0 || fd < 0) { TOK_CALL_ERROR(this, TOK_OTHER, REDIRECT_ERROR, this->buff); } else { this->buff += consumed; this->last_type = mode; this->last_token = to_string(fd); } break; } default: { // Maybe a redirection like '2>&1', maybe a pipe like 2>|, maybe just a string. const wchar_t *error_location = this->buff; size_t consumed = 0; enum token_type mode = TOK_NONE; int fd = -1; if (iswdigit(*this->buff)) { consumed = read_redirection_or_fd_pipe(this->buff, &mode, &fd); } if (consumed > 0) { // It looks like a redirection or a pipe. But we don't support piping fd 0. Note // that fd 0 may be -1, indicating overflow; but we don't treat that as a tokenizer // error. if (mode == TOK_PIPE && fd == 0) { TOK_CALL_ERROR(this, TOK_OTHER, PIPE_ERROR, error_location); } else { this->buff += consumed; this->last_type = mode; this->last_token = to_string(fd); } } else { // Not a redirection or pipe, so just a string. this->read_string(); } break; } } }
/** Read the next token as a string */ void tokenizer_t::read_string() { const wchar_t *start; long len; int do_loop=1; int paran_count=0; start = this->buff; bool is_first = true; enum tok_mode_t { mode_regular_text = 0, // regular text mode_subshell = 1, // inside of subshell mode_array_brackets = 2, // inside of array brackets mode_array_brackets_and_subshell = 3 // inside of array brackets and subshell, like in '$foo[(ech' } mode = mode_regular_text; while (1) { if (!myal(*this->buff)) { if (*this->buff == L'\\') { this->buff++; if (*this->buff == L'\0') { if ((!this->accept_unfinished)) { TOK_CALL_ERROR(this, TOK_UNTERMINATED_ESCAPE, QUOTE_ERROR); return; } else { /* Since we are about to increment tok->buff, decrement it first so the increment doesn't go past the end of the buffer. https://github.com/fish-shell/fish-shell/issues/389 */ this->buff--; do_loop = 0; } } this->buff++; continue; } switch (mode) { case mode_regular_text: { switch (*this->buff) { case L'(': { paran_count=1; mode = mode_subshell; break; } case L'[': { if (this->buff != start) mode = mode_array_brackets; break; } case L'\'': case L'"': { const wchar_t *end = quote_end(this->buff); if (end) { this->buff=end; } else { this->buff += wcslen(this->buff); if (! this->accept_unfinished) { TOK_CALL_ERROR(this, TOK_UNTERMINATED_QUOTE, QUOTE_ERROR); return; } do_loop = 0; } break; } default: { if (! tok_is_string_character(*(this->buff), is_first)) { do_loop=0; } } } break; } case mode_array_brackets_and_subshell: case mode_subshell: switch (*this->buff) { case L'\'': case L'\"': { const wchar_t *end = quote_end(this->buff); if (end) { this->buff = end; } else { this->buff += wcslen(this->buff); if ((!this->accept_unfinished)) { TOK_CALL_ERROR(this, TOK_UNTERMINATED_QUOTE, QUOTE_ERROR); return; } do_loop = 0; } break; } case L'(': paran_count++; break; case L')': paran_count--; if (paran_count == 0) { mode = (mode == mode_array_brackets_and_subshell ? mode_array_brackets : mode_regular_text); } break; case L'\0': do_loop = 0; break; } break; case mode_array_brackets: switch (*this->buff) { case L'(': paran_count=1; mode = mode_array_brackets_and_subshell; break; case L']': mode = mode_regular_text; break; case L'\0': do_loop = 0; break; } break; } } if (!do_loop) break; this->buff++; is_first = false; } if ((!this->accept_unfinished) && (mode != mode_regular_text)) { switch (mode) { case mode_subshell: TOK_CALL_ERROR(this, TOK_UNTERMINATED_SUBSHELL, PARAN_ERROR); break; case mode_array_brackets: case mode_array_brackets_and_subshell: TOK_CALL_ERROR(this, TOK_UNTERMINATED_SUBSHELL, SQUARE_BRACKET_ERROR); // TOK_UNTERMINATED_SUBSHELL is a lie but nobody actually looks at it break; default: assert(0 && "Unexpected mode in read_string"); break; } return; } len = this->buff - start; this->last_token.assign(start, len); this->last_type = TOK_STRING; }