/// Read the next token as a string.
void tokenizer_t::read_string() {
long len;
int do_loop = 1;
size_t paran_count = 0;
// Up to 96 open parens, before we give up on good error reporting.
const size_t paran_offsets_max = 96;
size_t paran_offsets[paran_offsets_max];
// Where the open bracket is.
size_t offset_of_bracket = 0;
const wchar_t *const start = this->buff;
bool is_first = true;
enum tok_mode_t {
mode_regular_text = 0, // regular text
mode_subshell = 1, // inside of subshell
mode_array_brackets = 2, // inside of array brackets
mode_array_brackets_and_subshell =
3 // inside of array brackets and subshell, like in '$foo[(ech'
} mode = mode_regular_text;
while (1) {
if (!myal(*this->buff)) {
if (*this->buff == L'\\') {
const wchar_t *error_location = this->buff;
this->buff++;
if (*this->buff == L'\0') {
if ((!this->accept_unfinished)) {
TOK_CALL_ERROR(this, TOK_UNTERMINATED_ESCAPE, UNTERMINATED_ESCAPE_ERROR,
error_location);
return;
}
// Since we are about to increment tok->buff, decrement it first so the
// increment doesn't go past the end of the buffer. See issue #389.
this->buff--;
do_loop = 0;
}
this->buff++;
continue;
}
switch (mode) {
case mode_regular_text: {
switch (*this->buff) {
case L'(': {
paran_count = 1;
paran_offsets[0] = this->buff - this->orig_buff;
mode = mode_subshell;
break;
}
case L'[': {
if (this->buff != start) {
mode = mode_array_brackets;
offset_of_bracket = this->buff - this->orig_buff;
}
break;
}
case L'\'':
case L'"': {
const wchar_t *end = quote_end(this->buff);
if (end) {
this->buff = end;
} else {
const wchar_t *error_loc = this->buff;
this->buff += wcslen(this->buff);
if (!this->accept_unfinished) {
TOK_CALL_ERROR(this, TOK_UNTERMINATED_QUOTE, QUOTE_ERROR,
error_loc);
return;
}
do_loop = 0;
}
break;
}
default: {
if (!tok_is_string_character(*(this->buff), is_first)) {
do_loop = 0;
}
}
}
break;
}
case mode_array_brackets_and_subshell:
case mode_subshell: {
switch (*this->buff) {
case L'\'':
case L'\"': {
const wchar_t *end = quote_end(this->buff);
if (end) {
this->buff = end;
} else {
const wchar_t *error_loc = this->buff;
this->buff += wcslen(this->buff);
if ((!this->accept_unfinished)) {
TOK_CALL_ERROR(this, TOK_UNTERMINATED_QUOTE, QUOTE_ERROR,
error_loc);
return;
}
do_loop = 0;
}
break;
}
case L'(': {
if (paran_count < paran_offsets_max) {
paran_offsets[paran_count] = this->buff - this->orig_buff;
}
paran_count++;
break;
}
case L')': {
assert(paran_count > 0);
paran_count--;
if (paran_count == 0) {
mode =
(mode == mode_array_brackets_and_subshell ? mode_array_brackets
: mode_regular_text);
}
break;
}
case L'\0': {
do_loop = 0;
break;
}
}
break;
}
case mode_array_brackets: {
switch (*this->buff) {
case L'(': {
paran_count = 1;
paran_offsets[0] = this->buff - this->orig_buff;
mode = mode_array_brackets_and_subshell;
break;
}
case L']': {
mode = mode_regular_text;
break;
}
case L'\0': {
do_loop = 0;
break;
}
}
break;
}
}
}
if (!do_loop) break;
this->buff++;
is_first = false;
}
if ((!this->accept_unfinished) && (mode != mode_regular_text)) {
switch (mode) {
case mode_subshell: {
// Determine the innermost opening paran offset by interrogating paran_offsets.
assert(paran_count > 0);
size_t offset_of_open_paran = 0;
if (paran_count <= paran_offsets_max) {
offset_of_open_paran = paran_offsets[paran_count - 1];
}
TOK_CALL_ERROR(this, TOK_UNTERMINATED_SUBSHELL, PARAN_ERROR,
this->orig_buff + offset_of_open_paran);
break;
}
case mode_array_brackets:
case mode_array_brackets_and_subshell: {
TOK_CALL_ERROR(this, TOK_UNTERMINATED_SLICE, SQUARE_BRACKET_ERROR,
this->orig_buff + offset_of_bracket);
break;
}
default: {
assert(0 && "Unexpected mode in read_string");
break;
}
}
return;
}
len = this->buff - start;
this->last_token.assign(start, len);
this->last_type = TOK_STRING;
}
void tokenizer_t::tok_next() {
if (this->last_type == TOK_ERROR) {
this->has_next = false;
return;
}
if (!this->has_next) {
// wprintf( L"EOL\n" );
this->last_type = TOK_END;
return;
}
while (1) {
if (this->buff[0] == L'\\' && this->buff[1] == L'\n') {
this->buff += 2;
this->continue_line_after_comment = true;
} else if (my_iswspace(this->buff[0])) {
this->buff++;
} else {
break;
}
}
while (*this->buff == L'#') {
if (this->show_comments) {
this->last_pos = this->buff - this->orig_buff;
this->read_comment();
if (this->buff[0] == L'\n' && this->continue_line_after_comment) this->buff++;
return;
}
while (*(this->buff) != L'\n' && *(this->buff) != L'\0') this->buff++;
if (this->buff[0] == L'\n' && this->continue_line_after_comment) this->buff++;
while (my_iswspace(*(this->buff))) this->buff++;
}
this->continue_line_after_comment = false;
this->last_pos = this->buff - this->orig_buff;
switch (*this->buff) {
case L'\0': {
this->last_type = TOK_END;
// fwprintf( stderr, L"End of string\n" );
this->has_next = false;
break;
}
case L'\r': // carriage-return
case L'\n': // newline
case L';': {
this->last_type = TOK_END;
this->buff++;
// Hack: when we get a newline, swallow as many as we can. This compresses multiple
// subsequent newlines into a single one.
if (!this->show_blank_lines) {
while (*this->buff == L'\n' || *this->buff == 13 /* CR */ || *this->buff == ' ' ||
*this->buff == '\t') {
this->buff++;
}
}
this->last_token.clear();
break;
}
case L'&': {
this->last_type = TOK_BACKGROUND;
this->buff++;
break;
}
case L'|': {
this->last_token = L"1";
this->last_type = TOK_PIPE;
this->buff++;
break;
}
case L'>':
case L'<':
case L'^': {
// There's some duplication with the code in the default case below. The key difference
// here is that we must never parse these as a string; a failed redirection is an error!
enum token_type mode = TOK_NONE;
int fd = -1;
size_t consumed = read_redirection_or_fd_pipe(this->buff, &mode, &fd);
if (consumed == 0 || fd < 0) {
TOK_CALL_ERROR(this, TOK_OTHER, REDIRECT_ERROR, this->buff);
} else {
this->buff += consumed;
this->last_type = mode;
this->last_token = to_string(fd);
}
break;
}
default: {
// Maybe a redirection like '2>&1', maybe a pipe like 2>|, maybe just a string.
const wchar_t *error_location = this->buff;
size_t consumed = 0;
enum token_type mode = TOK_NONE;
int fd = -1;
if (iswdigit(*this->buff)) {
consumed = read_redirection_or_fd_pipe(this->buff, &mode, &fd);
}
if (consumed > 0) {
// It looks like a redirection or a pipe. But we don't support piping fd 0. Note
// that fd 0 may be -1, indicating overflow; but we don't treat that as a tokenizer
// error.
if (mode == TOK_PIPE && fd == 0) {
TOK_CALL_ERROR(this, TOK_OTHER, PIPE_ERROR, error_location);
} else {
this->buff += consumed;
this->last_type = mode;
this->last_token = to_string(fd);
}
} else {
// Not a redirection or pipe, so just a string.
this->read_string();
}
break;
}
}
}
/**
Read the next token as a string
*/
void tokenizer_t::read_string()
{
const wchar_t *start;
long len;
int do_loop=1;
int paran_count=0;
start = this->buff;
bool is_first = true;
enum tok_mode_t
{
mode_regular_text = 0, // regular text
mode_subshell = 1, // inside of subshell
mode_array_brackets = 2, // inside of array brackets
mode_array_brackets_and_subshell = 3 // inside of array brackets and subshell, like in '$foo[(ech'
} mode = mode_regular_text;
while (1)
{
if (!myal(*this->buff))
{
if (*this->buff == L'\\')
{
this->buff++;
if (*this->buff == L'\0')
{
if ((!this->accept_unfinished))
{
TOK_CALL_ERROR(this, TOK_UNTERMINATED_ESCAPE, QUOTE_ERROR);
return;
}
else
{
/* Since we are about to increment tok->buff, decrement it first so the increment doesn't go past the end of the buffer. https://github.com/fish-shell/fish-shell/issues/389 */
this->buff--;
do_loop = 0;
}
}
this->buff++;
continue;
}
switch (mode)
{
case mode_regular_text:
{
switch (*this->buff)
{
case L'(':
{
paran_count=1;
mode = mode_subshell;
break;
}
case L'[':
{
if (this->buff != start)
mode = mode_array_brackets;
break;
}
case L'\'':
case L'"':
{
const wchar_t *end = quote_end(this->buff);
if (end)
{
this->buff=end;
}
else
{
this->buff += wcslen(this->buff);
if (! this->accept_unfinished)
{
TOK_CALL_ERROR(this, TOK_UNTERMINATED_QUOTE, QUOTE_ERROR);
return;
}
do_loop = 0;
}
break;
}
default:
{
if (! tok_is_string_character(*(this->buff), is_first))
{
do_loop=0;
}
}
}
break;
}
case mode_array_brackets_and_subshell:
case mode_subshell:
switch (*this->buff)
{
case L'\'':
case L'\"':
{
const wchar_t *end = quote_end(this->buff);
if (end)
{
this->buff = end;
}
else
{
this->buff += wcslen(this->buff);
if ((!this->accept_unfinished))
{
TOK_CALL_ERROR(this, TOK_UNTERMINATED_QUOTE, QUOTE_ERROR);
return;
}
do_loop = 0;
}
break;
}
case L'(':
paran_count++;
break;
case L')':
paran_count--;
if (paran_count == 0)
{
mode = (mode == mode_array_brackets_and_subshell ? mode_array_brackets : mode_regular_text);
}
break;
case L'\0':
do_loop = 0;
break;
}
break;
case mode_array_brackets:
switch (*this->buff)
{
case L'(':
paran_count=1;
mode = mode_array_brackets_and_subshell;
break;
case L']':
mode = mode_regular_text;
break;
case L'\0':
do_loop = 0;
break;
}
break;
}
}
if (!do_loop)
break;
this->buff++;
is_first = false;
}
if ((!this->accept_unfinished) && (mode != mode_regular_text))
{
switch (mode)
{
case mode_subshell:
TOK_CALL_ERROR(this, TOK_UNTERMINATED_SUBSHELL, PARAN_ERROR);
break;
case mode_array_brackets:
case mode_array_brackets_and_subshell:
TOK_CALL_ERROR(this, TOK_UNTERMINATED_SUBSHELL, SQUARE_BRACKET_ERROR); // TOK_UNTERMINATED_SUBSHELL is a lie but nobody actually looks at it
break;
default:
assert(0 && "Unexpected mode in read_string");
break;
}
return;
}
len = this->buff - start;
this->last_token.assign(start, len);
this->last_type = TOK_STRING;
}