/// Read the next token as a string.
tok_t tokenizer_t::read_string() {
tok_mode mode { tok_mode::regular_text };
std::vector<int> paran_offsets;
std::vector<int> brace_offsets;
std::vector<char> expecting;
int slice_offset = 0;
const wchar_t *const buff_start = this->buff;
bool is_first = true;
while (true) {
wchar_t c = *this->buff;
#if false
wcstring msg = L"Handling 0x%x (%lc)";
tok_mode mode_begin = mode;
#endif
if (c == L'\0') {
break;
}
// Make sure this character isn't being escaped before anything else
if ((mode & tok_mode::char_escape) == tok_mode::char_escape) {
mode &= ~(tok_mode::char_escape);
// and do nothing more
}
else if (myal(c)) {
// Early exit optimization in case the character is just a letter,
// which has no special meaning to the tokenizer, i.e. the same mode continues.
}
// Now proceed with the evaluation of the token, first checking to see if the token
// has been explicitly ignored (escaped).
else if (c == L'\\') {
mode |= tok_mode::char_escape;
}
else if (c == L'(') {
paran_offsets.push_back(this->buff - this->start);
expecting.push_back(L')');
mode |= tok_mode::subshell;
}
else if (c == L'{') {
brace_offsets.push_back(this->buff - this->start);
expecting.push_back(L'}');
mode |= tok_mode::curly_braces;
}
else if (c == L')') {
if (expecting.size() > 0 && expecting.back() == L'}') {
return this->call_error(TOK_EXPECTED_BCLOSE_FOUND_PCLOSE, this->start, this->buff);
}
switch (paran_offsets.size()) {
case 0:
return this->call_error(TOK_CLOSING_UNOPENED_SUBSHELL, this->start, this->buff);
case 1:
mode &= ~(tok_mode::subshell);
default:
paran_offsets.pop_back();
}
expecting.pop_back();
}
else if (c == L'}') {
if (expecting.size() > 0 && expecting.back() == L')') {
return this->call_error(TOK_EXPECTED_PCLOSE_FOUND_BCLOSE, this->start, this->buff);
}
switch (brace_offsets.size()) {
case 0:
return this->call_error(TOK_CLOSING_UNOPENED_BRACE, this->start, this->buff);
case 1:
mode &= ~(tok_mode::curly_braces);
default:
brace_offsets.pop_back();
}
expecting.pop_back();
}
else if (c == L'[') {
if (this->buff != buff_start) {
if ((mode & tok_mode::array_brackets) == tok_mode::array_brackets) {
// Nested brackets should not overwrite the existing slice_offset
//mqudsi: TOK_ILLEGAL_SLICE is the right error here, but the shell
//prints an error message with the caret pointing at token_start,
//not err_loc, making the TOK_ILLEGAL_SLICE message misleading.
// return call_error(TOK_ILLEGAL_SLICE, buff_start, this->buff);
return this->call_error(TOK_UNTERMINATED_SLICE, this->start, this->buff);
}
slice_offset = this->buff - this->start;
mode |= tok_mode::array_brackets;
}
else {
// This is actually allowed so the test operator `[` can be used as the head of a command
}
}
// Only exit bracket mode if we are in bracket mode.
// Reason: `]` can be a parameter, e.g. last parameter to `[` test alias.
// e.g. echo $argv[([ $x -eq $y ])] # must not end bracket mode on first bracket
else if (c == L']' && ((mode & tok_mode::array_brackets) == tok_mode::array_brackets)) {
mode &= ~(tok_mode::array_brackets);
}
else if (c == L'\'' || c == L'"') {
const wchar_t *end = quote_end(this->buff);
if (end) {
this->buff = end;
} else {
const wchar_t *error_loc = this->buff;
this->buff += wcslen(this->buff);
if ((!this->accept_unfinished)) {
return this->call_error(TOK_UNTERMINATED_QUOTE, buff_start, error_loc);
}
break;
}
} else if (mode == tok_mode::regular_text && !tok_is_string_character(c, is_first)) {
break;
}
#if false
if (mode != mode_begin) {
msg.append(L": mode 0x%x -> 0x%x\n");
} else {
msg.push_back(L'\n');
}
debug(0, msg.c_str(), c, c, int(mode_begin), int(mode));
#endif
this->buff++;
is_first = false;
}
if ((!this->accept_unfinished) && (mode != tok_mode::regular_text)) {
tok_t error;
if ((mode & tok_mode::char_escape) == tok_mode::char_escape) {
error = this->call_error(TOK_UNTERMINATED_ESCAPE, buff_start,
this->buff - 1);
}
else if ((mode & tok_mode::array_brackets) == tok_mode::array_brackets) {
error = this->call_error(TOK_UNTERMINATED_SLICE, buff_start,
this->start + slice_offset);
}
else if ((mode & tok_mode::subshell) == tok_mode::subshell) {
assert(paran_offsets.size() > 0);
size_t offset_of_open_paran = paran_offsets.back();
error = this->call_error(TOK_UNTERMINATED_SUBSHELL, buff_start,
this->start + offset_of_open_paran);
}
else if ((mode & tok_mode::curly_braces) == tok_mode::curly_braces) {
assert(brace_offsets.size() > 0);
size_t offset_of_open_brace = brace_offsets.back();
error = this->call_error(TOK_UNTERMINATED_BRACE, buff_start,
this->start + offset_of_open_brace);
}
return error;
}
tok_t result;
result.type = TOK_STRING;
result.offset = buff_start - this->start;
result.length = this->buff - buff_start;
return result;
}
/// Read the next token as a string.
void tokenizer_t::read_string() {
long len;
int do_loop = 1;
size_t paran_count = 0;
// Up to 96 open parens, before we give up on good error reporting.
const size_t paran_offsets_max = 96;
size_t paran_offsets[paran_offsets_max];
// Where the open bracket is.
size_t offset_of_bracket = 0;
const wchar_t *const start = this->buff;
bool is_first = true;
enum tok_mode_t {
mode_regular_text = 0, // regular text
mode_subshell = 1, // inside of subshell
mode_array_brackets = 2, // inside of array brackets
mode_array_brackets_and_subshell =
3 // inside of array brackets and subshell, like in '$foo[(ech'
} mode = mode_regular_text;
while (1) {
if (!myal(*this->buff)) {
if (*this->buff == L'\\') {
const wchar_t *error_location = this->buff;
this->buff++;
if (*this->buff == L'\0') {
if ((!this->accept_unfinished)) {
TOK_CALL_ERROR(this, TOK_UNTERMINATED_ESCAPE, UNTERMINATED_ESCAPE_ERROR,
error_location);
return;
}
// Since we are about to increment tok->buff, decrement it first so the
// increment doesn't go past the end of the buffer. See issue #389.
this->buff--;
do_loop = 0;
}
this->buff++;
continue;
}
switch (mode) {
case mode_regular_text: {
switch (*this->buff) {
case L'(': {
paran_count = 1;
paran_offsets[0] = this->buff - this->orig_buff;
mode = mode_subshell;
break;
}
case L'[': {
if (this->buff != start) {
mode = mode_array_brackets;
offset_of_bracket = this->buff - this->orig_buff;
}
break;
}
case L'\'':
case L'"': {
const wchar_t *end = quote_end(this->buff);
if (end) {
this->buff = end;
} else {
const wchar_t *error_loc = this->buff;
this->buff += wcslen(this->buff);
if (!this->accept_unfinished) {
TOK_CALL_ERROR(this, TOK_UNTERMINATED_QUOTE, QUOTE_ERROR,
error_loc);
return;
}
do_loop = 0;
}
break;
}
default: {
if (!tok_is_string_character(*(this->buff), is_first)) {
do_loop = 0;
}
}
}
break;
}
case mode_array_brackets_and_subshell:
case mode_subshell: {
switch (*this->buff) {
case L'\'':
case L'\"': {
const wchar_t *end = quote_end(this->buff);
if (end) {
this->buff = end;
} else {
const wchar_t *error_loc = this->buff;
this->buff += wcslen(this->buff);
if ((!this->accept_unfinished)) {
TOK_CALL_ERROR(this, TOK_UNTERMINATED_QUOTE, QUOTE_ERROR,
error_loc);
return;
}
do_loop = 0;
}
break;
}
case L'(': {
if (paran_count < paran_offsets_max) {
paran_offsets[paran_count] = this->buff - this->orig_buff;
}
paran_count++;
break;
}
case L')': {
assert(paran_count > 0);
paran_count--;
if (paran_count == 0) {
mode =
(mode == mode_array_brackets_and_subshell ? mode_array_brackets
: mode_regular_text);
}
break;
}
case L'\0': {
do_loop = 0;
break;
}
}
break;
}
case mode_array_brackets: {
switch (*this->buff) {
case L'(': {
paran_count = 1;
paran_offsets[0] = this->buff - this->orig_buff;
mode = mode_array_brackets_and_subshell;
break;
}
case L']': {
mode = mode_regular_text;
break;
}
case L'\0': {
do_loop = 0;
break;
}
}
break;
}
}
}
if (!do_loop) break;
this->buff++;
is_first = false;
}
if ((!this->accept_unfinished) && (mode != mode_regular_text)) {
switch (mode) {
case mode_subshell: {
// Determine the innermost opening paran offset by interrogating paran_offsets.
assert(paran_count > 0);
size_t offset_of_open_paran = 0;
if (paran_count <= paran_offsets_max) {
offset_of_open_paran = paran_offsets[paran_count - 1];
}
TOK_CALL_ERROR(this, TOK_UNTERMINATED_SUBSHELL, PARAN_ERROR,
this->orig_buff + offset_of_open_paran);
break;
}
case mode_array_brackets:
case mode_array_brackets_and_subshell: {
TOK_CALL_ERROR(this, TOK_UNTERMINATED_SLICE, SQUARE_BRACKET_ERROR,
this->orig_buff + offset_of_bracket);
break;
}
default: {
assert(0 && "Unexpected mode in read_string");
break;
}
}
return;
}
len = this->buff - start;
this->last_token.assign(start, len);
this->last_type = TOK_STRING;
}
/**
Read the next token as a string
*/
void tokenizer_t::read_string()
{
const wchar_t *start;
long len;
int do_loop=1;
int paran_count=0;
start = this->buff;
bool is_first = true;
enum tok_mode_t
{
mode_regular_text = 0, // regular text
mode_subshell = 1, // inside of subshell
mode_array_brackets = 2, // inside of array brackets
mode_array_brackets_and_subshell = 3 // inside of array brackets and subshell, like in '$foo[(ech'
} mode = mode_regular_text;
while (1)
{
if (!myal(*this->buff))
{
if (*this->buff == L'\\')
{
this->buff++;
if (*this->buff == L'\0')
{
if ((!this->accept_unfinished))
{
TOK_CALL_ERROR(this, TOK_UNTERMINATED_ESCAPE, QUOTE_ERROR);
return;
}
else
{
/* Since we are about to increment tok->buff, decrement it first so the increment doesn't go past the end of the buffer. https://github.com/fish-shell/fish-shell/issues/389 */
this->buff--;
do_loop = 0;
}
}
this->buff++;
continue;
}
switch (mode)
{
case mode_regular_text:
{
switch (*this->buff)
{
case L'(':
{
paran_count=1;
mode = mode_subshell;
break;
}
case L'[':
{
if (this->buff != start)
mode = mode_array_brackets;
break;
}
case L'\'':
case L'"':
{
const wchar_t *end = quote_end(this->buff);
if (end)
{
this->buff=end;
}
else
{
this->buff += wcslen(this->buff);
if (! this->accept_unfinished)
{
TOK_CALL_ERROR(this, TOK_UNTERMINATED_QUOTE, QUOTE_ERROR);
return;
}
do_loop = 0;
}
break;
}
default:
{
if (! tok_is_string_character(*(this->buff), is_first))
{
do_loop=0;
}
}
}
break;
}
case mode_array_brackets_and_subshell:
case mode_subshell:
switch (*this->buff)
{
case L'\'':
case L'\"':
{
const wchar_t *end = quote_end(this->buff);
if (end)
{
this->buff = end;
}
else
{
this->buff += wcslen(this->buff);
if ((!this->accept_unfinished))
{
TOK_CALL_ERROR(this, TOK_UNTERMINATED_QUOTE, QUOTE_ERROR);
return;
}
do_loop = 0;
}
break;
}
case L'(':
paran_count++;
break;
case L')':
paran_count--;
if (paran_count == 0)
{
mode = (mode == mode_array_brackets_and_subshell ? mode_array_brackets : mode_regular_text);
}
break;
case L'\0':
do_loop = 0;
break;
}
break;
case mode_array_brackets:
switch (*this->buff)
{
case L'(':
paran_count=1;
mode = mode_array_brackets_and_subshell;
break;
case L']':
mode = mode_regular_text;
break;
case L'\0':
do_loop = 0;
break;
}
break;
}
}
if (!do_loop)
break;
this->buff++;
is_first = false;
}
if ((!this->accept_unfinished) && (mode != mode_regular_text))
{
switch (mode)
{
case mode_subshell:
TOK_CALL_ERROR(this, TOK_UNTERMINATED_SUBSHELL, PARAN_ERROR);
break;
case mode_array_brackets:
case mode_array_brackets_and_subshell:
TOK_CALL_ERROR(this, TOK_UNTERMINATED_SUBSHELL, SQUARE_BRACKET_ERROR); // TOK_UNTERMINATED_SUBSHELL is a lie but nobody actually looks at it
break;
default:
assert(0 && "Unexpected mode in read_string");
break;
}
return;
}
len = this->buff - start;
this->last_token.assign(start, len);
this->last_type = TOK_STRING;
}
