Esempio n. 1
0
bool move_word_state_machine_t::is_path_component_character(wchar_t c) {
    // Always treat separators as first. All this does is ensure that we treat ^ as a string
    // character instead of as stderr redirection, which I hypothesize is usually what is desired.
    return tok_is_string_character(c, true) && !wcschr(L"/={,}'\"", c);
}
Esempio n. 2
0
/// Read the next token as a string.
tok_t tokenizer_t::read_string() {
    tok_mode mode { tok_mode::regular_text };
    std::vector<int> paran_offsets;
    std::vector<int> brace_offsets;
    std::vector<char> expecting;
    int slice_offset = 0;
    const wchar_t *const buff_start = this->buff;
    bool is_first = true;

    while (true) {
        wchar_t c = *this->buff;
#if false
        wcstring msg = L"Handling 0x%x (%lc)";
        tok_mode mode_begin = mode;
#endif

        if (c == L'\0') {
            break;
        }

        // Make sure this character isn't being escaped before anything else
        if ((mode & tok_mode::char_escape) == tok_mode::char_escape) {
            mode &= ~(tok_mode::char_escape);
            // and do nothing more
        }
        else if (myal(c)) {
            // Early exit optimization in case the character is just a letter,
            // which has no special meaning to the tokenizer, i.e. the same mode continues.
        }

        // Now proceed with the evaluation of the token, first checking to see if the token
        // has been explicitly ignored (escaped).
        else if (c == L'\\') {
            mode |= tok_mode::char_escape;
        }
        else if (c == L'(') {
            paran_offsets.push_back(this->buff - this->start);
            expecting.push_back(L')');
            mode |= tok_mode::subshell;
        }
        else if (c == L'{') {
            brace_offsets.push_back(this->buff - this->start);
            expecting.push_back(L'}');
            mode |= tok_mode::curly_braces;
        }
        else if (c == L')') {
            if (expecting.size() > 0 && expecting.back() == L'}') {
                return this->call_error(TOK_EXPECTED_BCLOSE_FOUND_PCLOSE, this->start, this->buff);
            }
            switch (paran_offsets.size()) {
                case 0:
                    return this->call_error(TOK_CLOSING_UNOPENED_SUBSHELL, this->start, this->buff);
                case 1:
                    mode &= ~(tok_mode::subshell);
                default:
                    paran_offsets.pop_back();
            }
            expecting.pop_back();
        }
        else if (c == L'}') {
            if (expecting.size() > 0 && expecting.back() == L')') {
                return this->call_error(TOK_EXPECTED_PCLOSE_FOUND_BCLOSE, this->start, this->buff);
            }
            switch (brace_offsets.size()) {
                case 0:
                    return this->call_error(TOK_CLOSING_UNOPENED_BRACE, this->start, this->buff);
                case 1:
                    mode &= ~(tok_mode::curly_braces);
                default:
                    brace_offsets.pop_back();
            }
            expecting.pop_back();
        }
        else if (c == L'[') {
            if (this->buff != buff_start) {
                if ((mode & tok_mode::array_brackets) == tok_mode::array_brackets) {
                    // Nested brackets should not overwrite the existing slice_offset
                    //mqudsi: TOK_ILLEGAL_SLICE is the right error here, but the shell
                    //prints an error message with the caret pointing at token_start,
                    //not err_loc, making the TOK_ILLEGAL_SLICE message misleading.
                    // return call_error(TOK_ILLEGAL_SLICE, buff_start, this->buff);
                    return this->call_error(TOK_UNTERMINATED_SLICE, this->start, this->buff);
                }
                slice_offset = this->buff - this->start;
                mode |= tok_mode::array_brackets;
            }
            else {
                // This is actually allowed so the test operator `[` can be used as the head of a command
            }
        }
        // Only exit bracket mode if we are in bracket mode.
        // Reason: `]` can be a parameter, e.g. last parameter to `[` test alias.
        // e.g. echo $argv[([ $x -eq $y ])] # must not end bracket mode on first bracket
        else if (c == L']' && ((mode & tok_mode::array_brackets) == tok_mode::array_brackets)) {
            mode &= ~(tok_mode::array_brackets);
        }
        else if (c == L'\'' || c == L'"') {
            const wchar_t *end = quote_end(this->buff);
            if (end) {
                this->buff = end;
            } else {
                const wchar_t *error_loc = this->buff;
                this->buff += wcslen(this->buff);
                if ((!this->accept_unfinished)) {
                    return this->call_error(TOK_UNTERMINATED_QUOTE, buff_start, error_loc);
                }
                break;
            }
        } else if (mode == tok_mode::regular_text && !tok_is_string_character(c, is_first)) {
            break;
        }

#if false
        if (mode != mode_begin) {
            msg.append(L": mode 0x%x -> 0x%x\n");
        } else {
            msg.push_back(L'\n');
        }
        debug(0, msg.c_str(), c, c, int(mode_begin), int(mode));
#endif

        this->buff++;
        is_first = false;
    }

    if ((!this->accept_unfinished) && (mode != tok_mode::regular_text)) {
        tok_t error;
        if ((mode & tok_mode::char_escape) == tok_mode::char_escape) {
            error = this->call_error(TOK_UNTERMINATED_ESCAPE, buff_start,
                    this->buff - 1);
        }
        else if ((mode & tok_mode::array_brackets) == tok_mode::array_brackets) {
            error = this->call_error(TOK_UNTERMINATED_SLICE, buff_start,
                    this->start + slice_offset);
        }
        else if ((mode & tok_mode::subshell) == tok_mode::subshell) {
            assert(paran_offsets.size() > 0);
            size_t offset_of_open_paran = paran_offsets.back();

            error = this->call_error(TOK_UNTERMINATED_SUBSHELL, buff_start,
                    this->start + offset_of_open_paran);
        }
        else if ((mode & tok_mode::curly_braces) == tok_mode::curly_braces) {
            assert(brace_offsets.size() > 0);
            size_t offset_of_open_brace = brace_offsets.back();

            error = this->call_error(TOK_UNTERMINATED_BRACE, buff_start,
                    this->start + offset_of_open_brace);
        }
        return error;
    }

    tok_t result;
    result.type = TOK_STRING;
    result.offset = buff_start - this->start;
    result.length = this->buff - buff_start;
    return result;
}
Esempio n. 3
0
/// Read the next token as a string.
void tokenizer_t::read_string() {
    long len;
    int do_loop = 1;
    size_t paran_count = 0;
    // Up to 96 open parens, before we give up on good error reporting.
    const size_t paran_offsets_max = 96;
    size_t paran_offsets[paran_offsets_max];
    // Where the open bracket is.
    size_t offset_of_bracket = 0;
    const wchar_t *const start = this->buff;
    bool is_first = true;

    enum tok_mode_t {
        mode_regular_text = 0,    // regular text
        mode_subshell = 1,        // inside of subshell
        mode_array_brackets = 2,  // inside of array brackets
        mode_array_brackets_and_subshell =
            3  // inside of array brackets and subshell, like in '$foo[(ech'
    } mode = mode_regular_text;

    while (1) {
        if (!myal(*this->buff)) {
            if (*this->buff == L'\\') {
                const wchar_t *error_location = this->buff;
                this->buff++;
                if (*this->buff == L'\0') {
                    if ((!this->accept_unfinished)) {
                        TOK_CALL_ERROR(this, TOK_UNTERMINATED_ESCAPE, UNTERMINATED_ESCAPE_ERROR,
                                       error_location);
                        return;
                    }
                    // Since we are about to increment tok->buff, decrement it first so the
                    // increment doesn't go past the end of the buffer. See issue #389.
                    this->buff--;
                    do_loop = 0;
                }

                this->buff++;
                continue;
            }

            switch (mode) {
                case mode_regular_text: {
                    switch (*this->buff) {
                        case L'(': {
                            paran_count = 1;
                            paran_offsets[0] = this->buff - this->orig_buff;
                            mode = mode_subshell;
                            break;
                        }
                        case L'[': {
                            if (this->buff != start) {
                                mode = mode_array_brackets;
                                offset_of_bracket = this->buff - this->orig_buff;
                            }
                            break;
                        }
                        case L'\'':
                        case L'"': {
                            const wchar_t *end = quote_end(this->buff);
                            if (end) {
                                this->buff = end;
                            } else {
                                const wchar_t *error_loc = this->buff;
                                this->buff += wcslen(this->buff);

                                if (!this->accept_unfinished) {
                                    TOK_CALL_ERROR(this, TOK_UNTERMINATED_QUOTE, QUOTE_ERROR,
                                                   error_loc);
                                    return;
                                }
                                do_loop = 0;
                            }
                            break;
                        }
                        default: {
                            if (!tok_is_string_character(*(this->buff), is_first)) {
                                do_loop = 0;
                            }
                        }
                    }
                    break;
                }

                case mode_array_brackets_and_subshell:
                case mode_subshell: {
                    switch (*this->buff) {
                        case L'\'':
                        case L'\"': {
                            const wchar_t *end = quote_end(this->buff);
                            if (end) {
                                this->buff = end;
                            } else {
                                const wchar_t *error_loc = this->buff;
                                this->buff += wcslen(this->buff);
                                if ((!this->accept_unfinished)) {
                                    TOK_CALL_ERROR(this, TOK_UNTERMINATED_QUOTE, QUOTE_ERROR,
                                                   error_loc);
                                    return;
                                }
                                do_loop = 0;
                            }
                            break;
                        }
                        case L'(': {
                            if (paran_count < paran_offsets_max) {
                                paran_offsets[paran_count] = this->buff - this->orig_buff;
                            }
                            paran_count++;
                            break;
                        }
                        case L')': {
                            assert(paran_count > 0);
                            paran_count--;
                            if (paran_count == 0) {
                                mode =
                                    (mode == mode_array_brackets_and_subshell ? mode_array_brackets
                                                                              : mode_regular_text);
                            }
                            break;
                        }
                        case L'\0': {
                            do_loop = 0;
                            break;
                        }
                    }
                    break;
                }

                case mode_array_brackets: {
                    switch (*this->buff) {
                        case L'(': {
                            paran_count = 1;
                            paran_offsets[0] = this->buff - this->orig_buff;
                            mode = mode_array_brackets_and_subshell;
                            break;
                        }
                        case L']': {
                            mode = mode_regular_text;
                            break;
                        }
                        case L'\0': {
                            do_loop = 0;
                            break;
                        }
                    }
                    break;
                }
            }
        }

        if (!do_loop) break;

        this->buff++;
        is_first = false;
    }

    if ((!this->accept_unfinished) && (mode != mode_regular_text)) {
        switch (mode) {
            case mode_subshell: {
                // Determine the innermost opening paran offset by interrogating paran_offsets.
                assert(paran_count > 0);
                size_t offset_of_open_paran = 0;
                if (paran_count <= paran_offsets_max) {
                    offset_of_open_paran = paran_offsets[paran_count - 1];
                }

                TOK_CALL_ERROR(this, TOK_UNTERMINATED_SUBSHELL, PARAN_ERROR,
                               this->orig_buff + offset_of_open_paran);
                break;
            }
            case mode_array_brackets:
            case mode_array_brackets_and_subshell: {
                TOK_CALL_ERROR(this, TOK_UNTERMINATED_SLICE, SQUARE_BRACKET_ERROR,
                               this->orig_buff + offset_of_bracket);
                break;
            }
            default: {
                assert(0 && "Unexpected mode in read_string");
                break;
            }
        }
        return;
    }

    len = this->buff - start;

    this->last_token.assign(start, len);
    this->last_type = TOK_STRING;
}
Esempio n. 4
0
/**
   Read the next token as a string
*/
void tokenizer_t::read_string()
{
    const wchar_t *start;
    long len;
    int do_loop=1;
    int paran_count=0;

    start = this->buff;
    bool is_first = true;

    enum tok_mode_t
    {
        mode_regular_text = 0, // regular text
        mode_subshell = 1, // inside of subshell
        mode_array_brackets = 2, // inside of array brackets
        mode_array_brackets_and_subshell = 3 // inside of array brackets and subshell, like in '$foo[(ech'
    } mode = mode_regular_text;

    while (1)
    {
        if (!myal(*this->buff))
        {
            if (*this->buff == L'\\')
            {
                this->buff++;
                if (*this->buff == L'\0')
                {
                    if ((!this->accept_unfinished))
                    {
                        TOK_CALL_ERROR(this, TOK_UNTERMINATED_ESCAPE, QUOTE_ERROR);
                        return;
                    }
                    else
                    {
                        /* Since we are about to increment tok->buff, decrement it first so the increment doesn't go past the end of the buffer. https://github.com/fish-shell/fish-shell/issues/389 */
                        this->buff--;
                        do_loop = 0;
                    }
                }

                this->buff++;
                continue;
            }

            switch (mode)
            {
                case mode_regular_text:
                {
                    switch (*this->buff)
                    {
                        case L'(':
                        {
                            paran_count=1;
                            mode = mode_subshell;
                            break;
                        }

                        case L'[':
                        {
                            if (this->buff != start)
                                mode = mode_array_brackets;
                            break;
                        }

                        case L'\'':
                        case L'"':
                        {

                            const wchar_t *end = quote_end(this->buff);
                            if (end)
                            {
                                this->buff=end;
                            }
                            else
                            {
                                this->buff += wcslen(this->buff);

                                if (! this->accept_unfinished)
                                {
                                    TOK_CALL_ERROR(this, TOK_UNTERMINATED_QUOTE, QUOTE_ERROR);
                                    return;
                                }
                                do_loop = 0;

                            }
                            break;
                        }

                        default:
                        {
                            if (! tok_is_string_character(*(this->buff), is_first))
                            {
                                do_loop=0;
                            }
                        }
                    }
                    break;
                }

                case mode_array_brackets_and_subshell:
                case mode_subshell:
                    switch (*this->buff)
                    {
                        case L'\'':
                        case L'\"':
                        {
                            const wchar_t *end = quote_end(this->buff);
                            if (end)
                            {
                                this->buff = end;
                            }
                            else
                            {
                                this->buff += wcslen(this->buff);
                                if ((!this->accept_unfinished))
                                {
                                    TOK_CALL_ERROR(this, TOK_UNTERMINATED_QUOTE, QUOTE_ERROR);
                                    return;
                                }
                                do_loop = 0;
                            }

                            break;
                        }

                        case L'(':
                            paran_count++;
                            break;
                        case L')':
                            paran_count--;
                            if (paran_count == 0)
                            {
                                mode = (mode == mode_array_brackets_and_subshell ? mode_array_brackets : mode_regular_text);
                            }
                            break;
                        case L'\0':
                            do_loop = 0;
                            break;
                    }
                    break;

                case mode_array_brackets:
                    switch (*this->buff)
                    {
                        case L'(':
                            paran_count=1;
                            mode = mode_array_brackets_and_subshell;
                            break;

                        case L']':
                            mode = mode_regular_text;
                            break;

                        case L'\0':
                            do_loop = 0;
                            break;
                    }
                    break;
            }
        }


        if (!do_loop)
            break;

        this->buff++;
        is_first = false;
    }

    if ((!this->accept_unfinished) && (mode != mode_regular_text))
    {
        switch (mode)
        {
            case mode_subshell:
                TOK_CALL_ERROR(this, TOK_UNTERMINATED_SUBSHELL, PARAN_ERROR);
                break;
            case mode_array_brackets:
            case mode_array_brackets_and_subshell:
                TOK_CALL_ERROR(this, TOK_UNTERMINATED_SUBSHELL, SQUARE_BRACKET_ERROR); // TOK_UNTERMINATED_SUBSHELL is a lie but nobody actually looks at it
                break;
            default:
                assert(0 && "Unexpected mode in read_string");
                break;
        }
        return;
    }


    len = this->buff - start;

    this->last_token.assign(start, len);
    this->last_type = TOK_STRING;
}