// This function does I/O
static void tokenize( const wchar_t * const buff, std::vector<int> &color, const int pos, wcstring_list_t *error, const wcstring &working_directory, const env_vars_snapshot_t &vars) {
ASSERT_IS_BACKGROUND_THREAD();
wcstring cmd;
int had_cmd=0;
wcstring last_cmd;
int len;
int accept_switches = 1;
int use_function = 1;
int use_command = 1;
int use_builtin = 1;
CHECK( buff, );
len = wcslen(buff);
if( !len )
return;
std::fill(color.begin(), color.end(), -1);
tokenizer tok;
for( tok_init( &tok, buff, TOK_SHOW_COMMENTS | TOK_SQUASH_ERRORS );
tok_has_next( &tok );
tok_next( &tok ) )
{
int last_type = tok_last_type( &tok );
switch( last_type )
{
case TOK_STRING:
{
if( had_cmd )
{
/*Parameter */
wchar_t *param = tok_last( &tok );
if( param[0] == L'-' )
{
if (wcscmp( param, L"--" ) == 0 )
{
accept_switches = 0;
color.at(tok_get_pos( &tok )) = HIGHLIGHT_PARAM;
}
else if( accept_switches )
{
if( complete_is_valid_option( last_cmd.c_str(), param, error, false /* no autoload */ ) )
color.at(tok_get_pos( &tok )) = HIGHLIGHT_PARAM;
else
color.at(tok_get_pos( &tok )) = HIGHLIGHT_ERROR;
}
else
{
color.at(tok_get_pos( &tok )) = HIGHLIGHT_PARAM;
}
}
else
{
color.at(tok_get_pos( &tok )) = HIGHLIGHT_PARAM;
}
if( cmd == L"cd" )
{
wcstring dir = tok_last( &tok );
if (expand_one(dir, EXPAND_SKIP_CMDSUBST))
{
int is_help = string_prefixes_string(dir, L"--help") || string_prefixes_string(dir, L"-h");
if( !is_help && ! is_potential_cd_path(dir, working_directory, PATH_EXPAND_TILDE, NULL))
{
color.at(tok_get_pos( &tok )) = HIGHLIGHT_ERROR;
}
}
}
/* Highlight the parameter. highlight_param wants to write one more color than we have characters (hysterical raisins) so allocate one more in the vector. But don't copy it back. */
const wcstring param_str = param;
int tok_pos = tok_get_pos(&tok);
std::vector<int>::const_iterator where = color.begin() + tok_pos;
std::vector<int> subcolors(where, where + param_str.size());
subcolors.push_back(-1);
highlight_param(param_str, subcolors, pos-tok_pos, error);
/* Copy the subcolors back into our colors array */
std::copy(subcolors.begin(), subcolors.begin() + param_str.size(), color.begin() + tok_pos);
}
else
{
/*
Command. First check that the command actually exists.
*/
cmd = tok_last( &tok );
bool expanded = expand_one(cmd, EXPAND_SKIP_CMDSUBST | EXPAND_SKIP_VARIABLES);
if (! expanded || has_expand_reserved(cmd.c_str()))
{
color.at(tok_get_pos( &tok )) = HIGHLIGHT_ERROR;
}
else
{
bool is_cmd = false;
int is_subcommand = 0;
int mark = tok_get_pos( &tok );
color.at(tok_get_pos( &tok )) = HIGHLIGHT_COMMAND;
if( parser_keywords_is_subcommand( cmd ) )
{
int sw;
if( cmd == L"builtin")
{
use_function = 0;
use_command = 0;
use_builtin = 1;
}
else if( cmd == L"command")
{
use_command = 1;
use_function = 0;
use_builtin = 0;
}
tok_next( &tok );
sw = parser_keywords_is_switch( tok_last( &tok ) );
if( !parser_keywords_is_block( cmd ) &&
sw == ARG_SWITCH )
{
/*
The 'builtin' and 'command' builtins
are normally followed by another
command, but if they are invoked
with a switch, they aren't.
*/
use_command = 1;
use_function = 1;
use_builtin = 2;
}
else
{
if( sw == ARG_SKIP )
{
color.at(tok_get_pos( &tok )) = HIGHLIGHT_PARAM;
mark = tok_get_pos( &tok );
}
is_subcommand = 1;
}
tok_set_pos( &tok, mark );
}
if( !is_subcommand )
{
/*
OK, this is a command, it has been
successfully expanded and everything
looks ok. Lets check if the command
exists.
*/
/*
First check if it is a builtin or
function, since we don't have to stat
any files for that
*/
if (! is_cmd && use_builtin )
is_cmd = builtin_exists( cmd );
if (! is_cmd && use_function )
is_cmd = function_exists_no_autoload( cmd, vars );
/*
Moving on to expensive tests
*/
/*
Check if this is a regular command
*/
if (! is_cmd && use_command )
{
is_cmd = path_get_path( cmd, NULL, vars );
}
/* Maybe it is a path for a implicit cd command. */
if (! is_cmd)
{
if (use_builtin || (use_function && function_exists_no_autoload( L"cd", vars)))
is_cmd = path_can_be_implicit_cd(cmd, NULL, working_directory.c_str(), vars);
}
if( is_cmd )
{
color.at(tok_get_pos( &tok )) = HIGHLIGHT_COMMAND;
}
else
{
if( error ) {
error->push_back(format_string(L"Unknown command \'%ls\'", cmd.c_str()));
}
color.at(tok_get_pos( &tok )) = (HIGHLIGHT_ERROR);
}
had_cmd = 1;
}
if( had_cmd )
{
last_cmd = tok_last( &tok );
}
}
}
break;
}
case TOK_REDIRECT_NOCLOB:
case TOK_REDIRECT_OUT:
case TOK_REDIRECT_IN:
case TOK_REDIRECT_APPEND:
case TOK_REDIRECT_FD:
{
if( !had_cmd )
{
color.at(tok_get_pos( &tok )) = HIGHLIGHT_ERROR;
if( error )
error->push_back(L"Redirection without a command");
break;
}
wcstring target_str;
const wchar_t *target=NULL;
color.at(tok_get_pos( &tok )) = HIGHLIGHT_REDIRECTION;
tok_next( &tok );
/*
Check that we are redirecting into a file
*/
switch( tok_last_type( &tok ) )
{
case TOK_STRING:
{
target_str = tok_last( &tok );
if (expand_one(target_str, EXPAND_SKIP_CMDSUBST)) {
target = target_str.c_str();
}
/*
Redirect filename may contain a cmdsubst.
If so, it will be ignored/not flagged.
*/
}
break;
default:
{
size_t pos = tok_get_pos(&tok);
if (pos < color.size()) {
color.at(pos) = HIGHLIGHT_ERROR;
}
if( error )
error->push_back(L"Invalid redirection");
}
}
if( target != 0 )
{
wcstring dir = target;
size_t slash_idx = dir.find_last_of(L'/');
struct stat buff;
/*
If file is in directory other than '.', check
that the directory exists.
*/
if( slash_idx != wcstring::npos )
{
dir.resize(slash_idx);
if( wstat( dir, &buff ) == -1 )
{
color.at(tok_get_pos( &tok )) = HIGHLIGHT_ERROR;
if( error )
error->push_back(format_string(L"Directory \'%ls\' does not exist", dir.c_str()));
}
}
/*
If the file is read from or appended to, check
if it exists.
*/
if( last_type == TOK_REDIRECT_IN ||
last_type == TOK_REDIRECT_APPEND )
{
if( wstat( target, &buff ) == -1 )
{
color.at(tok_get_pos( &tok )) = HIGHLIGHT_ERROR;
if( error )
error->push_back(format_string(L"File \'%ls\' does not exist", target));
}
}
if( last_type == TOK_REDIRECT_NOCLOB )
{
if( wstat( target, &buff ) != -1 )
{
color.at(tok_get_pos( &tok )) = HIGHLIGHT_ERROR;
if( error )
error->push_back(format_string(L"File \'%ls\' exists", target));
}
}
}
break;
}
case TOK_PIPE:
case TOK_BACKGROUND:
{
if( had_cmd )
{
color.at(tok_get_pos( &tok )) = HIGHLIGHT_END;
had_cmd = 0;
use_command = 1;
use_function = 1;
use_builtin = 1;
accept_switches = 1;
}
else
{
color.at(tok_get_pos( &tok )) = HIGHLIGHT_ERROR;
if( error )
error->push_back(L"No job to put in background" );
}
break;
}
case TOK_END:
{
color.at(tok_get_pos( &tok )) = HIGHLIGHT_END;
had_cmd = 0;
use_command = 1;
use_function = 1;
use_builtin = 1;
accept_switches = 1;
break;
}
case TOK_COMMENT:
{
color.at(tok_get_pos( &tok )) = HIGHLIGHT_COMMENT;
break;
}
case TOK_ERROR:
default:
{
/*
If the tokenizer reports an error, highlight it as such.
*/
if( error )
error->push_back(tok_last( &tok));
color.at(tok_get_pos( &tok )) = HIGHLIGHT_ERROR;
break;
}
}
}
tok_destroy( &tok );
}
parser_test_error_bits_t parse_util_detect_errors(const wcstring &buff_src, parse_error_list_t *out_errors, bool allow_incomplete)
{
parse_node_tree_t node_tree;
parse_error_list_t parse_errors;
parser_test_error_bits_t res = 0;
// Whether we encountered a parse error
bool errored = false;
// Whether we encountered an unclosed block
// We detect this via an 'end_command' block without source
bool has_unclosed_block = false;
// Whether there's an unclosed quote, and therefore unfinished
// This is only set if allow_incomplete is set
bool has_unclosed_quote = false;
// Parse the input string into a parse tree
// Some errors are detected here
bool parsed = parse_tree_from_string(buff_src, allow_incomplete ? parse_flag_leave_unterminated : parse_flag_none, &node_tree, &parse_errors);
if (allow_incomplete)
{
for (size_t i=0; i < parse_errors.size(); i++)
{
if (parse_errors.at(i).code == parse_error_tokenizer_unterminated_quote)
{
// Remove this error, since we don't consider it a real error
has_unclosed_quote = true;
parse_errors.erase(parse_errors.begin() + i);
i--;
}
}
}
// #1238: If the only error was unterminated quote, then consider this to have parsed successfully. A better fix would be to have parse_tree_from_string return this information directly (but it would be a shame to munge up its nice bool return).
if (parse_errors.empty() && has_unclosed_quote)
{
parsed = true;
}
if (! parsed)
{
errored = true;
}
// has_unclosed_quote may only be set if allow_incomplete is true
assert(! has_unclosed_quote || allow_incomplete);
// Expand all commands
// Verify 'or' and 'and' not used inside pipelines
// Verify pipes via parser_is_pipe_forbidden
// Verify return only within a function
// Verify no variable expansions
if (! errored)
{
const size_t node_tree_size = node_tree.size();
for (size_t i=0; i < node_tree_size; i++)
{
const parse_node_t &node = node_tree.at(i);
if (node.type == symbol_end_command && ! node.has_source())
{
// an 'end' without source is an unclosed block
has_unclosed_block = true;
}
else if (node.type == symbol_boolean_statement)
{
// 'or' and 'and' can be in a pipeline, as long as they're first
parse_bool_statement_type_t type = parse_node_tree_t::statement_boolean_type(node);
if ((type == parse_bool_and || type == parse_bool_or) && node_tree.statement_is_in_pipeline(node, false /* don't count first */))
{
errored = append_syntax_error(&parse_errors, node, EXEC_ERR_MSG, (type == parse_bool_and) ? L"and" : L"or");
}
}
else if (node.type == symbol_argument)
{
const wcstring arg_src = node.get_source(buff_src);
res |= parse_util_detect_errors_in_argument(node, arg_src, &parse_errors);
}
else if (node.type == symbol_job)
{
if (node_tree.job_should_be_backgrounded(node))
{
/* Disallow background in the following cases:
foo & ; and bar
foo & ; or bar
if foo & ; end
while foo & ; end
*/
const parse_node_t *job_parent = node_tree.get_parent(node);
assert(job_parent != NULL);
switch (job_parent->type)
{
case symbol_if_clause:
case symbol_while_header:
{
assert(node_tree.get_child(*job_parent, 1) == &node);
errored = append_syntax_error(&parse_errors, node, BACKGROUND_IN_CONDITIONAL_ERROR_MSG);
break;
}
case symbol_job_list:
{
// This isn't very complete, e.g. we don't catch 'foo & ; not and bar'
assert(node_tree.get_child(*job_parent, 0) == &node);
const parse_node_t *next_job_list = node_tree.get_child(*job_parent, 1, symbol_job_list);
assert(next_job_list != NULL);
const parse_node_t *next_job = node_tree.next_node_in_node_list(*next_job_list, symbol_job, NULL);
if (next_job != NULL)
{
const parse_node_t *next_statement = node_tree.get_child(*next_job, 0, symbol_statement);
if (next_statement != NULL)
{
const parse_node_t *spec_statement = node_tree.get_child(*next_statement, 0);
if (spec_statement && spec_statement->type == symbol_boolean_statement)
{
switch (parse_node_tree_t::statement_boolean_type(*spec_statement))
{
// These are not allowed
case parse_bool_and:
errored = append_syntax_error(&parse_errors, *spec_statement, BOOL_AFTER_BACKGROUND_ERROR_MSG, L"and");
break;
case parse_bool_or:
errored = append_syntax_error(&parse_errors, *spec_statement, BOOL_AFTER_BACKGROUND_ERROR_MSG, L"or");
break;
case parse_bool_not:
// This one is OK
break;
}
}
}
}
break;
}
default:
break;
}
}
}
else if (node.type == symbol_plain_statement)
{
// In a few places below, we want to know if we are in a pipeline
const bool is_in_pipeline = node_tree.statement_is_in_pipeline(node, true /* count first */);
// We need to know the decoration
const enum parse_statement_decoration_t decoration = node_tree.decoration_for_plain_statement(node);
// Check that we don't try to pipe through exec
if (is_in_pipeline && decoration == parse_statement_decoration_exec)
{
errored = append_syntax_error(&parse_errors, node, EXEC_ERR_MSG, L"exec");
}
wcstring command;
if (node_tree.command_for_plain_statement(node, buff_src, &command))
{
// Check that we can expand the command
if (! expand_one(command, EXPAND_SKIP_CMDSUBST | EXPAND_SKIP_VARIABLES | EXPAND_SKIP_JOBS, NULL))
{
// TODO: leverage the resulting errors
errored = append_syntax_error(&parse_errors, node, ILLEGAL_CMD_ERR_MSG, command.c_str());
}
// Check that pipes are sound
if (! errored && parser_is_pipe_forbidden(command) && is_in_pipeline)
{
errored = append_syntax_error(&parse_errors, node, EXEC_ERR_MSG, command.c_str());
}
// Check that we don't return from outside a function
// But we allow it if it's 'return --help'
if (! errored && command == L"return")
{
const parse_node_t *ancestor = &node;
bool found_function = false;
while (ancestor != NULL)
{
const parse_node_t *possible_function_header = node_tree.header_node_for_block_statement(*ancestor);
if (possible_function_header != NULL && possible_function_header->type == symbol_function_header)
{
found_function = true;
break;
}
ancestor = node_tree.get_parent(*ancestor);
}
if (! found_function && ! first_argument_is_help(node_tree, node, buff_src))
{
errored = append_syntax_error(&parse_errors, node, INVALID_RETURN_ERR_MSG);
}
}
// Check that we don't break or continue from outside a loop
if (! errored && (command == L"break" || command == L"continue"))
{
// Walk up until we hit a 'for' or 'while' loop. If we hit a function first, stop the search; we can't break an outer loop from inside a function.
// This is a little funny because we can't tell if it's a 'for' or 'while' loop from the ancestor alone; we need the header. That is, we hit a block_statement, and have to check its header.
bool found_loop = false, end_search = false;
const parse_node_t *ancestor = &node;
while (ancestor != NULL && ! end_search)
{
const parse_node_t *loop_or_function_header = node_tree.header_node_for_block_statement(*ancestor);
if (loop_or_function_header != NULL)
{
switch (loop_or_function_header->type)
{
case symbol_while_header:
case symbol_for_header:
// this is a loop header, so we can break or continue
found_loop = true;
end_search = true;
break;
case symbol_function_header:
// this is a function header, so we cannot break or continue. We stop our search here.
found_loop = false;
end_search = true;
break;
default:
// most likely begin / end style block, which makes no difference
break;
}
}
ancestor = node_tree.get_parent(*ancestor);
}
if (! found_loop && ! first_argument_is_help(node_tree, node, buff_src))
{
errored = append_syntax_error(&parse_errors, node, (command == L"break" ? INVALID_BREAK_ERR_MSG : INVALID_CONTINUE_ERR_MSG));
}
}
// Check that we don't do an invalid builtin (#1252)
if (! errored && decoration == parse_statement_decoration_builtin && ! builtin_exists(command))
{
errored = append_syntax_error(&parse_errors, node, UNKNOWN_BUILTIN_ERR_MSG, command.c_str());
}
}
}
}
}
if (errored)
res |= PARSER_TEST_ERROR;
if (has_unclosed_block || has_unclosed_quote)
res |= PARSER_TEST_INCOMPLETE;
if (out_errors)
{
out_errors->swap(parse_errors);
}
return res;
}
bool autosuggest_validate_from_history(const history_item_t &item, file_detection_context_t &detector, const wcstring &working_directory, const env_vars_snapshot_t &vars) {
ASSERT_IS_BACKGROUND_THREAD();
bool handled = false, suggestionOK = false;
/* Parse the string */
wcstring parsed_command;
wcstring_list_t parsed_arguments;
int parsed_last_arg_pos = -1;
if (! autosuggest_parse_command(item.str(), &parsed_command, &parsed_arguments, &parsed_last_arg_pos))
return false;
if (parsed_command == L"cd" && ! parsed_arguments.empty()) {
/* We can possibly handle this specially */
wcstring dir = parsed_arguments.back();
if (expand_one(dir, EXPAND_SKIP_CMDSUBST))
{
handled = true;
bool is_help = string_prefixes_string(dir, L"--help") || string_prefixes_string(dir, L"-h");
if (is_help) {
suggestionOK = false;
} else {
wcstring path;
bool can_cd = path_get_cdpath(dir, &path, working_directory.c_str(), vars);
if (! can_cd) {
suggestionOK = false;
} else if (paths_are_same_file(working_directory, path)) {
/* Don't suggest the working directory as the path! */
suggestionOK = false;
} else {
suggestionOK = true;
}
}
}
}
/* If not handled specially, handle it here */
if (! handled) {
bool cmd_ok = false;
if (path_get_path(parsed_command, NULL))
{
cmd_ok = true;
}
else if (builtin_exists(parsed_command) || function_exists_no_autoload(parsed_command, vars))
{
cmd_ok = true;
}
if (cmd_ok) {
const path_list_t &paths = item.get_required_paths();
if (paths.empty()) {
suggestionOK= true;
}
else {
detector.potential_paths = paths;
suggestionOK = detector.paths_are_valid(paths);
}
}
}
return suggestionOK;
}
