maybe_t<redirection_type_t> redirection_type_for_string(const wcstring &str, int *out_fd) {
auto v = read_redirection_or_fd_pipe(str.c_str());
// Redirections only, no pipes.
if (!v || v->type != TOK_REDIRECT || v->fd < 0) return none();
if (out_fd) *out_fd = v->fd;
return v->redirection_mode;
}
int fd_redirected_by_pipe(const wcstring &str) {
// Hack for the common case.
if (str == L"|") {
return STDOUT_FILENO;
}
auto v = read_redirection_or_fd_pipe(str.c_str());
return (v && v->type == TOK_PIPE) ? v->fd : -1;
}
enum token_type redirection_type_for_string(const wcstring &str, int *out_fd) {
enum token_type mode = TOK_NONE;
int fd = 0;
read_redirection_or_fd_pipe(str.c_str(), &mode, &fd);
// Redirections only, no pipes.
if (mode == TOK_PIPE || fd < 0) mode = TOK_NONE;
if (out_fd != NULL) *out_fd = fd;
return mode;
}
int fd_redirected_by_pipe(const wcstring &str) {
// Hack for the common case.
if (str == L"|") {
return STDOUT_FILENO;
}
enum token_type mode = TOK_NONE;
int fd = 0;
read_redirection_or_fd_pipe(str.c_str(), &mode, &fd);
// Pipes only.
if (mode != TOK_PIPE || fd < 0) fd = -1;
return fd;
}
void tokenizer_t::tok_next() {
if (this->last_type == TOK_ERROR) {
this->has_next = false;
return;
}
if (!this->has_next) {
// wprintf( L"EOL\n" );
this->last_type = TOK_END;
return;
}
while (1) {
if (this->buff[0] == L'\\' && this->buff[1] == L'\n') {
this->buff += 2;
this->continue_line_after_comment = true;
} else if (my_iswspace(this->buff[0])) {
this->buff++;
} else {
break;
}
}
while (*this->buff == L'#') {
if (this->show_comments) {
this->last_pos = this->buff - this->orig_buff;
this->read_comment();
if (this->buff[0] == L'\n' && this->continue_line_after_comment) this->buff++;
return;
}
while (*(this->buff) != L'\n' && *(this->buff) != L'\0') this->buff++;
if (this->buff[0] == L'\n' && this->continue_line_after_comment) this->buff++;
while (my_iswspace(*(this->buff))) this->buff++;
}
this->continue_line_after_comment = false;
this->last_pos = this->buff - this->orig_buff;
switch (*this->buff) {
case L'\0': {
this->last_type = TOK_END;
// fwprintf( stderr, L"End of string\n" );
this->has_next = false;
break;
}
case L'\r': // carriage-return
case L'\n': // newline
case L';': {
this->last_type = TOK_END;
this->buff++;
// Hack: when we get a newline, swallow as many as we can. This compresses multiple
// subsequent newlines into a single one.
if (!this->show_blank_lines) {
while (*this->buff == L'\n' || *this->buff == 13 /* CR */ || *this->buff == ' ' ||
*this->buff == '\t') {
this->buff++;
}
}
this->last_token.clear();
break;
}
case L'&': {
this->last_type = TOK_BACKGROUND;
this->buff++;
break;
}
case L'|': {
this->last_token = L"1";
this->last_type = TOK_PIPE;
this->buff++;
break;
}
case L'>':
case L'<':
case L'^': {
// There's some duplication with the code in the default case below. The key difference
// here is that we must never parse these as a string; a failed redirection is an error!
enum token_type mode = TOK_NONE;
int fd = -1;
size_t consumed = read_redirection_or_fd_pipe(this->buff, &mode, &fd);
if (consumed == 0 || fd < 0) {
TOK_CALL_ERROR(this, TOK_OTHER, REDIRECT_ERROR, this->buff);
} else {
this->buff += consumed;
this->last_type = mode;
this->last_token = to_string(fd);
}
break;
}
default: {
// Maybe a redirection like '2>&1', maybe a pipe like 2>|, maybe just a string.
const wchar_t *error_location = this->buff;
size_t consumed = 0;
enum token_type mode = TOK_NONE;
int fd = -1;
if (iswdigit(*this->buff)) {
consumed = read_redirection_or_fd_pipe(this->buff, &mode, &fd);
}
if (consumed > 0) {
// It looks like a redirection or a pipe. But we don't support piping fd 0. Note
// that fd 0 may be -1, indicating overflow; but we don't treat that as a tokenizer
// error.
if (mode == TOK_PIPE && fd == 0) {
TOK_CALL_ERROR(this, TOK_OTHER, PIPE_ERROR, error_location);
} else {
this->buff += consumed;
this->last_type = mode;
this->last_token = to_string(fd);
}
} else {
// Not a redirection or pipe, so just a string.
this->read_string();
}
break;
}
}
}
maybe_t<tok_t> tokenizer_t::tok_next() {
if (!this->has_next) {
return none();
}
// Consume non-newline whitespace. If we get an escaped newline, mark it and continue past it.
bool preceding_escaped_nl = false;
for (;;) {
if (this->buff[0] == L'\\' && this->buff[1] == L'\n') {
this->buff += 2;
this->continue_line_after_comment = true;
preceding_escaped_nl = true;
} else if (iswspace_not_nl(this->buff[0])) {
this->buff++;
} else {
break;
}
}
while (*this->buff == L'#') {
// We have a comment, walk over the comment.
const wchar_t *comment_start = this->buff;
while (this->buff[0] != L'\n' && this->buff[0] != L'\0') this->buff++;
size_t comment_len = this->buff - comment_start;
// If we are going to continue after the comment, skip any trailing newline.
if (this->buff[0] == L'\n' && this->continue_line_after_comment) this->buff++;
// Maybe return the comment.
if (this->show_comments) {
tok_t result;
result.type = TOK_COMMENT;
result.offset = comment_start - this->start;
result.length = comment_len;
result.preceding_escaped_nl = preceding_escaped_nl;
return result;
}
while (iswspace_not_nl(this->buff[0])) this->buff++;
}
// We made it past the comments and ate any trailing newlines we wanted to ignore.
this->continue_line_after_comment = false;
size_t start_pos = this->buff - this->start;
tok_t result;
result.offset = start_pos;
switch (*this->buff) {
case L'\0': {
this->has_next = false;
return none();
}
case L'\r': // carriage-return
case L'\n': // newline
case L';': {
result.type = TOK_END;
result.length = 1;
this->buff++;
// Hack: when we get a newline, swallow as many as we can. This compresses multiple
// subsequent newlines into a single one.
if (!this->show_blank_lines) {
while (*this->buff == L'\n' || *this->buff == 13 /* CR */ || *this->buff == ' ' ||
*this->buff == '\t') {
this->buff++;
}
}
break;
}
case L'&': {
if (this->buff[1] == L'&') {
result.type = TOK_ANDAND;
result.length = 2;
this->buff += 2;
} else {
result.type = TOK_BACKGROUND;
result.length = 1;
this->buff++;
}
break;
}
case L'|': {
if (this->buff[1] == L'|') {
result.type = TOK_OROR;
result.length = 2;
this->buff += 2;
} else {
result.type = TOK_PIPE;
result.redirected_fd = 1;
result.length = 1;
this->buff++;
}
break;
}
case L'>':
case L'<': {
// There's some duplication with the code in the default case below. The key difference
// here is that we must never parse these as a string; a failed redirection is an error!
auto redir_or_pipe = read_redirection_or_fd_pipe(this->buff);
if (!redir_or_pipe || redir_or_pipe->fd < 0) {
return this->call_error(TOK_INVALID_REDIRECT, this->buff, this->buff);
}
result.type = redir_or_pipe->type;
result.redirected_fd = redir_or_pipe->fd;
result.length = redir_or_pipe->consumed;
this->buff += redir_or_pipe->consumed;
break;
}
default: {
// Maybe a redirection like '2>&1', maybe a pipe like 2>|, maybe just a string.
const wchar_t *error_location = this->buff;
maybe_t<parsed_redir_or_pipe_t> redir_or_pipe;
if (iswdigit(*this->buff) || (*this->buff == L'^' && caret_redirs())) {
redir_or_pipe = read_redirection_or_fd_pipe(this->buff);
}
if (redir_or_pipe && redir_or_pipe->consumed > 0) {
// It looks like a redirection or a pipe. But we don't support piping fd 0. Note
// that fd 0 may be -1, indicating overflow; but we don't treat that as a tokenizer
// error.
if (redir_or_pipe->type == TOK_PIPE && redir_or_pipe->fd == 0) {
return this->call_error(TOK_INVALID_PIPE, error_location, error_location);
}
result.type = redir_or_pipe->type;
result.redirected_fd = redir_or_pipe->fd;
result.length = redir_or_pipe->consumed;
this->buff += redir_or_pipe->consumed;
} else {
// Not a redirection or pipe, so just a string.
result = this->read_string();
}
break;
}
}
result.preceding_escaped_nl = preceding_escaped_nl;
return result;
}
