Пример #1
0
void
yylex(void)
{
    if (tok == LEXERR)
	return;
    do
	tok = gettok();
    while (tok != ENDINPUT && exalias());
    if (tok == NEWLIN || tok == ENDINPUT) {
	while (hdocs) {
	    struct heredocs *next = hdocs->next;
	    char *name;

	    hwbegin(0);
	    cmdpush(hdocs->type == REDIR_HEREDOC ? CS_HEREDOC : CS_HEREDOCD);
	    STOPHIST
	    name = gethere(hdocs->str, hdocs->type);
	    ALLOWHIST
	    cmdpop();
	    hwend();
	    setheredoc(hdocs->pc, REDIR_HERESTR, name);
	    zfree(hdocs, sizeof(struct heredocs));
	    hdocs = next;
	}
    }
    if (tok != NEWLIN)
	isnewlin = 0;
    else
	isnewlin = (inbufct) ? -1 : 1;
    if (tok == SEMI || tok == NEWLIN)
	tok = SEPER;
}
Пример #2
0
Файл: lex.c Проект: AMDmi3/zsh
void
zshlex(void)
{
    if (tok == LEXERR)
	return;
    do {
	if (inrepeat_)
	    ++inrepeat_;
	if (inrepeat_ == 3 && isset(SHORTLOOPS))
	    incmdpos = 1;
	tok = gettok();
    } while (tok != ENDINPUT && exalias());
    nocorrect &= 1;
    if (tok == NEWLIN || tok == ENDINPUT) {
	while (hdocs) {
	    struct heredocs *next = hdocs->next;
	    char *doc, *munged_term;

	    hwbegin(0);
	    cmdpush(hdocs->type == REDIR_HEREDOC ? CS_HEREDOC : CS_HEREDOCD);
	    munged_term = dupstring(hdocs->str);
	    STOPHIST
	    doc = gethere(&munged_term, hdocs->type);
	    ALLOWHIST
	    cmdpop();
	    hwend();
	    if (!doc) {
		zerr("here document too large");
		while (hdocs) {
		    next = hdocs->next;
		    zfree(hdocs, sizeof(struct heredocs));
		    hdocs = next;
		}
		tok = LEXERR;
		break;
	    }
	    setheredoc(hdocs->pc, REDIR_HERESTR, doc, hdocs->str,
		       munged_term);
	    zfree(hdocs, sizeof(struct heredocs));
	    hdocs = next;
	}
    }
    if (tok != NEWLIN)
	isnewlin = 0;
    else
	isnewlin = (inbufct) ? -1 : 1;
    if (tok == SEMI || (tok == NEWLIN && !(lexflags & LEXFLAGS_NEWLINE)))
	tok = SEPER;
}
Пример #3
0
int
yylex(int cf)
{
	Lex_state states[STATE_BSIZE], *statep;
	State_info state_info;
	int c, state;
	XString ws;		/* expandable output word */
	char *wp;		/* output word pointer */
	char *sp, *dp;
	int c2;


  Again:
	states[0].ls_state = -1;
	states[0].ls_info.base = (Lex_state *) 0;
	statep = &states[1];
	state_info.base = states;
	state_info.end = &states[STATE_BSIZE];

	Xinit(ws, wp, 64, ATEMP);

	backslash_skip = 0;
	ignore_backslash_newline = 0;

	if (cf&ONEWORD)
		state = SWORD;
	else if (cf&LETEXPR) {
		*wp++ = OQUOTE;	 /* enclose arguments in (double) quotes */
		state = SLETPAREN;
		statep->ls_sletparen.nparen = 0;
	} else {		/* normal lexing */
		state = (cf & HEREDELIM) ? SHEREDELIM : SBASE;
		while ((c = getsc()) == ' ' || c == '\t')
			;
		if (c == '#') {
			ignore_backslash_newline++;
			while ((c = getsc()) != '\0' && c != '\n')
				;
			ignore_backslash_newline--;
		}
		ungetsc(c);
	}
	if (source->flags & SF_ALIAS) {	/* trailing ' ' in alias definition */
		source->flags &= ~SF_ALIAS;
		/* In POSIX mode, a trailing space only counts if we are
		 * parsing a simple command
		 */
		if (!Flag(FPOSIX) || (cf & CMDWORD))
			cf |= ALIAS;
	}

	/* Initial state: one of SBASE SHEREDELIM SWORD SASPAREN */
	statep->ls_state = state;

	/* collect non-special or quoted characters to form word */
	while (!((c = getsc()) == 0
		 || ((state == SBASE || state == SHEREDELIM)
		     && ctype(c, C_LEX1))))
	{
		Xcheck(ws, wp);
		switch (state) {
		  case SBASE:
			if (Flag(FCSHHISTORY) && (source->flags & SF_TTY) && c == '!') {
				char **replace = NULL;

				c2 = getsc();
				if (c2 == '\0')
					;
				else if (c2 == ' ' || c2 == '\t')
					ungetsc(c2);
				else if (c2 == '!')
					replace = hist_get_newest(0);
				else if (isdigit(c2) || c2 == '-' ||
				    isalpha(c2)) {
					int get = !isalpha(c2);
					char match[200], *str = match;

					*str++ = c2;
					do {
						if ((c2 = getsc()) == '\0')
							break;
						if (c2 == '\t' || c2 == ' ' ||
						    c2 == '\n') {
							ungetsc(c2);
							break;
						}
						*str++ = c2;
					} while (str < &match[sizeof(match)-1]);
					*str = '\0';

					if (get) {
						int h = findhistrel(match);
						if (h >= 0)
							replace = &history[h];
					} else {
						int h = findhist(-1, 0, match, true);
						if (h >= 0)
							replace = &history[h];
					}
				}

				/*
				 * XXX ksh history buffer saves un-expanded
				 * commands. Until the history buffer code is
				 * changed to contain expanded commands, we
				 * ignore the bad commands (spinning sucks)
				 */
				if (replace && **replace == '!')
					ungetsc(c2);
				else if (replace) {
					Source *s;

					/* do not strdup replacement via alloc */
					s = pushs(SREREAD,
					      source->areap);
					s->start = s->str = *replace;
					s->next = source;
					source = s;
					continue;
				} else
					ungetsc(c2);
			}
			if (c == '[' && (cf & (VARASN|ARRAYVAR))) {
				*wp = EOS; /* temporary */
				if (is_wdvarname(Xstring(ws, wp), false))
				{
					char *p, *tmp;

					if (arraysub(&tmp)) {
						*wp++ = CHAR;
						*wp++ = c;
						for (p = tmp; *p; ) {
							Xcheck(ws, wp);
							*wp++ = CHAR;
							*wp++ = *p++;
						}
						afree(tmp, ATEMP);
						break;
					} else {
						Source *s;

						s = pushs(SREREAD,
							  source->areap);
						s->start = s->str
							= s->u.freeme = tmp;
						s->next = source;
						source = s;
					}
				}
				*wp++ = CHAR;
				*wp++ = c;
				break;
			}
			/* fall through.. */
		  Sbase1:	/* includes *(...|...) pattern (*+?@!) */
			if (c == '*' || c == '@' || c == '+' || c == '?'
			    || c == '!')
			{
				c2 = getsc();
				if (c2 == '(' /*)*/ ) {
					*wp++ = OPAT;
					*wp++ = c;
					PUSH_STATE(SPATTERN);
					break;
				}
				ungetsc(c2);
			}
			/* fall through.. */
		  Sbase2:	/* doesn't include *(...|...) pattern (*+?@!) */
			switch (c) {
			  case '\\':
				c = getsc();
				if (c) /* trailing \ is lost */
					*wp++ = QCHAR, *wp++ = c;
				break;
			  case '\'':
				*wp++ = OQUOTE;
				ignore_backslash_newline++;
				PUSH_STATE(SSQUOTE);
				break;
			  case '"':
				*wp++ = OQUOTE;
				PUSH_STATE(SDQUOTE);
				break;
			  default:
				goto Subst;
			}
			break;

		  Subst:
			switch (c) {
			  case '\\':
				c = getsc();
				switch (c) {
				  case '"': case '\\':
				  case '$': case '`':
					*wp++ = QCHAR, *wp++ = c;
					break;
				  default:
					Xcheck(ws, wp);
					if (c) { /* trailing \ is lost */
						*wp++ = CHAR, *wp++ = '\\';
						*wp++ = CHAR, *wp++ = c;
					}
					break;
				}
				break;
			  case '$':
				c = getsc();
				if (c == '(') /*)*/ {
					c = getsc();
					if (c == '(') /*)*/ {
						PUSH_STATE(SASPAREN);
						statep->ls_sasparen.nparen = 2;
						statep->ls_sasparen.start =
							Xsavepos(ws, wp);
						*wp++ = EXPRSUB;
					} else {
						ungetsc(c);
						PUSH_STATE(SCSPAREN);
						statep->ls_scsparen.nparen = 1;
						statep->ls_scsparen.csstate = 0;
						*wp++ = COMSUB;
					}
				} else if (c == '{') /*}*/ {
					*wp++ = OSUBST;
					*wp++ = '{'; /*}*/
					wp = get_brace_var(&ws, wp);
					c = getsc();
					/* allow :# and :% (ksh88 compat) */
					if (c == ':') {
						*wp++ = CHAR, *wp++ = c;
						c = getsc();
					}
					/* If this is a trim operation,
					 * treat (,|,) specially in STBRACE.
					 */
					if (c == '#' || c == '%') {
						ungetsc(c);
						PUSH_STATE(STBRACE);
					} else {
						ungetsc(c);
						PUSH_STATE(SBRACE);
					}
				} else if (ctype(c, C_ALPHA)) {
					*wp++ = OSUBST;
					*wp++ = 'X';
					do {
						Xcheck(ws, wp);
						*wp++ = c;
						c = getsc();
					} while (ctype(c, C_ALPHA|C_DIGIT));
					*wp++ = '\0';
					*wp++ = CSUBST;
					*wp++ = 'X';
					ungetsc(c);
				} else if (ctype(c, C_DIGIT|C_VAR1)) {
					Xcheck(ws, wp);
					*wp++ = OSUBST;
					*wp++ = 'X';
					*wp++ = c;
					*wp++ = '\0';
					*wp++ = CSUBST;
					*wp++ = 'X';
				} else {
					*wp++ = CHAR, *wp++ = '$';
					ungetsc(c);
				}
				break;
			  case '`':
				PUSH_STATE(SBQUOTE);
				*wp++ = COMSUB;
				/* Need to know if we are inside double quotes
				 * since sh/at&t-ksh translate the \" to " in
				 * "`..\"..`".
				 * This is not done in posix mode (section
				 * 3.2.3, Double Quotes: "The backquote shall
				 * retain its special meaning introducing the
				 * other form of command substitution (see
				 * 3.6.3). The portion of the quoted string
				 * from the initial backquote and the
				 * characters up to the next backquote that
				 * is not preceded by a backslash (having
				 * escape characters removed) defines that
				 * command whose output replaces `...` when
				 * the word is expanded."
				 * Section 3.6.3, Command Substitution:
				 * "Within the backquoted style of command
				 * substitution, backslash shall retain its
				 * literal meaning, except when followed by
				 * $ ` \.").
				 */
				statep->ls_sbquote.indquotes = 0;
				if (!Flag(FPOSIX)) {
					Lex_state *s = statep;
					Lex_state *base = state_info.base;
					while (1) {
						for (; s != base; s--) {
							if (s->ls_state == SDQUOTE) {
								statep->ls_sbquote.indquotes = 1;
								break;
							}
						}
						if (s != base)
							break;
						if (!(s = s->ls_info.base))
							break;
						base = s-- - STATE_BSIZE;
					}
				}
				break;
			  default:
				*wp++ = CHAR, *wp++ = c;
			}
			break;

		  case SSQUOTE:
			if (c == '\'') {
				POP_STATE();
				*wp++ = CQUOTE;
				ignore_backslash_newline--;
			} else
				*wp++ = QCHAR, *wp++ = c;
			break;

		  case SDQUOTE:
			if (c == '"') {
				POP_STATE();
				*wp++ = CQUOTE;
			} else
				goto Subst;
			break;

		  case SCSPAREN: /* $( .. ) */
			/* todo: deal with $(...) quoting properly
			 * kludge to partly fake quoting inside $(..): doesn't
			 * really work because nested $(..) or ${..} inside
			 * double quotes aren't dealt with.
			 */
			switch (statep->ls_scsparen.csstate) {
			  case 0: /* normal */
				switch (c) {
				  case '(':
					statep->ls_scsparen.nparen++;
					break;
				  case ')':
					statep->ls_scsparen.nparen--;
					break;
				  case '\\':
					statep->ls_scsparen.csstate = 1;
					break;
				  case '"':
					statep->ls_scsparen.csstate = 2;
					break;
				  case '\'':
					statep->ls_scsparen.csstate = 4;
					ignore_backslash_newline++;
					break;
				}
				break;

			  case 1: /* backslash in normal mode */
			  case 3: /* backslash in double quotes */
				--statep->ls_scsparen.csstate;
				break;

			  case 2: /* double quotes */
				if (c == '"')
					statep->ls_scsparen.csstate = 0;
				else if (c == '\\')
					statep->ls_scsparen.csstate = 3;
				break;

			  case 4: /* single quotes */
				if (c == '\'') {
					statep->ls_scsparen.csstate = 0;
					ignore_backslash_newline--;
				}
				break;
			}
			if (statep->ls_scsparen.nparen == 0) {
				POP_STATE();
				*wp++ = 0; /* end of COMSUB */
			} else
				*wp++ = c;
			break;

		  case SASPAREN: /* $(( .. )) */
			/* todo: deal with $((...); (...)) properly */
			/* XXX should nest using existing state machine
			 *     (embed "..", $(...), etc.) */
			if (c == '(')
				statep->ls_sasparen.nparen++;
			else if (c == ')') {
				statep->ls_sasparen.nparen--;
				if (statep->ls_sasparen.nparen == 1) {
					/*(*/
					if ((c2 = getsc()) == ')') {
						POP_STATE();
						*wp++ = 0; /* end of EXPRSUB */
						break;
					} else {
						char *s;

						ungetsc(c2);
						/* mismatched parenthesis -
						 * assume we were really
						 * parsing a $(..) expression
						 */
						s = Xrestpos(ws, wp,
						     statep->ls_sasparen.start);
						memmove(s + 1, s, wp - s);
						*s++ = COMSUB;
						*s = '('; /*)*/
						wp++;
						statep->ls_scsparen.nparen = 1;
						statep->ls_scsparen.csstate = 0;
						state = statep->ls_state
							= SCSPAREN;
					}
				}
			}
			*wp++ = c;
			break;

		  case SBRACE:
			/*{*/
			if (c == '}') {
				POP_STATE();
				*wp++ = CSUBST;
				*wp++ = /*{*/ '}';
			} else
				goto Sbase1;
			break;

		  case STBRACE:
			/* Same as SBRACE, except (,|,) treated specially */
			/*{*/
			if (c == '}') {
				POP_STATE();
				*wp++ = CSUBST;
				*wp++ = /*{*/ '}';
			} else if (c == '|') {
				*wp++ = SPAT;
			} else if (c == '(') {
				*wp++ = OPAT;
				*wp++ = ' ';	/* simile for @ */
				PUSH_STATE(SPATTERN);
			} else
				goto Sbase1;
			break;

		  case SBQUOTE:
			if (c == '`') {
				*wp++ = 0;
				POP_STATE();
			} else if (c == '\\') {
				switch (c = getsc()) {
				  case '\\':
				  case '$': case '`':
					*wp++ = c;
					break;
				  case '"':
					if (statep->ls_sbquote.indquotes) {
						*wp++ = c;
						break;
					}
					/* fall through.. */
				  default:
					if (c) { /* trailing \ is lost */
						*wp++ = '\\';
						*wp++ = c;
					}
					break;
				}
			} else
				*wp++ = c;
			break;

		  case SWORD:	/* ONEWORD */
			goto Subst;

		  case SLETPAREN:	/* LETEXPR: (( ... )) */
			/*(*/
			if (c == ')') {
				if (statep->ls_sletparen.nparen > 0)
				    --statep->ls_sletparen.nparen;
				/*(*/
				else if ((c2 = getsc()) == ')') {
					c = 0;
					*wp++ = CQUOTE;
					goto Done;
				} else
					ungetsc(c2);
			} else if (c == '(')
				/* parenthesis inside quotes and backslashes
				 * are lost, but at&t ksh doesn't count them
				 * either
				 */
				++statep->ls_sletparen.nparen;
			goto Sbase2;

		  case SHEREDELIM:	/* <<,<<- delimiter */
			/* XXX chuck this state (and the next) - use
			 * the existing states ($ and \`..` should be
			 * stripped of their specialness after the
			 * fact).
			 */
			/* here delimiters need a special case since
			 * $ and `..` are not to be treated specially
			 */
			if (c == '\\') {
				c = getsc();
				if (c) { /* trailing \ is lost */
					*wp++ = QCHAR;
					*wp++ = c;
				}
			} else if (c == '\'') {
				PUSH_STATE(SSQUOTE);
				*wp++ = OQUOTE;
				ignore_backslash_newline++;
			} else if (c == '"') {
				state = statep->ls_state = SHEREDQUOTE;
				*wp++ = OQUOTE;
			} else {
				*wp++ = CHAR;
				*wp++ = c;
			}
			break;

		  case SHEREDQUOTE:	/* " in <<,<<- delimiter */
			if (c == '"') {
				*wp++ = CQUOTE;
				state = statep->ls_state = SHEREDELIM;
			} else {
				if (c == '\\') {
					switch (c = getsc()) {
					  case '\\': case '"':
					  case '$': case '`':
						break;
					  default:
						if (c) { /* trailing \ lost */
							*wp++ = CHAR;
							*wp++ = '\\';
						}
						break;
					}
				}
				*wp++ = CHAR;
				*wp++ = c;
			}
			break;

		  case SPATTERN:	/* in *(...|...) pattern (*+?@!) */
			if ( /*(*/ c == ')') {
				*wp++ = CPAT;
				POP_STATE();
			} else if (c == '|') {
				*wp++ = SPAT;
			} else if (c == '(') {
				*wp++ = OPAT;
				*wp++ = ' ';	/* simile for @ */
				PUSH_STATE(SPATTERN);
			} else
				goto Sbase1;
			break;
		}
	}
Done:
	Xcheck(ws, wp);
	if (statep != &states[1])
		/* XXX figure out what is missing */
		yyerror("no closing quote\n");

	/* This done to avoid tests for SHEREDELIM wherever SBASE tested */
	if (state == SHEREDELIM)
		state = SBASE;

	dp = Xstring(ws, wp);
	if ((c == '<' || c == '>') && state == SBASE
	    && ((c2 = Xlength(ws, wp)) == 0
	        || (c2 == 2 && dp[0] == CHAR && digit(dp[1]))))
	{
		struct ioword *iop =
				(struct ioword *) alloc(sizeof(*iop), ATEMP);

		if (c2 == 2)
			iop->unit = dp[1] - '0';
		else
			iop->unit = c == '>'; /* 0 for <, 1 for > */

		c2 = getsc();
		/* <<, >>, <> are ok, >< is not */
		if (c == c2 || (c == '<' && c2 == '>')) {
			iop->flag = c == c2 ?
				  (c == '>' ? IOCAT : IOHERE) : IORDWR;
			if (iop->flag == IOHERE) {
				if ((c2 = getsc()) == '-')
					iop->flag |= IOSKIP;
				else
					ungetsc(c2);
			}
		} else if (c2 == '&')
			iop->flag = IODUP | (c == '<' ? IORDUP : 0);
		else {
			iop->flag = c == '>' ? IOWRITE : IOREAD;
			if (c == '>' && c2 == '|')
				iop->flag |= IOCLOB;
			else
				ungetsc(c2);
		}

		iop->name = (char *) 0;
		iop->delim = (char *) 0;
		iop->heredoc = (char *) 0;
		Xfree(ws, wp);	/* free word */
		yylval.iop = iop;
		return REDIR;
	}

	if (wp == dp && state == SBASE) {
		Xfree(ws, wp);	/* free word */
		/* no word, process LEX1 character */
		switch (c) {
		  default:
			return c;

		  case '|':
		  case '&':
		  case ';':
			if ((c2 = getsc()) == c)
				c = (c == ';') ? BREAK :
				    (c == '|') ? LOGOR :
				    (c == '&') ? LOGAND :
				    YYERRCODE;
			else if (c == '|' && c2 == '&')
				c = COPROC;
			else
				ungetsc(c2);
			return c;

		  case '\n':
			gethere();
			if (cf & CONTIN)
				goto Again;
			return c;

		  case '(':  /*)*/
			if (!Flag(FSH)) {
				if ((c2 = getsc()) == '(') /*)*/
					/* XXX need to handle ((...); (...)) */
					c = MDPAREN;
				else
					ungetsc(c2);
			}
			return c;
		  /*(*/
		  case ')':
			return c;
		}
	}

	*wp++ = EOS;		/* terminate word */
	yylval.cp = Xclose(ws, wp);
	if (state == SWORD || state == SLETPAREN)	/* ONEWORD? */
		return LWORD;
	ungetsc(c);		/* unget terminator */

	/* copy word to unprefixed string ident */
	for (sp = yylval.cp, dp = ident; dp < ident+IDENT && (c = *sp++) == CHAR; )
		*dp++ = *sp++;
	/* Make sure the ident array stays '\0' padded */
	memset(dp, 0, (ident+IDENT) - dp + 1);
	if (c != EOS)
		*ident = '\0';	/* word is not unquoted */

	if (*ident != '\0' && (cf&(KEYWORD|ALIAS))) {
		struct tbl *p;
		int h = hash(ident);

		/* { */
		if ((cf & KEYWORD) && (p = tsearch(&keywords, ident, h))
		    && (!(cf & ESACONLY) || p->val.i == ESAC || p->val.i == '}'))
		{
			afree(yylval.cp, ATEMP);
			return p->val.i;
		}
		if ((cf & ALIAS) && (p = tsearch(&aliases, ident, h))
		    && (p->flag & ISSET))
		{
			Source *s;

			for (s = source; s->type == SALIAS; s = s->next)
				if (s->u.tblp == p)
					return LWORD;
			/* push alias expansion */
			s = pushs(SALIAS, source->areap);
			s->start = s->str = p->val.s;
			s->u.tblp = p;
			s->next = source;
			source = s;
			afree(yylval.cp, ATEMP);
			goto Again;
		}
	}

	return LWORD;
}
Пример #4
0
int
yylex(int cf)
{
	Lex_state states[STATE_BSIZE], *statep, *s2, *base;
	State_info state_info;
	int c, c2, state;
	size_t cz;
	XString ws;		/* expandable output word */
	char *wp;		/* output word pointer */
	char *sp, *dp;

 Again:
	states[0].type = SINVALID;
	states[0].ls_base = NULL;
	statep = &states[1];
	state_info.base = states;
	state_info.end = &state_info.base[STATE_BSIZE];

	Xinit(ws, wp, 64, ATEMP);

	backslash_skip = 0;
	ignore_backslash_newline = 0;

	if (cf & ONEWORD)
		state = SWORD;
	else if (cf & LETEXPR) {
		/* enclose arguments in (double) quotes */
		*wp++ = OQUOTE;
		state = SLETPAREN;
		statep->nparen = 0;
	} else {
		/* normal lexing */
		state = (cf & HEREDELIM) ? SHEREDELIM : SBASE;
		while ((c = getsc()) == ' ' || c == '\t')
			;
		if (c == '#') {
			ignore_backslash_newline++;
			while ((c = getsc()) != '\0' && c != '\n')
				;
			ignore_backslash_newline--;
		}
		ungetsc(c);
	}
	if (source->flags & SF_ALIAS) {
		/* trailing ' ' in alias definition */
		source->flags &= ~SF_ALIAS;
		/* POSIX: trailing space only counts if parsing simple cmd */
		if (!Flag(FPOSIX) || (cf & CMDWORD))
			cf |= ALIAS;
	}

	/* Initial state: one of SWORD SLETPAREN SHEREDELIM SBASE */
	statep->type = state;

	/* collect non-special or quoted characters to form word */
	while (!((c = getsc()) == 0 ||
	    ((state == SBASE || state == SHEREDELIM) && ctype(c, C_LEX1)))) {
		if (state == SBASE &&
		    subshell_nesting_type == /*{*/ '}' &&
		    c == /*{*/ '}')
			/* possibly end ${ :;} */
			break;
		Xcheck(ws, wp);
		switch (state) {
		case SADELIM:
			if (c == '(')
				statep->nparen++;
			else if (c == ')')
				statep->nparen--;
			else if (statep->nparen == 0 && (c == /*{*/ '}' ||
			    c == (int)statep->ls_adelim.delimiter)) {
				*wp++ = ADELIM;
				*wp++ = c;
				if (c == /*{*/ '}' || --statep->ls_adelim.num == 0)
					POP_STATE();
				if (c == /*{*/ '}')
					POP_STATE();
				break;
			}
			/* FALLTHROUGH */
		case SBASE:
			if (c == '[' && (cf & (VARASN|ARRAYVAR))) {
				/* temporary */
				*wp = EOS;
				if (is_wdvarname(Xstring(ws, wp), false)) {
					char *p, *tmp;

					if (arraysub(&tmp)) {
						*wp++ = CHAR;
						*wp++ = c;
						for (p = tmp; *p; ) {
							Xcheck(ws, wp);
							*wp++ = CHAR;
							*wp++ = *p++;
						}
						afree(tmp, ATEMP);
						break;
					} else {
						Source *s;

						s = pushs(SREREAD,
						    source->areap);
						s->start = s->str =
						    s->u.freeme = tmp;
						s->next = source;
						source = s;
					}
				}
				*wp++ = CHAR;
				*wp++ = c;
				break;
			}
			/* FALLTHROUGH */
 Sbase1:		/* includes *(...|...) pattern (*+?@!) */
			if (c == '*' || c == '@' || c == '+' || c == '?' ||
			    c == '!') {
				c2 = getsc();
				if (c2 == '(' /*)*/ ) {
					*wp++ = OPAT;
					*wp++ = c;
					PUSH_STATE(SPATTERN);
					break;
				}
				ungetsc(c2);
			}
			/* FALLTHROUGH */
 Sbase2:		/* doesn't include *(...|...) pattern (*+?@!) */
			switch (c) {
			case '\\':
 getsc_qchar:
				if ((c = getsc())) {
					/* trailing \ is lost */
					*wp++ = QCHAR;
					*wp++ = c;
				}
				break;
			case '\'':
 open_ssquote_unless_heredoc:
				if ((cf & HEREDOC))
					goto store_char;
				*wp++ = OQUOTE;
				ignore_backslash_newline++;
				PUSH_STATE(SSQUOTE);
				break;
			case '"':
 open_sdquote:
				*wp++ = OQUOTE;
				PUSH_STATE(SDQUOTE);
				break;
			case '$':
				/*
				 * processing of dollar sign belongs into
				 * Subst, except for those which can open
				 * a string: $'…' and $"…"
				 */
 subst_dollar_ex:
				c = getsc();
				switch (c) {
				case '"':
					goto open_sdquote;
				case '\'':
					goto open_sequote;
				default:
					goto SubstS;
				}
			default:
				goto Subst;
			}
			break;

 Subst:
			switch (c) {
			case '\\':
				c = getsc();
				switch (c) {
				case '"':
					if ((cf & HEREDOC))
						goto heredocquote;
					/* FALLTHROUGH */
				case '\\':
				case '$': case '`':
 store_qchar:
					*wp++ = QCHAR;
					*wp++ = c;
					break;
				default:
 heredocquote:
					Xcheck(ws, wp);
					if (c) {
						/* trailing \ is lost */
						*wp++ = CHAR;
						*wp++ = '\\';
						*wp++ = CHAR;
						*wp++ = c;
					}
					break;
				}
				break;
			case '$':
				c = getsc();
 SubstS:
				if (c == '(') /*)*/ {
					c = getsc();
					if (c == '(') /*)*/ {
						*wp++ = EXPRSUB;
						PUSH_SRETRACE(SASPAREN);
						statep->nparen = 2;
						*retrace_info->xp++ = '(';
					} else {
						ungetsc(c);
 subst_command:
						c = COMSUB;
 subst_command2:
						sp = yyrecursive(c);
						cz = strlen(sp) + 1;
						XcheckN(ws, wp, cz);
						*wp++ = c;
						memcpy(wp, sp, cz);
						wp += cz;
					}
				} else if (c == '{') /*}*/ {
					if ((c = getsc()) == '|') {
						/*
						 * non-subenvironment
						 * value substitution
						 */
						c = VALSUB;
						goto subst_command2;
					} else if (ctype(c, C_IFSWS)) {
						/*
						 * non-subenvironment
						 * "command" substitution
						 */
						c = FUNSUB;
						goto subst_command2;
					}
					ungetsc(c);
					*wp++ = OSUBST;
					*wp++ = '{'; /*}*/
					wp = get_brace_var(&ws, wp);
					c = getsc();
					/* allow :# and :% (ksh88 compat) */
					if (c == ':') {
						*wp++ = CHAR;
						*wp++ = c;
						c = getsc();
						if (c == ':') {
							*wp++ = CHAR;
							*wp++ = '0';
							*wp++ = ADELIM;
							*wp++ = ':';
							PUSH_STATE(SBRACE);
							PUSH_STATE(SADELIM);
							statep->ls_adelim.delimiter = ':';
							statep->ls_adelim.num = 1;
							statep->nparen = 0;
							break;
						} else if (ksh_isdigit(c) ||
						    c == '('/*)*/ || c == ' ' ||
						    /*XXX what else? */
						    c == '$') {
							/* substring subst. */
							if (c != ' ') {
								*wp++ = CHAR;
								*wp++ = ' ';
							}
							ungetsc(c);
							PUSH_STATE(SBRACE);
							PUSH_STATE(SADELIM);
							statep->ls_adelim.delimiter = ':';
							statep->ls_adelim.num = 2;
							statep->nparen = 0;
							break;
						}
					} else if (c == '/') {
						*wp++ = CHAR;
						*wp++ = c;
						if ((c = getsc()) == '/') {
							*wp++ = ADELIM;
							*wp++ = c;
						} else
							ungetsc(c);
						PUSH_STATE(SBRACE);
						PUSH_STATE(SADELIM);
						statep->ls_adelim.delimiter = '/';
						statep->ls_adelim.num = 1;
						statep->nparen = 0;
						break;
					}
					/*
					 * If this is a trim operation,
					 * treat (,|,) specially in STBRACE.
					 */
					if (ctype(c, C_SUBOP2)) {
						ungetsc(c);
						if (Flag(FSH))
							PUSH_STATE(STBRACEBOURNE);
						else
							PUSH_STATE(STBRACEKORN);
					} else {
						ungetsc(c);
						if (state == SDQUOTE ||
						    state == SQBRACE)
							PUSH_STATE(SQBRACE);
						else
							PUSH_STATE(SBRACE);
					}
				} else if (ksh_isalphx(c)) {
					*wp++ = OSUBST;
					*wp++ = 'X';
					do {
						Xcheck(ws, wp);
						*wp++ = c;
						c = getsc();
					} while (ksh_isalnux(c));
					*wp++ = '\0';
					*wp++ = CSUBST;
					*wp++ = 'X';
					ungetsc(c);
				} else if (ctype(c, C_VAR1 | C_DIGIT)) {
					Xcheck(ws, wp);
					*wp++ = OSUBST;
					*wp++ = 'X';
					*wp++ = c;
					*wp++ = '\0';
					*wp++ = CSUBST;
					*wp++ = 'X';
				} else {
					*wp++ = CHAR;
					*wp++ = '$';
					ungetsc(c);
				}
				break;
			case '`':
 subst_gravis:
				PUSH_STATE(SBQUOTE);
				*wp++ = COMSUB;
				/*
				 * We need to know whether we are within double
				 * quotes, since most shells translate \" to "
				 * within "…`…\"…`…". This is not done in POSIX
				 * mode (§2.2.3 Double-Quotes: “The backquote
				 * shall retain its special meaning introducing
				 * the other form of command substitution (see
				 * Command Substitution). The portion of the
				 * quoted string from the initial backquote and
				 * the characters up to the next backquote that
				 * is not preceded by a <backslash>, having
				 * escape characters removed, defines that
				 * command whose output replaces "`...`" when
				 * the word is expanded.”; §2.6.3 Command
				 * Substitution: “Within the backquoted style
				 * of command substitution, <backslash> shall
				 * retain its literal meaning, except when
				 * followed by: '$', '`', or <backslash>. The
				 * search for the matching backquote shall be
				 * satisfied by the first unquoted non-escaped
				 * backquote; during this search, if a
				 * non-escaped backquote is encountered[…],
				 * undefined results occur.”).
				 */
				statep->ls_bool = false;
				if (Flag(FPOSIX))
					break;
				s2 = statep;
				base = state_info.base;
				while (/* CONSTCOND */ 1) {
					for (; s2 != base; s2--) {
						if (s2->type == SDQUOTE) {
							statep->ls_bool = true;
							break;
						}
					}
					if (s2 != base)
						break;
					if (!(s2 = s2->ls_base))
						break;
					base = s2-- - STATE_BSIZE;
				}
				break;
			case QCHAR:
				if (cf & LQCHAR) {
					*wp++ = QCHAR;
					*wp++ = getsc();
					break;
				}
				/* FALLTHROUGH */
			default:
 store_char:
				*wp++ = CHAR;
				*wp++ = c;
			}
			break;

		case SEQUOTE:
			if (c == '\'') {
				POP_STATE();
				*wp++ = CQUOTE;
				ignore_backslash_newline--;
			} else if (c == '\\') {
				if ((c2 = unbksl(true, s_get, s_put)) == -1)
					c2 = s_get();
				if (c2 == 0)
					statep->ls_bool = true;
				if (!statep->ls_bool) {
					char ts[4];

					if ((unsigned int)c2 < 0x100) {
						*wp++ = QCHAR;
						*wp++ = c2;
					} else {
						cz = utf_wctomb(ts, c2 - 0x100);
						ts[cz] = 0;
						cz = 0;
						do {
							*wp++ = QCHAR;
							*wp++ = ts[cz];
						} while (ts[++cz]);
					}
				}
			} else if (!statep->ls_bool) {
				*wp++ = QCHAR;
				*wp++ = c;
			}
			break;

		case SSQUOTE:
			if (c == '\'') {
				POP_STATE();
				if ((cf & HEREDOC) || state == SQBRACE)
					goto store_char;
				*wp++ = CQUOTE;
				ignore_backslash_newline--;
			} else {
				*wp++ = QCHAR;
				*wp++ = c;
			}
			break;

		case SDQUOTE:
			if (c == '"') {
				POP_STATE();
				*wp++ = CQUOTE;
			} else
				goto Subst;
			break;

		/* $(( ... )) */
		case SASPAREN:
			if (c == '(')
				statep->nparen++;
			else if (c == ')') {
				statep->nparen--;
				if (statep->nparen == 1) {
					/* end of EXPRSUB */
					POP_SRETRACE();

					if ((c2 = getsc()) == /*(*/ ')') {
						cz = strlen(sp) - 2;
						XcheckN(ws, wp, cz);
						memcpy(wp, sp + 1, cz);
						wp += cz;
						afree(sp, ATEMP);
						*wp++ = '\0';
						break;
					} else {
						Source *s;

						ungetsc(c2);
						/*
						 * mismatched parenthesis -
						 * assume we were really
						 * parsing a $(...) expression
						 */
						--wp;
						s = pushs(SREREAD,
						    source->areap);
						s->start = s->str =
						    s->u.freeme = sp;
						s->next = source;
						source = s;
						goto subst_command;
					}
				}
			}
			/* reuse existing state machine */
			goto Sbase2;

		case SQBRACE:
			if (c == '\\') {
				/*
				 * perform POSIX "quote removal" if the back-
				 * slash is "special", i.e. same cases as the
				 * {case '\\':} in Subst: plus closing brace;
				 * in mksh code "quote removal" on '\c' means
				 * write QCHAR+c, otherwise CHAR+\+CHAR+c are
				 * emitted (in heredocquote:)
				 */
				if ((c = getsc()) == '"' || c == '\\' ||
				    c == '$' || c == '`' || c == /*{*/'}')
					goto store_qchar;
				goto heredocquote;
			}
			goto common_SQBRACE;

		case SBRACE:
			if (c == '\'')
				goto open_ssquote_unless_heredoc;
			else if (c == '\\')
				goto getsc_qchar;
 common_SQBRACE:
			if (c == '"')
				goto open_sdquote;
			else if (c == '$')
				goto subst_dollar_ex;
			else if (c == '`')
				goto subst_gravis;
			else if (c != /*{*/ '}')
				goto store_char;
			POP_STATE();
			*wp++ = CSUBST;
			*wp++ = /*{*/ '}';
			break;

		/* Same as SBASE, except (,|,) treated specially */
		case STBRACEKORN:
			if (c == '|')
				*wp++ = SPAT;
			else if (c == '(') {
				*wp++ = OPAT;
				/* simile for @ */
				*wp++ = ' ';
				PUSH_STATE(SPATTERN);
			} else /* FALLTHROUGH */
		case STBRACEBOURNE:
			  if (c == /*{*/ '}') {
				POP_STATE();
				*wp++ = CSUBST;
				*wp++ = /*{*/ '}';
			} else
				goto Sbase1;
			break;

		case SBQUOTE:
			if (c == '`') {
				*wp++ = 0;
				POP_STATE();
			} else if (c == '\\') {
				switch (c = getsc()) {
				case 0:
					/* trailing \ is lost */
					break;
				case '$':
				case '`':
				case '\\':
					*wp++ = c;
					break;
				case '"':
					if (statep->ls_bool) {
						*wp++ = c;
						break;
					}
					/* FALLTHROUGH */
				default:
					*wp++ = '\\';
					*wp++ = c;
					break;
				}
			} else
				*wp++ = c;
			break;

		/* ONEWORD */
		case SWORD:
			goto Subst;

		/* LETEXPR: (( ... )) */
		case SLETPAREN:
			if (c == /*(*/ ')') {
				if (statep->nparen > 0)
					--statep->nparen;
				else if ((c2 = getsc()) == /*(*/ ')') {
					c = 0;
					*wp++ = CQUOTE;
					goto Done;
				} else {
					Source *s;

					ungetsc(c2);
					/*
					 * mismatched parenthesis -
					 * assume we were really
					 * parsing a (...) expression
					 */
					*wp = EOS;
					sp = Xstring(ws, wp);
					dp = wdstrip(sp + 1, WDS_TPUTS);
					s = pushs(SREREAD, source->areap);
					s->start = s->str = s->u.freeme = dp;
					s->next = source;
					source = s;
					return ('('/*)*/);
				}
			} else if (c == '(')
				/*
				 * parentheses inside quotes and
				 * backslashes are lost, but AT&T ksh
				 * doesn't count them either
				 */
				++statep->nparen;
			goto Sbase2;

		/* << or <<- delimiter */
		case SHEREDELIM:
			/*
			 * here delimiters need a special case since
			 * $ and `...` are not to be treated specially
			 */
			switch (c) {
			case '\\':
				if ((c = getsc())) {
					/* trailing \ is lost */
					*wp++ = QCHAR;
					*wp++ = c;
				}
				break;
			case '\'':
				goto open_ssquote_unless_heredoc;
			case '$':
				if ((c2 = getsc()) == '\'') {
 open_sequote:
					*wp++ = OQUOTE;
					ignore_backslash_newline++;
					PUSH_STATE(SEQUOTE);
					statep->ls_bool = false;
					break;
				} else if (c2 == '"') {
					/* FALLTHROUGH */
			case '"':
					PUSH_SRETRACE(SHEREDQUOTE);
					break;
				}
				ungetsc(c2);
				/* FALLTHROUGH */
			default:
				*wp++ = CHAR;
				*wp++ = c;
			}
			break;

		/* " in << or <<- delimiter */
		case SHEREDQUOTE:
			if (c != '"')
				goto Subst;
			POP_SRETRACE();
			dp = strnul(sp) - 1;
			/* remove the trailing double quote */
			*dp = '\0';
			/* store the quoted string */
			*wp++ = OQUOTE;
			XcheckN(ws, wp, (dp - sp) * 2);
			dp = sp;
			while ((c = *dp++)) {
				if (c == '\\') {
					switch ((c = *dp++)) {
					case '\\':
					case '"':
					case '$':
					case '`':
						break;
					default:
						*wp++ = CHAR;
						*wp++ = '\\';
						break;
					}
				}
				*wp++ = CHAR;
				*wp++ = c;
			}
			afree(sp, ATEMP);
			*wp++ = CQUOTE;
			state = statep->type = SHEREDELIM;
			break;

		/* in *(...|...) pattern (*+?@!) */
		case SPATTERN:
			if (c == /*(*/ ')') {
				*wp++ = CPAT;
				POP_STATE();
			} else if (c == '|') {
				*wp++ = SPAT;
			} else if (c == '(') {
				*wp++ = OPAT;
				/* simile for @ */
				*wp++ = ' ';
				PUSH_STATE(SPATTERN);
			} else
				goto Sbase1;
			break;
		}
	}
 Done:
	Xcheck(ws, wp);
	if (statep != &states[1])
		/* XXX figure out what is missing */
		yyerror("no closing quote\n");

	/* This done to avoid tests for SHEREDELIM wherever SBASE tested */
	if (state == SHEREDELIM)
		state = SBASE;

	dp = Xstring(ws, wp);
	if (state == SBASE && (
#ifndef MKSH_LEGACY_MODE
	    (c == '&' && !Flag(FSH) && !Flag(FPOSIX)) ||
#endif
	    c == '<' || c == '>')) {
		struct ioword *iop = alloc(sizeof(struct ioword), ATEMP);

		if (Xlength(ws, wp) == 0)
			iop->unit = c == '<' ? 0 : 1;
		else for (iop->unit = 0, c2 = 0; c2 < Xlength(ws, wp); c2 += 2) {
			if (dp[c2] != CHAR)
				goto no_iop;
			if (!ksh_isdigit(dp[c2 + 1]))
				goto no_iop;
			iop->unit = iop->unit * 10 + ksh_numdig(dp[c2 + 1]);
			if (iop->unit >= FDBASE)
				goto no_iop;
		}

		if (c == '&') {
			if ((c2 = getsc()) != '>') {
				ungetsc(c2);
				goto no_iop;
			}
			c = c2;
			iop->ioflag = IOBASH;
		} else
			iop->ioflag = 0;

		c2 = getsc();
		/* <<, >>, <> are ok, >< is not */
		if (c == c2 || (c == '<' && c2 == '>')) {
			iop->ioflag |= c == c2 ?
			    (c == '>' ? IOCAT : IOHERE) : IORDWR;
			if (iop->ioflag == IOHERE) {
				if ((c2 = getsc()) == '-')
					iop->ioflag |= IOSKIP;
				else if (c2 == '<')
					iop->ioflag |= IOHERESTR;
				else
					ungetsc(c2);
			}
		} else if (c2 == '&')
			iop->ioflag |= IODUP | (c == '<' ? IORDUP : 0);
		else {
			iop->ioflag |= c == '>' ? IOWRITE : IOREAD;
			if (c == '>' && c2 == '|')
				iop->ioflag |= IOCLOB;
			else
				ungetsc(c2);
		}

		iop->ioname = NULL;
		iop->delim = NULL;
		iop->heredoc = NULL;
		/* free word */
		Xfree(ws, wp);
		yylval.iop = iop;
		return (REDIR);
 no_iop:
		afree(iop, ATEMP);
	}

	if (wp == dp && state == SBASE) {
		/* free word */
		Xfree(ws, wp);
		/* no word, process LEX1 character */
		if ((c == '|') || (c == '&') || (c == ';') || (c == '('/*)*/)) {
			if ((c2 = getsc()) == c)
				c = (c == ';') ? BREAK :
				    (c == '|') ? LOGOR :
				    (c == '&') ? LOGAND :
				    /* c == '(' ) */ MDPAREN;
			else if (c == '|' && c2 == '&')
				c = COPROC;
			else if (c == ';' && c2 == '|')
				c = BRKEV;
			else if (c == ';' && c2 == '&')
				c = BRKFT;
			else
				ungetsc(c2);
#ifndef MKSH_SMALL
			if (c == BREAK) {
				if ((c2 = getsc()) == '&')
					c = BRKEV;
				else
					ungetsc(c2);
			}
#endif
		} else if (c == '\n') {
			if (cf & HEREDELIM)
				ungetsc(c);
			else {
				gethere();
				if (cf & CONTIN)
					goto Again;
			}
		}
		return (c);
	}

	/* terminate word */
	*wp++ = EOS;
	yylval.cp = Xclose(ws, wp);
	if (state == SWORD || state == SLETPAREN
	    /* XXX ONEWORD? */)
		return (LWORD);

	/* unget terminator */
	ungetsc(c);

	/*
	 * note: the alias-vs-function code below depends on several
	 * interna: starting from here, source->str is not modified;
	 * the way getsc() and ungetsc() operate; etc.
	 */

	/* copy word to unprefixed string ident */
	sp = yylval.cp;
	dp = ident;
	while ((dp - ident) < IDENT && (c = *sp++) == CHAR)
		*dp++ = *sp++;
	if (c != EOS)
		/* word is not unquoted */
		dp = ident;
	/* make sure the ident array stays NUL padded */
	memset(dp, 0, (ident + IDENT) - dp + 1);

	if (!(cf & (KEYWORD | ALIAS)))
		return (LWORD);

	if (*ident != '\0') {
		struct tbl *p;
		uint32_t h = hash(ident);

		if ((cf & KEYWORD) && (p = ktsearch(&keywords, ident, h)) &&
		    (!(cf & ESACONLY) || p->val.i == ESAC ||
		    p->val.i == /*{*/ '}')) {
			afree(yylval.cp, ATEMP);
			return (p->val.i);
		}
		if ((cf & ALIAS) && (p = ktsearch(&aliases, ident, h)) &&
		    (p->flag & ISSET)) {
			/*
			 * this still points to the same character as the
			 * ungetsc'd terminator from above
			 */
			const char *cp = source->str;

			/* prefer POSIX but not Korn functions over aliases */
			while (*cp == ' ' || *cp == '\t')
				/*
				 * this is like getsc() without skipping
				 * over Source boundaries (including not
				 * parsing ungetsc'd characters that got
				 * pushed into an SREREAD) which is what
				 * we want here anyway: find out whether
				 * the alias name is followed by a POSIX
				 * function definition
				 */
				++cp;
			/* prefer functions over aliases */
			if (cp[0] != '(' || cp[1] != ')') {
				Source *s = source;

				while (s && (s->flags & SF_HASALIAS))
					if (s->u.tblp == p)
						return (LWORD);
					else
						s = s->next;
				/* push alias expansion */
				s = pushs(SALIAS, source->areap);
				s->start = s->str = p->val.s;
				s->u.tblp = p;
				s->flags |= SF_HASALIAS;
				s->next = source;
				if (source->type == SEOF) {
					/* prevent infinite recursion at EOS */
					source->u.tblp = p;
					source->flags |= SF_HASALIAS;
				}
				source = s;
				afree(yylval.cp, ATEMP);
				goto Again;
			}
		}
	} else if (cf & ALIAS) {
		/* retain typeset et al. even when quoted */
		if (assign_command((dp = wdstrip(yylval.cp, 0))))
			strlcpy(ident, dp, sizeof(ident));
		afree(dp, ATEMP);
	}

	return (LWORD);
}