int GetNumberToken(
char cCharacter,
char *pcToken)
/* DEC_NUMBER_TOKEN :== [digit] */
/* HEX_NUMBER_TOKEN :== '0' 'x' [hexdigit] */
/* BIN_NUMBER_TOKEN :== '0' 'b' [bindigit] */
/* FLOAT_TOKEN :== <digit> '.' [digit] */
{
char *pDigitClass;
int iResult = DEC_NUMBER_TOKEN;
*(pcToken++) = cCharacter;
cCharacter = GetChar();
if (cCharacter == 'x')
{
*(pcToken++) = cCharacter;
pDigitClass = pHexDigits;
cCharacter = GetChar();
iResult = HEX_NUMBER_TOKEN;
}
else if (cCharacter == 'b')
{
*(pcToken++) = cCharacter;
pDigitClass = pBinDigits;
cCharacter = GetChar();
iResult = BIN_NUMBER_TOKEN;
}
else
pDigitClass = pDecDigits;
while (chrpos(cCharacter, pDigitClass) != NULL)
{
*(pcToken++) = cCharacter;
cCharacter = GetChar();
}
if ((iResult == DEC_NUMBER_TOKEN) && (cCharacter == '.'))
{
*(pcToken++) = cCharacter;
cCharacter = GetChar();
iResult = FLOAT_TOKEN;
while (chrpos(cCharacter, pDigitClass) != NULL)
{
*(pcToken++) = cCharacter;
cCharacter = GetChar();
}
UngetChar();
}
*pcToken = 0;
UngetChar();
return iResult;
}
int GetOperatorToken(
char cCharacter,
char *pcToken)
/* OPERATOR_TOKEN :== '=' */
/* :== '=' '=' */
/* :== '<' */
/* :== '>' */
/* :== '<' '=' */
/* :== '>' '=' */
/* :== '<' '>' */
/* :== '+' */
/* :== '-' */
/* :== '*' */
/* :== '/' */
/* :== '<' '<' */
/* :== '>' '>' */
/* :== '&' */
/* :== '|' */
/* :== '[' ']' */
/* The following two tokens are read as OPERATOR_TOKEN, but are translated */
/* to LEFT_ARROW_TOKEN or RIGHT_ARROW_TOKEN before leaving the scanner. */
/* :== '-' '>' */
/* :== '<' '-' */
{
char *pcCompleteToken;
pcCompleteToken = pcToken; /* store pointer to first token char */
*(pcToken++) = cCharacter; /* assign first character to token */
if (chrpos(cCharacter, "-=<>[/.") != NULL) /* if second character possible */
{
cCharacter = GetChar(); /* read second character */
if (chrpos(cCharacter, "-=<>]/.") != NULL) /* if valid second character */
{
*pcToken = cCharacter; /* add character */
pcToken[1] = 0; /* and make it zero terminated */
/* now a 2 byte operator is read. Check if this is a valid two byte */
/* operator. If not, put back the second character in the input stream. */
if (strpos(pcCompleteToken, "== <= >= <> << >> [] -> <- // ..") == NULL)
{
UngetChar();
}
else
{
pcToken++;
}
}
else
{
UngetChar(); /* else put second character back */
}
}
*pcToken = 0; /* terminate token */
return OPERATOR_TOKEN;
}
int GetStringToken(
char cCharacter,
char *pcToken)
/* STRING_TOKEN :== dquote <any> dquote */
{
int iResult;
int iCurrentLine;
iCurrentLine = CurrentLine();
cCharacter = GetChar();
while ((cCharacter != '"') && (iCurrentLine == CurrentLine()))
{
*(pcToken++) = cCharacter;
cCharacter = GetChar();
}
/* end of line, or " is found. Check for error and try to recover */
if (iCurrentLine == CurrentLine())
{
iResult = STRING_TOKEN;
}
else
{
iResult = UNTERMINATED_STRING_READ;
UngetChar();
}
*pcToken = 0;
return iResult;
}
//------------------------------------------------------------------------------
bool FBasicTokenParser::GetRawTokenRespectingQuotes(FBasicToken& Token, TCHAR StopChar/* = TCHAR('\n')*/)
{
// if the parser is in a bad state, then don't continue parsing (who
// knows what will happen!?)
if (!IsValid())
{
return false;
}
// Get token after whitespace.
TCHAR Temp[MAX_STRING_CONST_SIZE];
int32 Length=0;
TCHAR c = GetLeadingChar();
bool bInQuote = false;
while( !IsEOL(c) && ((c != StopChar) || bInQuote) )
{
if( (c=='/' && PeekChar()=='/') || (c=='/' && PeekChar()=='*') )
{
break;
}
if (c == '"')
{
bInQuote = !bInQuote;
}
Temp[Length++] = c;
if( Length >= MAX_STRING_CONST_SIZE )
{
Length = ((int32)MAX_STRING_CONST_SIZE) - 1;
Temp[Length]=0; // needs to happen for the error description below
FText ErrorDesc = FText::Format(LOCTEXT("IdTooLong", "Identifer ({0}...) exceeds maximum length of {1}"), FText::FromString(Temp), FText::AsNumber((int32)MAX_STRING_CONST_SIZE));
SetError(FErrorState::ParseError, ErrorDesc);
c = GetChar(true);
break;
}
c = GetChar(true);
}
UngetChar();
// Get rid of trailing whitespace.
while( Length>0 && (Temp[Length-1]==' ' || Temp[Length-1]==9 ) )
{
Length--;
}
Temp[Length]=0;
if (bInQuote)
{
FText ErrorDesc = FText::Format(LOCTEXT("NoClosingQuote", "Unterminated quoted string ({0})"), FText::FromString(Temp));
SetError(FErrorState::ParseError, ErrorDesc);
}
Token.SetConstString(Temp);
return Length>0 && IsValid();
}
/**
* Put all text from the current position up to either EOL or the StopToken
* into Token. Advances the compiler's current position.
*
* @param Token [out] will contain the text that was parsed
* @param StopChar stop processing when this character is reached
*
* @return the number of character parsed
*/
bool FBaseParser::GetRawToken( FToken& Token, TCHAR StopChar /* = TCHAR('\n') */ )
{
// Get token after whitespace.
TCHAR Temp[MAX_STRING_CONST_SIZE];
int32 Length=0;
TCHAR c = GetLeadingChar();
while( !IsEOL(c) && c != StopChar )
{
if( (c=='/' && PeekChar()=='/') || (c=='/' && PeekChar()=='*') )
{
break;
}
Temp[Length++] = c;
if( Length >= MAX_STRING_CONST_SIZE )
{
FError::Throwf(TEXT("Identifier exceeds maximum of %i characters"), (int32)MAX_STRING_CONST_SIZE );
}
c = GetChar(true);
}
UngetChar();
// Get rid of trailing whitespace.
while( Length>0 && (Temp[Length-1]==' ' || Temp[Length-1]==9 ) )
{
Length--;
}
Temp[Length]=0;
Token.SetConstString(Temp);
return Length>0;
}
static int32
PeekChar(JSTokenStream *ts)
{
int32 c;
c = GetChar(ts);
UngetChar(ts, c);
return c;
}
static JSBool
MatchChar(JSTokenStream *ts, int32 expect)
{
int32 c;
c = GetChar(ts);
if (c == expect)
return JS_TRUE;
UngetChar(ts, c);
return JS_FALSE;
}
bool FBaseParser::GetRawTokenRespectingQuotes( FToken& Token, TCHAR StopChar /* = TCHAR('\n') */ )
{
// Get token after whitespace.
TCHAR Temp[MAX_STRING_CONST_SIZE];
int32 Length=0;
TCHAR c = GetLeadingChar();
bool bInQuote = false;
while( !IsEOL(c) && ((c != StopChar) || bInQuote) )
{
if( (c=='/' && PeekChar()=='/') || (c=='/' && PeekChar()=='*') )
{
break;
}
if (c == '"')
{
bInQuote = !bInQuote;
}
Temp[Length++] = c;
if( Length >= MAX_STRING_CONST_SIZE )
{
FError::Throwf(TEXT("Identifier exceeds maximum of %i characters"), (int32)MAX_STRING_CONST_SIZE );
c = GetChar(true);
Length = ((int32)MAX_STRING_CONST_SIZE) - 1;
break;
}
c = GetChar(true);
}
UngetChar();
if (bInQuote)
{
FError::Throwf(TEXT("Unterminated quoted string"));
}
// Get rid of trailing whitespace.
while( Length>0 && (Temp[Length-1]==' ' || Temp[Length-1]==9 ) )
{
Length--;
}
Temp[Length]=0;
Token.SetConstString(Temp);
return Length>0;
}
static JSBool
PeekChars(JSTokenStream *ts, intN n, jschar *cp)
{
intN i, j;
int32 c;
for (i = 0; i < n; i++) {
c = GetChar(ts);
if (c == EOF)
break;
cp[i] = (jschar)c;
}
for (j = i - 1; j >= 0; j--)
UngetChar(ts, cp[j]);
return i == n;
}
int GetPrecompilerToken(
char cCharacter,
char *pcToken)
/* PRECOMPILER_TOKEN :== '#' [alpha] */
{
*(pcToken++) = cCharacter;
cCharacter = GetChar();
while (isAlpha(cCharacter))
{
*(pcToken++) = cCharacter;
cCharacter = GetChar();
}
UngetChar();
*pcToken = 0;
return PRECOMPILER_TOKEN;
}
int GetIdentifierToken(
char cCharacter,
char *pcToken)
/* IDENTIFIER_TOKEN :== alpha <alphanum> */
{
*(pcToken++) = cCharacter;
cCharacter = GetChar();
while (isAlpha(cCharacter) || isDigit(cCharacter) || (cCharacter == '_'))
{
*(pcToken++) = cCharacter;
cCharacter = GetChar();
}
*pcToken = 0;
UngetChar();
return IDENTIFIER_TOKEN;
}
//------------------------------------------------------------------------------
bool FBasicTokenParser::GetRawToken(FBasicToken& Token, TCHAR StopChar/* = TCHAR('\n')*/)
{
// if the parser is in a bad state, then don't continue parsing (who
// knows what will happen!?)
if (!IsValid())
{
return false;
}
// Get token after whitespace.
TCHAR Temp[MAX_STRING_CONST_SIZE];
int32 Length=0;
TCHAR c = GetLeadingChar();
while( !IsEOL(c) && c != StopChar )
{
if( (c=='/' && PeekChar()=='/') || (c=='/' && PeekChar()=='*') )
{
break;
}
Temp[Length++] = c;
if( Length >= MAX_STRING_CONST_SIZE )
{
Temp[Length] = 0;
FText ErrorDesc = FText::Format(LOCTEXT("IdTooLong", "Identifer ({0}...) exceeds maximum length of {1}"), FText::FromString(Temp), FText::AsNumber((int32)MAX_STRING_CONST_SIZE));
SetError(FErrorState::ParseError, ErrorDesc);
}
c = GetChar(true);
}
UngetChar();
// Get rid of trailing whitespace.
while( Length>0 && (Temp[Length-1]==' ' || Temp[Length-1]==9 ) )
{
Length--;
}
Temp[Length]=0;
Token.SetConstString(Temp);
return Length>0;
}
char GetChar(
void)
/******************************************************************************/
/* get a char from the input file. Return a put back char if available. */
/******************************************************************************/
{
char cReturnValue;
if (bPutBackCharValid == TRUE)
{
/* "re-read" the ungetted char. Correct the linecounter if nessecary. */
cReturnValue = cPutBackChar;
bPutBackCharValid = FALSE;
if (((cPutBackChar == LF) && (bLfCounting == TRUE))
|| ((cPutBackChar == CR) && (bCrCounting == TRUE)))
{
iCurrentLineNumber++;
}
}
else
{
cReturnValue = GetNewChar();
}
if ((cReturnValue == CR) || (cReturnValue == LF))
{
if (bBlockCrLf)
{
UngetChar();
cReturnValue = ILLEGAL_CHAR;
}
}
return cReturnValue;
}
UINT CLiteHTMLReader::parseDocument(void)
{
ASSERT(m_lpszBuffer != NULL);
bool bAbort = false; // continue parsing or abort?
bool bIsClosingTag = false; // tag parsed is a closing tag?
bool bIsOpeningTag = false; // tag parsed is an opening tag?
CString strCharacters; // character data
CString strComment; // comment data
CString strT; // temporary storage
DWORD dwCharDataStart = 0L; // starting position of character data
DWORD dwCharDataLen = 0L; // length of character data
LONG lTemp = 0L; // temporary storage
TCHAR ch = 0; // character at current buffer position
CLiteHTMLTag oTag; // tag information
if ( (!m_lpszBuffer) || (!m_dwBufLen) )
return (0U);
// reset seek pointer to beginning
ResetSeekPointer();
// notify event handler about parsing startup
if (getEventNotify(notifyStartStop))
{
bAbort = false;
m_pEventHandler->BeginParse(m_dwAppData, bAbort);
if (bAbort) goto LEndParse;
}
// skip leading white-space characters
while (isWhiteSpace(ReadChar()))
;
ch = UngetChar();
while ((ch = ReadChar()) != NULL)
{
switch (ch)
{
// tag starting delimeter?
case _T('<'):
{
UngetChar();
strComment.Empty();
if (!parseComment(strComment))
{
bIsOpeningTag = false;
bIsClosingTag = false;
if (!parseTag(oTag, bIsOpeningTag, bIsClosingTag))
{
++dwCharDataLen;
// manually advance buffer position
// because the last call to UngetChar()
// moved it back one character
ch = ReadChar();
break;
}
}
// clear pending notifications
if ( (dwCharDataLen) || (strCharacters.GetLength()) )
{
strCharacters += CString(&m_lpszBuffer[dwCharDataStart], dwCharDataLen);
NormalizeCharacters(strCharacters);
if ( (strCharacters.GetLength()) &&
(getEventNotify(notifyCharacters)) )
{
bAbort = false;
m_pEventHandler->Characters(strCharacters, m_dwAppData, bAbort);
if (bAbort) goto LEndParse;
}
strCharacters.Empty();
}
dwCharDataLen = 0L;
dwCharDataStart = m_dwBufPos;
if (strComment.GetLength())
{
if (getEventNotify(notifyComment))
{
bAbort = false;
m_pEventHandler->Comment(strComment, m_dwAppData, bAbort);
if (bAbort) goto LEndParse;
}
}
else
{
if ( (bIsOpeningTag) && (getEventNotify(notifyTagStart)) )
{
bAbort = false;
m_pEventHandler->StartTag(&oTag, m_dwAppData, bAbort);
if (bAbort) goto LEndParse;
}
if ( (bIsClosingTag) && (getEventNotify(notifyTagEnd)) )
{
bAbort = false;
m_pEventHandler->EndTag(&oTag, m_dwAppData, bAbort);
if (bAbort) goto LEndParse;
}
}
break;
}
// entity reference beginning delimeter?
case _T('&'):
{
UngetChar();
lTemp = 0;
if (m_bResolveEntities)
lTemp = CLiteHTMLEntityResolver::resolveEntity(&m_lpszBuffer[m_dwBufPos], ch);
if (lTemp)
{
strCharacters += CString(&m_lpszBuffer[dwCharDataStart], dwCharDataLen) + ch;
m_dwBufPos += lTemp;
dwCharDataStart = m_dwBufPos;
dwCharDataLen = 0L;
}
else
{
ch = ReadChar();
++dwCharDataLen;
}
break;
}
// any other character
default:
{
++dwCharDataLen;
break;
}
}
}
// clear pending notifications
if ( (dwCharDataLen) || (strCharacters.GetLength()) )
{
strCharacters += CString(&m_lpszBuffer[dwCharDataStart], dwCharDataLen) + ch;
NormalizeCharacters(strCharacters);
strCharacters.TrimRight(); // explicit trailing white-space removal
if ( (strCharacters.GetLength()) &&
(getEventNotify(notifyCharacters)) )
{
bAbort = false;
m_pEventHandler->Characters(strCharacters, m_dwAppData, bAbort);
if (bAbort) goto LEndParse;
}
}
LEndParse:
// notify event handler about parsing completion
if (getEventNotify(notifyStartStop))
m_pEventHandler->EndParse(m_dwAppData, bAbort);
m_lpszBuffer = NULL;
m_dwBufLen = 0L;
return (m_dwBufPos);
}
JSTokenType
js_GetToken(JSContext *cx, JSTokenStream *ts)
{
JSTokenType tt;
JSToken *tp;
int32 c;
JSAtom *atom;
JSBool hadUnicodeEscape;
#define INIT_TOKENBUF(tb) ((tb)->ptr = (tb)->base)
#define FINISH_TOKENBUF(tb) if (!AddToTokenBuf(cx, tb, 0)) RETURN(TOK_ERROR)
#define TOKEN_LENGTH(tb) ((tb)->ptr - (tb)->base - 1)
#define RETURN(tt) { if (tt == TOK_ERROR) ts->flags |= TSF_ERROR; \
tp->pos.end.index = ts->linepos + \
(ts->linebuf.ptr - ts->linebuf.base) - \
ts->ungetpos; \
return (tp->type = tt); }
/* If there was a fatal error, keep returning TOK_ERROR. */
if (ts->flags & TSF_ERROR)
return TOK_ERROR;
/* Check for a pushed-back token resulting from mismatching lookahead. */
while (ts->lookahead != 0) {
ts->lookahead--;
ts->cursor = (ts->cursor + 1) & NTOKENS_MASK;
tt = CURRENT_TOKEN(ts).type;
if (tt != TOK_EOL || (ts->flags & TSF_NEWLINES))
return tt;
}
retry:
do {
c = GetChar(ts);
if (c == '\n') {
ts->flags &= ~TSF_DIRTYLINE;
if (ts->flags & TSF_NEWLINES)
break;
}
} while (JS_ISSPACE(c));
ts->cursor = (ts->cursor + 1) & NTOKENS_MASK;
tp = &CURRENT_TOKEN(ts);
tp->ptr = ts->linebuf.ptr - 1;
tp->pos.begin.index = ts->linepos + (tp->ptr - ts->linebuf.base);
tp->pos.begin.lineno = tp->pos.end.lineno = ts->lineno;
if (c == EOF)
RETURN(TOK_EOF);
if (c != '-' && c != '\n')
ts->flags |= TSF_DIRTYLINE;
hadUnicodeEscape = JS_FALSE;
if (JS_ISIDENT_START(c) ||
(c == '\\' &&
(c = GetUnicodeEscape(ts),
hadUnicodeEscape = JS_ISIDENT_START(c)))) {
INIT_TOKENBUF(&ts->tokenbuf);
for (;;) {
if (!AddToTokenBuf(cx, &ts->tokenbuf, (jschar)c))
RETURN(TOK_ERROR);
c = GetChar(ts);
if (c == '\\') {
c = GetUnicodeEscape(ts);
if (!JS_ISIDENT(c))
break;
hadUnicodeEscape = JS_TRUE;
} else {
if (!JS_ISIDENT(c))
break;
}
}
UngetChar(ts, c);
FINISH_TOKENBUF(&ts->tokenbuf);
atom = js_AtomizeChars(cx,
ts->tokenbuf.base,
TOKEN_LENGTH(&ts->tokenbuf),
0);
if (!atom)
RETURN(TOK_ERROR);
if (!hadUnicodeEscape && ATOM_KEYWORD(atom)) {
struct keyword *kw = ATOM_KEYWORD(atom);
if (JSVERSION_IS_ECMA(cx->version) || kw->version <= cx->version) {
tp->t_op = (JSOp) kw->op;
RETURN(kw->tokentype);
}
}
tp->t_op = JSOP_NAME;
tp->t_atom = atom;
RETURN(TOK_NAME);
}
if (JS7_ISDEC(c) || (c == '.' && JS7_ISDEC(PeekChar(ts)))) {
jsint radix;
const jschar *endptr;
jsdouble dval;
radix = 10;
INIT_TOKENBUF(&ts->tokenbuf);
if (c == '0') {
if (!AddToTokenBuf(cx, &ts->tokenbuf, (jschar)c))
RETURN(TOK_ERROR);
c = GetChar(ts);
if (JS_TOLOWER(c) == 'x') {
if (!AddToTokenBuf(cx, &ts->tokenbuf, (jschar)c))
RETURN(TOK_ERROR);
c = GetChar(ts);
radix = 16;
} else if (JS7_ISDEC(c)) {
radix = 8;
}
}
while (JS7_ISHEX(c)) {
if (radix < 16) {
if (JS7_ISLET(c))
break;
/*
* We permit 08 and 09 as decimal numbers, which makes our
* behaviour a superset of the ECMA numeric grammar. We might
* not always be so permissive, so we warn about it.
*/
if (radix == 8 && c >= '8') {
if (!js_ReportCompileErrorNumber(cx, ts, NULL,
JSREPORT_WARNING,
JSMSG_BAD_OCTAL,
c == '8' ? "08" : "09")) {
RETURN(TOK_ERROR);
}
radix = 10;
}
}
if (!AddToTokenBuf(cx, &ts->tokenbuf, (jschar)c))
RETURN(TOK_ERROR);
c = GetChar(ts);
}
if (radix == 10 && (c == '.' || JS_TOLOWER(c) == 'e')) {
if (c == '.') {
do {
if (!AddToTokenBuf(cx, &ts->tokenbuf, (jschar)c))
RETURN(TOK_ERROR);
c = GetChar(ts);
} while (JS7_ISDEC(c));
}
if (JS_TOLOWER(c) == 'e') {
if (!AddToTokenBuf(cx, &ts->tokenbuf, (jschar)c))
RETURN(TOK_ERROR);
c = GetChar(ts);
if (c == '+' || c == '-') {
if (!AddToTokenBuf(cx, &ts->tokenbuf, (jschar)c))
RETURN(TOK_ERROR);
c = GetChar(ts);
}
if (!JS7_ISDEC(c)) {
js_ReportCompileErrorNumber(cx, ts, NULL, JSREPORT_ERROR,
JSMSG_MISSING_EXPONENT);
RETURN(TOK_ERROR);
}
do {
if (!AddToTokenBuf(cx, &ts->tokenbuf, (jschar)c))
RETURN(TOK_ERROR);
c = GetChar(ts);
} while (JS7_ISDEC(c));
}
}
UngetChar(ts, c);
FINISH_TOKENBUF(&ts->tokenbuf);
if (radix == 10) {
if (!js_strtod(cx, ts->tokenbuf.base, &endptr, &dval)) {
js_ReportCompileErrorNumber(cx, ts, NULL, JSREPORT_ERROR,
JSMSG_OUT_OF_MEMORY);
RETURN(TOK_ERROR);
}
} else {
if (!js_strtointeger(cx, ts->tokenbuf.base, &endptr, radix, &dval)) {
js_ReportCompileErrorNumber(cx, ts, NULL, JSREPORT_ERROR,
JSMSG_OUT_OF_MEMORY);
RETURN(TOK_ERROR);
}
}
tp->t_dval = dval;
RETURN(TOK_NUMBER);
}
if (c == '"' || c == '\'') {
int32 val, qc = c;
INIT_TOKENBUF(&ts->tokenbuf);
while ((c = GetChar(ts)) != qc) {
if (c == '\n' || c == EOF) {
UngetChar(ts, c);
js_ReportCompileErrorNumber(cx, ts, NULL, JSREPORT_ERROR,
JSMSG_UNTERMINATED_STRING);
RETURN(TOK_ERROR);
}
if (c == '\\') {
switch (c = GetChar(ts)) {
case 'b': c = '\b'; break;
case 'f': c = '\f'; break;
case 'n': c = '\n'; break;
case 'r': c = '\r'; break;
case 't': c = '\t'; break;
case 'v': c = '\v'; break;
default:
if ('0' <= c && c < '8') {
val = JS7_UNDEC(c);
c = PeekChar(ts);
if ('0' <= c && c < '8') {
val = 8 * val + JS7_UNDEC(c);
GetChar(ts);
c = PeekChar(ts);
if ('0' <= c && c < '8') {
int32 save = val;
val = 8 * val + JS7_UNDEC(c);
if (val <= 0377)
GetChar(ts);
else
val = save;
}
}
c = (jschar)val;
} else if (c == 'u') {
jschar cp[4];
if (PeekChars(ts, 4, cp) &&
JS7_ISHEX(cp[0]) && JS7_ISHEX(cp[1]) &&
JS7_ISHEX(cp[2]) && JS7_ISHEX(cp[3])) {
c = (((((JS7_UNHEX(cp[0]) << 4)
+ JS7_UNHEX(cp[1])) << 4)
+ JS7_UNHEX(cp[2])) << 4)
+ JS7_UNHEX(cp[3]);
SkipChars(ts, 4);
}
} else if (c == 'x') {
jschar cp[2];
if (PeekChars(ts, 2, cp) &&
JS7_ISHEX(cp[0]) && JS7_ISHEX(cp[1])) {
c = (JS7_UNHEX(cp[0]) << 4) + JS7_UNHEX(cp[1]);
SkipChars(ts, 2);
}
} else if (c == '\n' && JSVERSION_IS_ECMA(cx->version)) {
/* ECMA follows C by removing escaped newlines. */
continue;
}
break;
}
}
if (!AddToTokenBuf(cx, &ts->tokenbuf, (jschar)c))
RETURN(TOK_ERROR);
}
FINISH_TOKENBUF(&ts->tokenbuf);
atom = js_AtomizeChars(cx,
ts->tokenbuf.base,
TOKEN_LENGTH(&ts->tokenbuf),
0);
if (!atom)
RETURN(TOK_ERROR);
tp->pos.end.lineno = ts->lineno;
tp->t_op = JSOP_STRING;
tp->t_atom = atom;
RETURN(TOK_STRING);
}
switch (c) {
case '\n':
c = TOK_EOL;
break;
case ';': c = TOK_SEMI; break;
case '.': c = TOK_DOT; break;
case '[': c = TOK_LB; break;
case ']': c = TOK_RB; break;
case '{': c = TOK_LC; break;
case '}': c = TOK_RC; break;
case '(': c = TOK_LP; break;
case ')': c = TOK_RP; break;
case ',': c = TOK_COMMA; break;
case '?': c = TOK_HOOK; break;
case ':':
/*
* Default so compiler can modify to JSOP_GETTER if 'p getter: v' in an
* object initializer, likewise for setter.
*/
tp->t_op = JSOP_NOP;
c = TOK_COLON;
break;
case '|':
if (MatchChar(ts, c)) {
c = TOK_OR;
} else if (MatchChar(ts, '=')) {
tp->t_op = JSOP_BITOR;
c = TOK_ASSIGN;
} else {
c = TOK_BITOR;
}
break;
case '^':
if (MatchChar(ts, '=')) {
tp->t_op = JSOP_BITXOR;
c = TOK_ASSIGN;
} else {
c = TOK_BITXOR;
}
break;
case '&':
if (MatchChar(ts, c)) {
c = TOK_AND;
} else if (MatchChar(ts, '=')) {
tp->t_op = JSOP_BITAND;
c = TOK_ASSIGN;
} else {
c = TOK_BITAND;
}
break;
case '=':
if (MatchChar(ts, c)) {
#if JS_HAS_TRIPLE_EQOPS
tp->t_op = MatchChar(ts, c) ? JSOP_NEW_EQ : (JSOp)cx->jsop_eq;
#else
tp->t_op = cx->jsop_eq;
#endif
c = TOK_EQOP;
} else {
tp->t_op = JSOP_NOP;
c = TOK_ASSIGN;
}
break;
case '!':
if (MatchChar(ts, '=')) {
#if JS_HAS_TRIPLE_EQOPS
tp->t_op = MatchChar(ts, '=') ? JSOP_NEW_NE : (JSOp)cx->jsop_ne;
#else
tp->t_op = cx->jsop_ne;
#endif
c = TOK_EQOP;
} else {
tp->t_op = JSOP_NOT;
c = TOK_UNARYOP;
}
break;
case '<':
/* NB: treat HTML begin-comment as comment-till-end-of-line */
if (MatchChar(ts, '!')) {
if (MatchChar(ts, '-')) {
if (MatchChar(ts, '-'))
goto skipline;
UngetChar(ts, '-');
}
UngetChar(ts, '!');
}
if (MatchChar(ts, c)) {
tp->t_op = JSOP_LSH;
c = MatchChar(ts, '=') ? TOK_ASSIGN : TOK_SHOP;
} else {
tp->t_op = MatchChar(ts, '=') ? JSOP_LE : JSOP_LT;
c = TOK_RELOP;
}
break;
case '>':
if (MatchChar(ts, c)) {
tp->t_op = MatchChar(ts, c) ? JSOP_URSH : JSOP_RSH;
c = MatchChar(ts, '=') ? TOK_ASSIGN : TOK_SHOP;
} else {
tp->t_op = MatchChar(ts, '=') ? JSOP_GE : JSOP_GT;
c = TOK_RELOP;
}
break;
case '*':
tp->t_op = JSOP_MUL;
c = MatchChar(ts, '=') ? TOK_ASSIGN : TOK_STAR;
break;
case '/':
if (MatchChar(ts, '/')) {
skipline:
while ((c = GetChar(ts)) != EOF && c != '\n')
/* skip to end of line */;
UngetChar(ts, c);
goto retry;
}
if (MatchChar(ts, '*')) {
while ((c = GetChar(ts)) != EOF &&
!(c == '*' && MatchChar(ts, '/'))) {
/* Ignore all characters until comment close. */
}
if (c == EOF) {
js_ReportCompileErrorNumber(cx, ts, NULL, JSREPORT_ERROR,
JSMSG_UNTERMINATED_COMMENT);
RETURN(TOK_ERROR);
}
goto retry;
}
#if JS_HAS_REGEXPS
if (ts->flags & TSF_REGEXP) {
JSObject *obj;
uintN flags;
INIT_TOKENBUF(&ts->tokenbuf);
while ((c = GetChar(ts)) != '/') {
if (c == '\n' || c == EOF) {
UngetChar(ts, c);
js_ReportCompileErrorNumber(cx, ts, NULL, JSREPORT_ERROR,
JSMSG_UNTERMINATED_REGEXP);
RETURN(TOK_ERROR);
}
if (c == '\\') {
if (!AddToTokenBuf(cx, &ts->tokenbuf, (jschar)c))
RETURN(TOK_ERROR);
c = GetChar(ts);
}
if (!AddToTokenBuf(cx, &ts->tokenbuf, (jschar)c))
RETURN(TOK_ERROR);
}
FINISH_TOKENBUF(&ts->tokenbuf);
for (flags = 0; ; ) {
if (MatchChar(ts, 'g'))
flags |= JSREG_GLOB;
else if (MatchChar(ts, 'i'))
flags |= JSREG_FOLD;
else if (MatchChar(ts, 'm'))
flags |= JSREG_MULTILINE;
else
break;
}
c = PeekChar(ts);
if (JS7_ISLET(c)) {
tp->ptr = ts->linebuf.ptr - 1;
js_ReportCompileErrorNumber(cx, ts, NULL, JSREPORT_ERROR,
JSMSG_BAD_REGEXP_FLAG);
(void) GetChar(ts);
RETURN(TOK_ERROR);
}
obj = js_NewRegExpObject(cx, ts,
ts->tokenbuf.base,
TOKEN_LENGTH(&ts->tokenbuf),
flags);
if (!obj)
RETURN(TOK_ERROR);
atom = js_AtomizeObject(cx, obj, 0);
if (!atom)
RETURN(TOK_ERROR);
tp->t_op = JSOP_OBJECT;
tp->t_atom = atom;
RETURN(TOK_OBJECT);
}
#endif /* JS_HAS_REGEXPS */
tp->t_op = JSOP_DIV;
c = MatchChar(ts, '=') ? TOK_ASSIGN : TOK_DIVOP;
break;
case '%':
tp->t_op = JSOP_MOD;
c = MatchChar(ts, '=') ? TOK_ASSIGN : TOK_DIVOP;
break;
case '~':
tp->t_op = JSOP_BITNOT;
c = TOK_UNARYOP;
break;
case '+':
if (MatchChar(ts, '=')) {
tp->t_op = JSOP_ADD;
c = TOK_ASSIGN;
} else if (MatchChar(ts, c)) {
c = TOK_INC;
} else {
tp->t_op = JSOP_POS;
c = TOK_PLUS;
}
break;
case '-':
if (MatchChar(ts, '=')) {
tp->t_op = JSOP_SUB;
c = TOK_ASSIGN;
} else if (MatchChar(ts, c)) {
if (PeekChar(ts) == '>' && !(ts->flags & TSF_DIRTYLINE))
goto skipline;
c = TOK_DEC;
} else {
tp->t_op = JSOP_NEG;
c = TOK_MINUS;
}
ts->flags |= TSF_DIRTYLINE;
break;
#if JS_HAS_SHARP_VARS
case '#':
{
uint32 n;
c = GetChar(ts);
if (!JS7_ISDEC(c)) {
UngetChar(ts, c);
goto badchar;
}
n = (uint32)JS7_UNDEC(c);
for (;;) {
c = GetChar(ts);
if (!JS7_ISDEC(c))
break;
n = 10 * n + JS7_UNDEC(c);
if (n >= ATOM_INDEX_LIMIT) {
js_ReportCompileErrorNumber(cx, ts, NULL, JSREPORT_ERROR,
JSMSG_SHARPVAR_TOO_BIG);
RETURN(TOK_ERROR);
}
}
tp->t_dval = (jsdouble) n;
if (JS_HAS_STRICT_OPTION(cx) &&
(c == '=' || c == '#')) {
char buf[20];
JS_snprintf(buf, sizeof buf, "#%u%c", n, c);
if (!js_ReportCompileErrorNumber(cx, ts, NULL,
JSREPORT_WARNING |
JSREPORT_STRICT,
JSMSG_DEPRECATED_USAGE,
buf)) {
RETURN(TOK_ERROR);
}
}
if (c == '=')
RETURN(TOK_DEFSHARP);
if (c == '#')
RETURN(TOK_USESHARP);
goto badchar;
}
badchar:
#endif /* JS_HAS_SHARP_VARS */
default:
js_ReportCompileErrorNumber(cx, ts, NULL, JSREPORT_ERROR,
JSMSG_ILLEGAL_CHARACTER);
RETURN(TOK_ERROR);
}
JS_ASSERT(c < TOK_LIMIT);
RETURN((JSTokenType)c);
#undef INIT_TOKENBUF
#undef FINISH_TOKENBUF
#undef TOKEN_LENGTH
#undef RETURN
}
/* a space or comma separated list of tag names */
Bool ParseTagNames( TidyDocImpl* doc, const TidyOptionImpl* option )
{
TidyConfigImpl* cfg = &doc->config;
tmbchar buf[1024];
uint i = 0, nTags = 0;
uint c = SkipWhite( cfg );
UserTagType ttyp = tagtype_null;
switch ( option->id )
{
case TidyInlineTags: ttyp = tagtype_inline; break;
case TidyBlockTags: ttyp = tagtype_block; break;
case TidyEmptyTags: ttyp = tagtype_empty; break;
case TidyPreTags: ttyp = tagtype_pre; break;
default:
ReportUnknownOption( doc, option->name );
return no;
}
SetOptionValue( doc, option->id, NULL );
FreeDeclaredTags( doc, ttyp );
cfg->defined_tags |= ttyp;
do
{
if (c == ' ' || c == '\t' || c == ',')
{
c = AdvanceChar( cfg );
continue;
}
if ( c == '\r' || c == '\n' )
{
uint c2 = AdvanceChar( cfg );
if ( c == '\r' && c2 == '\n' )
c = AdvanceChar( cfg );
else
c = c2;
if ( !IsWhite(c) )
{
buf[i] = 0;
UngetChar( c, cfg->cfgIn );
UngetChar( '\n', cfg->cfgIn );
break;
}
}
/*
if ( c == '\n' )
{
c = AdvanceChar( cfg );
if ( !IsWhite(c) )
{
buf[i] = 0;
UngetChar( c, cfg->cfgIn );
UngetChar( '\n', cfg->cfgIn );
break;
}
}
*/
while ( i < sizeof(buf)-2 && c != EndOfStream && !IsWhite(c) && c != ',' )
{
buf[i++] = (tmbchar) c;
c = AdvanceChar( cfg );
}
buf[i] = '\0';
if (i == 0) /* Skip empty tag definition. Possible when */
continue; /* there is a trailing space on the line. */
/* add tag to dictionary */
DeclareUserTag( doc, option->id, ttyp, buf );
i = 0;
++nTags;
}
while ( c != EndOfStream );
if ( i > 0 )
DeclareUserTag( doc, option->id, ttyp, buf );
return ( nTags > 0 );
}
// Gets the next token from the input stream, advancing the variables which keep track of the current input position and line.
bool FBaseParser::GetToken( FToken& Token, bool bNoConsts/*=false*/, ESymbolParseOption bParseTemplateClosingBracket/*=ESymbolParseOption::Normal*/ )
{
Token.TokenName = NAME_None;
TCHAR c = GetLeadingChar();
TCHAR p = PeekChar();
if( c == 0 )
{
UngetChar();
return 0;
}
Token.StartPos = PrevPos;
Token.StartLine = PrevLine;
if( (c>='A' && c<='Z') || (c>='a' && c<='z') || (c=='_') )
{
// Alphanumeric token.
int32 Length=0;
do
{
Token.Identifier[Length++] = c;
if( Length >= NAME_SIZE )
{
FError::Throwf(TEXT("Identifer length exceeds maximum of %i"), (int32)NAME_SIZE);
Length = ((int32)NAME_SIZE) - 1;
break;
}
c = GetChar();
} while( ((c>='A')&&(c<='Z')) || ((c>='a')&&(c<='z')) || ((c>='0')&&(c<='9')) || (c=='_') );
UngetChar();
Token.Identifier[Length]=0;
// Assume this is an identifier unless we find otherwise.
Token.TokenType = TOKEN_Identifier;
// Lookup the token's global name.
Token.TokenName = FName( Token.Identifier, FNAME_Find, true );
// If const values are allowed, determine whether the identifier represents a constant
if ( !bNoConsts )
{
// See if the identifier is part of a vector, rotation or other struct constant.
// boolean true/false
if( Token.Matches(TEXT("true")) )
{
Token.SetConstBool(true);
return true;
}
else if( Token.Matches(TEXT("false")) )
{
Token.SetConstBool(false);
return true;
}
}
return true;
}
// if const values are allowed, determine whether the non-identifier token represents a const
else if ( !bNoConsts && ((c>='0' && c<='9') || ((c=='+' || c=='-') && (p>='0' && p<='9'))) )
{
// Integer or floating point constant.
bool bIsFloat = 0;
int32 Length = 0;
bool bIsHex = 0;
do
{
if( c==TEXT('.') )
{
bIsFloat = true;
}
if( c==TEXT('X') || c == TEXT('x') )
{
bIsHex = true;
}
Token.Identifier[Length++] = c;
if( Length >= NAME_SIZE )
{
FError::Throwf(TEXT("Number length exceeds maximum of %i "), (int32)NAME_SIZE );
Length = ((int32)NAME_SIZE) - 1;
break;
}
c = FChar::ToUpper(GetChar());
} while ((c >= TEXT('0') && c <= TEXT('9')) || (!bIsFloat && c == TEXT('.')) || (!bIsHex && c == TEXT('X')) || (bIsHex && c >= TEXT('A') && c <= TEXT('F')));
Token.Identifier[Length]=0;
if (!bIsFloat || c != 'F')
{
UngetChar();
}
if (bIsFloat)
{
Token.SetConstFloat( FCString::Atof(Token.Identifier) );
}
else if (bIsHex)
{
TCHAR* End = Token.Identifier + FCString::Strlen(Token.Identifier);
Token.SetConstInt( FCString::Strtoi(Token.Identifier,&End,0) );
}
else
{
Token.SetConstInt( FCString::Atoi(Token.Identifier) );
}
return true;
}
else if (c == '\'')
{
TCHAR ActualCharLiteral = GetChar(/*bLiteral=*/ true);
if (ActualCharLiteral == '\\')
{
ActualCharLiteral = GetChar(/*bLiteral=*/ true);
switch (ActualCharLiteral)
{
case TCHAR('t'):
ActualCharLiteral = '\t';
break;
case TCHAR('n'):
ActualCharLiteral = '\n';
break;
case TCHAR('r'):
ActualCharLiteral = '\r';
break;
}
}
c = GetChar(/*bLiteral=*/ true);
if (c != '\'')
{
FError::Throwf(TEXT("Unterminated character constant"));
UngetChar();
}
Token.SetConstChar(ActualCharLiteral);
return true;
}
else if (c == '"')
{
// String constant.
TCHAR Temp[MAX_STRING_CONST_SIZE];
int32 Length=0;
c = GetChar(/*bLiteral=*/ true);
while( (c!='"') && !IsEOL(c) )
{
if( c=='\\' )
{
c = GetChar(/*bLiteral=*/ true);
if( IsEOL(c) )
{
break;
}
else if(c == 'n')
{
// Newline escape sequence.
c = '\n';
}
}
Temp[Length++] = c;
if( Length >= MAX_STRING_CONST_SIZE )
{
FError::Throwf(TEXT("String constant exceeds maximum of %i characters"), (int32)MAX_STRING_CONST_SIZE );
c = TEXT('\"');
Length = ((int32)MAX_STRING_CONST_SIZE) - 1;
break;
}
c = GetChar(/*bLiteral=*/ true);
}
Temp[Length]=0;
if( c != '"' )
{
FError::Throwf(TEXT("Unterminated string constant: %s"), Temp);
UngetChar();
}
Token.SetConstString(Temp);
return true;
}
else
{
// Symbol.
int32 Length=0;
Token.Identifier[Length++] = c;
// Handle special 2-character symbols.
#define PAIR(cc,dd) ((c==cc)&&(d==dd)) /* Comparison macro for convenience */
TCHAR d = GetChar();
if
( PAIR('<','<')
|| (PAIR('>','>') && (bParseTemplateClosingBracket != ESymbolParseOption::CloseTemplateBracket))
|| PAIR('!','=')
|| PAIR('<','=')
|| PAIR('>','=')
|| PAIR('+','+')
|| PAIR('-','-')
|| PAIR('+','=')
|| PAIR('-','=')
|| PAIR('*','=')
|| PAIR('/','=')
|| PAIR('&','&')
|| PAIR('|','|')
|| PAIR('^','^')
|| PAIR('=','=')
|| PAIR('*','*')
|| PAIR('~','=')
|| PAIR(':',':')
)
{
Token.Identifier[Length++] = d;
if( c=='>' && d=='>' )
{
if( GetChar()=='>' )
Token.Identifier[Length++] = '>';
else
UngetChar();
}
}
else UngetChar();
#undef PAIR
Token.Identifier[Length] = 0;
Token.TokenType = TOKEN_Symbol;
// Lookup the token's global name.
Token.TokenName = FName( Token.Identifier, FNAME_Find, true );
return true;
}
}
//
// Skip past all spaces and tabs in the input stream.
//
TCHAR FBaseParser::GetLeadingChar()
{
TCHAR TrailingCommentNewline = 0;
for (;;)
{
bool MultipleNewlines = false;
TCHAR c;
// Skip blanks.
do
{
c = GetChar();
// Check if we've encountered another newline since the last one
if (c == TrailingCommentNewline)
{
MultipleNewlines = true;
}
} while (IsWhitespace(c));
if (c != TEXT('/') || PeekChar() != TEXT('/'))
{
return c;
}
// Clear the comment if we've encountered newlines since the last comment
if (MultipleNewlines)
{
ClearComment();
}
// Record the first slash. The first iteration of the loop will get the second slash.
PrevComment += c;
do
{
c = GetChar(true);
if (c == 0)
return c;
PrevComment += c;
} while (!IsEOL(c));
TrailingCommentNewline = c;
for (;;)
{
c = GetChar();
if (c == 0)
return c;
if (c == TrailingCommentNewline || !IsEOL(c))
{
UngetChar();
break;
}
PrevComment += c;
}
}
}
//------------------------------------------------------------------------------
bool FBasicTokenParser::GetToken(FBasicToken& Token, bool bNoConsts/* = false*/)
{
// if the parser is in a bad state, then don't continue parsing (who
// knows what will happen!?)
if (!IsValid())
{
return false;
}
Token.TokenName = NAME_None;
TCHAR c = GetLeadingChar();
TCHAR p = PeekChar();
if( c == 0 )
{
UngetChar();
return 0;
}
Token.StartPos = PrevPos;
Token.StartLine = PrevLine;
if( (c>='A' && c<='Z') || (c>='a' && c<='z') || (c=='_') )
{
// Alphanumeric token.
int32 Length=0;
do
{
Token.Identifier[Length++] = c;
if( Length >= NAME_SIZE )
{
Length = ((int32)NAME_SIZE) - 1;
Token.Identifier[Length]=0; // need this for the error description
FText ErrorDesc = FText::Format(LOCTEXT("IdTooLong", "Identifer ({0}...) exceeds maximum length of {1}"), FText::FromString(Token.Identifier), FText::AsNumber((int32)NAME_SIZE));
SetError(FErrorState::ParseError, ErrorDesc);
break;
}
c = GetChar();
} while( ((c>='A')&&(c<='Z')) || ((c>='a')&&(c<='z')) || ((c>='0')&&(c<='9')) || (c=='_') );
UngetChar();
Token.Identifier[Length]=0;
// Assume this is an identifier unless we find otherwise.
Token.TokenType = FBasicToken::TOKEN_Identifier;
// Lookup the token's global name.
Token.TokenName = FName( Token.Identifier, FNAME_Find, true );
// If const values are allowed, determine whether the identifier represents a constant
if ( !bNoConsts )
{
// See if the identifier is part of a vector, rotation or other struct constant.
// boolean true/false
if( Token.Matches(TEXT("true")) )
{
Token.SetConstBool(true);
return true;
}
else if( Token.Matches(TEXT("false")) )
{
Token.SetConstBool(false);
return true;
}
}
return IsValid();
}
// if const values are allowed, determine whether the non-identifier token represents a const
else if ( !bNoConsts && ((c>='0' && c<='9') || ((c=='+' || c=='-') && (p>='0' && p<='9'))) )
{
// Integer or floating point constant.
bool bIsFloat = 0;
int32 Length = 0;
bool bIsHex = 0;
do
{
if( c==TEXT('.') )
{
bIsFloat = true;
}
if( c==TEXT('X') || c == TEXT('x') )
{
bIsHex = true;
}
Token.Identifier[Length++] = c;
if( Length >= NAME_SIZE )
{
Length = ((int32)NAME_SIZE) - 1;
Token.Identifier[Length]=0; // need this for the error description
FText ErrorDesc = FText::Format(LOCTEXT("IdTooLong", "Identifer ({0}...) exceeds maximum length of {1}"), FText::FromString(Token.Identifier), FText::AsNumber((int32)NAME_SIZE));
SetError(FErrorState::ParseError, ErrorDesc);
break;
}
c = FChar::ToUpper(GetChar());
} while ((c >= TEXT('0') && c <= TEXT('9')) || (!bIsFloat && c == TEXT('.')) || (!bIsHex && c == TEXT('X')) || (bIsHex && c >= TEXT('A') && c <= TEXT('F')));
Token.Identifier[Length]=0;
if (!bIsFloat || c != 'F')
{
UngetChar();
}
if (bIsFloat)
{
Token.SetConstFloat( FCString::Atof(Token.Identifier) );
}
else if (bIsHex)
{
TCHAR* End = Token.Identifier + FCString::Strlen(Token.Identifier);
Token.SetConstInt( FCString::Strtoi(Token.Identifier,&End,0) );
}
else
{
Token.SetConstInt( FCString::Atoi(Token.Identifier) );
}
return IsValid();
}
else if( c=='"' )
{
// String constant.
TCHAR Temp[MAX_STRING_CONST_SIZE];
int32 Length=0;
c = GetChar(1);
while( (c!='"') && !IsEOL(c) )
{
if( c=='\\' )
{
c = GetChar(1);
if( IsEOL(c) )
{
break;
}
else if(c == 'n')
{
// Newline escape sequence.
c = '\n';
}
}
Temp[Length++] = c;
if( Length >= MAX_STRING_CONST_SIZE )
{
Length = ((int32)MAX_STRING_CONST_SIZE) - 1;
Temp[Length]=0; // need this for the error description
FText ErrorDesc = FText::Format(LOCTEXT("StringConstTooLong", "String constant ({0}...) exceeds maximum of {1} characters"), FText::FromString(Temp), FText::AsNumber((int32)MAX_STRING_CONST_SIZE));
SetError(FErrorState::ParseError, ErrorDesc);
c = TEXT('\"');
break;
}
c = GetChar(1);
}
Temp[Length]=0;
if( c != '"' )
{
FText ErrorDesc = FText::Format(LOCTEXT("NoClosingQuote", "Unterminated quoted string ({0})"), FText::FromString(Temp));
SetError(FErrorState::ParseError, ErrorDesc);
UngetChar();
}
Token.SetConstString(Temp);
return IsValid();
}
else
{
// Symbol.
int32 Length=0;
Token.Identifier[Length++] = c;
// Handle special 2-character symbols.
#define PAIR(cc,dd) ((c==cc)&&(d==dd)) /* Comparison macro for convenience */
TCHAR d = GetChar();
if
( PAIR('<','<')
|| PAIR('>','>')
|| PAIR('!','=')
|| PAIR('<','=')
|| PAIR('>','=')
|| PAIR('+','+')
|| PAIR('-','-')
|| PAIR('+','=')
|| PAIR('-','=')
|| PAIR('*','=')
|| PAIR('/','=')
|| PAIR('&','&')
|| PAIR('|','|')
|| PAIR('^','^')
|| PAIR('=','=')
|| PAIR('*','*')
|| PAIR('~','=')
|| PAIR(':',':')
)
{
Token.Identifier[Length++] = d;
if( c=='>' && d=='>' )
{
if( GetChar()=='>' )
Token.Identifier[Length++] = '>';
else
UngetChar();
}
}
else UngetChar();
#undef PAIR
Token.Identifier[Length] = 0;
Token.TokenType = FBasicToken::TOKEN_Symbol;
// Lookup the token's global name.
Token.TokenName = FName( Token.Identifier, FNAME_Find, true );
return true;
}
}
int GetNewToken(
char *pcToken)
/******************************************************************************/
/* get a new token from the input file. Don't check for a put back token */
/* characters are classified as: */
/* cr :== carriage return character */
/* lf :== line feed character */
/* tab :== the tab character */
/* dquote :== the " character */
/* digit :== 0123456789 */
/* bindigit :== 01 */
/* hexdigit :== 0123456789ABCDEF */
/* alpha :== _abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ */
/* any :== all prtble chars except cr, lf and dqoute */
/* tab :== the tab character */
/* alphanum :== alpha | digit */
/* */
/* <x> :== zero or more x */
/* [x] :== one or more x */
/* x :== exactly one x */
/* */
/* recognized tokens are : */
/* */
/* FLOAT_TOKEN :== <digit> '.' [digit] */
/* DEC_NUMBER_TOKEN :== [digit] */
/* HEX_NUMBER_TOKEN :== '0' 'x' [hexdigit] */
/* BIN_NUMBER_TOKEN :== '0' 'b' [bindigit] */
/* IDENTIFIER_TOKEN :== alpha <alphanum> */
/* STRING_TOKEN :== dquote <any> dquote */
/* OPERATOR_TOKEN :== '=' */
/* :== '=' '=' */
/* :== '<' */
/* :== '>' */
/* :== '<' '=' */
/* :== '>' '=' */
/* :== '<' '>' */
/* :== '+' */
/* :== '-' */
/* :== '*' */
/* :== '/' */
/* :== '<' '<' */
/* :== '>' '>' */
/* :== '&' */
/* :== '|' */
/* :== '[' ']' */
/* :== '[' */
/* :== ']' */
/* DOT_TOKEN :== '.' */
/* DOTDOT_TOKEN :== '.' '.' */
/* COMMA_TOKEN :== ',' */
/* SEMICOLON_TOKEN :== ';' */
/* COLON_TOKEN :== ':' */
/* LEFT_ARROW_TOKEN :== '<' '-' */
/* RIGHT_ARROW_TOKEN :== '-' '>' */
/* OPEN_PARENTHESE_TOKEN :== '(' */
/* CLOSE_PARENTHESE_TOKEN :== ')' */
/* PRECOMPILER_TOKEN :== '#' [alpha] */
/* DELIMITER_TOKEN :== ' ' */
/* :== tab */
/* :== cr */
/* :== lf */
/******************************************************************************/
{
char cCharacter;
int iResult;
int iCurrentLine;
boolean bEoLn;
char *pcTokenWalker;
pcTokenWalker = pcToken;
cCharacter = GetChar();
/* first skip all delimiter chars */
while (chrpos((char) cCharacter, DELIMITER_CHARS) != NULL)
{
cCharacter = GetChar();
}
if (isDigit(cCharacter))
{
/* read a DEC_NUMBER_TOKEN */
/* or read a HEX_NUMBER_TOKEN */
/* or read a BIN_NUMBER_TOKEN */
iResult = GetNumberToken(cCharacter, pcToken);
}
else if (isAlpha(cCharacter))
{
/* read an IDENTIFIER_TOKEN */
iResult = GetIdentifierToken(cCharacter, pcToken);
}
else if (cCharacter == '"')
{
/* read a STRING_TOKEN */
iResult = GetStringToken(cCharacter, pcToken);
}
else if (chrpos(cCharacter, "=<>+-*/&|[].") != NULL)
{
/* read an OPERATOR_TOKEN */
iResult = GetOperatorToken(cCharacter, pcToken);
if (strcmp((char *) pcToken, "->") == 0)
iResult = RIGHT_ARROW_TOKEN;
else if (strcmp((char *) pcToken, "<-") == 0)
iResult = LEFT_ARROW_TOKEN;
else if (strcmp((char *) pcToken, "..") == 0)
iResult = DOTDOT_TOKEN;
else if (strcmp((char *) pcToken, "//") == 0)
{
iCurrentLine = CurrentLine();
/* bEoLn (EndOfLine) will be set to true if end of line is reached */
/* undetected because of end of file. (This will also happen in the */
/* unlikely situation that an inputfile contains something like: */
/* #define COMMENT //this is comment */
/* because while parsing a preprocessor macro, the scanner refuses to */
/* go to the next line. */
bEoLn = FALSE;
while ((iCurrentLine == CurrentLine()) && (!bEoLn))
{
cCharacter = GetChar();
if ((cCharacter != ILLEGAL_CHAR) && (cCharacter != '\0'))
{
*(pcToken++) = cCharacter;
}
else
{
bEoLn = TRUE;
}
}
iResult = COMMENT_TOKEN;
if (!bEoLn)
{
/* End of comment detected by reading a Cr/Lf character. This */
/* read character is added to pcToken. */
/* Put cr/lf back, only to adjust the line counter... */
/* (only needed if not in the middle of a macro.) */
UngetChar();
/* Delete cr or lf character from pcToken by overwriting it with */
/* the zero character (for terminating pcToken) */
*(--pcToken) = 0;
}
else
{
/* Terminate pcToken by adding a zero character */
*(pcToken) = 0;
}
}
}
else if (cCharacter == '#')
{
/* read a PRECOMPILER_TOKEN */
iResult = GetPrecompilerToken(cCharacter, pcToken);
}
else if (cCharacter == '{')
{
iResult = OPEN_BRACKETS_TOKEN;
(*pcTokenWalker++) = cCharacter;
*pcTokenWalker = 0;
}
else if (cCharacter == '}')
{
iResult = CLOSE_BRACKETS_TOKEN;
(*pcTokenWalker++) = cCharacter;
*pcTokenWalker = 0;
}
else if (cCharacter == '(')
{
iResult = OPEN_PARENTHESES_TOKEN;
(*pcTokenWalker++) = cCharacter;
*pcTokenWalker = 0;
}
else if (cCharacter == ')')
{
iResult = CLOSE_PARENTHESES_TOKEN;
(*pcTokenWalker++) = cCharacter;
*pcTokenWalker = 0;
}
else if (cCharacter == '.')
{
iResult = DOT_TOKEN;
(*pcTokenWalker++) = cCharacter;
cCharacter = GetChar();
while (isDigit(cCharacter))
{
iResult = FLOAT_TOKEN;
(*pcTokenWalker++) = cCharacter;
cCharacter = GetChar();
}
UngetChar();
*pcTokenWalker = 0;
}
else if (cCharacter == ',')
{
iResult = COMMA_TOKEN;
(*pcTokenWalker++) = cCharacter;
*pcTokenWalker = 0;
}
else if (cCharacter == ';')
{
iResult = SEMICOLON_TOKEN;
(*pcTokenWalker++) = cCharacter;
*pcTokenWalker = 0;
}
else if (cCharacter == ':')
{
iResult = COLON_TOKEN;
(*pcTokenWalker++) = cCharacter;
*pcTokenWalker = 0;
}
else if (cCharacter == '~')
{
iResult = TILDE_TOKEN;
(*pcTokenWalker++) = cCharacter;
*pcTokenWalker = 0;
}
else if (cCharacter == '\\')
{
iResult = BACKSLASH_TOKEN;
(*pcTokenWalker++) = cCharacter;
*pcTokenWalker = 0;
}
else if (cCharacter == 0)
{
/* End of file reached. */
iResult = END_OF_FILE;
(*pcTokenWalker++) = cCharacter;
}
else
{
/* unexpected character read */
iResult = UNEXPECTED_CHAR_READ;
(*pcTokenWalker++) = cCharacter;
*pcTokenWalker = 0;
}
return iResult;
}
//------------------------------------------------------------------------------
TCHAR FBasicTokenParser::GetLeadingChar()
{
// if the parser is in a bad state, then don't continue parsing (who
// knows what will happen!?)... return a char signaling the end-of-stream
if (!IsValid())
{
return 0;
}
TCHAR TrailingCommentNewline = 0;
for (;;)
{
bool MultipleNewlines = false;
TCHAR c;
// Skip blanks.
do
{
c = GetChar();
// Check if we've encountered another newline since the last one
if (c == TrailingCommentNewline)
{
MultipleNewlines = true;
}
} while (IsWhitespace(c));
if (c != TEXT('/') || PeekChar() != TEXT('/'))
{
return c;
}
// Clear the comment if we've encountered newlines since the last comment
if (MultipleNewlines)
{
ClearCachedComment();
}
// Record the first slash. The first iteration of the loop will get the second slash.
PrevComment += c;
do
{
c = GetChar(true);
if (c == 0)
return c;
PrevComment += c;
} while (!IsEOL(c));
TrailingCommentNewline = c;
for (;;)
{
c = GetChar();
if (c == 0)
return c;
if (c == TrailingCommentNewline || !IsEOL(c))
{
UngetChar();
break;
}
PrevComment += c;
}
}
}
int getLexBuf(struct lexBuf *lb) {
register int ch;
struct charBuf *cb;
register char *ccc, *cfin;
register char *cc, *dd, *lmax, *lexStartDd;
unsigned chval=0;
int rlex;
int cline,clo; /* current line, current line offset (for collumn)*/
char *clb; /* current line begin */
char oldCh;
int line,size,cfile,lexStartCol, lexStartFilePos, column, lexemlen;
S_position lexPos;
/* first test whether the input is cached */
if (s_cache.activeCache && inStacki==0 && macStacki==0) {
cacheInput();
s_cache.lexcc = lb->a;
}
lmax = lb->a + LEX_BUFF_SIZE - MAX_LEXEM_SIZE;
for(dd=lb->a,cc=lb->cc; cc<lb->fin; cc++,dd++) *dd = *cc;
lb->cc = lb->a;
cb = &lb->cb;
cline = cb->lineNum; clb = cb->lineBegin; clo = cb->collumnOffset;
ccc = cb->cc; cfin = cb->fin; cfile = cb->fileNumber;
GetChar(ch,ccc,cfin,cb,clb,clo);
contin:
DeleteBlank(ch,ccc,cfin,cb,clb,clo);
if (dd >= lmax) {
UngetChar(ch,ccc,cfin,cb);
goto finish;
}
NOTE_NEW_LEXEM_POSITION(ch,ccc,cfin,cb,lb,dd,cfile,cline,clb,clo);
/* yytext = ccc; */
lexStartDd = dd;
lexStartCol = COLUMN_POS(ccc,clb,clo);
if (ch == '_' || isalpha(ch) || (ch=='$' && (LANGUAGE(LAN_YACC)||LANGUAGE(LAN_JAVA)))) {
ProcessIdentifier(ch, ccc, cfin, cb, dd, cfile, cline, clb, clo, lab2);
goto nextLexem;
} else if (isdigit(ch)) {
/* *************** number ******************************* */
register long unsigned val=0;
lexStartFilePos = ABS_FILE_POS(cb,cfin,ccc);
if (ch=='0') {
GetChar(ch,ccc,cfin,cb,clb,clo);
if (ch=='x' || ch=='X') {
/* hexa */
GetChar(ch,ccc,cfin,cb,clb,clo);
while (isdigit(ch)||(ch>='a'&&ch<='f')||(ch>='A'&&ch<='F')) {
if (ch>='a') val = val*16+ch-'a'+10;
else if (ch>='A') val = val*16+ch-'A'+10;
else val = val*16+ch-'0';
GetChar(ch,ccc,cfin,cb,clb,clo);
}
} else {
/* octal */
while (isdigit(ch) && ch<='8') {
val = val*8+ch-'0';
GetChar(ch,ccc,cfin,cb,clb,clo);
}
}
} else {
/* decimal */
while (isdigit(ch)) {
val = val*10+ch-'0';
GetChar(ch,ccc,cfin,cb,clb,clo);
}
}
if (ch == '.' || ch=='e' || ch=='E'
|| ((ch=='d' || ch=='D'|| ch=='f' || ch=='F') && LANGUAGE(LAN_JAVA))) {
/* floating point */
fpConstFin(ch,ccc,cfin,cb,clb,clo,rlex);
PutLexToken(rlex,dd);
PutLexPosition(cfile, cline, lexStartCol, dd);
PutLexInt(ABS_FILE_POS(cb,cfin,ccc)-lexStartFilePos, dd);
goto nextLexem;
}
/* integer */
conType(val,ch,ccc,cfin,cb,clb,clo, rlex);
PutLexToken(rlex,dd);
PutLexInt(val,dd);
PutLexPosition(cfile, cline, lexStartCol, dd);
PutLexInt(ABS_FILE_POS(cb,cfin,ccc)-lexStartFilePos, dd);
goto nextLexem;
} else switch (ch) {
/* ************ special character ********************* */
case '.':
lexStartFilePos = ABS_FILE_POS(cb,cfin,ccc);
GetChar(ch,ccc,cfin,cb,clb,clo);
if (ch == '.' && LANGUAGE(LAN_C|LAN_YACC|LAN_CCC)) {
GetChar(ch,ccc,cfin,cb,clb,clo);
if (ch == '.') {
GetChar(ch,ccc,cfin,cb,clb,clo);
PutLexToken(ELIPSIS,dd);
PutLexPosition(cfile, cline, lexStartCol, dd);
goto nextLexem;
} else {
UngetChar(ch,ccc,cfin,cb);
ch = '.';
}
PutLexToken('.',dd);
PutLexPosition(cfile, cline, lexStartCol, dd);
goto nextLexem;
} else if (ch=='*' && LANGUAGE(LAN_CCC)) {
GetChar(ch,ccc,cfin,cb,clb,clo);
PutLexToken(POINTM_OP,dd);
PutLexPosition(cfile, cline, lexStartCol, dd);
goto nextLexem;
} else if (isdigit(ch)) {
/* floating point constant */
UngetChar(ch,ccc,cfin,cb);
ch = '.';
fpConstFin(ch,ccc,cfin,cb,clb,clo,rlex);
PutLexToken(rlex,dd);
PutLexPosition(cfile, cline, lexStartCol, dd);
PutLexInt(ABS_FILE_POS(cb,cfin,ccc)-lexStartFilePos, dd);
goto nextLexem;
} else {
PutLexToken('.',dd);
PutLexPosition(cfile, cline, lexStartCol, dd);
goto nextLexem;
}
case '-':
GetChar(ch,ccc,cfin,cb,clb,clo);
if (ch=='=') {
PutLexToken(SUB_ASSIGN,dd); PutLexPosition(cfile, cline, lexStartCol, dd); GetChar(ch,ccc,cfin,cb,clb,clo);
goto nextLexem;
} else if (ch=='-') {PutLexToken(DEC_OP,dd); PutLexPosition(cfile, cline, lexStartCol, dd);GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem;}
else if (ch=='>' && LANGUAGE(LAN_C|LAN_YACC|LAN_CCC)) {
GetChar(ch,ccc,cfin,cb,clb,clo);
if (ch=='*' && LANGUAGE(LAN_CCC)) {PutLexToken(PTRM_OP,dd); PutLexPosition(cfile, cline, lexStartCol, dd);GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem;}
else {PutLexToken(PTR_OP,dd); PutLexPosition(cfile, cline, lexStartCol, dd);goto nextLexem;}
} else {PutLexToken('-',dd); PutLexPosition(cfile, cline, lexStartCol, dd);goto nextLexem;}
case '+':
GetChar(ch,ccc,cfin,cb,clb,clo);
if (ch == '=') { PutLexToken(ADD_ASSIGN,dd); PutLexPosition(cfile, cline, lexStartCol, dd);GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem;}
else if (ch == '+') { PutLexToken(INC_OP,dd); PutLexPosition(cfile, cline, lexStartCol, dd);GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem; }
else {PutLexToken('+',dd); PutLexPosition(cfile, cline, lexStartCol, dd);goto nextLexem;}
case '>':
GetChar(ch,ccc,cfin,cb,clb,clo);
if (ch == '>') {
GetChar(ch,ccc,cfin,cb,clb,clo);
if(ch=='>' && LANGUAGE(LAN_JAVA)){
GetChar(ch,ccc,cfin,cb,clb,clo);
if(ch=='='){PutLexToken(URIGHT_ASSIGN,dd); PutLexPosition(cfile, cline, lexStartCol, dd);GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem;}
else {PutLexToken(URIGHT_OP,dd); PutLexPosition(cfile, cline, lexStartCol, dd);goto nextLexem;}
} else if(ch=='='){PutLexToken(RIGHT_ASSIGN,dd); PutLexPosition(cfile, cline, lexStartCol, dd);GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem;}
else {PutLexToken(RIGHT_OP,dd); PutLexPosition(cfile, cline, lexStartCol, dd);goto nextLexem;}
} else if (ch == '='){PutLexToken(GE_OP,dd); PutLexPosition(cfile, cline, lexStartCol, dd);GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem;}
else {PutLexToken('>',dd); PutLexPosition(cfile, cline, lexStartCol, dd);goto nextLexem;}
case '<':
GetChar(ch,ccc,cfin,cb,clb,clo);
if (ch == '<') {
GetChar(ch,ccc,cfin,cb,clb,clo);
if(ch=='='){PutLexToken(LEFT_ASSIGN,dd); PutLexPosition(cfile, cline, lexStartCol, dd);GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem;}
else {PutLexToken(LEFT_OP,dd); PutLexPosition(cfile, cline, lexStartCol, dd);goto nextLexem;}
} else if (ch == '='){PutLexToken(LE_OP,dd); PutLexPosition(cfile, cline, lexStartCol, dd);GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem;}
else {PutLexToken('<',dd); PutLexPosition(cfile, cline, lexStartCol, dd);goto nextLexem;}
case '*':
GetChar(ch,ccc,cfin,cb,clb,clo);
if (ch == '='){ PutLexToken(MUL_ASSIGN,dd); PutLexPosition(cfile, cline, lexStartCol, dd);GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem; }
else {PutLexToken('*',dd); PutLexPosition(cfile, cline, lexStartCol, dd);goto nextLexem;}
case '%':
GetChar(ch,ccc,cfin,cb,clb,clo);
if (ch == '='){ PutLexToken(MOD_ASSIGN,dd); PutLexPosition(cfile, cline, lexStartCol, dd);GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem; }
/*&
else if (LANGUAGE(LAN_YACC) && ch == '{'){ PutLexToken(YACC_PERC_LPAR,dd); PutLexPosition(cfile, cline, lexStartCol, dd);GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem; }
else if (LANGUAGE(LAN_YACC) && ch == '}'){ PutLexToken(YACC_PERC_RPAR,dd); PutLexPosition(cfile, cline, lexStartCol, dd);GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem; }
&*/
else {PutLexToken('%',dd); PutLexPosition(cfile, cline, lexStartCol, dd);goto nextLexem;}
case '&':
GetChar(ch,ccc,cfin,cb,clb,clo);
if (ch == '='){ PutLexToken(AND_ASSIGN,dd); PutLexPosition(cfile, cline, lexStartCol, dd);GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem; }
else if (ch == '&'){ PutLexToken(AND_OP,dd); PutLexPosition(cfile, cline, lexStartCol, dd);GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem; }
else if (ch == '*') {
GetChar(ch,ccc,cfin,cb,clb,clo);
if (ch == '/') {
/* a program commentary, ignore */
GetChar(ch,ccc,cfin,cb,clb,clo);
CommentaryEndRef(ch,ccc,cfin,cb,dd,cfile,cline,clb,clo,0);
goto nextLexem;
} else {
UngetChar(ch,ccc,cfin,cb); ch = '*';
PutLexToken('&',dd); PutLexPosition(cfile, cline, lexStartCol, dd);goto nextLexem;
}}
else {PutLexToken('&',dd); PutLexPosition(cfile, cline, lexStartCol, dd);goto nextLexem;}
case '^':
GetChar(ch,ccc,cfin,cb,clb,clo);
if (ch == '='){ PutLexToken(XOR_ASSIGN,dd); PutLexPosition(cfile, cline, lexStartCol, dd);GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem; }
else {PutLexToken('^',dd); PutLexPosition(cfile, cline, lexStartCol, dd);goto nextLexem;}
case '|':
GetChar(ch,ccc,cfin,cb,clb,clo);
if (ch == '='){ PutLexToken(OR_ASSIGN,dd); PutLexPosition(cfile, cline, lexStartCol, dd);GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem; }
else if (ch == '|'){ PutLexToken(OR_OP,dd); PutLexPosition(cfile, cline, lexStartCol, dd);GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem; }
else {PutLexToken('|',dd); PutLexPosition(cfile, cline, lexStartCol, dd);goto nextLexem;}
case '=':
GetChar(ch,ccc,cfin,cb,clb,clo);
if (ch == '='){ PutLexToken(EQ_OP,dd); PutLexPosition(cfile, cline, lexStartCol, dd);GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem; }
else {PutLexToken('=',dd); PutLexPosition(cfile, cline, lexStartCol, dd); goto nextLexem;}
case '!':
GetChar(ch,ccc,cfin,cb,clb,clo);
if (ch == '='){ PutLexToken(NE_OP,dd); PutLexPosition(cfile, cline, lexStartCol, dd);GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem; }
else {PutLexToken('!',dd); PutLexPosition(cfile, cline, lexStartCol, dd);goto nextLexem;}
case ':':
GetChar(ch,ccc,cfin,cb,clb,clo);
if (ch == ':' && LANGUAGE(LAN_CCC)){ PutLexToken(DPOINT,dd); PutLexPosition(cfile, cline, lexStartCol, dd);GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem; }
else {PutLexToken(':',dd); PutLexPosition(cfile, cline, lexStartCol, dd);goto nextLexem;}
case '\'':
chval = 0;
lexStartFilePos = ABS_FILE_POS(cb,cfin,ccc);
do {
GetChar(ch,ccc,cfin,cb,clb,clo);
while (ch=='\\') {
GetChar(ch,ccc,cfin,cb,clb,clo);
/* TODO escape sequences */
GetChar(ch,ccc,cfin,cb,clb,clo);
}
if (ch != '\'') chval = chval * 256 + ch;
} while (ch != '\'' && ch != '\n');
if (ch=='\'') {
PutLexToken(CHAR_LITERAL,dd);
PutLexInt(chval,dd);
PutLexPosition(cfile, cline, lexStartCol, dd);
PutLexInt(ABS_FILE_POS(cb,cfin,ccc)-lexStartFilePos, dd);
GetChar(ch,ccc,cfin,cb,clb,clo);
}
goto nextLexem;
case '\"':
line = cline; size = 0;
PutLexToken(STRING_LITERAL,dd);
do {
GetChar(ch,ccc,cfin,cb,clb,clo); size ++;
if (ch!='\"' && size<MAX_LEXEM_SIZE-10) PutLexChar(ch,dd);
if (ch=='\\') {
GetChar(ch,ccc,cfin,cb,clb,clo); size ++;
if (size < MAX_LEXEM_SIZE-10) PutLexChar(ch,dd);
/* TODO escape sequences */
if (ch == '\n') {cline ++; clb = ccc; clo = 0;}
ch = 0;
}
if (ch == '\n') {
cline ++; clb = ccc; clo = 0;
if (s_opt.strictAnsi && (s_opt.debug || s_opt.err)) {
warning(ERR_ST,"string constant through end of line");
}
}
// in Java CR LF can't be a part of string, even there
// are benchmarks making Xrefactory coredump if CR or LF
// is a part of strings
} while (ch != '\"' && (ch != '\n' || !s_opt.strictAnsi) && ch != -1);
if (ch == -1 && s_opt.taskRegime!=RegimeEditServer) {
warning(ERR_ST,"string constant through EOF");
}
PutLexChar(0,dd);
PutLexPosition(cfile, cline, lexStartCol, dd);
PutLexLine(cline-line,dd);
GetChar(ch,ccc,cfin,cb,clb,clo);
goto nextLexem;
case '/':
GetChar(ch,ccc,cfin,cb,clb,clo);
if (ch == '='){ PutLexToken(DIV_ASSIGN,dd); PutLexPosition(cfile, cline, lexStartCol, dd);GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem; }
else if (ch=='*') {
int javadoc=0;
CommentaryBegRef(ch,ccc,cfin,cb,dd,cfile,cline,clb,clo);
GetChar(ch,ccc,cfin,cb,clb,clo);
if (ch == '&') {
/* ****** a program comment, ignore */
GetChar(ch,ccc,cfin,cb,clb,clo);
goto nextLexem;
} else {
if (ch=='*' && LANGUAGE(LAN_JAVA)) javadoc = 1;
UngetChar(ch,ccc,cfin,cb); ch = '*';
} /* !!! COPY BLOCK TO '/n' */
PassComment(ch,oldCh,ccc,cfin,cb,dd,cline,clb,clo);
CommentaryEndRef(ch,ccc,cfin,cb,dd,cfile,cline,clb,clo,javadoc);
goto nextLexem;
} else if (ch=='/' && s_opt.cpp_comment) {
/* ******* a // comment ******* */
CommentaryBegRef(ch,ccc,cfin,cb,dd,cfile,cline,clb,clo);
GetChar(ch,ccc,cfin,cb,clb,clo);
if (ch == '&') {
/* ****** a program comment, ignore */
GetChar(ch,ccc,cfin,cb,clb,clo);
CommentaryEndRef(ch,ccc,cfin,cb,dd,cfile,cline,clb,clo,0);
goto nextLexem;
}
line = cline;
while (ch!='\n' && ch != -1) {
GetChar(ch,ccc,cfin,cb,clb,clo);
if (ch == '\\') {
GetChar(ch,ccc,cfin,cb,clb,clo);
if (ch=='\n') {cline ++; clb = ccc; clo = 0;}
GetChar(ch,ccc,cfin,cb,clb,clo);
}
}
CommentaryEndRef(ch,ccc,cfin,cb,dd,cfile,cline,clb,clo,0);
PutLexLine(cline-line,dd);
goto nextLexem;
} else {
PutLexToken('/',dd);
PutLexPosition(cfile, cline, lexStartCol, dd);
goto nextLexem;
}
case '\\':
GetChar(ch,ccc,cfin,cb,clb,clo);
if (ch == '\n') {
cline ++; clb = ccc; clo = 0;
PutLexLine(1, dd);
GetChar(ch,ccc,cfin,cb,clb,clo);
} else {
PutLexToken('\\',dd);
PutLexPosition(cfile, cline, lexStartCol, dd);
}
goto nextLexem;
case '\n':
column = COLUMN_POS(ccc,clb,clo);
if (column >= MAX_REFERENCABLE_COLUMN) {
fatalError(ERR_ST, "position over MAX_REFERENCABLE_COLUMN, read TROUBLES in README file", XREF_EXIT_ERR);
}
if (cline >= MAX_REFERENCABLE_LINE) {
fatalError(ERR_ST, "position over MAX_REFERENCABLE_LINE, read TROUBLES in README file", XREF_EXIT_ERR);
}
cline ++; clb = ccc; clo = 0;
GetChar(ch,ccc,cfin,cb,clb,clo);
DeleteBlank(ch,ccc,cfin,cb,clb,clo);
if (ch == '/') {
GetChar(ch,ccc,cfin,cb,clb,clo);
if (ch == '*') {
CommentaryBegRef(ch,ccc,cfin,cb,dd,cfile,cline,clb,clo);
GetChar(ch,ccc,cfin,cb,clb,clo);
if (ch == '&') {
/* ****** a program comment, ignore */
GetChar(ch,ccc,cfin,cb,clb,clo);
} else {
int javadoc=0;
if (ch == '*' && LANGUAGE(LAN_JAVA)) javadoc = 1;
UngetChar(ch,ccc,cfin,cb); ch = '*';
PassComment(ch,oldCh,ccc,cfin,cb,dd,cline,clb,clo);
CommentaryEndRef(ch,ccc,cfin,cb,dd,cfile,cline,clb,clo,javadoc);
DeleteBlank(ch,ccc,cfin,cb,clb,clo);
}
} else {
UngetChar(ch,ccc,cfin,cb);
ch = '/';
}
}
PutLexToken('\n',dd);
PutLexPosition(cfile, cline, lexStartCol, dd);
if (ch == '#' && LANGUAGE(LAN_C|LAN_CCC|LAN_YACC)) {
NOTE_NEW_LEXEM_POSITION(ch,ccc,cfin,cb,lb,dd,cfile,cline,clb,clo);
HandleCppToken(ch,ccc,cfin,cb,dd,cfile,cline,clb,clo);
}
goto nextLexem;
case '#':
GetChar(ch,ccc,cfin,cb,clb,clo);
if (ch == '#') {
GetChar(ch,ccc,cfin,cb,clb,clo);
PutLexToken(CPP_COLLATION,dd);
PutLexPosition(cfile, cline, lexStartCol, dd);
} else {
PutLexToken('#',dd);
PutLexPosition(cfile, cline, lexStartCol, dd);
}
goto nextLexem;
case -1:
/* ** probably end of file ** */
goto nextLexem;
#if ZERO
case '{': if (brack_deep==0) printf("/**/}");
brack_deep+=2;
case '}': brack_deep--;
#endif
default:
if (ch >= 32) { /* small chars ignored */
PutLexToken(ch,dd);
PutLexPosition(cfile, cline, lexStartCol, dd);
}
GetChar(ch,ccc,cfin,cb,clb,clo);
goto nextLexem;
}
assert(0);
nextLexem:
if (s_opt.taskRegime == RegimeEditServer) {
int pi,lpi,len,lastlex,parChar,apos,idcoll;
S_position *ps;
int pos0,pos1,currentLexemPosition;
pi = (lb->posi-1) % LEX_POSITIONS_RING_SIZE;
ps = & lb->pRing[pi];
currentLexemPosition = lb->fpRing[pi];
if ( cfile == s_olOriginalFileNumber
&& cfile != s_noneFileIndex
&& cfile != -1
&& s_jsl==NULL
) {
if (s_opt.cxrefs == OLO_EXTRACT && lb->posi>=2) {
DeleteBlank(ch,ccc,cfin,cb,clb,clo);
pos1 = ABS_FILE_POS(cb, cfin, ccc);
//&idcoll = COLUMN_POS(ccc,clb,clo);
//&fprintf(dumpOut,":pos1==%d, olCursorPos==%d, olMarkPos==%d\n",pos1,s_opt.olCursorPos,s_opt.olMarkPos);
// all this is very, very HACK!!!
if (pos1 >= s_opt.olCursorPos && ! s_cps.marker1Flag) {
if (LANGUAGE(LAN_JAVA)) parChar = ';';
else {
if (s_cps.marker2Flag) parChar='}';
else parChar = '{';
}
PutLexToken(parChar,dd);
PutLexPosition(ps->file,ps->line,ps->coll,dd);
PutLexToken(parChar,dd);
PutLexPosition(ps->file,ps->line,ps->coll,dd);
PutLexToken(';',dd);
PutLexPosition(ps->file,ps->line,ps->coll,dd);
PutLexToken(';',dd);
PutLexPosition(ps->file,ps->line,ps->coll,dd);
PutLexToken(OL_MARKER_TOKEN,dd);
PutLexPosition(ps->file,ps->line,ps->coll,dd);
PutLexToken(parChar,dd);
PutLexPosition(ps->file,ps->line,ps->coll,dd);
PutLexToken(parChar,dd);
PutLexPosition(ps->file,ps->line,ps->coll,dd);
s_cps.marker1Flag=1;
} else if (pos1 >= s_opt.olMarkPos && ! s_cps.marker2Flag){
if (LANGUAGE(LAN_JAVA)) parChar = ';';
else {
if (s_cps.marker1Flag) parChar='}';
else parChar = '{';
}
PutLexToken(parChar,dd);
PutLexPosition(ps->file,ps->line,ps->coll,dd);
PutLexToken(parChar,dd);
PutLexPosition(ps->file,ps->line,ps->coll,dd);
PutLexToken(';',dd);
PutLexPosition(ps->file,ps->line,ps->coll,dd);
PutLexToken(';',dd);
PutLexPosition(ps->file,ps->line,ps->coll,dd);
PutLexToken(OL_MARKER_TOKEN,dd);
PutLexPosition(ps->file,ps->line,ps->coll,dd);
PutLexToken(parChar,dd);
PutLexPosition(ps->file,ps->line,ps->coll,dd);
PutLexToken(parChar,dd);
PutLexPosition(ps->file,ps->line,ps->coll,dd);
s_cps.marker2Flag=1;
}
} else if ( s_opt.cxrefs == OLO_COMPLETION
|| s_opt.cxrefs == OLO_SEARCH) {
DeleteBlank(ch,ccc,cfin,cb,clb,clo);
apos = ABS_FILE_POS(cb, cfin, ccc);
if (currentLexemPosition < s_opt.olCursorPos
&& (apos >= s_opt.olCursorPos
|| (ch == -1 && apos+1 == s_opt.olCursorPos))) {
//&sprintf(tmpBuff,"currentLexemPosition, s_opt.olCursorPos, ABS_FILE_POS, ch == %d, %d, %d, %d\n",currentLexemPosition, s_opt.olCursorPos, apos, ch);ppcGenTmpBuff();
//&fprintf(dumpOut,":check\n");fflush(dumpOut);
lastlex = NextLexToken(lexStartDd);
if (lastlex == IDENTIFIER) {
len = s_opt.olCursorPos-currentLexemPosition;
//&fprintf(dumpOut,":check %s[%d] <-> %d\n", lexStartDd+TOKEN_SIZE, len,strlen(lexStartDd+TOKEN_SIZE));fflush(dumpOut);
if (len <= strlen(lexStartDd+TOKEN_SIZE)) {
if (s_opt.cxrefs == OLO_SEARCH) {
char *ddd;
ddd = lexStartDd;
PutLexToken(IDENT_TO_COMPLETE, ddd);
} else {
dd = lexStartDd;
PutLexToken(IDENT_TO_COMPLETE, dd);
dd += len;
PutLexChar(0,dd);
PutLexPosition(ps->file,ps->line,ps->coll,dd);
}
//&fprintf(dumpOut,":ress %s\n", lexStartDd+TOKEN_SIZE);fflush(dumpOut);
} else {
// completion after an identifier
PUT_EMPTY_COMPLETION_ID(ccc,dd,cline,clo,clb,
cfile,
apos-s_opt.olCursorPos);
}
} else if ((lastlex == LINE_TOK || lastlex == STRING_LITERAL)
&& (apos-s_opt.olCursorPos != 0)) {
// completion inside special lexems, do
// NO COMPLETION
} else {
// completion after another lexem
PUT_EMPTY_COMPLETION_ID(ccc,dd,cline,clo,clb,
cfile,
apos-s_opt.olCursorPos);
}
}
// TODO, make this in a more standard way, !!!
} else {
//&fprintf(dumpOut,":testing %d <= %d <= %d\n", currentLexemPosition, s_opt.olCursorPos, ABS_FILE_POS(cb, cfin, ccc));
if (currentLexemPosition <= s_opt.olCursorPos
&& ABS_FILE_POS(cb, cfin, ccc) >= s_opt.olCursorPos) {
gotOnLineCxRefs( ps);
lastlex = NextLexToken(lexStartDd);
if (lastlex == IDENTIFIER) {
strcpy(s_olstring, lexStartDd+TOKEN_SIZE);
}
}
if (LANGUAGE(LAN_JAVA)) {
// there is a problem with this, when browsing at CPP construction
// that is why I restrict it to Java language! It is usefull
// only for Java refactorings
DeleteBlank(ch,ccc,cfin,cb,clb,clo);
apos = ABS_FILE_POS(cb, cfin, ccc);
if (apos >= s_opt.olCursorPos && ! s_cps.marker1Flag) {
PutLexToken(OL_MARKER_TOKEN,dd);
PutLexPosition(ps->file,ps->line,ps->coll,dd);
s_cps.marker1Flag=1;
}
}
}
}
}
if (ch != -1) goto contin;
finish:
cb->cc = ccc; cb->fin = cfin;
cb->lineNum = cline; cb->lineBegin = clb; cb->collumnOffset = clo;
lb->fin = dd;
//&lexBufDump(lb);
if (lb->fin == lb->a) return(0);
return(1);
}
// Gets the next token from the input stream, advancing the variables which keep track of the current input position and line.
bool FBaseParser::GetToken( FToken& Token, bool bNoConsts/*=false*/ )
{
Token.TokenName = NAME_None;
TCHAR c = GetLeadingChar();
TCHAR p = PeekChar();
if( c == 0 )
{
UngetChar();
return 0;
}
Token.StartPos = PrevPos;
Token.StartLine = PrevLine;
if( (c>='A' && c<='Z') || (c>='a' && c<='z') || (c=='_') )
{
// Alphanumeric token.
int32 Length=0;
do
{
Token.Identifier[Length++] = c;
if( Length >= NAME_SIZE )
{
FError::Throwf(TEXT("Identifer length exceeds maximum of %i"), (int32)NAME_SIZE);
Length = ((int32)NAME_SIZE) - 1;
break;
}
c = GetChar();
} while( ((c>='A')&&(c<='Z')) || ((c>='a')&&(c<='z')) || ((c>='0')&&(c<='9')) || (c=='_') );
UngetChar();
Token.Identifier[Length]=0;
// Assume this is an identifier unless we find otherwise.
Token.TokenType = TOKEN_Identifier;
// Lookup the token's global name.
Token.TokenName = FName( Token.Identifier, FNAME_Find, true );
// If const values are allowed, determine whether the identifier represents a constant
if ( !bNoConsts )
{
// See if the identifier is part of a vector, rotation or other struct constant.
// boolean true/false
if( Token.Matches(TEXT("true")) )
{
Token.SetConstBool(true);
return true;
}
else if( Token.Matches(TEXT("false")) )
{
Token.SetConstBool(false);
return true;
}
}
return true;
}
// if const values are allowed, determine whether the non-identifier token represents a const
else if ( !bNoConsts && ((c>='0' && c<='9') || ((c=='+' || c=='-') && (p>='0' && p<='9'))) )
{
// Integer or floating point constant.
bool bIsFloat = 0;
int32 Length = 0;
bool bIsHex = 0;
do
{
if( c==TEXT('.') )
{
bIsFloat = true;
}
if( c==TEXT('X') || c == TEXT('x') )
{
bIsHex = true;
}
Token.Identifier[Length++] = c;
if( Length >= NAME_SIZE )
{
FError::Throwf(TEXT("Number length exceeds maximum of %i "), (int32)NAME_SIZE );
Length = ((int32)NAME_SIZE) - 1;
break;
}
c = FChar::ToUpper(GetChar());
} while ((c >= TEXT('0') && c <= TEXT('9')) || (!bIsFloat && c == TEXT('.')) || (!bIsHex && c == TEXT('X')) || (bIsHex && c >= TEXT('A') && c <= TEXT('F')));
Token.Identifier[Length]=0;
if (!bIsFloat || c != 'F')
{
UngetChar();
}
if (bIsFloat)
{
Token.SetConstFloat( FCString::Atof(Token.Identifier) );
}
else if (bIsHex)
{
TCHAR* End = Token.Identifier + FCString::Strlen(Token.Identifier);
Token.SetConstInt( FCString::Strtoi(Token.Identifier,&End,0) );
}
else
{
Token.SetConstInt( FCString::Atoi(Token.Identifier) );
}
return true;
}
//@TODO: 'z' is a character literal in C++, not a FName like it was in UnrealScript - do we need this code?
// else if( !bNoConsts && c=='\'' )
// {
// // Name constant.
// int32 Length=0;
// c = GetChar();
// while( (c>='A' && c<='Z') || (c>='a' && c<='z') || (c>='0' && c<='9') || (c=='_') || (c=='-') || (c==' ') ) //@FIXME: space in names should be illegal!
// {
// Token.Identifier[Length++] = c;
// if( Length >= NAME_SIZE )
// {
// FError::Throwf(TEXT("Name length exceeds maximum of %i"), (int32)NAME_SIZE );
// // trick the error a few lines down
// c = TEXT('\'');
// Length = ((int32)NAME_SIZE) - 1;
// break;
// }
// c = GetChar();
// }
// if( c != '\'' )
// {
// UngetChar();
// FError::Throwf(TEXT("Illegal character in name") );
// }
// Token.Identifier[Length]=0;
//
// // Make constant name.
// Token.SetConstName( FName(Token.Identifier) );
// return true;
// }
else if( c=='"' )
{
// String constant.
TCHAR Temp[MAX_STRING_CONST_SIZE];
int32 Length=0;
c = GetChar(1);
while( (c!='"') && !IsEOL(c) )
{
if( c=='\\' )
{
c = GetChar(1);
if( IsEOL(c) )
{
break;
}
else if(c == 'n')
{
// Newline escape sequence.
c = '\n';
}
}
Temp[Length++] = c;
if( Length >= MAX_STRING_CONST_SIZE )
{
FError::Throwf(TEXT("String constant exceeds maximum of %i characters"), (int32)MAX_STRING_CONST_SIZE );
c = TEXT('\"');
Length = ((int32)MAX_STRING_CONST_SIZE) - 1;
break;
}
c = GetChar(1);
}
Temp[Length]=0;
if( c != '"' )
{
FError::Throwf(TEXT("Unterminated string constant: %s"), Temp);
UngetChar();
}
Token.SetConstString(Temp);
return true;
}
else
{
// Symbol.
int32 Length=0;
Token.Identifier[Length++] = c;
// Handle special 2-character symbols.
#define PAIR(cc,dd) ((c==cc)&&(d==dd)) /* Comparison macro for convenience */
TCHAR d = GetChar();
if
( PAIR('<','<')
|| PAIR('>','>')
|| PAIR('!','=')
|| PAIR('<','=')
|| PAIR('>','=')
|| PAIR('+','+')
|| PAIR('-','-')
|| PAIR('+','=')
|| PAIR('-','=')
|| PAIR('*','=')
|| PAIR('/','=')
|| PAIR('&','&')
|| PAIR('|','|')
|| PAIR('^','^')
|| PAIR('=','=')
|| PAIR('*','*')
|| PAIR('~','=')
|| PAIR(':',':')
)
{
Token.Identifier[Length++] = d;
if( c=='>' && d=='>' )
{
if( GetChar()=='>' )
Token.Identifier[Length++] = '>';
else
UngetChar();
}
}
else UngetChar();
#undef PAIR
Token.Identifier[Length] = 0;
Token.TokenType = TOKEN_Symbol;
// Lookup the token's global name.
Token.TokenName = FName( Token.Identifier, FNAME_Find, true );
return true;
}
}
