int GetNumberToken( char cCharacter, char *pcToken) /* DEC_NUMBER_TOKEN :== [digit] */ /* HEX_NUMBER_TOKEN :== '0' 'x' [hexdigit] */ /* BIN_NUMBER_TOKEN :== '0' 'b' [bindigit] */ /* FLOAT_TOKEN :== <digit> '.' [digit] */ { char *pDigitClass; int iResult = DEC_NUMBER_TOKEN; *(pcToken++) = cCharacter; cCharacter = GetChar(); if (cCharacter == 'x') { *(pcToken++) = cCharacter; pDigitClass = pHexDigits; cCharacter = GetChar(); iResult = HEX_NUMBER_TOKEN; } else if (cCharacter == 'b') { *(pcToken++) = cCharacter; pDigitClass = pBinDigits; cCharacter = GetChar(); iResult = BIN_NUMBER_TOKEN; } else pDigitClass = pDecDigits; while (chrpos(cCharacter, pDigitClass) != NULL) { *(pcToken++) = cCharacter; cCharacter = GetChar(); } if ((iResult == DEC_NUMBER_TOKEN) && (cCharacter == '.')) { *(pcToken++) = cCharacter; cCharacter = GetChar(); iResult = FLOAT_TOKEN; while (chrpos(cCharacter, pDigitClass) != NULL) { *(pcToken++) = cCharacter; cCharacter = GetChar(); } UngetChar(); } *pcToken = 0; UngetChar(); return iResult; }
int GetOperatorToken( char cCharacter, char *pcToken) /* OPERATOR_TOKEN :== '=' */ /* :== '=' '=' */ /* :== '<' */ /* :== '>' */ /* :== '<' '=' */ /* :== '>' '=' */ /* :== '<' '>' */ /* :== '+' */ /* :== '-' */ /* :== '*' */ /* :== '/' */ /* :== '<' '<' */ /* :== '>' '>' */ /* :== '&' */ /* :== '|' */ /* :== '[' ']' */ /* The following two tokens are read as OPERATOR_TOKEN, but are translated */ /* to LEFT_ARROW_TOKEN or RIGHT_ARROW_TOKEN before leaving the scanner. */ /* :== '-' '>' */ /* :== '<' '-' */ { char *pcCompleteToken; pcCompleteToken = pcToken; /* store pointer to first token char */ *(pcToken++) = cCharacter; /* assign first character to token */ if (chrpos(cCharacter, "-=<>[/.") != NULL) /* if second character possible */ { cCharacter = GetChar(); /* read second character */ if (chrpos(cCharacter, "-=<>]/.") != NULL) /* if valid second character */ { *pcToken = cCharacter; /* add character */ pcToken[1] = 0; /* and make it zero terminated */ /* now a 2 byte operator is read. Check if this is a valid two byte */ /* operator. If not, put back the second character in the input stream. */ if (strpos(pcCompleteToken, "== <= >= <> << >> [] -> <- // ..") == NULL) { UngetChar(); } else { pcToken++; } } else { UngetChar(); /* else put second character back */ } } *pcToken = 0; /* terminate token */ return OPERATOR_TOKEN; }
int GetStringToken( char cCharacter, char *pcToken) /* STRING_TOKEN :== dquote <any> dquote */ { int iResult; int iCurrentLine; iCurrentLine = CurrentLine(); cCharacter = GetChar(); while ((cCharacter != '"') && (iCurrentLine == CurrentLine())) { *(pcToken++) = cCharacter; cCharacter = GetChar(); } /* end of line, or " is found. Check for error and try to recover */ if (iCurrentLine == CurrentLine()) { iResult = STRING_TOKEN; } else { iResult = UNTERMINATED_STRING_READ; UngetChar(); } *pcToken = 0; return iResult; }
//------------------------------------------------------------------------------ bool FBasicTokenParser::GetRawTokenRespectingQuotes(FBasicToken& Token, TCHAR StopChar/* = TCHAR('\n')*/) { // if the parser is in a bad state, then don't continue parsing (who // knows what will happen!?) if (!IsValid()) { return false; } // Get token after whitespace. TCHAR Temp[MAX_STRING_CONST_SIZE]; int32 Length=0; TCHAR c = GetLeadingChar(); bool bInQuote = false; while( !IsEOL(c) && ((c != StopChar) || bInQuote) ) { if( (c=='/' && PeekChar()=='/') || (c=='/' && PeekChar()=='*') ) { break; } if (c == '"') { bInQuote = !bInQuote; } Temp[Length++] = c; if( Length >= MAX_STRING_CONST_SIZE ) { Length = ((int32)MAX_STRING_CONST_SIZE) - 1; Temp[Length]=0; // needs to happen for the error description below FText ErrorDesc = FText::Format(LOCTEXT("IdTooLong", "Identifer ({0}...) exceeds maximum length of {1}"), FText::FromString(Temp), FText::AsNumber((int32)MAX_STRING_CONST_SIZE)); SetError(FErrorState::ParseError, ErrorDesc); c = GetChar(true); break; } c = GetChar(true); } UngetChar(); // Get rid of trailing whitespace. while( Length>0 && (Temp[Length-1]==' ' || Temp[Length-1]==9 ) ) { Length--; } Temp[Length]=0; if (bInQuote) { FText ErrorDesc = FText::Format(LOCTEXT("NoClosingQuote", "Unterminated quoted string ({0})"), FText::FromString(Temp)); SetError(FErrorState::ParseError, ErrorDesc); } Token.SetConstString(Temp); return Length>0 && IsValid(); }
/** * Put all text from the current position up to either EOL or the StopToken * into Token. Advances the compiler's current position. * * @param Token [out] will contain the text that was parsed * @param StopChar stop processing when this character is reached * * @return the number of character parsed */ bool FBaseParser::GetRawToken( FToken& Token, TCHAR StopChar /* = TCHAR('\n') */ ) { // Get token after whitespace. TCHAR Temp[MAX_STRING_CONST_SIZE]; int32 Length=0; TCHAR c = GetLeadingChar(); while( !IsEOL(c) && c != StopChar ) { if( (c=='/' && PeekChar()=='/') || (c=='/' && PeekChar()=='*') ) { break; } Temp[Length++] = c; if( Length >= MAX_STRING_CONST_SIZE ) { FError::Throwf(TEXT("Identifier exceeds maximum of %i characters"), (int32)MAX_STRING_CONST_SIZE ); } c = GetChar(true); } UngetChar(); // Get rid of trailing whitespace. while( Length>0 && (Temp[Length-1]==' ' || Temp[Length-1]==9 ) ) { Length--; } Temp[Length]=0; Token.SetConstString(Temp); return Length>0; }
static int32 PeekChar(JSTokenStream *ts) { int32 c; c = GetChar(ts); UngetChar(ts, c); return c; }
static JSBool MatchChar(JSTokenStream *ts, int32 expect) { int32 c; c = GetChar(ts); if (c == expect) return JS_TRUE; UngetChar(ts, c); return JS_FALSE; }
bool FBaseParser::GetRawTokenRespectingQuotes( FToken& Token, TCHAR StopChar /* = TCHAR('\n') */ ) { // Get token after whitespace. TCHAR Temp[MAX_STRING_CONST_SIZE]; int32 Length=0; TCHAR c = GetLeadingChar(); bool bInQuote = false; while( !IsEOL(c) && ((c != StopChar) || bInQuote) ) { if( (c=='/' && PeekChar()=='/') || (c=='/' && PeekChar()=='*') ) { break; } if (c == '"') { bInQuote = !bInQuote; } Temp[Length++] = c; if( Length >= MAX_STRING_CONST_SIZE ) { FError::Throwf(TEXT("Identifier exceeds maximum of %i characters"), (int32)MAX_STRING_CONST_SIZE ); c = GetChar(true); Length = ((int32)MAX_STRING_CONST_SIZE) - 1; break; } c = GetChar(true); } UngetChar(); if (bInQuote) { FError::Throwf(TEXT("Unterminated quoted string")); } // Get rid of trailing whitespace. while( Length>0 && (Temp[Length-1]==' ' || Temp[Length-1]==9 ) ) { Length--; } Temp[Length]=0; Token.SetConstString(Temp); return Length>0; }
static JSBool PeekChars(JSTokenStream *ts, intN n, jschar *cp) { intN i, j; int32 c; for (i = 0; i < n; i++) { c = GetChar(ts); if (c == EOF) break; cp[i] = (jschar)c; } for (j = i - 1; j >= 0; j--) UngetChar(ts, cp[j]); return i == n; }
int GetPrecompilerToken( char cCharacter, char *pcToken) /* PRECOMPILER_TOKEN :== '#' [alpha] */ { *(pcToken++) = cCharacter; cCharacter = GetChar(); while (isAlpha(cCharacter)) { *(pcToken++) = cCharacter; cCharacter = GetChar(); } UngetChar(); *pcToken = 0; return PRECOMPILER_TOKEN; }
int GetIdentifierToken( char cCharacter, char *pcToken) /* IDENTIFIER_TOKEN :== alpha <alphanum> */ { *(pcToken++) = cCharacter; cCharacter = GetChar(); while (isAlpha(cCharacter) || isDigit(cCharacter) || (cCharacter == '_')) { *(pcToken++) = cCharacter; cCharacter = GetChar(); } *pcToken = 0; UngetChar(); return IDENTIFIER_TOKEN; }
//------------------------------------------------------------------------------ bool FBasicTokenParser::GetRawToken(FBasicToken& Token, TCHAR StopChar/* = TCHAR('\n')*/) { // if the parser is in a bad state, then don't continue parsing (who // knows what will happen!?) if (!IsValid()) { return false; } // Get token after whitespace. TCHAR Temp[MAX_STRING_CONST_SIZE]; int32 Length=0; TCHAR c = GetLeadingChar(); while( !IsEOL(c) && c != StopChar ) { if( (c=='/' && PeekChar()=='/') || (c=='/' && PeekChar()=='*') ) { break; } Temp[Length++] = c; if( Length >= MAX_STRING_CONST_SIZE ) { Temp[Length] = 0; FText ErrorDesc = FText::Format(LOCTEXT("IdTooLong", "Identifer ({0}...) exceeds maximum length of {1}"), FText::FromString(Temp), FText::AsNumber((int32)MAX_STRING_CONST_SIZE)); SetError(FErrorState::ParseError, ErrorDesc); } c = GetChar(true); } UngetChar(); // Get rid of trailing whitespace. while( Length>0 && (Temp[Length-1]==' ' || Temp[Length-1]==9 ) ) { Length--; } Temp[Length]=0; Token.SetConstString(Temp); return Length>0; }
char GetChar( void) /******************************************************************************/ /* get a char from the input file. Return a put back char if available. */ /******************************************************************************/ { char cReturnValue; if (bPutBackCharValid == TRUE) { /* "re-read" the ungetted char. Correct the linecounter if nessecary. */ cReturnValue = cPutBackChar; bPutBackCharValid = FALSE; if (((cPutBackChar == LF) && (bLfCounting == TRUE)) || ((cPutBackChar == CR) && (bCrCounting == TRUE))) { iCurrentLineNumber++; } } else { cReturnValue = GetNewChar(); } if ((cReturnValue == CR) || (cReturnValue == LF)) { if (bBlockCrLf) { UngetChar(); cReturnValue = ILLEGAL_CHAR; } } return cReturnValue; }
UINT CLiteHTMLReader::parseDocument(void) { ASSERT(m_lpszBuffer != NULL); bool bAbort = false; // continue parsing or abort? bool bIsClosingTag = false; // tag parsed is a closing tag? bool bIsOpeningTag = false; // tag parsed is an opening tag? CString strCharacters; // character data CString strComment; // comment data CString strT; // temporary storage DWORD dwCharDataStart = 0L; // starting position of character data DWORD dwCharDataLen = 0L; // length of character data LONG lTemp = 0L; // temporary storage TCHAR ch = 0; // character at current buffer position CLiteHTMLTag oTag; // tag information if ( (!m_lpszBuffer) || (!m_dwBufLen) ) return (0U); // reset seek pointer to beginning ResetSeekPointer(); // notify event handler about parsing startup if (getEventNotify(notifyStartStop)) { bAbort = false; m_pEventHandler->BeginParse(m_dwAppData, bAbort); if (bAbort) goto LEndParse; } // skip leading white-space characters while (isWhiteSpace(ReadChar())) ; ch = UngetChar(); while ((ch = ReadChar()) != NULL) { switch (ch) { // tag starting delimeter? case _T('<'): { UngetChar(); strComment.Empty(); if (!parseComment(strComment)) { bIsOpeningTag = false; bIsClosingTag = false; if (!parseTag(oTag, bIsOpeningTag, bIsClosingTag)) { ++dwCharDataLen; // manually advance buffer position // because the last call to UngetChar() // moved it back one character ch = ReadChar(); break; } } // clear pending notifications if ( (dwCharDataLen) || (strCharacters.GetLength()) ) { strCharacters += CString(&m_lpszBuffer[dwCharDataStart], dwCharDataLen); NormalizeCharacters(strCharacters); if ( (strCharacters.GetLength()) && (getEventNotify(notifyCharacters)) ) { bAbort = false; m_pEventHandler->Characters(strCharacters, m_dwAppData, bAbort); if (bAbort) goto LEndParse; } strCharacters.Empty(); } dwCharDataLen = 0L; dwCharDataStart = m_dwBufPos; if (strComment.GetLength()) { if (getEventNotify(notifyComment)) { bAbort = false; m_pEventHandler->Comment(strComment, m_dwAppData, bAbort); if (bAbort) goto LEndParse; } } else { if ( (bIsOpeningTag) && (getEventNotify(notifyTagStart)) ) { bAbort = false; m_pEventHandler->StartTag(&oTag, m_dwAppData, bAbort); if (bAbort) goto LEndParse; } if ( (bIsClosingTag) && (getEventNotify(notifyTagEnd)) ) { bAbort = false; m_pEventHandler->EndTag(&oTag, m_dwAppData, bAbort); if (bAbort) goto LEndParse; } } break; } // entity reference beginning delimeter? case _T('&'): { UngetChar(); lTemp = 0; if (m_bResolveEntities) lTemp = CLiteHTMLEntityResolver::resolveEntity(&m_lpszBuffer[m_dwBufPos], ch); if (lTemp) { strCharacters += CString(&m_lpszBuffer[dwCharDataStart], dwCharDataLen) + ch; m_dwBufPos += lTemp; dwCharDataStart = m_dwBufPos; dwCharDataLen = 0L; } else { ch = ReadChar(); ++dwCharDataLen; } break; } // any other character default: { ++dwCharDataLen; break; } } } // clear pending notifications if ( (dwCharDataLen) || (strCharacters.GetLength()) ) { strCharacters += CString(&m_lpszBuffer[dwCharDataStart], dwCharDataLen) + ch; NormalizeCharacters(strCharacters); strCharacters.TrimRight(); // explicit trailing white-space removal if ( (strCharacters.GetLength()) && (getEventNotify(notifyCharacters)) ) { bAbort = false; m_pEventHandler->Characters(strCharacters, m_dwAppData, bAbort); if (bAbort) goto LEndParse; } } LEndParse: // notify event handler about parsing completion if (getEventNotify(notifyStartStop)) m_pEventHandler->EndParse(m_dwAppData, bAbort); m_lpszBuffer = NULL; m_dwBufLen = 0L; return (m_dwBufPos); }
JSTokenType js_GetToken(JSContext *cx, JSTokenStream *ts) { JSTokenType tt; JSToken *tp; int32 c; JSAtom *atom; JSBool hadUnicodeEscape; #define INIT_TOKENBUF(tb) ((tb)->ptr = (tb)->base) #define FINISH_TOKENBUF(tb) if (!AddToTokenBuf(cx, tb, 0)) RETURN(TOK_ERROR) #define TOKEN_LENGTH(tb) ((tb)->ptr - (tb)->base - 1) #define RETURN(tt) { if (tt == TOK_ERROR) ts->flags |= TSF_ERROR; \ tp->pos.end.index = ts->linepos + \ (ts->linebuf.ptr - ts->linebuf.base) - \ ts->ungetpos; \ return (tp->type = tt); } /* If there was a fatal error, keep returning TOK_ERROR. */ if (ts->flags & TSF_ERROR) return TOK_ERROR; /* Check for a pushed-back token resulting from mismatching lookahead. */ while (ts->lookahead != 0) { ts->lookahead--; ts->cursor = (ts->cursor + 1) & NTOKENS_MASK; tt = CURRENT_TOKEN(ts).type; if (tt != TOK_EOL || (ts->flags & TSF_NEWLINES)) return tt; } retry: do { c = GetChar(ts); if (c == '\n') { ts->flags &= ~TSF_DIRTYLINE; if (ts->flags & TSF_NEWLINES) break; } } while (JS_ISSPACE(c)); ts->cursor = (ts->cursor + 1) & NTOKENS_MASK; tp = &CURRENT_TOKEN(ts); tp->ptr = ts->linebuf.ptr - 1; tp->pos.begin.index = ts->linepos + (tp->ptr - ts->linebuf.base); tp->pos.begin.lineno = tp->pos.end.lineno = ts->lineno; if (c == EOF) RETURN(TOK_EOF); if (c != '-' && c != '\n') ts->flags |= TSF_DIRTYLINE; hadUnicodeEscape = JS_FALSE; if (JS_ISIDENT_START(c) || (c == '\\' && (c = GetUnicodeEscape(ts), hadUnicodeEscape = JS_ISIDENT_START(c)))) { INIT_TOKENBUF(&ts->tokenbuf); for (;;) { if (!AddToTokenBuf(cx, &ts->tokenbuf, (jschar)c)) RETURN(TOK_ERROR); c = GetChar(ts); if (c == '\\') { c = GetUnicodeEscape(ts); if (!JS_ISIDENT(c)) break; hadUnicodeEscape = JS_TRUE; } else { if (!JS_ISIDENT(c)) break; } } UngetChar(ts, c); FINISH_TOKENBUF(&ts->tokenbuf); atom = js_AtomizeChars(cx, ts->tokenbuf.base, TOKEN_LENGTH(&ts->tokenbuf), 0); if (!atom) RETURN(TOK_ERROR); if (!hadUnicodeEscape && ATOM_KEYWORD(atom)) { struct keyword *kw = ATOM_KEYWORD(atom); if (JSVERSION_IS_ECMA(cx->version) || kw->version <= cx->version) { tp->t_op = (JSOp) kw->op; RETURN(kw->tokentype); } } tp->t_op = JSOP_NAME; tp->t_atom = atom; RETURN(TOK_NAME); } if (JS7_ISDEC(c) || (c == '.' && JS7_ISDEC(PeekChar(ts)))) { jsint radix; const jschar *endptr; jsdouble dval; radix = 10; INIT_TOKENBUF(&ts->tokenbuf); if (c == '0') { if (!AddToTokenBuf(cx, &ts->tokenbuf, (jschar)c)) RETURN(TOK_ERROR); c = GetChar(ts); if (JS_TOLOWER(c) == 'x') { if (!AddToTokenBuf(cx, &ts->tokenbuf, (jschar)c)) RETURN(TOK_ERROR); c = GetChar(ts); radix = 16; } else if (JS7_ISDEC(c)) { radix = 8; } } while (JS7_ISHEX(c)) { if (radix < 16) { if (JS7_ISLET(c)) break; /* * We permit 08 and 09 as decimal numbers, which makes our * behaviour a superset of the ECMA numeric grammar. We might * not always be so permissive, so we warn about it. */ if (radix == 8 && c >= '8') { if (!js_ReportCompileErrorNumber(cx, ts, NULL, JSREPORT_WARNING, JSMSG_BAD_OCTAL, c == '8' ? "08" : "09")) { RETURN(TOK_ERROR); } radix = 10; } } if (!AddToTokenBuf(cx, &ts->tokenbuf, (jschar)c)) RETURN(TOK_ERROR); c = GetChar(ts); } if (radix == 10 && (c == '.' || JS_TOLOWER(c) == 'e')) { if (c == '.') { do { if (!AddToTokenBuf(cx, &ts->tokenbuf, (jschar)c)) RETURN(TOK_ERROR); c = GetChar(ts); } while (JS7_ISDEC(c)); } if (JS_TOLOWER(c) == 'e') { if (!AddToTokenBuf(cx, &ts->tokenbuf, (jschar)c)) RETURN(TOK_ERROR); c = GetChar(ts); if (c == '+' || c == '-') { if (!AddToTokenBuf(cx, &ts->tokenbuf, (jschar)c)) RETURN(TOK_ERROR); c = GetChar(ts); } if (!JS7_ISDEC(c)) { js_ReportCompileErrorNumber(cx, ts, NULL, JSREPORT_ERROR, JSMSG_MISSING_EXPONENT); RETURN(TOK_ERROR); } do { if (!AddToTokenBuf(cx, &ts->tokenbuf, (jschar)c)) RETURN(TOK_ERROR); c = GetChar(ts); } while (JS7_ISDEC(c)); } } UngetChar(ts, c); FINISH_TOKENBUF(&ts->tokenbuf); if (radix == 10) { if (!js_strtod(cx, ts->tokenbuf.base, &endptr, &dval)) { js_ReportCompileErrorNumber(cx, ts, NULL, JSREPORT_ERROR, JSMSG_OUT_OF_MEMORY); RETURN(TOK_ERROR); } } else { if (!js_strtointeger(cx, ts->tokenbuf.base, &endptr, radix, &dval)) { js_ReportCompileErrorNumber(cx, ts, NULL, JSREPORT_ERROR, JSMSG_OUT_OF_MEMORY); RETURN(TOK_ERROR); } } tp->t_dval = dval; RETURN(TOK_NUMBER); } if (c == '"' || c == '\'') { int32 val, qc = c; INIT_TOKENBUF(&ts->tokenbuf); while ((c = GetChar(ts)) != qc) { if (c == '\n' || c == EOF) { UngetChar(ts, c); js_ReportCompileErrorNumber(cx, ts, NULL, JSREPORT_ERROR, JSMSG_UNTERMINATED_STRING); RETURN(TOK_ERROR); } if (c == '\\') { switch (c = GetChar(ts)) { case 'b': c = '\b'; break; case 'f': c = '\f'; break; case 'n': c = '\n'; break; case 'r': c = '\r'; break; case 't': c = '\t'; break; case 'v': c = '\v'; break; default: if ('0' <= c && c < '8') { val = JS7_UNDEC(c); c = PeekChar(ts); if ('0' <= c && c < '8') { val = 8 * val + JS7_UNDEC(c); GetChar(ts); c = PeekChar(ts); if ('0' <= c && c < '8') { int32 save = val; val = 8 * val + JS7_UNDEC(c); if (val <= 0377) GetChar(ts); else val = save; } } c = (jschar)val; } else if (c == 'u') { jschar cp[4]; if (PeekChars(ts, 4, cp) && JS7_ISHEX(cp[0]) && JS7_ISHEX(cp[1]) && JS7_ISHEX(cp[2]) && JS7_ISHEX(cp[3])) { c = (((((JS7_UNHEX(cp[0]) << 4) + JS7_UNHEX(cp[1])) << 4) + JS7_UNHEX(cp[2])) << 4) + JS7_UNHEX(cp[3]); SkipChars(ts, 4); } } else if (c == 'x') { jschar cp[2]; if (PeekChars(ts, 2, cp) && JS7_ISHEX(cp[0]) && JS7_ISHEX(cp[1])) { c = (JS7_UNHEX(cp[0]) << 4) + JS7_UNHEX(cp[1]); SkipChars(ts, 2); } } else if (c == '\n' && JSVERSION_IS_ECMA(cx->version)) { /* ECMA follows C by removing escaped newlines. */ continue; } break; } } if (!AddToTokenBuf(cx, &ts->tokenbuf, (jschar)c)) RETURN(TOK_ERROR); } FINISH_TOKENBUF(&ts->tokenbuf); atom = js_AtomizeChars(cx, ts->tokenbuf.base, TOKEN_LENGTH(&ts->tokenbuf), 0); if (!atom) RETURN(TOK_ERROR); tp->pos.end.lineno = ts->lineno; tp->t_op = JSOP_STRING; tp->t_atom = atom; RETURN(TOK_STRING); } switch (c) { case '\n': c = TOK_EOL; break; case ';': c = TOK_SEMI; break; case '.': c = TOK_DOT; break; case '[': c = TOK_LB; break; case ']': c = TOK_RB; break; case '{': c = TOK_LC; break; case '}': c = TOK_RC; break; case '(': c = TOK_LP; break; case ')': c = TOK_RP; break; case ',': c = TOK_COMMA; break; case '?': c = TOK_HOOK; break; case ':': /* * Default so compiler can modify to JSOP_GETTER if 'p getter: v' in an * object initializer, likewise for setter. */ tp->t_op = JSOP_NOP; c = TOK_COLON; break; case '|': if (MatchChar(ts, c)) { c = TOK_OR; } else if (MatchChar(ts, '=')) { tp->t_op = JSOP_BITOR; c = TOK_ASSIGN; } else { c = TOK_BITOR; } break; case '^': if (MatchChar(ts, '=')) { tp->t_op = JSOP_BITXOR; c = TOK_ASSIGN; } else { c = TOK_BITXOR; } break; case '&': if (MatchChar(ts, c)) { c = TOK_AND; } else if (MatchChar(ts, '=')) { tp->t_op = JSOP_BITAND; c = TOK_ASSIGN; } else { c = TOK_BITAND; } break; case '=': if (MatchChar(ts, c)) { #if JS_HAS_TRIPLE_EQOPS tp->t_op = MatchChar(ts, c) ? JSOP_NEW_EQ : (JSOp)cx->jsop_eq; #else tp->t_op = cx->jsop_eq; #endif c = TOK_EQOP; } else { tp->t_op = JSOP_NOP; c = TOK_ASSIGN; } break; case '!': if (MatchChar(ts, '=')) { #if JS_HAS_TRIPLE_EQOPS tp->t_op = MatchChar(ts, '=') ? JSOP_NEW_NE : (JSOp)cx->jsop_ne; #else tp->t_op = cx->jsop_ne; #endif c = TOK_EQOP; } else { tp->t_op = JSOP_NOT; c = TOK_UNARYOP; } break; case '<': /* NB: treat HTML begin-comment as comment-till-end-of-line */ if (MatchChar(ts, '!')) { if (MatchChar(ts, '-')) { if (MatchChar(ts, '-')) goto skipline; UngetChar(ts, '-'); } UngetChar(ts, '!'); } if (MatchChar(ts, c)) { tp->t_op = JSOP_LSH; c = MatchChar(ts, '=') ? TOK_ASSIGN : TOK_SHOP; } else { tp->t_op = MatchChar(ts, '=') ? JSOP_LE : JSOP_LT; c = TOK_RELOP; } break; case '>': if (MatchChar(ts, c)) { tp->t_op = MatchChar(ts, c) ? JSOP_URSH : JSOP_RSH; c = MatchChar(ts, '=') ? TOK_ASSIGN : TOK_SHOP; } else { tp->t_op = MatchChar(ts, '=') ? JSOP_GE : JSOP_GT; c = TOK_RELOP; } break; case '*': tp->t_op = JSOP_MUL; c = MatchChar(ts, '=') ? TOK_ASSIGN : TOK_STAR; break; case '/': if (MatchChar(ts, '/')) { skipline: while ((c = GetChar(ts)) != EOF && c != '\n') /* skip to end of line */; UngetChar(ts, c); goto retry; } if (MatchChar(ts, '*')) { while ((c = GetChar(ts)) != EOF && !(c == '*' && MatchChar(ts, '/'))) { /* Ignore all characters until comment close. */ } if (c == EOF) { js_ReportCompileErrorNumber(cx, ts, NULL, JSREPORT_ERROR, JSMSG_UNTERMINATED_COMMENT); RETURN(TOK_ERROR); } goto retry; } #if JS_HAS_REGEXPS if (ts->flags & TSF_REGEXP) { JSObject *obj; uintN flags; INIT_TOKENBUF(&ts->tokenbuf); while ((c = GetChar(ts)) != '/') { if (c == '\n' || c == EOF) { UngetChar(ts, c); js_ReportCompileErrorNumber(cx, ts, NULL, JSREPORT_ERROR, JSMSG_UNTERMINATED_REGEXP); RETURN(TOK_ERROR); } if (c == '\\') { if (!AddToTokenBuf(cx, &ts->tokenbuf, (jschar)c)) RETURN(TOK_ERROR); c = GetChar(ts); } if (!AddToTokenBuf(cx, &ts->tokenbuf, (jschar)c)) RETURN(TOK_ERROR); } FINISH_TOKENBUF(&ts->tokenbuf); for (flags = 0; ; ) { if (MatchChar(ts, 'g')) flags |= JSREG_GLOB; else if (MatchChar(ts, 'i')) flags |= JSREG_FOLD; else if (MatchChar(ts, 'm')) flags |= JSREG_MULTILINE; else break; } c = PeekChar(ts); if (JS7_ISLET(c)) { tp->ptr = ts->linebuf.ptr - 1; js_ReportCompileErrorNumber(cx, ts, NULL, JSREPORT_ERROR, JSMSG_BAD_REGEXP_FLAG); (void) GetChar(ts); RETURN(TOK_ERROR); } obj = js_NewRegExpObject(cx, ts, ts->tokenbuf.base, TOKEN_LENGTH(&ts->tokenbuf), flags); if (!obj) RETURN(TOK_ERROR); atom = js_AtomizeObject(cx, obj, 0); if (!atom) RETURN(TOK_ERROR); tp->t_op = JSOP_OBJECT; tp->t_atom = atom; RETURN(TOK_OBJECT); } #endif /* JS_HAS_REGEXPS */ tp->t_op = JSOP_DIV; c = MatchChar(ts, '=') ? TOK_ASSIGN : TOK_DIVOP; break; case '%': tp->t_op = JSOP_MOD; c = MatchChar(ts, '=') ? TOK_ASSIGN : TOK_DIVOP; break; case '~': tp->t_op = JSOP_BITNOT; c = TOK_UNARYOP; break; case '+': if (MatchChar(ts, '=')) { tp->t_op = JSOP_ADD; c = TOK_ASSIGN; } else if (MatchChar(ts, c)) { c = TOK_INC; } else { tp->t_op = JSOP_POS; c = TOK_PLUS; } break; case '-': if (MatchChar(ts, '=')) { tp->t_op = JSOP_SUB; c = TOK_ASSIGN; } else if (MatchChar(ts, c)) { if (PeekChar(ts) == '>' && !(ts->flags & TSF_DIRTYLINE)) goto skipline; c = TOK_DEC; } else { tp->t_op = JSOP_NEG; c = TOK_MINUS; } ts->flags |= TSF_DIRTYLINE; break; #if JS_HAS_SHARP_VARS case '#': { uint32 n; c = GetChar(ts); if (!JS7_ISDEC(c)) { UngetChar(ts, c); goto badchar; } n = (uint32)JS7_UNDEC(c); for (;;) { c = GetChar(ts); if (!JS7_ISDEC(c)) break; n = 10 * n + JS7_UNDEC(c); if (n >= ATOM_INDEX_LIMIT) { js_ReportCompileErrorNumber(cx, ts, NULL, JSREPORT_ERROR, JSMSG_SHARPVAR_TOO_BIG); RETURN(TOK_ERROR); } } tp->t_dval = (jsdouble) n; if (JS_HAS_STRICT_OPTION(cx) && (c == '=' || c == '#')) { char buf[20]; JS_snprintf(buf, sizeof buf, "#%u%c", n, c); if (!js_ReportCompileErrorNumber(cx, ts, NULL, JSREPORT_WARNING | JSREPORT_STRICT, JSMSG_DEPRECATED_USAGE, buf)) { RETURN(TOK_ERROR); } } if (c == '=') RETURN(TOK_DEFSHARP); if (c == '#') RETURN(TOK_USESHARP); goto badchar; } badchar: #endif /* JS_HAS_SHARP_VARS */ default: js_ReportCompileErrorNumber(cx, ts, NULL, JSREPORT_ERROR, JSMSG_ILLEGAL_CHARACTER); RETURN(TOK_ERROR); } JS_ASSERT(c < TOK_LIMIT); RETURN((JSTokenType)c); #undef INIT_TOKENBUF #undef FINISH_TOKENBUF #undef TOKEN_LENGTH #undef RETURN }
/* a space or comma separated list of tag names */ Bool ParseTagNames( TidyDocImpl* doc, const TidyOptionImpl* option ) { TidyConfigImpl* cfg = &doc->config; tmbchar buf[1024]; uint i = 0, nTags = 0; uint c = SkipWhite( cfg ); UserTagType ttyp = tagtype_null; switch ( option->id ) { case TidyInlineTags: ttyp = tagtype_inline; break; case TidyBlockTags: ttyp = tagtype_block; break; case TidyEmptyTags: ttyp = tagtype_empty; break; case TidyPreTags: ttyp = tagtype_pre; break; default: ReportUnknownOption( doc, option->name ); return no; } SetOptionValue( doc, option->id, NULL ); FreeDeclaredTags( doc, ttyp ); cfg->defined_tags |= ttyp; do { if (c == ' ' || c == '\t' || c == ',') { c = AdvanceChar( cfg ); continue; } if ( c == '\r' || c == '\n' ) { uint c2 = AdvanceChar( cfg ); if ( c == '\r' && c2 == '\n' ) c = AdvanceChar( cfg ); else c = c2; if ( !IsWhite(c) ) { buf[i] = 0; UngetChar( c, cfg->cfgIn ); UngetChar( '\n', cfg->cfgIn ); break; } } /* if ( c == '\n' ) { c = AdvanceChar( cfg ); if ( !IsWhite(c) ) { buf[i] = 0; UngetChar( c, cfg->cfgIn ); UngetChar( '\n', cfg->cfgIn ); break; } } */ while ( i < sizeof(buf)-2 && c != EndOfStream && !IsWhite(c) && c != ',' ) { buf[i++] = (tmbchar) c; c = AdvanceChar( cfg ); } buf[i] = '\0'; if (i == 0) /* Skip empty tag definition. Possible when */ continue; /* there is a trailing space on the line. */ /* add tag to dictionary */ DeclareUserTag( doc, option->id, ttyp, buf ); i = 0; ++nTags; } while ( c != EndOfStream ); if ( i > 0 ) DeclareUserTag( doc, option->id, ttyp, buf ); return ( nTags > 0 ); }
// Gets the next token from the input stream, advancing the variables which keep track of the current input position and line. bool FBaseParser::GetToken( FToken& Token, bool bNoConsts/*=false*/, ESymbolParseOption bParseTemplateClosingBracket/*=ESymbolParseOption::Normal*/ ) { Token.TokenName = NAME_None; TCHAR c = GetLeadingChar(); TCHAR p = PeekChar(); if( c == 0 ) { UngetChar(); return 0; } Token.StartPos = PrevPos; Token.StartLine = PrevLine; if( (c>='A' && c<='Z') || (c>='a' && c<='z') || (c=='_') ) { // Alphanumeric token. int32 Length=0; do { Token.Identifier[Length++] = c; if( Length >= NAME_SIZE ) { FError::Throwf(TEXT("Identifer length exceeds maximum of %i"), (int32)NAME_SIZE); Length = ((int32)NAME_SIZE) - 1; break; } c = GetChar(); } while( ((c>='A')&&(c<='Z')) || ((c>='a')&&(c<='z')) || ((c>='0')&&(c<='9')) || (c=='_') ); UngetChar(); Token.Identifier[Length]=0; // Assume this is an identifier unless we find otherwise. Token.TokenType = TOKEN_Identifier; // Lookup the token's global name. Token.TokenName = FName( Token.Identifier, FNAME_Find, true ); // If const values are allowed, determine whether the identifier represents a constant if ( !bNoConsts ) { // See if the identifier is part of a vector, rotation or other struct constant. // boolean true/false if( Token.Matches(TEXT("true")) ) { Token.SetConstBool(true); return true; } else if( Token.Matches(TEXT("false")) ) { Token.SetConstBool(false); return true; } } return true; } // if const values are allowed, determine whether the non-identifier token represents a const else if ( !bNoConsts && ((c>='0' && c<='9') || ((c=='+' || c=='-') && (p>='0' && p<='9'))) ) { // Integer or floating point constant. bool bIsFloat = 0; int32 Length = 0; bool bIsHex = 0; do { if( c==TEXT('.') ) { bIsFloat = true; } if( c==TEXT('X') || c == TEXT('x') ) { bIsHex = true; } Token.Identifier[Length++] = c; if( Length >= NAME_SIZE ) { FError::Throwf(TEXT("Number length exceeds maximum of %i "), (int32)NAME_SIZE ); Length = ((int32)NAME_SIZE) - 1; break; } c = FChar::ToUpper(GetChar()); } while ((c >= TEXT('0') && c <= TEXT('9')) || (!bIsFloat && c == TEXT('.')) || (!bIsHex && c == TEXT('X')) || (bIsHex && c >= TEXT('A') && c <= TEXT('F'))); Token.Identifier[Length]=0; if (!bIsFloat || c != 'F') { UngetChar(); } if (bIsFloat) { Token.SetConstFloat( FCString::Atof(Token.Identifier) ); } else if (bIsHex) { TCHAR* End = Token.Identifier + FCString::Strlen(Token.Identifier); Token.SetConstInt( FCString::Strtoi(Token.Identifier,&End,0) ); } else { Token.SetConstInt( FCString::Atoi(Token.Identifier) ); } return true; } else if (c == '\'') { TCHAR ActualCharLiteral = GetChar(/*bLiteral=*/ true); if (ActualCharLiteral == '\\') { ActualCharLiteral = GetChar(/*bLiteral=*/ true); switch (ActualCharLiteral) { case TCHAR('t'): ActualCharLiteral = '\t'; break; case TCHAR('n'): ActualCharLiteral = '\n'; break; case TCHAR('r'): ActualCharLiteral = '\r'; break; } } c = GetChar(/*bLiteral=*/ true); if (c != '\'') { FError::Throwf(TEXT("Unterminated character constant")); UngetChar(); } Token.SetConstChar(ActualCharLiteral); return true; } else if (c == '"') { // String constant. TCHAR Temp[MAX_STRING_CONST_SIZE]; int32 Length=0; c = GetChar(/*bLiteral=*/ true); while( (c!='"') && !IsEOL(c) ) { if( c=='\\' ) { c = GetChar(/*bLiteral=*/ true); if( IsEOL(c) ) { break; } else if(c == 'n') { // Newline escape sequence. c = '\n'; } } Temp[Length++] = c; if( Length >= MAX_STRING_CONST_SIZE ) { FError::Throwf(TEXT("String constant exceeds maximum of %i characters"), (int32)MAX_STRING_CONST_SIZE ); c = TEXT('\"'); Length = ((int32)MAX_STRING_CONST_SIZE) - 1; break; } c = GetChar(/*bLiteral=*/ true); } Temp[Length]=0; if( c != '"' ) { FError::Throwf(TEXT("Unterminated string constant: %s"), Temp); UngetChar(); } Token.SetConstString(Temp); return true; } else { // Symbol. int32 Length=0; Token.Identifier[Length++] = c; // Handle special 2-character symbols. #define PAIR(cc,dd) ((c==cc)&&(d==dd)) /* Comparison macro for convenience */ TCHAR d = GetChar(); if ( PAIR('<','<') || (PAIR('>','>') && (bParseTemplateClosingBracket != ESymbolParseOption::CloseTemplateBracket)) || PAIR('!','=') || PAIR('<','=') || PAIR('>','=') || PAIR('+','+') || PAIR('-','-') || PAIR('+','=') || PAIR('-','=') || PAIR('*','=') || PAIR('/','=') || PAIR('&','&') || PAIR('|','|') || PAIR('^','^') || PAIR('=','=') || PAIR('*','*') || PAIR('~','=') || PAIR(':',':') ) { Token.Identifier[Length++] = d; if( c=='>' && d=='>' ) { if( GetChar()=='>' ) Token.Identifier[Length++] = '>'; else UngetChar(); } } else UngetChar(); #undef PAIR Token.Identifier[Length] = 0; Token.TokenType = TOKEN_Symbol; // Lookup the token's global name. Token.TokenName = FName( Token.Identifier, FNAME_Find, true ); return true; } }
// // Skip past all spaces and tabs in the input stream. // TCHAR FBaseParser::GetLeadingChar() { TCHAR TrailingCommentNewline = 0; for (;;) { bool MultipleNewlines = false; TCHAR c; // Skip blanks. do { c = GetChar(); // Check if we've encountered another newline since the last one if (c == TrailingCommentNewline) { MultipleNewlines = true; } } while (IsWhitespace(c)); if (c != TEXT('/') || PeekChar() != TEXT('/')) { return c; } // Clear the comment if we've encountered newlines since the last comment if (MultipleNewlines) { ClearComment(); } // Record the first slash. The first iteration of the loop will get the second slash. PrevComment += c; do { c = GetChar(true); if (c == 0) return c; PrevComment += c; } while (!IsEOL(c)); TrailingCommentNewline = c; for (;;) { c = GetChar(); if (c == 0) return c; if (c == TrailingCommentNewline || !IsEOL(c)) { UngetChar(); break; } PrevComment += c; } } }
//------------------------------------------------------------------------------ bool FBasicTokenParser::GetToken(FBasicToken& Token, bool bNoConsts/* = false*/) { // if the parser is in a bad state, then don't continue parsing (who // knows what will happen!?) if (!IsValid()) { return false; } Token.TokenName = NAME_None; TCHAR c = GetLeadingChar(); TCHAR p = PeekChar(); if( c == 0 ) { UngetChar(); return 0; } Token.StartPos = PrevPos; Token.StartLine = PrevLine; if( (c>='A' && c<='Z') || (c>='a' && c<='z') || (c=='_') ) { // Alphanumeric token. int32 Length=0; do { Token.Identifier[Length++] = c; if( Length >= NAME_SIZE ) { Length = ((int32)NAME_SIZE) - 1; Token.Identifier[Length]=0; // need this for the error description FText ErrorDesc = FText::Format(LOCTEXT("IdTooLong", "Identifer ({0}...) exceeds maximum length of {1}"), FText::FromString(Token.Identifier), FText::AsNumber((int32)NAME_SIZE)); SetError(FErrorState::ParseError, ErrorDesc); break; } c = GetChar(); } while( ((c>='A')&&(c<='Z')) || ((c>='a')&&(c<='z')) || ((c>='0')&&(c<='9')) || (c=='_') ); UngetChar(); Token.Identifier[Length]=0; // Assume this is an identifier unless we find otherwise. Token.TokenType = FBasicToken::TOKEN_Identifier; // Lookup the token's global name. Token.TokenName = FName( Token.Identifier, FNAME_Find, true ); // If const values are allowed, determine whether the identifier represents a constant if ( !bNoConsts ) { // See if the identifier is part of a vector, rotation or other struct constant. // boolean true/false if( Token.Matches(TEXT("true")) ) { Token.SetConstBool(true); return true; } else if( Token.Matches(TEXT("false")) ) { Token.SetConstBool(false); return true; } } return IsValid(); } // if const values are allowed, determine whether the non-identifier token represents a const else if ( !bNoConsts && ((c>='0' && c<='9') || ((c=='+' || c=='-') && (p>='0' && p<='9'))) ) { // Integer or floating point constant. bool bIsFloat = 0; int32 Length = 0; bool bIsHex = 0; do { if( c==TEXT('.') ) { bIsFloat = true; } if( c==TEXT('X') || c == TEXT('x') ) { bIsHex = true; } Token.Identifier[Length++] = c; if( Length >= NAME_SIZE ) { Length = ((int32)NAME_SIZE) - 1; Token.Identifier[Length]=0; // need this for the error description FText ErrorDesc = FText::Format(LOCTEXT("IdTooLong", "Identifer ({0}...) exceeds maximum length of {1}"), FText::FromString(Token.Identifier), FText::AsNumber((int32)NAME_SIZE)); SetError(FErrorState::ParseError, ErrorDesc); break; } c = FChar::ToUpper(GetChar()); } while ((c >= TEXT('0') && c <= TEXT('9')) || (!bIsFloat && c == TEXT('.')) || (!bIsHex && c == TEXT('X')) || (bIsHex && c >= TEXT('A') && c <= TEXT('F'))); Token.Identifier[Length]=0; if (!bIsFloat || c != 'F') { UngetChar(); } if (bIsFloat) { Token.SetConstFloat( FCString::Atof(Token.Identifier) ); } else if (bIsHex) { TCHAR* End = Token.Identifier + FCString::Strlen(Token.Identifier); Token.SetConstInt( FCString::Strtoi(Token.Identifier,&End,0) ); } else { Token.SetConstInt( FCString::Atoi(Token.Identifier) ); } return IsValid(); } else if( c=='"' ) { // String constant. TCHAR Temp[MAX_STRING_CONST_SIZE]; int32 Length=0; c = GetChar(1); while( (c!='"') && !IsEOL(c) ) { if( c=='\\' ) { c = GetChar(1); if( IsEOL(c) ) { break; } else if(c == 'n') { // Newline escape sequence. c = '\n'; } } Temp[Length++] = c; if( Length >= MAX_STRING_CONST_SIZE ) { Length = ((int32)MAX_STRING_CONST_SIZE) - 1; Temp[Length]=0; // need this for the error description FText ErrorDesc = FText::Format(LOCTEXT("StringConstTooLong", "String constant ({0}...) exceeds maximum of {1} characters"), FText::FromString(Temp), FText::AsNumber((int32)MAX_STRING_CONST_SIZE)); SetError(FErrorState::ParseError, ErrorDesc); c = TEXT('\"'); break; } c = GetChar(1); } Temp[Length]=0; if( c != '"' ) { FText ErrorDesc = FText::Format(LOCTEXT("NoClosingQuote", "Unterminated quoted string ({0})"), FText::FromString(Temp)); SetError(FErrorState::ParseError, ErrorDesc); UngetChar(); } Token.SetConstString(Temp); return IsValid(); } else { // Symbol. int32 Length=0; Token.Identifier[Length++] = c; // Handle special 2-character symbols. #define PAIR(cc,dd) ((c==cc)&&(d==dd)) /* Comparison macro for convenience */ TCHAR d = GetChar(); if ( PAIR('<','<') || PAIR('>','>') || PAIR('!','=') || PAIR('<','=') || PAIR('>','=') || PAIR('+','+') || PAIR('-','-') || PAIR('+','=') || PAIR('-','=') || PAIR('*','=') || PAIR('/','=') || PAIR('&','&') || PAIR('|','|') || PAIR('^','^') || PAIR('=','=') || PAIR('*','*') || PAIR('~','=') || PAIR(':',':') ) { Token.Identifier[Length++] = d; if( c=='>' && d=='>' ) { if( GetChar()=='>' ) Token.Identifier[Length++] = '>'; else UngetChar(); } } else UngetChar(); #undef PAIR Token.Identifier[Length] = 0; Token.TokenType = FBasicToken::TOKEN_Symbol; // Lookup the token's global name. Token.TokenName = FName( Token.Identifier, FNAME_Find, true ); return true; } }
int GetNewToken( char *pcToken) /******************************************************************************/ /* get a new token from the input file. Don't check for a put back token */ /* characters are classified as: */ /* cr :== carriage return character */ /* lf :== line feed character */ /* tab :== the tab character */ /* dquote :== the " character */ /* digit :== 0123456789 */ /* bindigit :== 01 */ /* hexdigit :== 0123456789ABCDEF */ /* alpha :== _abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ */ /* any :== all prtble chars except cr, lf and dqoute */ /* tab :== the tab character */ /* alphanum :== alpha | digit */ /* */ /* <x> :== zero or more x */ /* [x] :== one or more x */ /* x :== exactly one x */ /* */ /* recognized tokens are : */ /* */ /* FLOAT_TOKEN :== <digit> '.' [digit] */ /* DEC_NUMBER_TOKEN :== [digit] */ /* HEX_NUMBER_TOKEN :== '0' 'x' [hexdigit] */ /* BIN_NUMBER_TOKEN :== '0' 'b' [bindigit] */ /* IDENTIFIER_TOKEN :== alpha <alphanum> */ /* STRING_TOKEN :== dquote <any> dquote */ /* OPERATOR_TOKEN :== '=' */ /* :== '=' '=' */ /* :== '<' */ /* :== '>' */ /* :== '<' '=' */ /* :== '>' '=' */ /* :== '<' '>' */ /* :== '+' */ /* :== '-' */ /* :== '*' */ /* :== '/' */ /* :== '<' '<' */ /* :== '>' '>' */ /* :== '&' */ /* :== '|' */ /* :== '[' ']' */ /* :== '[' */ /* :== ']' */ /* DOT_TOKEN :== '.' */ /* DOTDOT_TOKEN :== '.' '.' */ /* COMMA_TOKEN :== ',' */ /* SEMICOLON_TOKEN :== ';' */ /* COLON_TOKEN :== ':' */ /* LEFT_ARROW_TOKEN :== '<' '-' */ /* RIGHT_ARROW_TOKEN :== '-' '>' */ /* OPEN_PARENTHESE_TOKEN :== '(' */ /* CLOSE_PARENTHESE_TOKEN :== ')' */ /* PRECOMPILER_TOKEN :== '#' [alpha] */ /* DELIMITER_TOKEN :== ' ' */ /* :== tab */ /* :== cr */ /* :== lf */ /******************************************************************************/ { char cCharacter; int iResult; int iCurrentLine; boolean bEoLn; char *pcTokenWalker; pcTokenWalker = pcToken; cCharacter = GetChar(); /* first skip all delimiter chars */ while (chrpos((char) cCharacter, DELIMITER_CHARS) != NULL) { cCharacter = GetChar(); } if (isDigit(cCharacter)) { /* read a DEC_NUMBER_TOKEN */ /* or read a HEX_NUMBER_TOKEN */ /* or read a BIN_NUMBER_TOKEN */ iResult = GetNumberToken(cCharacter, pcToken); } else if (isAlpha(cCharacter)) { /* read an IDENTIFIER_TOKEN */ iResult = GetIdentifierToken(cCharacter, pcToken); } else if (cCharacter == '"') { /* read a STRING_TOKEN */ iResult = GetStringToken(cCharacter, pcToken); } else if (chrpos(cCharacter, "=<>+-*/&|[].") != NULL) { /* read an OPERATOR_TOKEN */ iResult = GetOperatorToken(cCharacter, pcToken); if (strcmp((char *) pcToken, "->") == 0) iResult = RIGHT_ARROW_TOKEN; else if (strcmp((char *) pcToken, "<-") == 0) iResult = LEFT_ARROW_TOKEN; else if (strcmp((char *) pcToken, "..") == 0) iResult = DOTDOT_TOKEN; else if (strcmp((char *) pcToken, "//") == 0) { iCurrentLine = CurrentLine(); /* bEoLn (EndOfLine) will be set to true if end of line is reached */ /* undetected because of end of file. (This will also happen in the */ /* unlikely situation that an inputfile contains something like: */ /* #define COMMENT //this is comment */ /* because while parsing a preprocessor macro, the scanner refuses to */ /* go to the next line. */ bEoLn = FALSE; while ((iCurrentLine == CurrentLine()) && (!bEoLn)) { cCharacter = GetChar(); if ((cCharacter != ILLEGAL_CHAR) && (cCharacter != '\0')) { *(pcToken++) = cCharacter; } else { bEoLn = TRUE; } } iResult = COMMENT_TOKEN; if (!bEoLn) { /* End of comment detected by reading a Cr/Lf character. This */ /* read character is added to pcToken. */ /* Put cr/lf back, only to adjust the line counter... */ /* (only needed if not in the middle of a macro.) */ UngetChar(); /* Delete cr or lf character from pcToken by overwriting it with */ /* the zero character (for terminating pcToken) */ *(--pcToken) = 0; } else { /* Terminate pcToken by adding a zero character */ *(pcToken) = 0; } } } else if (cCharacter == '#') { /* read a PRECOMPILER_TOKEN */ iResult = GetPrecompilerToken(cCharacter, pcToken); } else if (cCharacter == '{') { iResult = OPEN_BRACKETS_TOKEN; (*pcTokenWalker++) = cCharacter; *pcTokenWalker = 0; } else if (cCharacter == '}') { iResult = CLOSE_BRACKETS_TOKEN; (*pcTokenWalker++) = cCharacter; *pcTokenWalker = 0; } else if (cCharacter == '(') { iResult = OPEN_PARENTHESES_TOKEN; (*pcTokenWalker++) = cCharacter; *pcTokenWalker = 0; } else if (cCharacter == ')') { iResult = CLOSE_PARENTHESES_TOKEN; (*pcTokenWalker++) = cCharacter; *pcTokenWalker = 0; } else if (cCharacter == '.') { iResult = DOT_TOKEN; (*pcTokenWalker++) = cCharacter; cCharacter = GetChar(); while (isDigit(cCharacter)) { iResult = FLOAT_TOKEN; (*pcTokenWalker++) = cCharacter; cCharacter = GetChar(); } UngetChar(); *pcTokenWalker = 0; } else if (cCharacter == ',') { iResult = COMMA_TOKEN; (*pcTokenWalker++) = cCharacter; *pcTokenWalker = 0; } else if (cCharacter == ';') { iResult = SEMICOLON_TOKEN; (*pcTokenWalker++) = cCharacter; *pcTokenWalker = 0; } else if (cCharacter == ':') { iResult = COLON_TOKEN; (*pcTokenWalker++) = cCharacter; *pcTokenWalker = 0; } else if (cCharacter == '~') { iResult = TILDE_TOKEN; (*pcTokenWalker++) = cCharacter; *pcTokenWalker = 0; } else if (cCharacter == '\\') { iResult = BACKSLASH_TOKEN; (*pcTokenWalker++) = cCharacter; *pcTokenWalker = 0; } else if (cCharacter == 0) { /* End of file reached. */ iResult = END_OF_FILE; (*pcTokenWalker++) = cCharacter; } else { /* unexpected character read */ iResult = UNEXPECTED_CHAR_READ; (*pcTokenWalker++) = cCharacter; *pcTokenWalker = 0; } return iResult; }
//------------------------------------------------------------------------------ TCHAR FBasicTokenParser::GetLeadingChar() { // if the parser is in a bad state, then don't continue parsing (who // knows what will happen!?)... return a char signaling the end-of-stream if (!IsValid()) { return 0; } TCHAR TrailingCommentNewline = 0; for (;;) { bool MultipleNewlines = false; TCHAR c; // Skip blanks. do { c = GetChar(); // Check if we've encountered another newline since the last one if (c == TrailingCommentNewline) { MultipleNewlines = true; } } while (IsWhitespace(c)); if (c != TEXT('/') || PeekChar() != TEXT('/')) { return c; } // Clear the comment if we've encountered newlines since the last comment if (MultipleNewlines) { ClearCachedComment(); } // Record the first slash. The first iteration of the loop will get the second slash. PrevComment += c; do { c = GetChar(true); if (c == 0) return c; PrevComment += c; } while (!IsEOL(c)); TrailingCommentNewline = c; for (;;) { c = GetChar(); if (c == 0) return c; if (c == TrailingCommentNewline || !IsEOL(c)) { UngetChar(); break; } PrevComment += c; } } }
int getLexBuf(struct lexBuf *lb) { register int ch; struct charBuf *cb; register char *ccc, *cfin; register char *cc, *dd, *lmax, *lexStartDd; unsigned chval=0; int rlex; int cline,clo; /* current line, current line offset (for collumn)*/ char *clb; /* current line begin */ char oldCh; int line,size,cfile,lexStartCol, lexStartFilePos, column, lexemlen; S_position lexPos; /* first test whether the input is cached */ if (s_cache.activeCache && inStacki==0 && macStacki==0) { cacheInput(); s_cache.lexcc = lb->a; } lmax = lb->a + LEX_BUFF_SIZE - MAX_LEXEM_SIZE; for(dd=lb->a,cc=lb->cc; cc<lb->fin; cc++,dd++) *dd = *cc; lb->cc = lb->a; cb = &lb->cb; cline = cb->lineNum; clb = cb->lineBegin; clo = cb->collumnOffset; ccc = cb->cc; cfin = cb->fin; cfile = cb->fileNumber; GetChar(ch,ccc,cfin,cb,clb,clo); contin: DeleteBlank(ch,ccc,cfin,cb,clb,clo); if (dd >= lmax) { UngetChar(ch,ccc,cfin,cb); goto finish; } NOTE_NEW_LEXEM_POSITION(ch,ccc,cfin,cb,lb,dd,cfile,cline,clb,clo); /* yytext = ccc; */ lexStartDd = dd; lexStartCol = COLUMN_POS(ccc,clb,clo); if (ch == '_' || isalpha(ch) || (ch=='$' && (LANGUAGE(LAN_YACC)||LANGUAGE(LAN_JAVA)))) { ProcessIdentifier(ch, ccc, cfin, cb, dd, cfile, cline, clb, clo, lab2); goto nextLexem; } else if (isdigit(ch)) { /* *************** number ******************************* */ register long unsigned val=0; lexStartFilePos = ABS_FILE_POS(cb,cfin,ccc); if (ch=='0') { GetChar(ch,ccc,cfin,cb,clb,clo); if (ch=='x' || ch=='X') { /* hexa */ GetChar(ch,ccc,cfin,cb,clb,clo); while (isdigit(ch)||(ch>='a'&&ch<='f')||(ch>='A'&&ch<='F')) { if (ch>='a') val = val*16+ch-'a'+10; else if (ch>='A') val = val*16+ch-'A'+10; else val = val*16+ch-'0'; GetChar(ch,ccc,cfin,cb,clb,clo); } } else { /* octal */ while (isdigit(ch) && ch<='8') { val = val*8+ch-'0'; GetChar(ch,ccc,cfin,cb,clb,clo); } } } else { /* decimal */ while (isdigit(ch)) { val = val*10+ch-'0'; GetChar(ch,ccc,cfin,cb,clb,clo); } } if (ch == '.' || ch=='e' || ch=='E' || ((ch=='d' || ch=='D'|| ch=='f' || ch=='F') && LANGUAGE(LAN_JAVA))) { /* floating point */ fpConstFin(ch,ccc,cfin,cb,clb,clo,rlex); PutLexToken(rlex,dd); PutLexPosition(cfile, cline, lexStartCol, dd); PutLexInt(ABS_FILE_POS(cb,cfin,ccc)-lexStartFilePos, dd); goto nextLexem; } /* integer */ conType(val,ch,ccc,cfin,cb,clb,clo, rlex); PutLexToken(rlex,dd); PutLexInt(val,dd); PutLexPosition(cfile, cline, lexStartCol, dd); PutLexInt(ABS_FILE_POS(cb,cfin,ccc)-lexStartFilePos, dd); goto nextLexem; } else switch (ch) { /* ************ special character ********************* */ case '.': lexStartFilePos = ABS_FILE_POS(cb,cfin,ccc); GetChar(ch,ccc,cfin,cb,clb,clo); if (ch == '.' && LANGUAGE(LAN_C|LAN_YACC|LAN_CCC)) { GetChar(ch,ccc,cfin,cb,clb,clo); if (ch == '.') { GetChar(ch,ccc,cfin,cb,clb,clo); PutLexToken(ELIPSIS,dd); PutLexPosition(cfile, cline, lexStartCol, dd); goto nextLexem; } else { UngetChar(ch,ccc,cfin,cb); ch = '.'; } PutLexToken('.',dd); PutLexPosition(cfile, cline, lexStartCol, dd); goto nextLexem; } else if (ch=='*' && LANGUAGE(LAN_CCC)) { GetChar(ch,ccc,cfin,cb,clb,clo); PutLexToken(POINTM_OP,dd); PutLexPosition(cfile, cline, lexStartCol, dd); goto nextLexem; } else if (isdigit(ch)) { /* floating point constant */ UngetChar(ch,ccc,cfin,cb); ch = '.'; fpConstFin(ch,ccc,cfin,cb,clb,clo,rlex); PutLexToken(rlex,dd); PutLexPosition(cfile, cline, lexStartCol, dd); PutLexInt(ABS_FILE_POS(cb,cfin,ccc)-lexStartFilePos, dd); goto nextLexem; } else { PutLexToken('.',dd); PutLexPosition(cfile, cline, lexStartCol, dd); goto nextLexem; } case '-': GetChar(ch,ccc,cfin,cb,clb,clo); if (ch=='=') { PutLexToken(SUB_ASSIGN,dd); PutLexPosition(cfile, cline, lexStartCol, dd); GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem; } else if (ch=='-') {PutLexToken(DEC_OP,dd); PutLexPosition(cfile, cline, lexStartCol, dd);GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem;} else if (ch=='>' && LANGUAGE(LAN_C|LAN_YACC|LAN_CCC)) { GetChar(ch,ccc,cfin,cb,clb,clo); if (ch=='*' && LANGUAGE(LAN_CCC)) {PutLexToken(PTRM_OP,dd); PutLexPosition(cfile, cline, lexStartCol, dd);GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem;} else {PutLexToken(PTR_OP,dd); PutLexPosition(cfile, cline, lexStartCol, dd);goto nextLexem;} } else {PutLexToken('-',dd); PutLexPosition(cfile, cline, lexStartCol, dd);goto nextLexem;} case '+': GetChar(ch,ccc,cfin,cb,clb,clo); if (ch == '=') { PutLexToken(ADD_ASSIGN,dd); PutLexPosition(cfile, cline, lexStartCol, dd);GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem;} else if (ch == '+') { PutLexToken(INC_OP,dd); PutLexPosition(cfile, cline, lexStartCol, dd);GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem; } else {PutLexToken('+',dd); PutLexPosition(cfile, cline, lexStartCol, dd);goto nextLexem;} case '>': GetChar(ch,ccc,cfin,cb,clb,clo); if (ch == '>') { GetChar(ch,ccc,cfin,cb,clb,clo); if(ch=='>' && LANGUAGE(LAN_JAVA)){ GetChar(ch,ccc,cfin,cb,clb,clo); if(ch=='='){PutLexToken(URIGHT_ASSIGN,dd); PutLexPosition(cfile, cline, lexStartCol, dd);GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem;} else {PutLexToken(URIGHT_OP,dd); PutLexPosition(cfile, cline, lexStartCol, dd);goto nextLexem;} } else if(ch=='='){PutLexToken(RIGHT_ASSIGN,dd); PutLexPosition(cfile, cline, lexStartCol, dd);GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem;} else {PutLexToken(RIGHT_OP,dd); PutLexPosition(cfile, cline, lexStartCol, dd);goto nextLexem;} } else if (ch == '='){PutLexToken(GE_OP,dd); PutLexPosition(cfile, cline, lexStartCol, dd);GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem;} else {PutLexToken('>',dd); PutLexPosition(cfile, cline, lexStartCol, dd);goto nextLexem;} case '<': GetChar(ch,ccc,cfin,cb,clb,clo); if (ch == '<') { GetChar(ch,ccc,cfin,cb,clb,clo); if(ch=='='){PutLexToken(LEFT_ASSIGN,dd); PutLexPosition(cfile, cline, lexStartCol, dd);GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem;} else {PutLexToken(LEFT_OP,dd); PutLexPosition(cfile, cline, lexStartCol, dd);goto nextLexem;} } else if (ch == '='){PutLexToken(LE_OP,dd); PutLexPosition(cfile, cline, lexStartCol, dd);GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem;} else {PutLexToken('<',dd); PutLexPosition(cfile, cline, lexStartCol, dd);goto nextLexem;} case '*': GetChar(ch,ccc,cfin,cb,clb,clo); if (ch == '='){ PutLexToken(MUL_ASSIGN,dd); PutLexPosition(cfile, cline, lexStartCol, dd);GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem; } else {PutLexToken('*',dd); PutLexPosition(cfile, cline, lexStartCol, dd);goto nextLexem;} case '%': GetChar(ch,ccc,cfin,cb,clb,clo); if (ch == '='){ PutLexToken(MOD_ASSIGN,dd); PutLexPosition(cfile, cline, lexStartCol, dd);GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem; } /*& else if (LANGUAGE(LAN_YACC) && ch == '{'){ PutLexToken(YACC_PERC_LPAR,dd); PutLexPosition(cfile, cline, lexStartCol, dd);GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem; } else if (LANGUAGE(LAN_YACC) && ch == '}'){ PutLexToken(YACC_PERC_RPAR,dd); PutLexPosition(cfile, cline, lexStartCol, dd);GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem; } &*/ else {PutLexToken('%',dd); PutLexPosition(cfile, cline, lexStartCol, dd);goto nextLexem;} case '&': GetChar(ch,ccc,cfin,cb,clb,clo); if (ch == '='){ PutLexToken(AND_ASSIGN,dd); PutLexPosition(cfile, cline, lexStartCol, dd);GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem; } else if (ch == '&'){ PutLexToken(AND_OP,dd); PutLexPosition(cfile, cline, lexStartCol, dd);GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem; } else if (ch == '*') { GetChar(ch,ccc,cfin,cb,clb,clo); if (ch == '/') { /* a program commentary, ignore */ GetChar(ch,ccc,cfin,cb,clb,clo); CommentaryEndRef(ch,ccc,cfin,cb,dd,cfile,cline,clb,clo,0); goto nextLexem; } else { UngetChar(ch,ccc,cfin,cb); ch = '*'; PutLexToken('&',dd); PutLexPosition(cfile, cline, lexStartCol, dd);goto nextLexem; }} else {PutLexToken('&',dd); PutLexPosition(cfile, cline, lexStartCol, dd);goto nextLexem;} case '^': GetChar(ch,ccc,cfin,cb,clb,clo); if (ch == '='){ PutLexToken(XOR_ASSIGN,dd); PutLexPosition(cfile, cline, lexStartCol, dd);GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem; } else {PutLexToken('^',dd); PutLexPosition(cfile, cline, lexStartCol, dd);goto nextLexem;} case '|': GetChar(ch,ccc,cfin,cb,clb,clo); if (ch == '='){ PutLexToken(OR_ASSIGN,dd); PutLexPosition(cfile, cline, lexStartCol, dd);GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem; } else if (ch == '|'){ PutLexToken(OR_OP,dd); PutLexPosition(cfile, cline, lexStartCol, dd);GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem; } else {PutLexToken('|',dd); PutLexPosition(cfile, cline, lexStartCol, dd);goto nextLexem;} case '=': GetChar(ch,ccc,cfin,cb,clb,clo); if (ch == '='){ PutLexToken(EQ_OP,dd); PutLexPosition(cfile, cline, lexStartCol, dd);GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem; } else {PutLexToken('=',dd); PutLexPosition(cfile, cline, lexStartCol, dd); goto nextLexem;} case '!': GetChar(ch,ccc,cfin,cb,clb,clo); if (ch == '='){ PutLexToken(NE_OP,dd); PutLexPosition(cfile, cline, lexStartCol, dd);GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem; } else {PutLexToken('!',dd); PutLexPosition(cfile, cline, lexStartCol, dd);goto nextLexem;} case ':': GetChar(ch,ccc,cfin,cb,clb,clo); if (ch == ':' && LANGUAGE(LAN_CCC)){ PutLexToken(DPOINT,dd); PutLexPosition(cfile, cline, lexStartCol, dd);GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem; } else {PutLexToken(':',dd); PutLexPosition(cfile, cline, lexStartCol, dd);goto nextLexem;} case '\'': chval = 0; lexStartFilePos = ABS_FILE_POS(cb,cfin,ccc); do { GetChar(ch,ccc,cfin,cb,clb,clo); while (ch=='\\') { GetChar(ch,ccc,cfin,cb,clb,clo); /* TODO escape sequences */ GetChar(ch,ccc,cfin,cb,clb,clo); } if (ch != '\'') chval = chval * 256 + ch; } while (ch != '\'' && ch != '\n'); if (ch=='\'') { PutLexToken(CHAR_LITERAL,dd); PutLexInt(chval,dd); PutLexPosition(cfile, cline, lexStartCol, dd); PutLexInt(ABS_FILE_POS(cb,cfin,ccc)-lexStartFilePos, dd); GetChar(ch,ccc,cfin,cb,clb,clo); } goto nextLexem; case '\"': line = cline; size = 0; PutLexToken(STRING_LITERAL,dd); do { GetChar(ch,ccc,cfin,cb,clb,clo); size ++; if (ch!='\"' && size<MAX_LEXEM_SIZE-10) PutLexChar(ch,dd); if (ch=='\\') { GetChar(ch,ccc,cfin,cb,clb,clo); size ++; if (size < MAX_LEXEM_SIZE-10) PutLexChar(ch,dd); /* TODO escape sequences */ if (ch == '\n') {cline ++; clb = ccc; clo = 0;} ch = 0; } if (ch == '\n') { cline ++; clb = ccc; clo = 0; if (s_opt.strictAnsi && (s_opt.debug || s_opt.err)) { warning(ERR_ST,"string constant through end of line"); } } // in Java CR LF can't be a part of string, even there // are benchmarks making Xrefactory coredump if CR or LF // is a part of strings } while (ch != '\"' && (ch != '\n' || !s_opt.strictAnsi) && ch != -1); if (ch == -1 && s_opt.taskRegime!=RegimeEditServer) { warning(ERR_ST,"string constant through EOF"); } PutLexChar(0,dd); PutLexPosition(cfile, cline, lexStartCol, dd); PutLexLine(cline-line,dd); GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem; case '/': GetChar(ch,ccc,cfin,cb,clb,clo); if (ch == '='){ PutLexToken(DIV_ASSIGN,dd); PutLexPosition(cfile, cline, lexStartCol, dd);GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem; } else if (ch=='*') { int javadoc=0; CommentaryBegRef(ch,ccc,cfin,cb,dd,cfile,cline,clb,clo); GetChar(ch,ccc,cfin,cb,clb,clo); if (ch == '&') { /* ****** a program comment, ignore */ GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem; } else { if (ch=='*' && LANGUAGE(LAN_JAVA)) javadoc = 1; UngetChar(ch,ccc,cfin,cb); ch = '*'; } /* !!! COPY BLOCK TO '/n' */ PassComment(ch,oldCh,ccc,cfin,cb,dd,cline,clb,clo); CommentaryEndRef(ch,ccc,cfin,cb,dd,cfile,cline,clb,clo,javadoc); goto nextLexem; } else if (ch=='/' && s_opt.cpp_comment) { /* ******* a // comment ******* */ CommentaryBegRef(ch,ccc,cfin,cb,dd,cfile,cline,clb,clo); GetChar(ch,ccc,cfin,cb,clb,clo); if (ch == '&') { /* ****** a program comment, ignore */ GetChar(ch,ccc,cfin,cb,clb,clo); CommentaryEndRef(ch,ccc,cfin,cb,dd,cfile,cline,clb,clo,0); goto nextLexem; } line = cline; while (ch!='\n' && ch != -1) { GetChar(ch,ccc,cfin,cb,clb,clo); if (ch == '\\') { GetChar(ch,ccc,cfin,cb,clb,clo); if (ch=='\n') {cline ++; clb = ccc; clo = 0;} GetChar(ch,ccc,cfin,cb,clb,clo); } } CommentaryEndRef(ch,ccc,cfin,cb,dd,cfile,cline,clb,clo,0); PutLexLine(cline-line,dd); goto nextLexem; } else { PutLexToken('/',dd); PutLexPosition(cfile, cline, lexStartCol, dd); goto nextLexem; } case '\\': GetChar(ch,ccc,cfin,cb,clb,clo); if (ch == '\n') { cline ++; clb = ccc; clo = 0; PutLexLine(1, dd); GetChar(ch,ccc,cfin,cb,clb,clo); } else { PutLexToken('\\',dd); PutLexPosition(cfile, cline, lexStartCol, dd); } goto nextLexem; case '\n': column = COLUMN_POS(ccc,clb,clo); if (column >= MAX_REFERENCABLE_COLUMN) { fatalError(ERR_ST, "position over MAX_REFERENCABLE_COLUMN, read TROUBLES in README file", XREF_EXIT_ERR); } if (cline >= MAX_REFERENCABLE_LINE) { fatalError(ERR_ST, "position over MAX_REFERENCABLE_LINE, read TROUBLES in README file", XREF_EXIT_ERR); } cline ++; clb = ccc; clo = 0; GetChar(ch,ccc,cfin,cb,clb,clo); DeleteBlank(ch,ccc,cfin,cb,clb,clo); if (ch == '/') { GetChar(ch,ccc,cfin,cb,clb,clo); if (ch == '*') { CommentaryBegRef(ch,ccc,cfin,cb,dd,cfile,cline,clb,clo); GetChar(ch,ccc,cfin,cb,clb,clo); if (ch == '&') { /* ****** a program comment, ignore */ GetChar(ch,ccc,cfin,cb,clb,clo); } else { int javadoc=0; if (ch == '*' && LANGUAGE(LAN_JAVA)) javadoc = 1; UngetChar(ch,ccc,cfin,cb); ch = '*'; PassComment(ch,oldCh,ccc,cfin,cb,dd,cline,clb,clo); CommentaryEndRef(ch,ccc,cfin,cb,dd,cfile,cline,clb,clo,javadoc); DeleteBlank(ch,ccc,cfin,cb,clb,clo); } } else { UngetChar(ch,ccc,cfin,cb); ch = '/'; } } PutLexToken('\n',dd); PutLexPosition(cfile, cline, lexStartCol, dd); if (ch == '#' && LANGUAGE(LAN_C|LAN_CCC|LAN_YACC)) { NOTE_NEW_LEXEM_POSITION(ch,ccc,cfin,cb,lb,dd,cfile,cline,clb,clo); HandleCppToken(ch,ccc,cfin,cb,dd,cfile,cline,clb,clo); } goto nextLexem; case '#': GetChar(ch,ccc,cfin,cb,clb,clo); if (ch == '#') { GetChar(ch,ccc,cfin,cb,clb,clo); PutLexToken(CPP_COLLATION,dd); PutLexPosition(cfile, cline, lexStartCol, dd); } else { PutLexToken('#',dd); PutLexPosition(cfile, cline, lexStartCol, dd); } goto nextLexem; case -1: /* ** probably end of file ** */ goto nextLexem; #if ZERO case '{': if (brack_deep==0) printf("/**/}"); brack_deep+=2; case '}': brack_deep--; #endif default: if (ch >= 32) { /* small chars ignored */ PutLexToken(ch,dd); PutLexPosition(cfile, cline, lexStartCol, dd); } GetChar(ch,ccc,cfin,cb,clb,clo); goto nextLexem; } assert(0); nextLexem: if (s_opt.taskRegime == RegimeEditServer) { int pi,lpi,len,lastlex,parChar,apos,idcoll; S_position *ps; int pos0,pos1,currentLexemPosition; pi = (lb->posi-1) % LEX_POSITIONS_RING_SIZE; ps = & lb->pRing[pi]; currentLexemPosition = lb->fpRing[pi]; if ( cfile == s_olOriginalFileNumber && cfile != s_noneFileIndex && cfile != -1 && s_jsl==NULL ) { if (s_opt.cxrefs == OLO_EXTRACT && lb->posi>=2) { DeleteBlank(ch,ccc,cfin,cb,clb,clo); pos1 = ABS_FILE_POS(cb, cfin, ccc); //&idcoll = COLUMN_POS(ccc,clb,clo); //&fprintf(dumpOut,":pos1==%d, olCursorPos==%d, olMarkPos==%d\n",pos1,s_opt.olCursorPos,s_opt.olMarkPos); // all this is very, very HACK!!! if (pos1 >= s_opt.olCursorPos && ! s_cps.marker1Flag) { if (LANGUAGE(LAN_JAVA)) parChar = ';'; else { if (s_cps.marker2Flag) parChar='}'; else parChar = '{'; } PutLexToken(parChar,dd); PutLexPosition(ps->file,ps->line,ps->coll,dd); PutLexToken(parChar,dd); PutLexPosition(ps->file,ps->line,ps->coll,dd); PutLexToken(';',dd); PutLexPosition(ps->file,ps->line,ps->coll,dd); PutLexToken(';',dd); PutLexPosition(ps->file,ps->line,ps->coll,dd); PutLexToken(OL_MARKER_TOKEN,dd); PutLexPosition(ps->file,ps->line,ps->coll,dd); PutLexToken(parChar,dd); PutLexPosition(ps->file,ps->line,ps->coll,dd); PutLexToken(parChar,dd); PutLexPosition(ps->file,ps->line,ps->coll,dd); s_cps.marker1Flag=1; } else if (pos1 >= s_opt.olMarkPos && ! s_cps.marker2Flag){ if (LANGUAGE(LAN_JAVA)) parChar = ';'; else { if (s_cps.marker1Flag) parChar='}'; else parChar = '{'; } PutLexToken(parChar,dd); PutLexPosition(ps->file,ps->line,ps->coll,dd); PutLexToken(parChar,dd); PutLexPosition(ps->file,ps->line,ps->coll,dd); PutLexToken(';',dd); PutLexPosition(ps->file,ps->line,ps->coll,dd); PutLexToken(';',dd); PutLexPosition(ps->file,ps->line,ps->coll,dd); PutLexToken(OL_MARKER_TOKEN,dd); PutLexPosition(ps->file,ps->line,ps->coll,dd); PutLexToken(parChar,dd); PutLexPosition(ps->file,ps->line,ps->coll,dd); PutLexToken(parChar,dd); PutLexPosition(ps->file,ps->line,ps->coll,dd); s_cps.marker2Flag=1; } } else if ( s_opt.cxrefs == OLO_COMPLETION || s_opt.cxrefs == OLO_SEARCH) { DeleteBlank(ch,ccc,cfin,cb,clb,clo); apos = ABS_FILE_POS(cb, cfin, ccc); if (currentLexemPosition < s_opt.olCursorPos && (apos >= s_opt.olCursorPos || (ch == -1 && apos+1 == s_opt.olCursorPos))) { //&sprintf(tmpBuff,"currentLexemPosition, s_opt.olCursorPos, ABS_FILE_POS, ch == %d, %d, %d, %d\n",currentLexemPosition, s_opt.olCursorPos, apos, ch);ppcGenTmpBuff(); //&fprintf(dumpOut,":check\n");fflush(dumpOut); lastlex = NextLexToken(lexStartDd); if (lastlex == IDENTIFIER) { len = s_opt.olCursorPos-currentLexemPosition; //&fprintf(dumpOut,":check %s[%d] <-> %d\n", lexStartDd+TOKEN_SIZE, len,strlen(lexStartDd+TOKEN_SIZE));fflush(dumpOut); if (len <= strlen(lexStartDd+TOKEN_SIZE)) { if (s_opt.cxrefs == OLO_SEARCH) { char *ddd; ddd = lexStartDd; PutLexToken(IDENT_TO_COMPLETE, ddd); } else { dd = lexStartDd; PutLexToken(IDENT_TO_COMPLETE, dd); dd += len; PutLexChar(0,dd); PutLexPosition(ps->file,ps->line,ps->coll,dd); } //&fprintf(dumpOut,":ress %s\n", lexStartDd+TOKEN_SIZE);fflush(dumpOut); } else { // completion after an identifier PUT_EMPTY_COMPLETION_ID(ccc,dd,cline,clo,clb, cfile, apos-s_opt.olCursorPos); } } else if ((lastlex == LINE_TOK || lastlex == STRING_LITERAL) && (apos-s_opt.olCursorPos != 0)) { // completion inside special lexems, do // NO COMPLETION } else { // completion after another lexem PUT_EMPTY_COMPLETION_ID(ccc,dd,cline,clo,clb, cfile, apos-s_opt.olCursorPos); } } // TODO, make this in a more standard way, !!! } else { //&fprintf(dumpOut,":testing %d <= %d <= %d\n", currentLexemPosition, s_opt.olCursorPos, ABS_FILE_POS(cb, cfin, ccc)); if (currentLexemPosition <= s_opt.olCursorPos && ABS_FILE_POS(cb, cfin, ccc) >= s_opt.olCursorPos) { gotOnLineCxRefs( ps); lastlex = NextLexToken(lexStartDd); if (lastlex == IDENTIFIER) { strcpy(s_olstring, lexStartDd+TOKEN_SIZE); } } if (LANGUAGE(LAN_JAVA)) { // there is a problem with this, when browsing at CPP construction // that is why I restrict it to Java language! It is usefull // only for Java refactorings DeleteBlank(ch,ccc,cfin,cb,clb,clo); apos = ABS_FILE_POS(cb, cfin, ccc); if (apos >= s_opt.olCursorPos && ! s_cps.marker1Flag) { PutLexToken(OL_MARKER_TOKEN,dd); PutLexPosition(ps->file,ps->line,ps->coll,dd); s_cps.marker1Flag=1; } } } } } if (ch != -1) goto contin; finish: cb->cc = ccc; cb->fin = cfin; cb->lineNum = cline; cb->lineBegin = clb; cb->collumnOffset = clo; lb->fin = dd; //&lexBufDump(lb); if (lb->fin == lb->a) return(0); return(1); }
// Gets the next token from the input stream, advancing the variables which keep track of the current input position and line. bool FBaseParser::GetToken( FToken& Token, bool bNoConsts/*=false*/ ) { Token.TokenName = NAME_None; TCHAR c = GetLeadingChar(); TCHAR p = PeekChar(); if( c == 0 ) { UngetChar(); return 0; } Token.StartPos = PrevPos; Token.StartLine = PrevLine; if( (c>='A' && c<='Z') || (c>='a' && c<='z') || (c=='_') ) { // Alphanumeric token. int32 Length=0; do { Token.Identifier[Length++] = c; if( Length >= NAME_SIZE ) { FError::Throwf(TEXT("Identifer length exceeds maximum of %i"), (int32)NAME_SIZE); Length = ((int32)NAME_SIZE) - 1; break; } c = GetChar(); } while( ((c>='A')&&(c<='Z')) || ((c>='a')&&(c<='z')) || ((c>='0')&&(c<='9')) || (c=='_') ); UngetChar(); Token.Identifier[Length]=0; // Assume this is an identifier unless we find otherwise. Token.TokenType = TOKEN_Identifier; // Lookup the token's global name. Token.TokenName = FName( Token.Identifier, FNAME_Find, true ); // If const values are allowed, determine whether the identifier represents a constant if ( !bNoConsts ) { // See if the identifier is part of a vector, rotation or other struct constant. // boolean true/false if( Token.Matches(TEXT("true")) ) { Token.SetConstBool(true); return true; } else if( Token.Matches(TEXT("false")) ) { Token.SetConstBool(false); return true; } } return true; } // if const values are allowed, determine whether the non-identifier token represents a const else if ( !bNoConsts && ((c>='0' && c<='9') || ((c=='+' || c=='-') && (p>='0' && p<='9'))) ) { // Integer or floating point constant. bool bIsFloat = 0; int32 Length = 0; bool bIsHex = 0; do { if( c==TEXT('.') ) { bIsFloat = true; } if( c==TEXT('X') || c == TEXT('x') ) { bIsHex = true; } Token.Identifier[Length++] = c; if( Length >= NAME_SIZE ) { FError::Throwf(TEXT("Number length exceeds maximum of %i "), (int32)NAME_SIZE ); Length = ((int32)NAME_SIZE) - 1; break; } c = FChar::ToUpper(GetChar()); } while ((c >= TEXT('0') && c <= TEXT('9')) || (!bIsFloat && c == TEXT('.')) || (!bIsHex && c == TEXT('X')) || (bIsHex && c >= TEXT('A') && c <= TEXT('F'))); Token.Identifier[Length]=0; if (!bIsFloat || c != 'F') { UngetChar(); } if (bIsFloat) { Token.SetConstFloat( FCString::Atof(Token.Identifier) ); } else if (bIsHex) { TCHAR* End = Token.Identifier + FCString::Strlen(Token.Identifier); Token.SetConstInt( FCString::Strtoi(Token.Identifier,&End,0) ); } else { Token.SetConstInt( FCString::Atoi(Token.Identifier) ); } return true; } //@TODO: 'z' is a character literal in C++, not a FName like it was in UnrealScript - do we need this code? // else if( !bNoConsts && c=='\'' ) // { // // Name constant. // int32 Length=0; // c = GetChar(); // while( (c>='A' && c<='Z') || (c>='a' && c<='z') || (c>='0' && c<='9') || (c=='_') || (c=='-') || (c==' ') ) //@FIXME: space in names should be illegal! // { // Token.Identifier[Length++] = c; // if( Length >= NAME_SIZE ) // { // FError::Throwf(TEXT("Name length exceeds maximum of %i"), (int32)NAME_SIZE ); // // trick the error a few lines down // c = TEXT('\''); // Length = ((int32)NAME_SIZE) - 1; // break; // } // c = GetChar(); // } // if( c != '\'' ) // { // UngetChar(); // FError::Throwf(TEXT("Illegal character in name") ); // } // Token.Identifier[Length]=0; // // // Make constant name. // Token.SetConstName( FName(Token.Identifier) ); // return true; // } else if( c=='"' ) { // String constant. TCHAR Temp[MAX_STRING_CONST_SIZE]; int32 Length=0; c = GetChar(1); while( (c!='"') && !IsEOL(c) ) { if( c=='\\' ) { c = GetChar(1); if( IsEOL(c) ) { break; } else if(c == 'n') { // Newline escape sequence. c = '\n'; } } Temp[Length++] = c; if( Length >= MAX_STRING_CONST_SIZE ) { FError::Throwf(TEXT("String constant exceeds maximum of %i characters"), (int32)MAX_STRING_CONST_SIZE ); c = TEXT('\"'); Length = ((int32)MAX_STRING_CONST_SIZE) - 1; break; } c = GetChar(1); } Temp[Length]=0; if( c != '"' ) { FError::Throwf(TEXT("Unterminated string constant: %s"), Temp); UngetChar(); } Token.SetConstString(Temp); return true; } else { // Symbol. int32 Length=0; Token.Identifier[Length++] = c; // Handle special 2-character symbols. #define PAIR(cc,dd) ((c==cc)&&(d==dd)) /* Comparison macro for convenience */ TCHAR d = GetChar(); if ( PAIR('<','<') || PAIR('>','>') || PAIR('!','=') || PAIR('<','=') || PAIR('>','=') || PAIR('+','+') || PAIR('-','-') || PAIR('+','=') || PAIR('-','=') || PAIR('*','=') || PAIR('/','=') || PAIR('&','&') || PAIR('|','|') || PAIR('^','^') || PAIR('=','=') || PAIR('*','*') || PAIR('~','=') || PAIR(':',':') ) { Token.Identifier[Length++] = d; if( c=='>' && d=='>' ) { if( GetChar()=='>' ) Token.Identifier[Length++] = '>'; else UngetChar(); } } else UngetChar(); #undef PAIR Token.Identifier[Length] = 0; Token.TokenType = TOKEN_Symbol; // Lookup the token's global name. Token.TokenName = FName( Token.Identifier, FNAME_Find, true ); return true; } }