/// Lex - Lex and return a token from this macro stream.
///
bool TokenLexer::Lex(Token &Tok) {
// Lexing off the end of the macro, pop this macro off the expansion stack.
if (isAtEnd()) {
// If this is a macro (not a token stream), mark the macro enabled now
// that it is no longer being expanded.
if (Macro) Macro->EnableMacro();
Tok.startToken();
Tok.setFlagValue(Token::StartOfLine , AtStartOfLine);
Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace || NextTokGetsSpace);
if (CurToken == 0)
Tok.setFlag(Token::LeadingEmptyMacro);
return PP.HandleEndOfTokenLexer(Tok);
}
SourceManager &SM = PP.getSourceManager();
// If this is the first token of the expanded result, we inherit spacing
// properties later.
bool isFirstToken = CurToken == 0;
// Get the next token to return.
Tok = Tokens[CurToken++];
bool TokenIsFromPaste = false;
// If this token is followed by a token paste (##) operator, paste the tokens!
// Note that ## is a normal token when not expanding a macro.
if (!isAtEnd() && Macro &&
(Tokens[CurToken].is(tok::hashhash) ||
// Special processing of L#x macros in -fms-compatibility mode.
// Microsoft compiler is able to form a wide string literal from
// 'L#macro_arg' construct in a function-like macro.
(PP.getLangOpts().MSVCCompat &&
isWideStringLiteralFromMacro(Tok, Tokens[CurToken])))) {
// When handling the microsoft /##/ extension, the final token is
// returned by PasteTokens, not the pasted token.
if (PasteTokens(Tok))
return true;
TokenIsFromPaste = true;
}
// The token's current location indicate where the token was lexed from. We
// need this information to compute the spelling of the token, but any
// diagnostics for the expanded token should appear as if they came from
// ExpansionLoc. Pull this information together into a new SourceLocation
// that captures all of this.
if (ExpandLocStart.isValid() && // Don't do this for token streams.
// Check that the token's location was not already set properly.
SM.isBeforeInSLocAddrSpace(Tok.getLocation(), MacroStartSLocOffset)) {
SourceLocation instLoc;
if (Tok.is(tok::comment)) {
instLoc = SM.createExpansionLoc(Tok.getLocation(),
ExpandLocStart,
ExpandLocEnd,
Tok.getLength());
} else {
instLoc = getExpansionLocForMacroDefLoc(Tok.getLocation());
}
Tok.setLocation(instLoc);
}
// If this is the first token, set the lexical properties of the token to
// match the lexical properties of the macro identifier.
if (isFirstToken) {
Tok.setFlagValue(Token::StartOfLine , AtStartOfLine);
Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace);
} else {
// If this is not the first token, we may still need to pass through
// leading whitespace if we've expanded a macro.
if (AtStartOfLine) Tok.setFlag(Token::StartOfLine);
if (HasLeadingSpace) Tok.setFlag(Token::LeadingSpace);
}
AtStartOfLine = false;
HasLeadingSpace = false;
// Handle recursive expansion!
if (!Tok.isAnnotation() && Tok.getIdentifierInfo() != nullptr) {
// Change the kind of this identifier to the appropriate token kind, e.g.
// turning "for" into a keyword.
IdentifierInfo *II = Tok.getIdentifierInfo();
Tok.setKind(II->getTokenID());
// If this identifier was poisoned and from a paste, emit an error. This
// won't be handled by Preprocessor::HandleIdentifier because this is coming
// from a macro expansion.
if (II->isPoisoned() && TokenIsFromPaste) {
PP.HandlePoisonedIdentifier(Tok);
}
if (!DisableMacroExpansion && II->isHandleIdentifierCase())
return PP.HandleIdentifier(Tok);
}
// Otherwise, return a normal token.
return true;
}
/// Lex - Lex and return a token from this macro stream.
///
void TokenLexer::Lex(Token &Tok) {
// Lexing off the end of the macro, pop this macro off the expansion stack.
if (isAtEnd()) {
// If this is a macro (not a token stream), mark the macro enabled now
// that it is no longer being expanded.
if (Macro) Macro->EnableMacro();
// Pop this context off the preprocessors lexer stack and get the next
// token. This will delete "this" so remember the PP instance var.
Preprocessor &PPCache = PP;
if (PP.HandleEndOfTokenLexer(Tok))
return;
// HandleEndOfTokenLexer may not return a token. If it doesn't, lex
// whatever is next.
return PPCache.Lex(Tok);
}
SourceManager &SM = PP.getSourceManager();
// If this is the first token of the expanded result, we inherit spacing
// properties later.
bool isFirstToken = CurToken == 0;
// Get the next token to return.
Tok = Tokens[CurToken++];
bool TokenIsFromPaste = false;
// If this token is followed by a token paste (##) operator, paste the tokens!
// Note that ## is a normal token when not expanding a macro.
if (!isAtEnd() && Tokens[CurToken].is(tok::hashhash) && Macro) {
// When handling the microsoft /##/ extension, the final token is
// returned by PasteTokens, not the pasted token.
if (PasteTokens(Tok))
return;
TokenIsFromPaste = true;
}
// The token's current location indicate where the token was lexed from. We
// need this information to compute the spelling of the token, but any
// diagnostics for the expanded token should appear as if they came from
// ExpansionLoc. Pull this information together into a new SourceLocation
// that captures all of this.
if (ExpandLocStart.isValid() && // Don't do this for token streams.
// Check that the token's location was not already set properly.
SM.isBeforeInSLocAddrSpace(Tok.getLocation(), MacroStartSLocOffset)) {
SourceLocation instLoc;
if (Tok.is(tok::comment)) {
instLoc = SM.createExpansionLoc(Tok.getLocation(),
ExpandLocStart,
ExpandLocEnd,
Tok.getLength());
} else {
instLoc = getExpansionLocForMacroDefLoc(Tok.getLocation());
}
Tok.setLocation(instLoc);
}
// If this is the first token, set the lexical properties of the token to
// match the lexical properties of the macro identifier.
if (isFirstToken) {
Tok.setFlagValue(Token::StartOfLine , AtStartOfLine);
Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace);
}
// Handle recursive expansion!
if (!Tok.isAnnotation() && Tok.getIdentifierInfo() != 0) {
// Change the kind of this identifier to the appropriate token kind, e.g.
// turning "for" into a keyword.
IdentifierInfo *II = Tok.getIdentifierInfo();
Tok.setKind(II->getTokenID());
// If this identifier was poisoned and from a paste, emit an error. This
// won't be handled by Preprocessor::HandleIdentifier because this is coming
// from a macro expansion.
if (II->isPoisoned() && TokenIsFromPaste) {
PP.HandlePoisonedIdentifier(Tok);
}
if (!DisableMacroExpansion && II->isHandleIdentifierCase())
PP.HandleIdentifier(Tok);
}
// Otherwise, return a normal token.
}
void
NasmLexer::LexIdentifier(Token* result, const char* cur_ptr, bool is_label)
{
// Match [_$#@~.?A-Za-z0-9]*, we have already matched [_?@A-Za-z]
unsigned int size;
unsigned char ch = *cur_ptr++;
while (isIdentifierBody(ch))
ch = *cur_ptr++;
--cur_ptr; // Back up over the skipped character.
// Fast path, no \ in identifier found. '\' might be an escaped newline.
if (ch != '\\')
{
FinishIdentifier:
const char* id_start = m_buf_ptr;
if (is_label)
FormTokenWithChars(result, cur_ptr, NasmToken::label);
else
FormTokenWithChars(result, cur_ptr, NasmToken::identifier);
// If we are in raw mode, return this identifier raw. There is no need
// to look up identifier information or attempt to macro expand it.
if (isLexingRawMode())
return;
// Fill in result.IdentifierInfo, looking up the identifier in the
// identifier table.
IdentifierInfo* ii = m_preproc->LookUpIdentifierInfo(result, id_start);
// Change the kind of this identifier to the appropriate token kind,
// e.g. turning "for" into a keyword.
unsigned int newtokkind = ii->getTokenKind();
if (newtokkind != Token::unknown)
result->setKind(newtokkind);
#if 0
// Finally, now that we know we have an identifier, pass this off to
// the preprocessor, which may macro expand it or something.
if (ii->isHandleIdentifierCase())
m_preproc->HandleIdentifier(result);
#endif
++num_identifier;
return;
}
// Otherwise, \ in identifier found. Enter slower path.
ch = getCharAndSize(cur_ptr, &size);
for (;;)
{
if (!isIdentifierBody(ch))
goto FinishIdentifier; // Found end of identifier.
// Otherwise, this character is good, consume it.
cur_ptr = ConsumeChar(cur_ptr, size, result);
ch = getCharAndSize(cur_ptr, &size);
while (isIdentifierBody(ch))
{
cur_ptr = ConsumeChar(cur_ptr, size, result);
ch = getCharAndSize(cur_ptr, &size);
}
}
}
