void PragmaOpenCLExtensionHandler::HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer, Token &Tok) { PP.LexUnexpandedToken(Tok); if (Tok.isNot(tok::identifier)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_identifier) << "OPENCL"; return; } IdentifierInfo *ename = Tok.getIdentifierInfo(); SourceLocation NameLoc = Tok.getLocation(); PP.Lex(Tok); if (Tok.isNot(tok::colon)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_colon) << ename; return; } PP.Lex(Tok); if (Tok.isNot(tok::identifier)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_enable_disable); return; } IdentifierInfo *op = Tok.getIdentifierInfo(); unsigned state; if (op->isStr("enable")) { state = 1; } else if (op->isStr("disable")) { state = 0; } else { PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_enable_disable); return; } SourceLocation StateLoc = Tok.getLocation(); PP.Lex(Tok); if (Tok.isNot(tok::eod)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_extra_tokens_at_eol) << "OPENCL EXTENSION"; return; } OpenCLExtData data(ename, state); Token *Toks = (Token*) PP.getPreprocessorAllocator().Allocate( sizeof(Token) * 1, llvm::alignOf<Token>()); new (Toks) Token(); Toks[0].startToken(); Toks[0].setKind(tok::annot_pragma_opencl_extension); Toks[0].setLocation(NameLoc); Toks[0].setAnnotationValue(data.getOpaqueValue()); PP.EnterTokenStream(Toks, 1, /*DisableMacroExpansion=*/true, /*OwnsTokens=*/false); if (PPCallbacks *Callbacks = PP.getPPCallbacks()) { Callbacks->PragmaOpenCLExtension(NameLoc, ename, StateLoc, state); } }
// #pragma ms_struct on // #pragma ms_struct off void PragmaMSStructHandler::HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer, Token &MSStructTok) { Sema::PragmaMSStructKind Kind = Sema::PMSST_OFF; Token Tok; PP.Lex(Tok); if (Tok.isNot(tok::identifier)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_ms_struct); return; } const IdentifierInfo *II = Tok.getIdentifierInfo(); if (II->isStr("on")) { Kind = Sema::PMSST_ON; PP.Lex(Tok); } else if (II->isStr("off") || II->isStr("reset")) PP.Lex(Tok); else { PP.Diag(Tok.getLocation(), diag::warn_pragma_ms_struct); return; } if (Tok.isNot(tok::eod)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_extra_tokens_at_eol) << "ms_struct"; return; } Actions.ActOnPragmaMSStruct(Kind); }
// #pragma weak identifier // #pragma weak identifier '=' identifier void PragmaWeakHandler::HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer, Token &WeakTok) { SourceLocation WeakLoc = WeakTok.getLocation(); Token Tok; PP.Lex(Tok); if (Tok.isNot(tok::identifier)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_identifier) << "weak"; return; } Token WeakName = Tok; bool HasAlias = false; Token AliasName; PP.Lex(Tok); if (Tok.is(tok::equal)) { HasAlias = true; PP.Lex(Tok); if (Tok.isNot(tok::identifier)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_identifier) << "weak"; return; } AliasName = Tok; PP.Lex(Tok); } if (Tok.isNot(tok::eod)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_extra_tokens_at_eol) << "weak"; return; } if (HasAlias) { Token *Toks = (Token*) PP.getPreprocessorAllocator().Allocate( sizeof(Token) * 3, llvm::alignOf<Token>()); Token &pragmaUnusedTok = Toks[0]; pragmaUnusedTok.startToken(); pragmaUnusedTok.setKind(tok::annot_pragma_weakalias); pragmaUnusedTok.setLocation(WeakLoc); Toks[1] = WeakName; Toks[2] = AliasName; PP.EnterTokenStream(Toks, 3, /*DisableMacroExpansion=*/true, /*OwnsTokens=*/false); } else { Token *Toks = (Token*) PP.getPreprocessorAllocator().Allocate( sizeof(Token) * 2, llvm::alignOf<Token>()); Token &pragmaUnusedTok = Toks[0]; pragmaUnusedTok.startToken(); pragmaUnusedTok.setKind(tok::annot_pragma_weak); pragmaUnusedTok.setLocation(WeakLoc); Toks[1] = WeakName; PP.EnterTokenStream(Toks, 2, /*DisableMacroExpansion=*/true, /*OwnsTokens=*/false); } }
// #pragma 'align' '=' {'native','natural','mac68k','power','reset'} // #pragma 'options 'align' '=' {'native','natural','mac68k','power','reset'} static void ParseAlignPragma(Sema &Actions, Preprocessor &PP, Token &FirstTok, bool IsOptions) { Token Tok; if (IsOptions) { PP.Lex(Tok); if (Tok.isNot(tok::identifier) || !Tok.getIdentifierInfo()->isStr("align")) { PP.Diag(Tok.getLocation(), diag::warn_pragma_options_expected_align); return; } } PP.Lex(Tok); if (Tok.isNot(tok::equal)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_align_expected_equal) << IsOptions; return; } PP.Lex(Tok); if (Tok.isNot(tok::identifier)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_identifier) << (IsOptions ? "options" : "align"); return; } Sema::PragmaOptionsAlignKind Kind = Sema::POAK_Natural; const IdentifierInfo *II = Tok.getIdentifierInfo(); if (II->isStr("native")) Kind = Sema::POAK_Native; else if (II->isStr("natural")) Kind = Sema::POAK_Natural; else if (II->isStr("packed")) Kind = Sema::POAK_Packed; else if (II->isStr("power")) Kind = Sema::POAK_Power; else if (II->isStr("mac68k")) Kind = Sema::POAK_Mac68k; else if (II->isStr("reset")) Kind = Sema::POAK_Reset; else { PP.Diag(Tok.getLocation(), diag::warn_pragma_align_invalid_option) << IsOptions; return; } SourceLocation KindLoc = Tok.getLocation(); PP.Lex(Tok); if (Tok.isNot(tok::eod)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_extra_tokens_at_eol) << (IsOptions ? "options" : "align"); return; } Actions.ActOnPragmaOptionsAlign(Kind, FirstTok.getLocation(), KindLoc); }
void PragmaOpenCLExtensionHandler::HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer, Token &Tok) { PP.LexUnexpandedToken(Tok); if (Tok.isNot(tok::identifier)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_identifier) << "OPENCL"; return; } IdentifierInfo *ename = Tok.getIdentifierInfo(); SourceLocation NameLoc = Tok.getLocation(); PP.Lex(Tok); if (Tok.isNot(tok::colon)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_colon) << ename; return; } PP.Lex(Tok); if (Tok.isNot(tok::identifier)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_enable_disable); return; } IdentifierInfo *op = Tok.getIdentifierInfo(); unsigned state; if (op->isStr("enable")) { state = 1; } else if (op->isStr("disable")) { state = 0; } else { PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_enable_disable); return; } OpenCLOptions &f = Actions.getOpenCLOptions(); // OpenCL 1.1 9.1: "The all variant sets the behavior for all extensions, // overriding all previously issued extension directives, but only if the // behavior is set to disable." if (state == 0 && ename->isStr("all")) { #define OPENCLEXT(nm) f.nm = 0; #include "clang/Basic/OpenCLExtensions.def" } #define OPENCLEXT(nm) else if (ename->isStr(#nm)) { f.nm = state; } #include "clang/Basic/OpenCLExtensions.def" else { PP.Diag(NameLoc, diag::warn_pragma_unknown_extension) << ename; return; } }
// #pragma GCC visibility comes in two variants: // 'push' '(' [visibility] ')' // 'pop' void PragmaGCCVisibilityHandler::HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer, Token &VisTok) { SourceLocation VisLoc = VisTok.getLocation(); Token Tok; PP.Lex(Tok); const IdentifierInfo *PushPop = Tok.getIdentifierInfo(); bool IsPush; const IdentifierInfo *VisType; if (PushPop && PushPop->isStr("pop")) { IsPush = false; VisType = 0; } else if (PushPop && PushPop->isStr("push")) { IsPush = true; PP.Lex(Tok); if (Tok.isNot(tok::l_paren)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_lparen) << "visibility"; return; } PP.Lex(Tok); VisType = Tok.getIdentifierInfo(); if (!VisType) { PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_identifier) << "visibility"; return; } PP.Lex(Tok); if (Tok.isNot(tok::r_paren)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_rparen) << "visibility"; return; } } else { PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_identifier) << "visibility"; return; } PP.Lex(Tok); if (Tok.isNot(tok::eom)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_extra_tokens_at_eol) << "visibility"; return; } Actions.ActOnPragmaVisibility(IsPush, VisType, VisLoc); }
void OMPPragmaHandler::LexUntil(Preprocessor &PP, Token &Tok, TokenKind Kind) { while (Tok.isNot(Kind) && Tok.isNot(clang::tok::eod)) { PP.Lex(Tok); } }
void clang::CacheTokens(Preprocessor &PP, llvm::raw_fd_ostream* OS) { // Get the name of the main file. const SourceManager &SrcMgr = PP.getSourceManager(); const FileEntry *MainFile = SrcMgr.getFileEntryForID(SrcMgr.getMainFileID()); SmallString<128> MainFilePath(MainFile->getName()); llvm::sys::fs::make_absolute(MainFilePath); // Create the PTHWriter. PTHWriter PW(*OS, PP); // Install the 'stat' system call listener in the FileManager. StatListener *StatCache = new StatListener(PW.getPM()); PP.getFileManager().addStatCache(StatCache, /*AtBeginning=*/true); // Lex through the entire file. This will populate SourceManager with // all of the header information. Token Tok; PP.EnterMainSourceFile(); do { PP.Lex(Tok); } while (Tok.isNot(tok::eof)); // Generate the PTH file. PP.getFileManager().removeStatCache(StatCache); PW.GeneratePTH(MainFilePath.str()); }
/// \brief Handle '#pragma omp ...' when OpenMP is enabled. /// void PragmaOpenMPHandler::HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer, Token &FirstTok) { SmallVector<Token, 16> Pragma; Token Tok; Tok.startToken(); Tok.setKind(tok::annot_pragma_openmp); Tok.setLocation(FirstTok.getLocation()); while (Tok.isNot(tok::eod)) { Pragma.push_back(Tok); PP.Lex(Tok); } SourceLocation EodLoc = Tok.getLocation(); Tok.startToken(); Tok.setKind(tok::annot_pragma_openmp_end); Tok.setLocation(EodLoc); Pragma.push_back(Tok); Token *Toks = new Token[Pragma.size()]; std::copy(Pragma.begin(), Pragma.end(), Toks); PP.EnterTokenStream(Toks, Pragma.size(), /*DisableMacroExpansion=*/true, /*OwnsTokens=*/true); }
/// InclusionRewriterInInput - Implement -frewrite-includes mode. void clang::RewriteIncludesInInput(Preprocessor &PP, raw_ostream *OS, const PreprocessorOutputOptions &Opts) { SourceManager &SM = PP.getSourceManager(); InclusionRewriter *Rewrite = new InclusionRewriter(PP, *OS, Opts.ShowLineMarkers); PP.addPPCallbacks(Rewrite); PP.IgnorePragmas(); // First let the preprocessor process the entire file and call callbacks. // Callbacks will record which #include's were actually performed. PP.EnterMainSourceFile(); Token Tok; // Only preprocessor directives matter here, so disable macro expansion // everywhere else as an optimization. // TODO: It would be even faster if the preprocessor could be switched // to a mode where it would parse only preprocessor directives and comments, // nothing else matters for parsing or processing. PP.SetMacroExpansionOnlyInDirectives(); do { PP.Lex(Tok); } while (Tok.isNot(tok::eof)); Rewrite->setPredefinesBuffer(SM.getBuffer(PP.getPredefinesFileID())); Rewrite->Process(PP.getPredefinesFileID(), SrcMgr::C_User); Rewrite->Process(SM.getMainFileID(), SrcMgr::C_User); OS->flush(); }
bool OMPPragmaHandler::handleList(Token &Tok, Preprocessor &PP, PragmaClause &C) { if (!Tok.is(clang::tok::l_paren)) return false; PP.Lex(Tok); if (C.Type == ReductionClause) { C.Op = Tok; PP.Lex(Tok); if (Tok.isNot(clang::tok::colon)) return false; PP.Lex(Tok); } else if (C.Type == IfClause || C.Type == NumThreadsClause || C.Type == DefaultClause || C.Type == ScheduleClause) { LexUntil(PP, Tok, clang::tok::r_paren); return Tok.is(clang::tok::r_paren); } while(Tok.isNot(clang::tok::eod)) { if (! (Tok.is(clang::tok::identifier))) return false; C.Options.push_back(Tok.getIdentifierInfo()); PP.Lex(Tok); if (Tok.is(clang::tok::r_paren)) { break; } if (Tok.isNot(clang::tok::comma)) return false; PP.Lex(Tok); } if (Tok.is(clang::tok::eod)) { return false; } return true; }
// #pragma redefine_extname identifier identifier void PragmaRedefineExtnameHandler::HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer, Token &RedefToken) { SourceLocation RedefLoc = RedefToken.getLocation(); Token Tok; PP.Lex(Tok); if (Tok.isNot(tok::identifier)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_identifier) << "redefine_extname"; return; } Token RedefName = Tok; PP.Lex(Tok); if (Tok.isNot(tok::identifier)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_identifier) << "redefine_extname"; return; } Token AliasName = Tok; PP.Lex(Tok); if (Tok.isNot(tok::eod)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_extra_tokens_at_eol) << "redefine_extname"; return; } Token *Toks = (Token*) PP.getPreprocessorAllocator().Allocate( sizeof(Token) * 3, llvm::alignOf<Token>()); Token &pragmaRedefTok = Toks[0]; pragmaRedefTok.startToken(); pragmaRedefTok.setKind(tok::annot_pragma_redefine_extname); pragmaRedefTok.setLocation(RedefLoc); Toks[1] = RedefName; Toks[2] = AliasName; PP.EnterTokenStream(Toks, 3, /*DisableMacroExpansion=*/true, /*OwnsTokens=*/false); }
// #pragma ms_struct on // #pragma ms_struct off void PragmaMSStructHandler::HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer, Token &MSStructTok) { Sema::PragmaMSStructKind Kind = Sema::PMSST_OFF; Token Tok; PP.Lex(Tok); if (Tok.isNot(tok::identifier)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_ms_struct); return; } const IdentifierInfo *II = Tok.getIdentifierInfo(); if (II->isStr("on")) { Kind = Sema::PMSST_ON; PP.Lex(Tok); } else if (II->isStr("off") || II->isStr("reset")) PP.Lex(Tok); else { PP.Diag(Tok.getLocation(), diag::warn_pragma_ms_struct); return; } if (Tok.isNot(tok::eod)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_extra_tokens_at_eol) << "ms_struct"; return; } Token *Toks = (Token*) PP.getPreprocessorAllocator().Allocate( sizeof(Token) * 1, llvm::alignOf<Token>()); new (Toks) Token(); Toks[0].startToken(); Toks[0].setKind(tok::annot_pragma_msstruct); Toks[0].setLocation(MSStructTok.getLocation()); Toks[0].setAnnotationValue(reinterpret_cast<void*>( static_cast<uintptr_t>(Kind))); PP.EnterTokenStream(Toks, 1, /*DisableMacroExpansion=*/true, /*OwnsTokens=*/false); }
// #pragma weak identifier // #pragma weak identifier '=' identifier void PragmaWeakHandler::HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer, Token &WeakTok) { // FIXME: Should we be expanding macros here? My guess is no. SourceLocation WeakLoc = WeakTok.getLocation(); Token Tok; PP.Lex(Tok); if (Tok.isNot(tok::identifier)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_identifier) << "weak"; return; } IdentifierInfo *WeakName = Tok.getIdentifierInfo(), *AliasName = 0; SourceLocation WeakNameLoc = Tok.getLocation(), AliasNameLoc; PP.Lex(Tok); if (Tok.is(tok::equal)) { PP.Lex(Tok); if (Tok.isNot(tok::identifier)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_identifier) << "weak"; return; } AliasName = Tok.getIdentifierInfo(); AliasNameLoc = Tok.getLocation(); PP.Lex(Tok); } if (Tok.isNot(tok::eod)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_extra_tokens_at_eol) << "weak"; return; } if (AliasName) { Actions.ActOnPragmaWeakAlias(WeakName, AliasName, WeakLoc, WeakNameLoc, AliasNameLoc); } else { Actions.ActOnPragmaWeakID(WeakName, WeakLoc, WeakNameLoc); } }
// #pragma unused(identifier) void PragmaUnusedHandler::HandlePragma(Preprocessor &PP, Token &UnusedTok) { // FIXME: Should we be expanding macros here? My guess is no. SourceLocation UnusedLoc = UnusedTok.getLocation(); // Lex the left '('. Token Tok; PP.Lex(Tok); if (Tok.isNot(tok::l_paren)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_lparen) << "unused"; return; } SourceLocation LParenLoc = Tok.getLocation(); // Lex the declaration reference(s). llvm::SmallVector<Action::ExprTy*, 5> Ex; SourceLocation RParenLoc; bool LexID = true; while (true) { PP.Lex(Tok); if (LexID) { if (Tok.is(tok::identifier)) { Action::OwningExprResult Name = Actions.ActOnIdentifierExpr(parser.CurScope, Tok.getLocation(), *Tok.getIdentifierInfo(), false); if (Name.isInvalid()) { if (!Ex.empty()) Action::MultiExprArg Release(Actions, &Ex[0], Ex.size()); return; } Ex.push_back(Name.release()); LexID = false; continue; } // Illegal token! Release the parsed expressions (if any) and emit // a warning. if (!Ex.empty()) Action::MultiExprArg Release(Actions, &Ex[0], Ex.size()); PP.Diag(Tok.getLocation(), diag::warn_pragma_unused_expected_var); return; } // We are execting a ')' or a ','. if (Tok.is(tok::comma)) { LexID = true; continue; } if (Tok.is(tok::r_paren)) { RParenLoc = Tok.getLocation(); break; } // Illegal token! Release the parsed expressions (if any) and emit // a warning. if (!Ex.empty()) Action::MultiExprArg Release(Actions, &Ex[0], Ex.size()); PP.Diag(Tok.getLocation(), diag::warn_pragma_unused_expected_punc); return; } // Verify that we have a location for the right parenthesis. assert(RParenLoc.isValid() && "Valid '#pragma unused' must have ')'"); assert(!Ex.empty() && "Valid '#pragma unused' must have arguments"); // Perform the action to handle the pragma. Actions.ActOnPragmaUnused(&Ex[0], Ex.size(), UnusedLoc, LParenLoc, RParenLoc); }
// #pragma unused(identifier) void PragmaUnusedHandler::HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer, Token &UnusedTok) { // FIXME: Should we be expanding macros here? My guess is no. SourceLocation UnusedLoc = UnusedTok.getLocation(); // Lex the left '('. Token Tok; PP.Lex(Tok); if (Tok.isNot(tok::l_paren)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_lparen) << "unused"; return; } // Lex the declaration reference(s). SmallVector<Token, 5> Identifiers; SourceLocation RParenLoc; bool LexID = true; while (true) { PP.Lex(Tok); if (LexID) { if (Tok.is(tok::identifier)) { Identifiers.push_back(Tok); LexID = false; continue; } // Illegal token! PP.Diag(Tok.getLocation(), diag::warn_pragma_unused_expected_var); return; } // We are execting a ')' or a ','. if (Tok.is(tok::comma)) { LexID = true; continue; } if (Tok.is(tok::r_paren)) { RParenLoc = Tok.getLocation(); break; } // Illegal token! PP.Diag(Tok.getLocation(), diag::warn_pragma_unused_expected_punc); return; } PP.Lex(Tok); if (Tok.isNot(tok::eod)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_extra_tokens_at_eol) << "unused"; return; } // Verify that we have a location for the right parenthesis. assert(RParenLoc.isValid() && "Valid '#pragma unused' must have ')'"); assert(!Identifiers.empty() && "Valid '#pragma unused' must have arguments"); // For each identifier token, insert into the token stream a // annot_pragma_unused token followed by the identifier token. // This allows us to cache a "#pragma unused" that occurs inside an inline // C++ member function. Token *Toks = (Token*) PP.getPreprocessorAllocator().Allocate( sizeof(Token) * 2 * Identifiers.size(), llvm::alignOf<Token>()); for (unsigned i=0; i != Identifiers.size(); i++) { Token &pragmaUnusedTok = Toks[2*i], &idTok = Toks[2*i+1]; pragmaUnusedTok.startToken(); pragmaUnusedTok.setKind(tok::annot_pragma_unused); pragmaUnusedTok.setLocation(UnusedLoc); idTok = Identifiers[i]; } PP.EnterTokenStream(Toks, 2*Identifiers.size(), /*DisableMacroExpansion=*/true, /*OwnsTokens=*/false); }
// #pragma 'align' '=' {'native','natural','mac68k','power','reset'} // #pragma 'options 'align' '=' {'native','natural','mac68k','power','reset'} static void ParseAlignPragma(Preprocessor &PP, Token &FirstTok, bool IsOptions) { Token Tok; if (IsOptions) { PP.Lex(Tok); if (Tok.isNot(tok::identifier) || !Tok.getIdentifierInfo()->isStr("align")) { PP.Diag(Tok.getLocation(), diag::warn_pragma_options_expected_align); return; } } PP.Lex(Tok); if (Tok.isNot(tok::equal)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_align_expected_equal) << IsOptions; return; } PP.Lex(Tok); if (Tok.isNot(tok::identifier)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_identifier) << (IsOptions ? "options" : "align"); return; } Sema::PragmaOptionsAlignKind Kind = Sema::POAK_Natural; const IdentifierInfo *II = Tok.getIdentifierInfo(); if (II->isStr("native")) Kind = Sema::POAK_Native; else if (II->isStr("natural")) Kind = Sema::POAK_Natural; else if (II->isStr("packed")) Kind = Sema::POAK_Packed; else if (II->isStr("power")) Kind = Sema::POAK_Power; else if (II->isStr("mac68k")) Kind = Sema::POAK_Mac68k; else if (II->isStr("reset")) Kind = Sema::POAK_Reset; else { PP.Diag(Tok.getLocation(), diag::warn_pragma_align_invalid_option) << IsOptions; return; } PP.Lex(Tok); if (Tok.isNot(tok::eod)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_extra_tokens_at_eol) << (IsOptions ? "options" : "align"); return; } Token *Toks = (Token*) PP.getPreprocessorAllocator().Allocate( sizeof(Token) * 1, llvm::alignOf<Token>()); new (Toks) Token(); Toks[0].startToken(); Toks[0].setKind(tok::annot_pragma_align); Toks[0].setLocation(FirstTok.getLocation()); Toks[0].setAnnotationValue(reinterpret_cast<void*>( static_cast<uintptr_t>(Kind))); PP.EnterTokenStream(Toks, 1, /*DisableMacroExpansion=*/true, /*OwnsTokens=*/false); }
// #pragma pack(...) comes in the following delicious flavors: // pack '(' [integer] ')' // pack '(' 'show' ')' // pack '(' ('push' | 'pop') [',' identifier] [, integer] ')' void PragmaPackHandler::HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer, Token &PackTok) { SourceLocation PackLoc = PackTok.getLocation(); Token Tok; PP.Lex(Tok); if (Tok.isNot(tok::l_paren)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_lparen) << "pack"; return; } Sema::PragmaPackKind Kind = Sema::PPK_Default; IdentifierInfo *Name = 0; Token Alignment; Alignment.startToken(); SourceLocation LParenLoc = Tok.getLocation(); PP.Lex(Tok); if (Tok.is(tok::numeric_constant)) { Alignment = Tok; PP.Lex(Tok); // In MSVC/gcc, #pragma pack(4) sets the alignment without affecting // the push/pop stack. // In Apple gcc, #pragma pack(4) is equivalent to #pragma pack(push, 4) if (PP.getLangOpts().ApplePragmaPack) Kind = Sema::PPK_Push; } else if (Tok.is(tok::identifier)) { const IdentifierInfo *II = Tok.getIdentifierInfo(); if (II->isStr("show")) { Kind = Sema::PPK_Show; PP.Lex(Tok); } else { if (II->isStr("push")) { Kind = Sema::PPK_Push; } else if (II->isStr("pop")) { Kind = Sema::PPK_Pop; } else { PP.Diag(Tok.getLocation(), diag::warn_pragma_pack_invalid_action); return; } PP.Lex(Tok); if (Tok.is(tok::comma)) { PP.Lex(Tok); if (Tok.is(tok::numeric_constant)) { Alignment = Tok; PP.Lex(Tok); } else if (Tok.is(tok::identifier)) { Name = Tok.getIdentifierInfo(); PP.Lex(Tok); if (Tok.is(tok::comma)) { PP.Lex(Tok); if (Tok.isNot(tok::numeric_constant)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_pack_malformed); return; } Alignment = Tok; PP.Lex(Tok); } } else { PP.Diag(Tok.getLocation(), diag::warn_pragma_pack_malformed); return; } } } } else if (PP.getLangOpts().ApplePragmaPack) { // In MSVC/gcc, #pragma pack() resets the alignment without affecting // the push/pop stack. // In Apple gcc #pragma pack() is equivalent to #pragma pack(pop). Kind = Sema::PPK_Pop; } if (Tok.isNot(tok::r_paren)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_rparen) << "pack"; return; } SourceLocation RParenLoc = Tok.getLocation(); PP.Lex(Tok); if (Tok.isNot(tok::eod)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_extra_tokens_at_eol) << "pack"; return; } PragmaPackInfo *Info = (PragmaPackInfo*) PP.getPreprocessorAllocator().Allocate( sizeof(PragmaPackInfo), llvm::alignOf<PragmaPackInfo>()); new (Info) PragmaPackInfo(); Info->Kind = Kind; Info->Name = Name; Info->Alignment = Alignment; Info->LParenLoc = LParenLoc; Info->RParenLoc = RParenLoc; Token *Toks = (Token*) PP.getPreprocessorAllocator().Allocate( sizeof(Token) * 1, llvm::alignOf<Token>()); new (Toks) Token(); Toks[0].startToken(); Toks[0].setKind(tok::annot_pragma_pack); Toks[0].setLocation(PackLoc); Toks[0].setAnnotationValue(static_cast<void*>(Info)); PP.EnterTokenStream(Toks, 1, /*DisableMacroExpansion=*/true, /*OwnsTokens=*/false); }
/// RewriteMacrosInInput - Implement -rewrite-macros mode. void clang::RewriteMacrosInInput(Preprocessor &PP, raw_ostream *OS) { SourceManager &SM = PP.getSourceManager(); Rewriter Rewrite; Rewrite.setSourceMgr(SM, PP.getLangOpts()); RewriteBuffer &RB = Rewrite.getEditBuffer(SM.getMainFileID()); std::vector<Token> RawTokens; LexRawTokensFromMainFile(PP, RawTokens); unsigned CurRawTok = 0; Token RawTok = GetNextRawTok(RawTokens, CurRawTok, false); // Get the first preprocessing token. PP.EnterMainSourceFile(); Token PPTok; PP.Lex(PPTok); // Preprocess the input file in parallel with raw lexing the main file. Ignore // all tokens that are preprocessed from a file other than the main file (e.g. // a header). If we see tokens that are in the preprocessed file but not the // lexed file, we have a macro expansion. If we see tokens in the lexed file // that aren't in the preprocessed view, we have macros that expand to no // tokens, or macro arguments etc. while (RawTok.isNot(tok::eof) || PPTok.isNot(tok::eof)) { SourceLocation PPLoc = SM.getExpansionLoc(PPTok.getLocation()); // If PPTok is from a different source file, ignore it. if (!SM.isFromMainFile(PPLoc)) { PP.Lex(PPTok); continue; } // If the raw file hits a preprocessor directive, they will be extra tokens // in the raw file that don't exist in the preprocsesed file. However, we // choose to preserve them in the output file and otherwise handle them // specially. if (RawTok.is(tok::hash) && RawTok.isAtStartOfLine()) { // If this is a #warning directive or #pragma mark (GNU extensions), // comment the line out. if (RawTokens[CurRawTok].is(tok::identifier)) { const IdentifierInfo *II = RawTokens[CurRawTok].getIdentifierInfo(); if (II->getName() == "warning") { // Comment out #warning. RB.InsertTextAfter(SM.getFileOffset(RawTok.getLocation()), "//"); } else if (II->getName() == "pragma" && RawTokens[CurRawTok+1].is(tok::identifier) && (RawTokens[CurRawTok+1].getIdentifierInfo()->getName() == "mark")) { // Comment out #pragma mark. RB.InsertTextAfter(SM.getFileOffset(RawTok.getLocation()), "//"); } } // Otherwise, if this is a #include or some other directive, just leave it // in the file by skipping over the line. RawTok = GetNextRawTok(RawTokens, CurRawTok, false); while (!RawTok.isAtStartOfLine() && RawTok.isNot(tok::eof)) RawTok = GetNextRawTok(RawTokens, CurRawTok, false); continue; } // Okay, both tokens are from the same file. Get their offsets from the // start of the file. unsigned PPOffs = SM.getFileOffset(PPLoc); unsigned RawOffs = SM.getFileOffset(RawTok.getLocation()); // If the offsets are the same and the token kind is the same, ignore them. if (PPOffs == RawOffs && isSameToken(RawTok, PPTok)) { RawTok = GetNextRawTok(RawTokens, CurRawTok, false); PP.Lex(PPTok); continue; } // If the PP token is farther along than the raw token, something was // deleted. Comment out the raw token. if (RawOffs <= PPOffs) { // Comment out a whole run of tokens instead of bracketing each one with // comments. Add a leading space if RawTok didn't have one. bool HasSpace = RawTok.hasLeadingSpace(); RB.InsertTextAfter(RawOffs, &" /*"[HasSpace]); unsigned EndPos; do { EndPos = RawOffs+RawTok.getLength(); RawTok = GetNextRawTok(RawTokens, CurRawTok, true); RawOffs = SM.getFileOffset(RawTok.getLocation()); if (RawTok.is(tok::comment)) { // Skip past the comment. RawTok = GetNextRawTok(RawTokens, CurRawTok, false); break; } } while (RawOffs <= PPOffs && !RawTok.isAtStartOfLine() && (PPOffs != RawOffs || !isSameToken(RawTok, PPTok))); RB.InsertTextBefore(EndPos, "*/"); continue; } // Otherwise, there was a replacement an expansion. Insert the new token // in the output buffer. Insert the whole run of new tokens at once to get // them in the right order. unsigned InsertPos = PPOffs; std::string Expansion; while (PPOffs < RawOffs) { Expansion += ' ' + PP.getSpelling(PPTok); PP.Lex(PPTok); PPLoc = SM.getExpansionLoc(PPTok.getLocation()); PPOffs = SM.getFileOffset(PPLoc); } Expansion += ' '; RB.InsertTextBefore(InsertPos, Expansion); } // Get the buffer corresponding to MainFileID. If we haven't changed it, then // we are done. if (const RewriteBuffer *RewriteBuf = Rewrite.getRewriteBufferFor(SM.getMainFileID())) { //printf("Changed:\n"); *OS << std::string(RewriteBuf->begin(), RewriteBuf->end()); } else { fprintf(stderr, "No changes\n"); } OS->flush(); }
void OMPPragmaHandler::HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer, SourceRange IntroducerRange, Token &FirstTok) { Diags.Report(IntroducerRange.getBegin(), DiagFoundPragmaStmt); // TODO: Clean this up because I'm too lazy to now PragmaDirective * DirectivePointer = new PragmaDirective; PragmaDirective &Directive = *DirectivePointer; // First lex the pragma statement extracting the variable names SourceLocation Loc = IntroducerRange.getBegin(); Token Tok = FirstTok; StringRef ident = getIdentifier(Tok); if (ident != "omp") { LexUntil(PP, Tok, clang::tok::eod); return; } PP.Lex(Tok); ident = getIdentifier(Tok); bool isParallel = false; bool isThreadPrivate = false; if (ident == "parallel") { PragmaConstruct C; C.Type = ParallelConstruct; C.Range = getTokenRange(Tok, PP); Directive.insertConstruct(C); isParallel = true; } else if (ident == "sections" || ident == "section" || ident == "task" || ident == "taskyield" || ident == "taskwait" || ident == "atomic" || ident == "ordered") { Diags.Report(Tok.getLocation(), DiagUnsupportedConstruct); LexUntil(PP, Tok, clang::tok::eod); return; } else if (ident == "for") { PragmaConstruct C; C.Type = ForConstruct; C.Range = getTokenRange(Tok, PP); Directive.insertConstruct(C); } else if (ident == "threadprivate") { isThreadPrivate = true; PragmaConstruct C; C.Type = ThreadprivateConstruct; C.Range = getTokenRange(Tok, PP); Directive.insertConstruct(C); } else if (ident == "single") { PragmaConstruct C; C.Type = SingleConstruct; C.Range = getTokenRange(Tok, PP); Directive.insertConstruct(C); } else if (ident == "master") { PragmaConstruct C; C.Type = MasterConstruct; C.Range = getTokenRange(Tok, PP); Directive.insertConstruct(C); } else if (ident == "critical" || ident == "flush") { // Ignored Directive // (Critical, Flush) LexUntil(PP, Tok, clang::tok::eod); return; } else if (ident == "barrier") { PragmaConstruct C; C.Type = BarrierConstruct; C.Range = getTokenRange(Tok, PP); Directive.insertConstruct(C); } else { Diags.Report(Tok.getLocation(), DiagUnknownDirective); return; } if (!isThreadPrivate) { PP.Lex(Tok); } if (isParallel) { ident = getIdentifier(Tok); if (ident == "sections") { Diags.Report(Tok.getLocation(), DiagUnsupportedConstruct); LexUntil(PP, Tok, clang::tok::eod); return; } else if (ident == "for") { PragmaConstruct C; C.Type = ForConstruct; C.Range = getTokenRange(Tok, PP); Directive.insertConstruct(C); PP.Lex(Tok); } else { // Just a standard "#pragma omp parallel" clause if (Tok.isNot(clang::tok::eod) && PragmaDirective::getClauseType(ident) == UnknownClause) { Diags.Report(Tok.getLocation(), DiagUnknownClause); return; } } } // If we've made it this far then we either have: // "#pragma omp parallel", // "#pragma omp parallel for", // "#pragma omp for", // "#pragma omp threadprivate // Need to read in the options, if they exists // Don't really care about them unless there exists a private(...) list // In which case, get the variables inside that list // But we read them all in anyway. // There's also threadprivate, which won't have any clauses, but will have // a list of private variables just after the threadprivate directive // Treating threadprivate as a clause and directive at the same time. while(Tok.isNot(clang::tok::eod)) { PragmaClause C; ident = getIdentifier(Tok); C.Type = PragmaDirective::getClauseType(ident); if (C.Type == UnknownClause) { Diags.Report(Tok.getLocation(), DiagUnknownClause); return; } SourceLocation clauseStart = Tok.getLocation(); SourceLocation clauseEnd = PP.getLocForEndOfToken(clauseStart); PP.Lex(Tok); if (Tok.is(clang::tok::l_paren)) { if (!handleList(Tok, PP, C)) { Diags.Report(clauseStart, DiagMalformedStatement); LexUntil(PP, Tok, clang::tok::eod); return; } clauseEnd = PP.getLocForEndOfToken(Tok.getLocation()); // Eat the clang::tok::r_paren PP.Lex(Tok); } C.Range = SourceRange(clauseStart, clauseEnd); Directive.insertClause(C); } SourceLocation EndLoc = PP.getLocForEndOfToken(Tok.getLocation()); Directive.setRange(SourceRange(Loc, EndLoc)); Directives.insert(std::make_pair(Loc.getRawEncoding(), DirectivePointer)); // Then replace with parseable compound statement to catch in Sema, and // references to private variables; // { // i; // j; // k; // } // If it's a threadprivate directive, then we skip this completely if (isThreadPrivate) { return; } set<IdentifierInfo *> PrivateVars = Directive.getPrivateIdentifiers(); int tokenCount = 2 + 2 * PrivateVars.size(); int currentToken = 0; Token * Toks = new Token[tokenCount]; Toks[currentToken++] = createToken(Loc, clang::tok::l_brace); set<IdentifierInfo *>::iterator PrivIt; for (PrivIt = PrivateVars.begin(); PrivIt != PrivateVars.end(); PrivIt++) { Toks[currentToken++] = createToken(Loc, clang::tok::identifier, *PrivIt); Toks[currentToken++] = createToken(Loc, clang::tok::semi); } Toks[currentToken++] = createToken(EndLoc, clang::tok::r_brace); assert(currentToken == tokenCount); Diags.setDiagnosticGroupMapping("unused-value", clang::diag::MAP_IGNORE, Loc); Diags.setDiagnosticGroupMapping("unused-value", clang::diag::MAP_WARNING, EndLoc); PP.EnterTokenStream(Toks, tokenCount, true, true); }
// #pragma pack(...) comes in the following delicious flavors: // pack '(' [integer] ')' // pack '(' 'show' ')' // pack '(' ('push' | 'pop') [',' identifier] [, integer] ')' void PragmaPackHandler::HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer, Token &PackTok) { SourceLocation PackLoc = PackTok.getLocation(); Token Tok; PP.Lex(Tok); if (Tok.isNot(tok::l_paren)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_lparen) << "pack"; return; } Sema::PragmaPackKind Kind = Sema::PPK_Default; IdentifierInfo *Name = 0; ExprResult Alignment; SourceLocation LParenLoc = Tok.getLocation(); PP.Lex(Tok); if (Tok.is(tok::numeric_constant)) { Alignment = Actions.ActOnNumericConstant(Tok); if (Alignment.isInvalid()) return; PP.Lex(Tok); // In MSVC/gcc, #pragma pack(4) sets the alignment without affecting // the push/pop stack. // In Apple gcc, #pragma pack(4) is equivalent to #pragma pack(push, 4) if (PP.getLangOptions().ApplePragmaPack) Kind = Sema::PPK_Push; } else if (Tok.is(tok::identifier)) { const IdentifierInfo *II = Tok.getIdentifierInfo(); if (II->isStr("show")) { Kind = Sema::PPK_Show; PP.Lex(Tok); } else { if (II->isStr("push")) { Kind = Sema::PPK_Push; } else if (II->isStr("pop")) { Kind = Sema::PPK_Pop; } else { PP.Diag(Tok.getLocation(), diag::warn_pragma_pack_invalid_action); return; } PP.Lex(Tok); if (Tok.is(tok::comma)) { PP.Lex(Tok); if (Tok.is(tok::numeric_constant)) { Alignment = Actions.ActOnNumericConstant(Tok); if (Alignment.isInvalid()) return; PP.Lex(Tok); } else if (Tok.is(tok::identifier)) { Name = Tok.getIdentifierInfo(); PP.Lex(Tok); if (Tok.is(tok::comma)) { PP.Lex(Tok); if (Tok.isNot(tok::numeric_constant)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_pack_malformed); return; } Alignment = Actions.ActOnNumericConstant(Tok); if (Alignment.isInvalid()) return; PP.Lex(Tok); } } else { PP.Diag(Tok.getLocation(), diag::warn_pragma_pack_malformed); return; } } } } else if (PP.getLangOptions().ApplePragmaPack) { // In MSVC/gcc, #pragma pack() resets the alignment without affecting // the push/pop stack. // In Apple gcc #pragma pack() is equivalent to #pragma pack(pop). Kind = Sema::PPK_Pop; } if (Tok.isNot(tok::r_paren)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_rparen) << "pack"; return; } SourceLocation RParenLoc = Tok.getLocation(); PP.Lex(Tok); if (Tok.isNot(tok::eod)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_extra_tokens_at_eol) << "pack"; return; } Actions.ActOnPragmaPack(Kind, Name, Alignment.release(), PackLoc, LParenLoc, RParenLoc); }
// #pragma pack(...) comes in the following delicious flavors: // pack '(' [integer] ')' // pack '(' 'show' ')' // pack '(' ('push' | 'pop') [',' identifier] [, integer] ')' void PragmaPackHandler::HandlePragma(Preprocessor &PP, Token &PackTok) { // FIXME: Should we be expanding macros here? My guess is no. SourceLocation PackLoc = PackTok.getLocation(); Token Tok; PP.Lex(Tok); if (Tok.isNot(tok::l_paren)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_lparen) << "pack"; return; } Action::PragmaPackKind Kind = Action::PPK_Default; IdentifierInfo *Name = 0; Action::OwningExprResult Alignment(Actions); SourceLocation LParenLoc = Tok.getLocation(); PP.Lex(Tok); if (Tok.is(tok::numeric_constant)) { Alignment = Actions.ActOnNumericConstant(Tok); if (Alignment.isInvalid()) return; PP.Lex(Tok); } else if (Tok.is(tok::identifier)) { const IdentifierInfo *II = Tok.getIdentifierInfo(); if (II->isStr("show")) { Kind = Action::PPK_Show; PP.Lex(Tok); } else { if (II->isStr("push")) { Kind = Action::PPK_Push; } else if (II->isStr("pop")) { Kind = Action::PPK_Pop; } else { PP.Diag(Tok.getLocation(), diag::warn_pragma_pack_invalid_action); return; } PP.Lex(Tok); if (Tok.is(tok::comma)) { PP.Lex(Tok); if (Tok.is(tok::numeric_constant)) { Alignment = Actions.ActOnNumericConstant(Tok); if (Alignment.isInvalid()) return; PP.Lex(Tok); } else if (Tok.is(tok::identifier)) { Name = Tok.getIdentifierInfo(); PP.Lex(Tok); if (Tok.is(tok::comma)) { PP.Lex(Tok); if (Tok.isNot(tok::numeric_constant)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_pack_malformed); return; } Alignment = Actions.ActOnNumericConstant(Tok); if (Alignment.isInvalid()) return; PP.Lex(Tok); } } else { PP.Diag(Tok.getLocation(), diag::warn_pragma_pack_malformed); return; } } } } if (Tok.isNot(tok::r_paren)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_rparen) << "pack"; return; } PP.Lex(Tok); if (Tok.isNot(tok::eom)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_extra_tokens_at_eol) << "pack"; return; } SourceLocation RParenLoc = Tok.getLocation(); Actions.ActOnPragmaPack(Kind, Name, Alignment.release(), PackLoc, LParenLoc, RParenLoc); }
// #pragma unused(identifier) void PragmaUnusedHandler::HandlePragma(Preprocessor &PP, Token &UnusedTok) { // FIXME: Should we be expanding macros here? My guess is no. SourceLocation UnusedLoc = UnusedTok.getLocation(); // Lex the left '('. Token Tok; PP.Lex(Tok); if (Tok.isNot(tok::l_paren)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_lparen) << "unused"; return; } SourceLocation LParenLoc = Tok.getLocation(); // Lex the declaration reference(s). llvm::SmallVector<Token, 5> Identifiers; SourceLocation RParenLoc; bool LexID = true; while (true) { PP.Lex(Tok); if (LexID) { if (Tok.is(tok::identifier)) { Identifiers.push_back(Tok); LexID = false; continue; } // Illegal token! PP.Diag(Tok.getLocation(), diag::warn_pragma_unused_expected_var); return; } // We are execting a ')' or a ','. if (Tok.is(tok::comma)) { LexID = true; continue; } if (Tok.is(tok::r_paren)) { RParenLoc = Tok.getLocation(); break; } // Illegal token! PP.Diag(Tok.getLocation(), diag::warn_pragma_unused_expected_punc); return; } PP.Lex(Tok); if (Tok.isNot(tok::eom)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_extra_tokens_at_eol) << "unused"; return; } // Verify that we have a location for the right parenthesis. assert(RParenLoc.isValid() && "Valid '#pragma unused' must have ')'"); assert(!Identifiers.empty() && "Valid '#pragma unused' must have arguments"); // Perform the action to handle the pragma. Actions.ActOnPragmaUnused(Identifiers.data(), Identifiers.size(), parser.CurScope, UnusedLoc, LParenLoc, RParenLoc); }
/// HighlightMacros - This uses the macro table state from the end of the /// file, to re-expand macros and insert (into the HTML) information about the /// macro expansions. This won't be perfectly perfect, but it will be /// reasonably close. void html::HighlightMacros(Rewriter &R, FileID FID, Preprocessor& PP) { // Re-lex the raw token stream into a token buffer. const SourceManager &SM = PP.getSourceManager(); std::vector<Token> TokenStream; Lexer L(FID, SM, PP.getLangOptions()); // Lex all the tokens in raw mode, to avoid entering #includes or expanding // macros. while (1) { Token Tok; L.LexFromRawLexer(Tok); // If this is a # at the start of a line, discard it from the token stream. // We don't want the re-preprocess step to see #defines, #includes or other // preprocessor directives. if (Tok.is(tok::hash) && Tok.isAtStartOfLine()) continue; // If this is a ## token, change its kind to unknown so that repreprocessing // it will not produce an error. if (Tok.is(tok::hashhash)) Tok.setKind(tok::unknown); // If this raw token is an identifier, the raw lexer won't have looked up // the corresponding identifier info for it. Do this now so that it will be // macro expanded when we re-preprocess it. if (Tok.is(tok::identifier)) { // Change the kind of this identifier to the appropriate token kind, e.g. // turning "for" into a keyword. Tok.setKind(PP.LookUpIdentifierInfo(Tok)->getTokenID()); } TokenStream.push_back(Tok); if (Tok.is(tok::eof)) break; } // Temporarily change the diagnostics object so that we ignore any generated // diagnostics from this pass. IgnoringDiagClient TmpDC; Diagnostic TmpDiags(&TmpDC); Diagnostic *OldDiags = &PP.getDiagnostics(); PP.setDiagnostics(TmpDiags); // Inform the preprocessor that we don't want comments. PP.SetCommentRetentionState(false, false); // Enter the tokens we just lexed. This will cause them to be macro expanded // but won't enter sub-files (because we removed #'s). PP.EnterTokenStream(&TokenStream[0], TokenStream.size(), false, false); TokenConcatenation ConcatInfo(PP); // Lex all the tokens. Token Tok; PP.Lex(Tok); while (Tok.isNot(tok::eof)) { // Ignore non-macro tokens. if (!Tok.getLocation().isMacroID()) { PP.Lex(Tok); continue; } // Okay, we have the first token of a macro expansion: highlight the // instantiation by inserting a start tag before the macro instantiation and // end tag after it. std::pair<SourceLocation, SourceLocation> LLoc = SM.getInstantiationRange(Tok.getLocation()); // Ignore tokens whose instantiation location was not the main file. if (SM.getFileID(LLoc.first) != FID) { PP.Lex(Tok); continue; } assert(SM.getFileID(LLoc.second) == FID && "Start and end of expansion must be in the same ultimate file!"); std::string Expansion = PP.getSpelling(Tok); unsigned LineLen = Expansion.size(); Token PrevTok = Tok; // Okay, eat this token, getting the next one. PP.Lex(Tok); // Skip all the rest of the tokens that are part of this macro // instantiation. It would be really nice to pop up a window with all the // spelling of the tokens or something. while (!Tok.is(tok::eof) && SM.getInstantiationLoc(Tok.getLocation()) == LLoc.first) { // Insert a newline if the macro expansion is getting large. if (LineLen > 60) { Expansion += "<br>"; LineLen = 0; } LineLen -= Expansion.size(); // If the tokens were already space separated, or if they must be to avoid // them being implicitly pasted, add a space between them. if (Tok.hasLeadingSpace() || ConcatInfo.AvoidConcat(PrevTok, Tok)) Expansion += ' '; // Escape any special characters in the token text. Expansion += EscapeText(PP.getSpelling(Tok)); LineLen += Expansion.size(); PrevTok = Tok; PP.Lex(Tok); } // Insert the expansion as the end tag, so that multi-line macros all get // highlighted. Expansion = "<span class='expansion'>" + Expansion + "</span></span>"; HighlightRange(R, LLoc.first, LLoc.second, "<span class='macro'>", Expansion.c_str()); } // Restore diagnostics object back to its own thing. PP.setDiagnostics(*OldDiags); }
/// \brief Handle the microsoft \#pragma comment extension. /// /// The syntax is: /// \code /// #pragma comment(linker, "foo") /// \endcode /// 'linker' is one of five identifiers: compiler, exestr, lib, linker, user. /// "foo" is a string, which is fully macro expanded, and permits string /// concatenation, embedded escape characters etc. See MSDN for more details. void PragmaCommentHandler::HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer, Token &Tok) { SourceLocation CommentLoc = Tok.getLocation(); PP.Lex(Tok); if (Tok.isNot(tok::l_paren)) { PP.Diag(CommentLoc, diag::err_pragma_comment_malformed); return; } // Read the identifier. PP.Lex(Tok); if (Tok.isNot(tok::identifier)) { PP.Diag(CommentLoc, diag::err_pragma_comment_malformed); return; } // Verify that this is one of the 5 whitelisted options. IdentifierInfo *II = Tok.getIdentifierInfo(); Sema::PragmaMSCommentKind Kind = llvm::StringSwitch<Sema::PragmaMSCommentKind>(II->getName()) .Case("linker", Sema::PCK_Linker) .Case("lib", Sema::PCK_Lib) .Case("compiler", Sema::PCK_Compiler) .Case("exestr", Sema::PCK_ExeStr) .Case("user", Sema::PCK_User) .Default(Sema::PCK_Unknown); if (Kind == Sema::PCK_Unknown) { PP.Diag(Tok.getLocation(), diag::err_pragma_comment_unknown_kind); return; } // Read the optional string if present. PP.Lex(Tok); std::string ArgumentString; if (Tok.is(tok::comma) && !PP.LexStringLiteral(Tok, ArgumentString, "pragma comment", /*MacroExpansion=*/true)) return; // FIXME: warn that 'exestr' is deprecated. // FIXME: If the kind is "compiler" warn if the string is present (it is // ignored). // The MSDN docs say that "lib" and "linker" require a string and have a short // whitelist of linker options they support, but in practice MSVC doesn't // issue a diagnostic. Therefore neither does clang. if (Tok.isNot(tok::r_paren)) { PP.Diag(Tok.getLocation(), diag::err_pragma_comment_malformed); return; } PP.Lex(Tok); // eat the r_paren. if (Tok.isNot(tok::eod)) { PP.Diag(Tok.getLocation(), diag::err_pragma_comment_malformed); return; } // If the pragma is lexically sound, notify any interested PPCallbacks. if (PP.getPPCallbacks()) PP.getPPCallbacks()->PragmaComment(CommentLoc, II, ArgumentString); Actions.ActOnPragmaMSComment(Kind, ArgumentString); }
/// This implements the main NASM lexer. It is an extremely performance /// critical piece of code. This assumes that the buffer has a null /// character at the end of the file. Return true if an error /// occurred and compilation should terminate, false if normal. This /// returns a preprocessing token, not a normal token, as such, it is an /// internal interface. It assumes that the flags of result have been /// cleared before calling this. void NasmLexer::LexTokenInternal(Token* result) { LexNextToken: // New token, can't need cleaning yet. result->clearFlag(Token::NeedsCleaning); result->setIdentifierInfo(0); // Cache m_buf_ptr in an automatic variable. const char* cur_ptr = m_buf_ptr; // Small amounts of horizontal whitespace is very common between tokens. if ((*cur_ptr == ' ') || (*cur_ptr == '\t')) { ++cur_ptr; while ((*cur_ptr == ' ') || (*cur_ptr == '\t')) ++cur_ptr; #if 0 // If we are keeping whitespace and other tokens, just return what we // just skipped. The next lexer invocation will return the token after // the whitespace. if (is_keep_whitespace_mode()) { form_token_with_chars(result, cur_ptr, Token::unknown); return; } #endif m_buf_ptr = cur_ptr; result->setFlag(Token::LeadingSpace); } unsigned int size_tmp; // Temporary for use in cases below. // Read a character, advancing over it. char ch = getAndAdvanceChar(cur_ptr, result); unsigned int kind; switch (ch) { case 0: // Null. // Found end of file? if (cur_ptr-1 == m_buf_end) { // Read the PP instance variable into an automatic variable, because // LexEndOfFile will often delete 'this'. Preprocessor* PPCache = m_preproc; if (LexEndOfFile(result, cur_ptr-1))// Retreat back into the file return; // Got a token to return. assert(PPCache && "Raw buffer::LexEndOfFile should return a token"); return PPCache->Lex(result); } if (!isLexingRawMode()) Diag(cur_ptr-1, diag::null_in_file); result->setFlag(Token::LeadingSpace); if (SkipWhitespace(result, cur_ptr)) return; // KeepWhitespaceMode goto LexNextToken; // GCC isn't tail call eliminating. case '\n': case '\r': // If we are inside a preprocessor directive and we see the end of line, // we know we are done with the directive, so return an EOM token. if (m_parsing_preprocessor_directive) { // Done parsing the "line". m_parsing_preprocessor_directive = false; } // Since we consumed a newline, we are back at the start of a line. m_is_at_start_of_line = true; kind = Token::eol; result->setFlag(Token::EndOfStatement); break; case ' ': case '\t': case '\f': case '\v': SkipHorizontalWhitespace: result->setFlag(Token::LeadingSpace); if (SkipWhitespace(result, cur_ptr)) return; // KeepWhitespaceMode SkipIgnoredUnits: cur_ptr = m_buf_ptr; // If the next token is obviously a ; comment, skip it efficiently // too (without going through the big switch stmt). if (cur_ptr[0] == ';') { SkipLineComment(result, cur_ptr+1); goto SkipIgnoredUnits; } else if (isHorizontalWhitespace(*cur_ptr)) { goto SkipHorizontalWhitespace; } goto LexNextToken; // GCC isn't tail call eliminating. // Integer and Floating Point Constants case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': return LexNumericConstant(result, cur_ptr); // Identifiers case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': case 'Y': case 'Z': case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': case 's': case 't': case 'u': case 'v': case 'w': case 'x': case 'y': case 'z': return LexIdentifier(result, cur_ptr, false); case '_': case '?': case '@': case '.': return LexIdentifier(result, cur_ptr, true); case '$': ch = getCharAndSize(cur_ptr, &size_tmp); if (ch >= '0' && ch <= '9') { // If it starts with a digit it's a hexidecimal number return LexNumericConstant(result, ConsumeChar(cur_ptr, size_tmp, result)); } else if (ch == '$') { // $$ (start of current section) cur_ptr = ConsumeChar(cur_ptr, size_tmp, result); kind = NasmToken::dollardollar; } else if (isIdentifierBody(ch)) { // Forced identifier return LexIdentifier(result, cur_ptr, true); } else kind = NasmToken::dollar; // just a plain $ (current position) break; // String Literals. case '\'': case '"': case '`': return LexStringLiteral(result, cur_ptr, ch); // Punctuators. case '[': kind = NasmToken::l_square; break; case ']': kind = NasmToken::r_square; break; case '(': kind = NasmToken::l_paren; break; case ')': kind = NasmToken::r_paren; break; case '&': if (getCharAndSize(cur_ptr, &size_tmp) == '&') { cur_ptr = ConsumeChar(cur_ptr, size_tmp, result); kind = NasmToken::ampamp; } else kind = NasmToken::amp; break; case '*': kind = NasmToken::star; break; case '+': kind = NasmToken::plus; break; case '-': kind = NasmToken::minus; break; case '~': kind = NasmToken::tilde; break; case '!': ch = getCharAndSize(cur_ptr, &size_tmp); if (ch == '=') { cur_ptr = ConsumeChar(cur_ptr, size_tmp, result); kind = NasmToken::exclaimequal; } else if (ch == '?') { cur_ptr = ConsumeChar(cur_ptr, size_tmp, result); kind = NasmToken::tern; } else kind = NasmToken::exclaim; break; case '/': if (getCharAndSize(cur_ptr, &size_tmp) == '/') { cur_ptr = ConsumeChar(cur_ptr, size_tmp, result); kind = NasmToken::slashslash; } else kind = NasmToken::slash; break; case '%': ch = getCharAndSize(cur_ptr, &size_tmp); if (ch == '%') { cur_ptr = ConsumeChar(cur_ptr, size_tmp, result); kind = NasmToken::percentpercent; } else { #if 0 // We parsed a % character. If this occurs at the start of the // line, it's actually the start of a preprocessing directive. // Callback to the preprocessor to handle it. if (result->isAtStartOfLine() && !isLexingRawMode()) { FormTokenWithChars(result, cur_ptr, NasmToken::hash); m_preproc->HandleDirective(result); // As an optimization, if the preprocessor didn't switch // lexers, tail recurse. if (m_preproc->is_current_lexer(this)) { // Start a new token. If this is a %include or something, // the PP may want us starting at the beginning of the // line again. If so, set the start of line flag. if (m_is_at_start_of_line) { result->setFlag(Token::StartOfLine); m_is_at_start_of_line = false; } goto LexNextToken; // GCC isn't tail call eliminating. } return m_preproc->Lex(result); } #endif kind = NasmToken::percent; } break; case '<': ch = getCharAndSize(cur_ptr, &size_tmp); if (ch == '<') { cur_ptr = ConsumeChar(cur_ptr, size_tmp, result); kind = NasmToken::lessless; } else if (ch == '>') { cur_ptr = ConsumeChar(cur_ptr, size_tmp, result); kind = NasmToken::lessgreater; } else if (ch == '=') { cur_ptr = ConsumeChar(cur_ptr, size_tmp, result); kind = NasmToken::lessequal; } else kind = NasmToken::less; break; case '>': ch = getCharAndSize(cur_ptr, &size_tmp); if (ch == '=') { cur_ptr = ConsumeChar(cur_ptr, size_tmp, result); kind = NasmToken::greaterequal; } else if (ch == '>') { cur_ptr = ConsumeChar(cur_ptr, size_tmp, result); kind = NasmToken::greatergreater; } else kind = NasmToken::greater; break; case '^': if (getCharAndSize(cur_ptr, &size_tmp) == '^') { cur_ptr = ConsumeChar(cur_ptr, size_tmp, result); kind = NasmToken::caretcaret; } else kind = NasmToken::caret; break; case '|': if (getCharAndSize(cur_ptr, &size_tmp) == '|') { cur_ptr = ConsumeChar(cur_ptr, size_tmp, result); kind = NasmToken::pipepipe; } else kind = NasmToken::pipe; break; case ':': kind = NasmToken::colon; break; case ';': // EOL comment ++num_eol_comment; if (SkipLineComment(result, cur_ptr)) return; // KeepCommentMode // It is common for the tokens immediately after a ; comment to be // whitespace (indentation for the next line). Instead of going // through the big switch, handle it efficiently now. goto SkipIgnoredUnits; case '=': if (getCharAndSize(cur_ptr, &size_tmp) == '=') { cur_ptr = ConsumeChar(cur_ptr, size_tmp, result); kind = NasmToken::equalequal; } else kind = NasmToken::equal; break; case ',': kind = NasmToken::comma; break; default: kind = NasmToken::unknown; break; } // Update the location of token as well as m_buf_ptr. FormTokenWithChars(result, cur_ptr, kind); }