static best_guess num_type(clang::Preprocessor &pp, const clang::Token &t) { llvm::StringRef sr(t.getLiteralData(), t.getLength()); clang::NumericLiteralParser parser(sr, t.getLocation(), pp); if(parser.isIntegerLiteral()) return tok_int; if(parser.isFloatingLiteral()) return tok_float; return tok_invalid; }
void PreprocessorCallback::MacroUndefined(const clang::Token& MacroNameTok, PreprocessorCallback::MyMacroDefinition MD) { clang::SourceLocation loc = MacroNameTok.getLocation(); if (!loc.isValid() || !loc.isFileID()) return; clang::SourceManager &sm = annotator.getSourceMgr(); clang::FileID FID = sm.getFileID(loc); if (!annotator.shouldProcess(FID)) return; std::string ref = llvm::Twine("_M/", MacroNameTok.getIdentifierInfo()->getName()).str(); std::string link; std::string dataProj; clang::SourceLocation defLoc; clang::FileID defFID; if (MD) { #if CLANG_VERSION_MAJOR == 3 && CLANG_VERSION_MINOR >= 7 auto *MI = MD.getMacroInfo(); #else auto *MI = MD->getMacroInfo(); #endif if (MI) { defLoc = MI->getDefinitionLoc(); defFID = sm.getFileID(defLoc); } } if (defFID.isInvalid() || defFID != FID) { if (!defFID.isInvalid()) { link = annotator.pathTo(FID, defFID, &dataProj); } if (link.empty()) { std::string tag = "class=\"macro\" data-ref=\"" % ref % "\""; annotator.generator(FID).addTag("span", tag, sm.getFileOffset(loc), MacroNameTok.getLength()); return; } if (!dataProj.empty()) { dataProj = " data-proj=\"" % dataProj % "\""; } } if (sm.getMainFileID() != defFID) { annotator.registerMacro(ref, MacroNameTok.getLocation(), Annotator::Use_Write); } std::string tag = "class=\"macro\" href=\"" % link % "#" % llvm::Twine(sm.getExpansionLineNumber(defLoc)).str() % "\" data-ref=\"" % ref % "\"" % dataProj; annotator.generator(FID).addTag("a", tag, sm.getFileOffset(loc), MacroNameTok.getLength()); }
void PreprocessingRecord::InclusionDirective(SourceLocation HashLoc, const clang::Token &IncludeTok, llvm::StringRef FileName, bool IsAngled, const FileEntry *File, clang::SourceLocation EndLoc) { InclusionDirective::InclusionKind Kind = InclusionDirective::Include; switch (IncludeTok.getIdentifierInfo()->getPPKeywordID()) { case tok::pp_include: Kind = InclusionDirective::Include; break; case tok::pp_import: Kind = InclusionDirective::Import; break; case tok::pp_include_next: Kind = InclusionDirective::IncludeNext; break; case tok::pp___include_macros: Kind = InclusionDirective::IncludeMacros; break; default: llvm_unreachable("Unknown include directive kind"); return; } clang::InclusionDirective *ID = new (*this) clang::InclusionDirective(*this, Kind, FileName, !IsAngled, File, SourceRange(HashLoc, EndLoc)); PreprocessedEntities.push_back(ID); }
void PragmaRecorder::HandlePragma(clang::Preprocessor &PP, clang::PragmaIntroducerKind Introducer, clang::Token &FirstToken) { clang::Token &CurrentToken = FirstToken; std::string PragmaName, PragmaValue = ""; // Pragma in ACC should be a name/value pair if (GetPragmaNameFromToken(FirstToken, PragmaName)) { // start parsing the value '(' PragmaValue ')', if we have one. const clang::Token* NextToken = &PP.LookAhead(0); if (NextToken->is(clang::tok::l_paren)) PP.LexUnexpandedToken(CurrentToken); else goto end_parsing_pragma_value; NextToken = &PP.LookAhead(0); if (GetPragmaValueFromToken(*NextToken, PragmaValue)) { PP.Lex(CurrentToken); } else { PP.LexUnexpandedToken(CurrentToken); PP.Diag(NextToken->getLocation(), PP.getDiagnostics().getCustomDiagID( clang::DiagnosticsEngine::Error, "expected value after '#pragma %0('")) << PragmaName; return; } if (!NextToken->is(clang::tok::r_paren)) { NextToken = &PP.LookAhead(0); if (NextToken->is(clang::tok::r_paren)) { PP.LexUnexpandedToken(CurrentToken); } else { PP.LexUnexpandedToken(CurrentToken); PP.Diag(NextToken->getLocation(), PP.getDiagnostics().getCustomDiagID( clang::DiagnosticsEngine::Error, "missing ')' after '#pragma %0(%1'")) << PragmaName << PragmaValue; return; } } } else { PP.Diag(FirstToken.getLocation(), PP.getDiagnostics().getCustomDiagID( clang::DiagnosticsEngine::Error, "no pragma name or value")); return; } end_parsing_pragma_value: // PragmaValue may be an empty string. mPragmas->push_back(make_pair(PragmaName, PragmaValue)); // Inform lex to eat the token PP.LexUnexpandedToken(CurrentToken); return; }
void FMacroCallback::MacroExpands(const clang::Token &MacroNameTok, const clang::MacroDirective *MD, clang::SourceRange Range, const clang::MacroArgs *Args) { auto MacroName = MacroNameTok.getIdentifierInfo()->getName(); if (IsBuiltInMacro(MD) || IsMacroDefinedInCommandLine(MacroName) || IsPredefinedMacro(MacroName)) { return; } if (MacroDefinitionToFile.Contains(FString(MacroName.data()))) { auto MacroDefinitionFilename = FString(*MacroDefinitionToFile[MacroName.data()]); MacroDefinitionFilename.ReplaceInline(TEXT("\\"), TEXT("/")); auto MacroDefinitionFilenameHash = FIncludeSetMapKeyFuncs::GetKeyHash(MacroDefinitionFilename); auto MacroExpansionFilename = FString(SourceManager.getFilename(Range.getBegin()).data()); MacroExpansionFilename.ReplaceInline(TEXT("\\"), TEXT("/")); auto MacroExpansionFilenameHash = FIncludeSetMapKeyFuncs::GetKeyHash(MacroExpansionFilename); if (!Includes.Contains(MacroExpansionFilenameHash)) { Includes.FindOrAdd(MacroExpansionFilenameHash).Add(MacroDefinitionFilenameHash); } GHashToFilename.Add(MacroDefinitionFilenameHash, MacroDefinitionFilename); GHashToFilename.Add(MacroExpansionFilenameHash, MacroExpansionFilename); return; } OutputFileContents.Logf(TEXT("Internal error. Found usage of unknown macro %s."), ANSI_TO_TCHAR(MacroName.data())); }
void FMacroCallback::MacroDefined(const clang::Token &MacroNameTok, const clang::MacroDirective *MD) { auto MacroName = MacroNameTok.getIdentifierInfo()->getName(); if (IsMacroDefinedInCommandLine(MacroName) || IsBuiltInMacro(MD) || IsPredefinedMacro(MacroName)) { return; } auto MacroDefinitionFile = FString(SourceManager.getFilename(MacroNameTok.getLocation()).data()); MacroDefinitionFile.ReplaceInline(TEXT("\\"), TEXT("/")); if (MacroDefinitionFile.IsEmpty()) { OutputFileContents.Logf(TEXT("Found unexpected definition of macro %s."), ANSI_TO_TCHAR(MacroName.data())); } MacroDefinitionToFile.Add(MacroName.data(), MacroDefinitionFile); }
bool PragmaRecorder::GetPragmaValueFromToken(const clang::Token &Token, std::string &PragmaValue) { // Same as the GetPragmaName() if (Token.is(clang::tok::r_paren)) PragmaValue.clear(); else return GetPragmaNameFromToken(Token, PragmaValue); return true; }
void PreprocessorCallback::MacroDefined(const clang::Token& MacroNameTok, const clang::MacroDirective *MD) { clang::SourceLocation loc = MacroNameTok.getLocation(); if (!loc.isValid() || !loc.isFileID()) return; clang::SourceManager &sm = annotator.getSourceMgr(); clang::FileID FID = sm.getFileID(loc); if (!annotator.shouldProcess(FID)) return; std::string ref = llvm::Twine("_M/", MacroNameTok.getIdentifierInfo()->getName()).str(); if (sm.getMainFileID() != FID) { annotator.registerMacro(ref, MacroNameTok.getLocation(), Annotator::Declaration); } annotator.generator(FID).addTag("dfn", "class=\"macro\" id=\""% ref %"\" data-ref=\"" % ref % "\"", sm.getFileOffset(loc), MacroNameTok.getLength()); }
static Optional<std::pair<llvm::APSInt, Type>> getIntegerConstantForMacroToken(ClangImporter::Implementation &impl, DeclContext *DC, const clang::Token &token) { // Integer literal. if (token.is(clang::tok::numeric_constant)) { if (auto literal = parseNumericLiteral<clang::IntegerLiteral>(impl,token)) { auto value = llvm::APSInt { literal->getValue(), literal->getType()->isUnsignedIntegerType() }; auto type = impl.importType(literal->getType(), ImportTypeKind::Value, isInSystemModule(DC), /*isFullyBridgeable*/false); return {{ value, type }}; } // Macro identifier. } else if (token.is(clang::tok::identifier) && token.getIdentifierInfo()->hasMacroDefinition()) { auto rawID = token.getIdentifierInfo(); auto macroInfo = impl.getClangPreprocessor().getMacroInfo(rawID); auto importedID = impl.getNameImporter().importMacroName(rawID, macroInfo); impl.importMacro(importedID, macroInfo); auto searcher = impl.ImportedMacroConstants.find(macroInfo); if (searcher == impl.ImportedMacroConstants.end()) { return None; } auto importedConstant = searcher->second; if (!importedConstant.first.isInt()) { return None; } return {{ importedConstant.first.getInt(), importedConstant.second }}; } return None; }
void PreprocessingRecord::InclusionDirective( SourceLocation HashLoc, const clang::Token &IncludeTok, StringRef FileName, bool IsAngled, CharSourceRange FilenameRange, const FileEntry *File, StringRef SearchPath, StringRef RelativePath, const Module *Imported) { InclusionDirective::InclusionKind Kind = InclusionDirective::Include; switch (IncludeTok.getIdentifierInfo()->getPPKeywordID()) { case tok::pp_include: Kind = InclusionDirective::Include; break; case tok::pp_import: Kind = InclusionDirective::Import; break; case tok::pp_using: Kind = InclusionDirective::Using; break; case tok::pp_include_next: Kind = InclusionDirective::IncludeNext; break; case tok::pp___include_macros: Kind = InclusionDirective::IncludeMacros; break; default: llvm_unreachable("Unknown include directive kind"); } SourceLocation EndLoc; if (!IsAngled) { EndLoc = FilenameRange.getBegin(); } else { EndLoc = FilenameRange.getEnd(); if (FilenameRange.isCharRange()) EndLoc = EndLoc.getLocWithOffset(-1); // the InclusionDirective expects // a token range. } clang::InclusionDirective *ID = new (*this) clang::InclusionDirective(*this, Kind, FileName, !IsAngled, File, SourceRange(HashLoc, EndLoc)); addPreprocessedEntity(ID); }
static best_guess tok_type(clang::Preprocessor &pp, const char *macro_name, const clang::Token &t, StringSet *seen) { using namespace clang; tok::TokenKind k = t.getKind(); if(k == tok::identifier) { return tok_ok; IdentifierInfo *ii = t.getIdentifierInfo(); if(ii && !seen->count(ii->getNameStart())) return macro_type(pp, ii->getNameStart(), pp.getMacroInfo(ii), seen); return tok_invalid; } if (k == tok::l_paren || k == tok::r_paren || k == tok::amp || k == tok::plus || k == tok::star || k == tok::minus || k == tok::tilde || k == tok::slash || k == tok::percent || k == tok::lessless || k == tok::greatergreater || k == tok::caret || k == tok::pipe) return tok_ok; return tok_invalid; }
bool PragmaRecorder::GetPragmaNameFromToken(const clang::Token &Token, std::string &PragmaName) { if (Token.isLiteral()) PragmaName.assign(Token.getLiteralData(), Token.getLength()); else if (Token.is(clang::tok::identifier)) PragmaName.assign(Token.getIdentifierInfo()->getNameStart(), Token.getIdentifierInfo()->getLength()); else return false; return true; }
static Optional<clang::QualType> builtinTypeForToken(const clang::Token &tok, const clang::ASTContext &context) { switch (tok.getKind()) { case clang::tok::kw_short: return clang::QualType(context.ShortTy); case clang::tok::kw_long: return clang::QualType(context.LongTy); case clang::tok::kw___int64: return clang::QualType(context.LongLongTy); case clang::tok::kw___int128: return clang::QualType(context.Int128Ty); case clang::tok::kw_signed: return clang::QualType(context.IntTy); case clang::tok::kw_unsigned: return clang::QualType(context.UnsignedIntTy); case clang::tok::kw_void: return clang::QualType(context.VoidTy); case clang::tok::kw_char: return clang::QualType(context.CharTy); case clang::tok::kw_int: return clang::QualType(context.IntTy); case clang::tok::kw_float: return clang::QualType(context.FloatTy); case clang::tok::kw_double: return clang::QualType(context.DoubleTy); case clang::tok::kw_wchar_t: return clang::QualType(context.WCharTy); case clang::tok::kw_bool: return clang::QualType(context.BoolTy); case clang::tok::kw_char16_t: return clang::QualType(context.Char16Ty); case clang::tok::kw_char32_t: return clang::QualType(context.Char32Ty); default: return llvm::None; } }
void PreprocessorCallback::MacroExpands(const clang::Token& MacroNameTok, const clang::MacroInfo* MI, clang::SourceRange Range) { if (disabled) return; clang::SourceLocation loc = MacroNameTok.getLocation(); if (!loc.isValid() || !loc.isFileID()) return; clang::SourceManager &sm = annotator.getSourceMgr(); clang::FileID FID = sm.getFileID(loc); if (!annotator.shouldProcess(FID)) return; const char *begin = sm.getCharacterData(Range.getBegin()); int len = sm.getCharacterData(Range.getEnd()) - begin; len += clang::Lexer::MeasureTokenLength(Range.getEnd(), sm, PP.getLangOpts()); std::string copy(begin, len); begin = copy.c_str(); clang::Lexer lex(loc, PP.getLangOpts(), begin, begin, begin + len); std::vector<clang::Token> tokens; std::string expansion; //Lousely based on code from clang::html::HighlightMacros // Lex all the tokens in raw mode, to avoid entering #includes or expanding // macros. clang::Token tok; do { lex.LexFromRawLexer(tok); // If this is a # at the start of a line, discard it from the token stream. // We don't want the re-preprocess step to see #defines, #includes or other // preprocessor directives. if (tok.is(clang::tok::hash) && tok.isAtStartOfLine()) continue; // If this is a ## token, change its kind to unknown so that repreprocessing // it will not produce an error. if (tok.is(clang::tok::hashhash)) tok.setKind(clang::tok::unknown); // If this raw token is an identifier, the raw lexer won't have looked up // the corresponding identifier info for it. Do this now so that it will be // macro expanded when we re-preprocess it. if (tok.is(clang::tok::raw_identifier)) PP.LookUpIdentifierInfo(tok); tokens.push_back(tok); } while(!tok.is(clang::tok::eof)); // Temporarily change the diagnostics object so that we ignore any generated // diagnostics from this pass. clang::DiagnosticsEngine TmpDiags(PP.getDiagnostics().getDiagnosticIDs(), #if CLANG_VERSION_MAJOR!=3 || CLANG_VERSION_MINOR>=2 &PP.getDiagnostics().getDiagnosticOptions(), #endif new clang::IgnoringDiagConsumer); disabled = true; clang::DiagnosticsEngine *OldDiags = &PP.getDiagnostics(); PP.setDiagnostics(TmpDiags); PP.EnterTokenStream(tokens.data(), tokens.size(), false, false); PP.Lex(tok); while(tok.isNot(clang::tok::eof)) { // If the tokens were already space separated, or if they must be to avoid // them being implicitly pasted, add a space between them. if (tok.hasLeadingSpace()) expansion += ' '; // ConcatInfo.AvoidConcat(PrevPrevTok, PrevTok, Tok)) //FIXME // Escape any special characters in the token text. expansion += PP.getSpelling(tok); PP.Lex(tok); } PP.setDiagnostics(*OldDiags); disabled = false; expansion = Generator::escapeAttr(expansion); clang::SourceLocation defLoc = MI->getDefinitionLoc(); clang::FileID defFID = sm.getFileID(defLoc); std::string link; if (defFID != FID) link = annotator.pathTo(FID, defFID); std::string tag = "class=\"macro\" href=\"" % link % "#" % llvm::Twine(sm.getExpansionLineNumber(defLoc)).str() % "\" title=\"" % expansion % "\""; annotator.generator(FID).addTag("a", tag, sm.getFileOffset(loc), MacroNameTok.getLength()); }
static bool isSignToken(const clang::Token &tok) { return tok.is(clang::tok::plus) || tok.is(clang::tok::minus) || tok.is(clang::tok::tilde); }
static bool isStringToken(const clang::Token &tok) { return tok.is(clang::tok::string_literal) || tok.is(clang::tok::utf8_string_literal); }
// Here we blacklist certain tokens that are not usually the first token in an // unwrapped line. This is used in attempt to distinguish macro calls without // trailing semicolons from other constructs split to several lines. bool tokenCanStartNewLine(clang::Token Tok) { // Semicolon can be a null-statement, l_square can be a start of a macro or // a C++11 attribute, but this doesn't seem to be common. return Tok.isNot(tok::semi) && Tok.isNot(tok::l_brace) && Tok.isNot(tok::l_square) && // Tokens that can only be used as binary operators and a part of // overloaded operator names. Tok.isNot(tok::period) && Tok.isNot(tok::periodstar) && Tok.isNot(tok::arrow) && Tok.isNot(tok::arrowstar) && Tok.isNot(tok::less) && Tok.isNot(tok::greater) && Tok.isNot(tok::slash) && Tok.isNot(tok::percent) && Tok.isNot(tok::lessless) && Tok.isNot(tok::greatergreater) && Tok.isNot(tok::equal) && Tok.isNot(tok::plusequal) && Tok.isNot(tok::minusequal) && Tok.isNot(tok::starequal) && Tok.isNot(tok::slashequal) && Tok.isNot(tok::percentequal) && Tok.isNot(tok::ampequal) && Tok.isNot(tok::pipeequal) && Tok.isNot(tok::caretequal) && Tok.isNot(tok::greatergreaterequal) && Tok.isNot(tok::lesslessequal) && // Colon is used in labels, base class lists, initializer lists, // range-based for loops, ternary operator, but should never be the // first token in an unwrapped line. Tok.isNot(tok::colon) && // 'noexcept' is a trailing annotation. Tok.isNot(tok::kw_noexcept); }
static bool isBinaryOperator(const clang::Token &tok) { return tok.is(clang::tok::amp) || tok.is(clang::tok::pipe) || tok.is(clang::tok::ampamp) || tok.is(clang::tok::pipepipe); }
static clang::SourceLocation GetFromLiteral(clang::Token Tok, clang::StringLiteral *Lit, clang::Preprocessor &PP) { return Lit->getLocationOfByte(PP.getSourceManager().getFileOffset(Tok.getLocation()), PP.getSourceManager(), PP.getLangOpts(), PP.getTargetInfo()); }
void DeclCollector::MacroDefined(const clang::Token &MacroNameTok, const clang::MacroDirective *MD) { Transaction::MacroDirectiveInfo MDE(MacroNameTok.getIdentifierInfo(), MD); m_CurTransaction->append(MDE); }