static best_guess num_type(clang::Preprocessor &pp,
const clang::Token &t) {
llvm::StringRef sr(t.getLiteralData(), t.getLength());
clang::NumericLiteralParser parser(sr, t.getLocation(), pp);
if(parser.isIntegerLiteral())
return tok_int;
if(parser.isFloatingLiteral())
return tok_float;
return tok_invalid;
}
void PreprocessorCallback::MacroUndefined(const clang::Token& MacroNameTok, PreprocessorCallback::MyMacroDefinition MD)
{
clang::SourceLocation loc = MacroNameTok.getLocation();
if (!loc.isValid() || !loc.isFileID())
return;
clang::SourceManager &sm = annotator.getSourceMgr();
clang::FileID FID = sm.getFileID(loc);
if (!annotator.shouldProcess(FID))
return;
std::string ref = llvm::Twine("_M/", MacroNameTok.getIdentifierInfo()->getName()).str();
std::string link;
std::string dataProj;
clang::SourceLocation defLoc;
clang::FileID defFID;
if (MD) {
#if CLANG_VERSION_MAJOR == 3 && CLANG_VERSION_MINOR >= 7
auto *MI = MD.getMacroInfo();
#else
auto *MI = MD->getMacroInfo();
#endif
if (MI) {
defLoc = MI->getDefinitionLoc();
defFID = sm.getFileID(defLoc);
}
}
if (defFID.isInvalid() || defFID != FID) {
if (!defFID.isInvalid()) {
link = annotator.pathTo(FID, defFID, &dataProj);
}
if (link.empty()) {
std::string tag = "class=\"macro\" data-ref=\"" % ref % "\"";
annotator.generator(FID).addTag("span", tag, sm.getFileOffset(loc), MacroNameTok.getLength());
return;
}
if (!dataProj.empty()) {
dataProj = " data-proj=\"" % dataProj % "\"";
}
}
if (sm.getMainFileID() != defFID) {
annotator.registerMacro(ref, MacroNameTok.getLocation(), Annotator::Use_Write);
}
std::string tag = "class=\"macro\" href=\"" % link % "#" % llvm::Twine(sm.getExpansionLineNumber(defLoc)).str()
% "\" data-ref=\"" % ref % "\"" % dataProj;
annotator.generator(FID).addTag("a", tag, sm.getFileOffset(loc), MacroNameTok.getLength());
}
void PreprocessingRecord::InclusionDirective(SourceLocation HashLoc,
const clang::Token &IncludeTok,
llvm::StringRef FileName,
bool IsAngled,
const FileEntry *File,
clang::SourceLocation EndLoc) {
InclusionDirective::InclusionKind Kind = InclusionDirective::Include;
switch (IncludeTok.getIdentifierInfo()->getPPKeywordID()) {
case tok::pp_include:
Kind = InclusionDirective::Include;
break;
case tok::pp_import:
Kind = InclusionDirective::Import;
break;
case tok::pp_include_next:
Kind = InclusionDirective::IncludeNext;
break;
case tok::pp___include_macros:
Kind = InclusionDirective::IncludeMacros;
break;
default:
llvm_unreachable("Unknown include directive kind");
return;
}
clang::InclusionDirective *ID
= new (*this) clang::InclusionDirective(*this, Kind, FileName, !IsAngled,
File, SourceRange(HashLoc, EndLoc));
PreprocessedEntities.push_back(ID);
}
void PragmaRecorder::HandlePragma(clang::Preprocessor &PP,
clang::PragmaIntroducerKind Introducer,
clang::Token &FirstToken) {
clang::Token &CurrentToken = FirstToken;
std::string PragmaName, PragmaValue = "";
// Pragma in ACC should be a name/value pair
if (GetPragmaNameFromToken(FirstToken, PragmaName)) {
// start parsing the value '(' PragmaValue ')', if we have one.
const clang::Token* NextToken = &PP.LookAhead(0);
if (NextToken->is(clang::tok::l_paren))
PP.LexUnexpandedToken(CurrentToken);
else
goto end_parsing_pragma_value;
NextToken = &PP.LookAhead(0);
if (GetPragmaValueFromToken(*NextToken, PragmaValue)) {
PP.Lex(CurrentToken);
} else {
PP.LexUnexpandedToken(CurrentToken);
PP.Diag(NextToken->getLocation(),
PP.getDiagnostics().getCustomDiagID(
clang::DiagnosticsEngine::Error,
"expected value after '#pragma %0('")) << PragmaName;
return;
}
if (!NextToken->is(clang::tok::r_paren)) {
NextToken = &PP.LookAhead(0);
if (NextToken->is(clang::tok::r_paren)) {
PP.LexUnexpandedToken(CurrentToken);
} else {
PP.LexUnexpandedToken(CurrentToken);
PP.Diag(NextToken->getLocation(),
PP.getDiagnostics().getCustomDiagID(
clang::DiagnosticsEngine::Error,
"missing ')' after '#pragma %0(%1'"))
<< PragmaName << PragmaValue;
return;
}
}
} else {
PP.Diag(FirstToken.getLocation(),
PP.getDiagnostics().getCustomDiagID(
clang::DiagnosticsEngine::Error,
"no pragma name or value"));
return;
}
end_parsing_pragma_value:
// PragmaValue may be an empty string.
mPragmas->push_back(make_pair(PragmaName, PragmaValue));
// Inform lex to eat the token
PP.LexUnexpandedToken(CurrentToken);
return;
}
void FMacroCallback::MacroExpands(const clang::Token &MacroNameTok, const clang::MacroDirective *MD, clang::SourceRange Range, const clang::MacroArgs *Args)
{
auto MacroName = MacroNameTok.getIdentifierInfo()->getName();
if (IsBuiltInMacro(MD) || IsMacroDefinedInCommandLine(MacroName) || IsPredefinedMacro(MacroName))
{
return;
}
if (MacroDefinitionToFile.Contains(FString(MacroName.data())))
{
auto MacroDefinitionFilename = FString(*MacroDefinitionToFile[MacroName.data()]);
MacroDefinitionFilename.ReplaceInline(TEXT("\\"), TEXT("/"));
auto MacroDefinitionFilenameHash = FIncludeSetMapKeyFuncs::GetKeyHash(MacroDefinitionFilename);
auto MacroExpansionFilename = FString(SourceManager.getFilename(Range.getBegin()).data());
MacroExpansionFilename.ReplaceInline(TEXT("\\"), TEXT("/"));
auto MacroExpansionFilenameHash = FIncludeSetMapKeyFuncs::GetKeyHash(MacroExpansionFilename);
if (!Includes.Contains(MacroExpansionFilenameHash))
{
Includes.FindOrAdd(MacroExpansionFilenameHash).Add(MacroDefinitionFilenameHash);
}
GHashToFilename.Add(MacroDefinitionFilenameHash, MacroDefinitionFilename);
GHashToFilename.Add(MacroExpansionFilenameHash, MacroExpansionFilename);
return;
}
OutputFileContents.Logf(TEXT("Internal error. Found usage of unknown macro %s."), ANSI_TO_TCHAR(MacroName.data()));
}
void FMacroCallback::MacroDefined(const clang::Token &MacroNameTok, const clang::MacroDirective *MD)
{
auto MacroName = MacroNameTok.getIdentifierInfo()->getName();
if (IsMacroDefinedInCommandLine(MacroName) || IsBuiltInMacro(MD) || IsPredefinedMacro(MacroName))
{
return;
}
auto MacroDefinitionFile = FString(SourceManager.getFilename(MacroNameTok.getLocation()).data());
MacroDefinitionFile.ReplaceInline(TEXT("\\"), TEXT("/"));
if (MacroDefinitionFile.IsEmpty())
{
OutputFileContents.Logf(TEXT("Found unexpected definition of macro %s."), ANSI_TO_TCHAR(MacroName.data()));
}
MacroDefinitionToFile.Add(MacroName.data(), MacroDefinitionFile);
}
bool PragmaRecorder::GetPragmaValueFromToken(const clang::Token &Token,
std::string &PragmaValue) {
// Same as the GetPragmaName()
if (Token.is(clang::tok::r_paren))
PragmaValue.clear();
else
return GetPragmaNameFromToken(Token, PragmaValue);
return true;
}
void PreprocessorCallback::MacroDefined(const clang::Token& MacroNameTok, const clang::MacroDirective *MD)
{
clang::SourceLocation loc = MacroNameTok.getLocation();
if (!loc.isValid() || !loc.isFileID())
return;
clang::SourceManager &sm = annotator.getSourceMgr();
clang::FileID FID = sm.getFileID(loc);
if (!annotator.shouldProcess(FID))
return;
std::string ref = llvm::Twine("_M/", MacroNameTok.getIdentifierInfo()->getName()).str();
if (sm.getMainFileID() != FID) {
annotator.registerMacro(ref, MacroNameTok.getLocation(), Annotator::Declaration);
}
annotator.generator(FID).addTag("dfn", "class=\"macro\" id=\""% ref %"\" data-ref=\"" % ref % "\"", sm.getFileOffset(loc), MacroNameTok.getLength());
}
static Optional<std::pair<llvm::APSInt, Type>>
getIntegerConstantForMacroToken(ClangImporter::Implementation &impl,
DeclContext *DC,
const clang::Token &token) {
// Integer literal.
if (token.is(clang::tok::numeric_constant)) {
if (auto literal = parseNumericLiteral<clang::IntegerLiteral>(impl,token)) {
auto value = llvm::APSInt { literal->getValue(),
literal->getType()->isUnsignedIntegerType() };
auto type = impl.importType(literal->getType(),
ImportTypeKind::Value,
isInSystemModule(DC),
/*isFullyBridgeable*/false);
return {{ value, type }};
}
// Macro identifier.
} else if (token.is(clang::tok::identifier) &&
token.getIdentifierInfo()->hasMacroDefinition()) {
auto rawID = token.getIdentifierInfo();
auto macroInfo = impl.getClangPreprocessor().getMacroInfo(rawID);
auto importedID = impl.getNameImporter().importMacroName(rawID, macroInfo);
impl.importMacro(importedID, macroInfo);
auto searcher = impl.ImportedMacroConstants.find(macroInfo);
if (searcher == impl.ImportedMacroConstants.end()) {
return None;
}
auto importedConstant = searcher->second;
if (!importedConstant.first.isInt()) {
return None;
}
return {{ importedConstant.first.getInt(), importedConstant.second }};
}
return None;
}
void PreprocessingRecord::InclusionDirective(
SourceLocation HashLoc,
const clang::Token &IncludeTok,
StringRef FileName,
bool IsAngled,
CharSourceRange FilenameRange,
const FileEntry *File,
StringRef SearchPath,
StringRef RelativePath,
const Module *Imported) {
InclusionDirective::InclusionKind Kind = InclusionDirective::Include;
switch (IncludeTok.getIdentifierInfo()->getPPKeywordID()) {
case tok::pp_include:
Kind = InclusionDirective::Include;
break;
case tok::pp_import:
Kind = InclusionDirective::Import;
break;
case tok::pp_using:
Kind = InclusionDirective::Using;
break;
case tok::pp_include_next:
Kind = InclusionDirective::IncludeNext;
break;
case tok::pp___include_macros:
Kind = InclusionDirective::IncludeMacros;
break;
default:
llvm_unreachable("Unknown include directive kind");
}
SourceLocation EndLoc;
if (!IsAngled) {
EndLoc = FilenameRange.getBegin();
} else {
EndLoc = FilenameRange.getEnd();
if (FilenameRange.isCharRange())
EndLoc = EndLoc.getLocWithOffset(-1); // the InclusionDirective expects
// a token range.
}
clang::InclusionDirective *ID
= new (*this) clang::InclusionDirective(*this, Kind, FileName, !IsAngled,
File, SourceRange(HashLoc, EndLoc));
addPreprocessedEntity(ID);
}
static best_guess tok_type(clang::Preprocessor &pp, const char *macro_name,
const clang::Token &t, StringSet *seen) {
using namespace clang;
tok::TokenKind k = t.getKind();
if(k == tok::identifier) {
return tok_ok;
IdentifierInfo *ii = t.getIdentifierInfo();
if(ii && !seen->count(ii->getNameStart()))
return macro_type(pp, ii->getNameStart(),
pp.getMacroInfo(ii), seen);
return tok_invalid;
}
if (k == tok::l_paren || k == tok::r_paren || k == tok::amp || k == tok::plus ||
k == tok::star || k == tok::minus || k == tok::tilde || k == tok::slash ||
k == tok::percent || k == tok::lessless || k == tok::greatergreater ||
k == tok::caret || k == tok::pipe)
return tok_ok;
return tok_invalid;
}
bool PragmaRecorder::GetPragmaNameFromToken(const clang::Token &Token,
std::string &PragmaName) {
if (Token.isLiteral())
PragmaName.assign(Token.getLiteralData(), Token.getLength());
else if (Token.is(clang::tok::identifier))
PragmaName.assign(Token.getIdentifierInfo()->getNameStart(),
Token.getIdentifierInfo()->getLength());
else
return false;
return true;
}
static Optional<clang::QualType> builtinTypeForToken(const clang::Token &tok,
const clang::ASTContext &context) {
switch (tok.getKind()) {
case clang::tok::kw_short:
return clang::QualType(context.ShortTy);
case clang::tok::kw_long:
return clang::QualType(context.LongTy);
case clang::tok::kw___int64:
return clang::QualType(context.LongLongTy);
case clang::tok::kw___int128:
return clang::QualType(context.Int128Ty);
case clang::tok::kw_signed:
return clang::QualType(context.IntTy);
case clang::tok::kw_unsigned:
return clang::QualType(context.UnsignedIntTy);
case clang::tok::kw_void:
return clang::QualType(context.VoidTy);
case clang::tok::kw_char:
return clang::QualType(context.CharTy);
case clang::tok::kw_int:
return clang::QualType(context.IntTy);
case clang::tok::kw_float:
return clang::QualType(context.FloatTy);
case clang::tok::kw_double:
return clang::QualType(context.DoubleTy);
case clang::tok::kw_wchar_t:
return clang::QualType(context.WCharTy);
case clang::tok::kw_bool:
return clang::QualType(context.BoolTy);
case clang::tok::kw_char16_t:
return clang::QualType(context.Char16Ty);
case clang::tok::kw_char32_t:
return clang::QualType(context.Char32Ty);
default:
return llvm::None;
}
}
void PreprocessorCallback::MacroExpands(const clang::Token& MacroNameTok,
const clang::MacroInfo* MI,
clang::SourceRange Range)
{
if (disabled)
return;
clang::SourceLocation loc = MacroNameTok.getLocation();
if (!loc.isValid() || !loc.isFileID())
return;
clang::SourceManager &sm = annotator.getSourceMgr();
clang::FileID FID = sm.getFileID(loc);
if (!annotator.shouldProcess(FID))
return;
const char *begin = sm.getCharacterData(Range.getBegin());
int len = sm.getCharacterData(Range.getEnd()) - begin;
len += clang::Lexer::MeasureTokenLength(Range.getEnd(), sm, PP.getLangOpts());
std::string copy(begin, len);
begin = copy.c_str();
clang::Lexer lex(loc, PP.getLangOpts(), begin, begin, begin + len);
std::vector<clang::Token> tokens;
std::string expansion;
//Lousely based on code from clang::html::HighlightMacros
// Lex all the tokens in raw mode, to avoid entering #includes or expanding
// macros.
clang::Token tok;
do {
lex.LexFromRawLexer(tok);
// If this is a # at the start of a line, discard it from the token stream.
// We don't want the re-preprocess step to see #defines, #includes or other
// preprocessor directives.
if (tok.is(clang::tok::hash) && tok.isAtStartOfLine())
continue;
// If this is a ## token, change its kind to unknown so that repreprocessing
// it will not produce an error.
if (tok.is(clang::tok::hashhash))
tok.setKind(clang::tok::unknown);
// If this raw token is an identifier, the raw lexer won't have looked up
// the corresponding identifier info for it. Do this now so that it will be
// macro expanded when we re-preprocess it.
if (tok.is(clang::tok::raw_identifier))
PP.LookUpIdentifierInfo(tok);
tokens.push_back(tok);
} while(!tok.is(clang::tok::eof));
// Temporarily change the diagnostics object so that we ignore any generated
// diagnostics from this pass.
clang::DiagnosticsEngine TmpDiags(PP.getDiagnostics().getDiagnosticIDs(),
#if CLANG_VERSION_MAJOR!=3 || CLANG_VERSION_MINOR>=2
&PP.getDiagnostics().getDiagnosticOptions(),
#endif
new clang::IgnoringDiagConsumer);
disabled = true;
clang::DiagnosticsEngine *OldDiags = &PP.getDiagnostics();
PP.setDiagnostics(TmpDiags);
PP.EnterTokenStream(tokens.data(), tokens.size(), false, false);
PP.Lex(tok);
while(tok.isNot(clang::tok::eof)) {
// If the tokens were already space separated, or if they must be to avoid
// them being implicitly pasted, add a space between them.
if (tok.hasLeadingSpace())
expansion += ' ';
// ConcatInfo.AvoidConcat(PrevPrevTok, PrevTok, Tok)) //FIXME
// Escape any special characters in the token text.
expansion += PP.getSpelling(tok);
PP.Lex(tok);
}
PP.setDiagnostics(*OldDiags);
disabled = false;
expansion = Generator::escapeAttr(expansion);
clang::SourceLocation defLoc = MI->getDefinitionLoc();
clang::FileID defFID = sm.getFileID(defLoc);
std::string link;
if (defFID != FID)
link = annotator.pathTo(FID, defFID);
std::string tag = "class=\"macro\" href=\"" % link % "#" % llvm::Twine(sm.getExpansionLineNumber(defLoc)).str()
% "\" title=\"" % expansion % "\"";
annotator.generator(FID).addTag("a", tag, sm.getFileOffset(loc), MacroNameTok.getLength());
}
static bool isSignToken(const clang::Token &tok) {
return tok.is(clang::tok::plus) || tok.is(clang::tok::minus) ||
tok.is(clang::tok::tilde);
}
static bool isStringToken(const clang::Token &tok) {
return tok.is(clang::tok::string_literal) ||
tok.is(clang::tok::utf8_string_literal);
}
// Here we blacklist certain tokens that are not usually the first token in an
// unwrapped line. This is used in attempt to distinguish macro calls without
// trailing semicolons from other constructs split to several lines.
bool tokenCanStartNewLine(clang::Token Tok) {
// Semicolon can be a null-statement, l_square can be a start of a macro or
// a C++11 attribute, but this doesn't seem to be common.
return Tok.isNot(tok::semi) && Tok.isNot(tok::l_brace) &&
Tok.isNot(tok::l_square) &&
// Tokens that can only be used as binary operators and a part of
// overloaded operator names.
Tok.isNot(tok::period) && Tok.isNot(tok::periodstar) &&
Tok.isNot(tok::arrow) && Tok.isNot(tok::arrowstar) &&
Tok.isNot(tok::less) && Tok.isNot(tok::greater) &&
Tok.isNot(tok::slash) && Tok.isNot(tok::percent) &&
Tok.isNot(tok::lessless) && Tok.isNot(tok::greatergreater) &&
Tok.isNot(tok::equal) && Tok.isNot(tok::plusequal) &&
Tok.isNot(tok::minusequal) && Tok.isNot(tok::starequal) &&
Tok.isNot(tok::slashequal) && Tok.isNot(tok::percentequal) &&
Tok.isNot(tok::ampequal) && Tok.isNot(tok::pipeequal) &&
Tok.isNot(tok::caretequal) && Tok.isNot(tok::greatergreaterequal) &&
Tok.isNot(tok::lesslessequal) &&
// Colon is used in labels, base class lists, initializer lists,
// range-based for loops, ternary operator, but should never be the
// first token in an unwrapped line.
Tok.isNot(tok::colon) &&
// 'noexcept' is a trailing annotation.
Tok.isNot(tok::kw_noexcept);
}
static bool isBinaryOperator(const clang::Token &tok) {
return tok.is(clang::tok::amp) ||
tok.is(clang::tok::pipe) ||
tok.is(clang::tok::ampamp) ||
tok.is(clang::tok::pipepipe);
}
static clang::SourceLocation GetFromLiteral(clang::Token Tok, clang::StringLiteral *Lit, clang::Preprocessor &PP) {
return Lit->getLocationOfByte(PP.getSourceManager().getFileOffset(Tok.getLocation()),
PP.getSourceManager(), PP.getLangOpts(), PP.getTargetInfo());
}
void DeclCollector::MacroDefined(const clang::Token &MacroNameTok,
const clang::MacroDirective *MD) {
Transaction::MacroDirectiveInfo MDE(MacroNameTok.getIdentifierInfo(), MD);
m_CurTransaction->append(MDE);
}