bool TokenLexer::MaybeRemoveCommaBeforeVaArgs(
SmallVectorImpl<Token> &ResultToks, bool HasPasteOperator, MacroInfo *Macro,
unsigned MacroArgNo, Preprocessor &PP) {
// Is the macro argument __VA_ARGS__?
if (!Macro->isVariadic() || MacroArgNo != Macro->getNumArgs()-1)
return false;
// In Microsoft-compatibility mode, a comma is removed in the expansion
// of " ... , __VA_ARGS__ " if __VA_ARGS__ is empty. This extension is
// not supported by gcc.
if (!HasPasteOperator && !PP.getLangOpts().MSVCCompat)
return false;
// GCC removes the comma in the expansion of " ... , ## __VA_ARGS__ " if
// __VA_ARGS__ is empty, but not in strict C99 mode where there are no
// named arguments, where it remains. In all other modes, including C99
// with GNU extensions, it is removed regardless of named arguments.
// Microsoft also appears to support this extension, unofficially.
if (PP.getLangOpts().C99 && !PP.getLangOpts().GNUMode
&& Macro->getNumArgs() < 2)
return false;
// Is a comma available to be removed?
if (ResultToks.empty() || !ResultToks.back().is(tok::comma))
return false;
// Issue an extension diagnostic for the paste operator.
if (HasPasteOperator)
PP.Diag(ResultToks.back().getLocation(), diag::ext_paste_comma);
// Remove the comma.
ResultToks.pop_back();
if (!ResultToks.empty()) {
// If the comma was right after another paste (e.g. "X##,##__VA_ARGS__"),
// then removal of the comma should produce a placemarker token (in C99
// terms) which we model by popping off the previous ##, giving us a plain
// "X" when __VA_ARGS__ is empty.
if (ResultToks.back().is(tok::hashhash))
ResultToks.pop_back();
// Remember that this comma was elided.
ResultToks.back().setFlag(Token::CommaAfterElided);
}
// Never add a space, even if the comma, ##, or arg had a space.
NextTokGetsSpace = false;
return true;
}
Sema::Sema(Preprocessor &pp, ASTContext &ctxt, ASTConsumer &consumer,
TranslationUnitKind TUKind,
CodeCompleteConsumer *CodeCompleter)
: TheTargetAttributesSema(0), ExternalSource(0),
isMultiplexExternalSource(false), FPFeatures(pp.getLangOpts()),
LangOpts(pp.getLangOpts()), PP(pp), Context(ctxt), Consumer(consumer),
Diags(PP.getDiagnostics()), SourceMgr(PP.getSourceManager()),
CollectStats(false), CodeCompleter(CodeCompleter),
CurContext(0), OriginalLexicalContext(0),
PackContext(0), MSStructPragmaOn(false), VisContext(0),
IsBuildingRecoveryCallExpr(false),
ExprNeedsCleanups(false), LateTemplateParser(0), OpaqueParser(0),
IdResolver(pp), StdInitializerList(0), CXXTypeInfoDecl(0), MSVCGuidDecl(0),
NSNumberDecl(0),
NSStringDecl(0), StringWithUTF8StringMethod(0),
NSArrayDecl(0), ArrayWithObjectsMethod(0),
NSDictionaryDecl(0), DictionaryWithObjectsMethod(0),
GlobalNewDeleteDeclared(false),
TUKind(TUKind),
NumSFINAEErrors(0), IsTransformingLambdaCallOperatorProtoType(false),
InFunctionDeclarator(0),
AccessCheckingSFINAE(false), InNonInstantiationSFINAEContext(false),
NonInstantiationEntries(0), ArgumentPackSubstitutionIndex(-1),
CurrentInstantiationScope(0), TyposCorrected(0),
AnalysisWarnings(*this) {
TUScope = 0;
LoadedExternalKnownNamespaces = false;
for (unsigned I = 0; I != NSAPI::NumNSNumberLiteralMethods; ++I)
NSNumberLiteralMethods[I] = 0;
if (getLangOpts().ObjC1)
NSAPIObj.reset(new NSAPI(Context));
if (getLangOpts().CPlusPlus)
FieldCollector.reset(new CXXFieldCollector());
// Tell diagnostics how to render things from the AST library.
PP.getDiagnostics().SetArgToStringFn(&FormatASTNodeDiagnosticArgument,
&Context);
ExprEvalContexts.push_back(
ExpressionEvaluationContextRecord(PotentiallyEvaluated, 0,
false, 0, false));
FunctionScopes.push_back(new FunctionScopeInfo(Diags));
}
/// LexRawTokensFromMainFile - Lets all the raw tokens from the main file into
/// the specified vector.
static void LexRawTokensFromMainFile(Preprocessor &PP,
std::vector<Token> &RawTokens) {
SourceManager &SM = PP.getSourceManager();
// Create a lexer to lex all the tokens of the main file in raw mode. Even
// though it is in raw mode, it will not return comments.
const llvm::MemoryBuffer *FromFile = SM.getBuffer(SM.getMainFileID());
Lexer RawLex(SM.getMainFileID(), FromFile, SM, PP.getLangOpts());
// Switch on comment lexing because we really do want them.
RawLex.SetCommentRetentionState(true);
Token RawTok;
do {
RawLex.LexFromRawLexer(RawTok);
// If we have an identifier with no identifier info for our raw token, look
// up the indentifier info. This is important for equality comparison of
// identifier tokens.
if (RawTok.is(tok::raw_identifier))
PP.LookUpIdentifierInfo(RawTok);
RawTokens.push_back(RawTok);
} while (RawTok.isNot(tok::eof));
}
/// Returns true if the statement is expanded from a configuration macro.
static bool isExpandedFromConfigurationMacro(const Stmt *S,
Preprocessor &PP,
bool IgnoreYES_NO = false) {
// FIXME: This is not very precise. Here we just check to see if the
// value comes from a macro, but we can do much better. This is likely
// to be over conservative. This logic is factored into a separate function
// so that we can refine it later.
SourceLocation L = S->getBeginLoc();
if (L.isMacroID()) {
SourceManager &SM = PP.getSourceManager();
if (IgnoreYES_NO) {
// The Objective-C constant 'YES' and 'NO'
// are defined as macros. Do not treat them
// as configuration values.
SourceLocation TopL = getTopMostMacro(L, SM);
StringRef MacroName = PP.getImmediateMacroName(TopL);
if (MacroName == "YES" || MacroName == "NO")
return false;
} else if (!PP.getLangOpts().CPlusPlus) {
// Do not treat C 'false' and 'true' macros as configuration values.
SourceLocation TopL = getTopMostMacro(L, SM);
StringRef MacroName = PP.getImmediateMacroName(TopL);
if (MacroName == "false" || MacroName == "true")
return false;
}
return true;
}
return false;
}
/// FindExpectedDiags - Lex the main source file to find all of the
// expected errors and warnings.
static void FindExpectedDiags(const Preprocessor &PP, ExpectedData &ED,
FileID FID) {
// Create a raw lexer to pull all the comments out of FID.
if (FID.isInvalid())
return;
SourceManager& SM = PP.getSourceManager();
// Create a lexer to lex all the tokens of the main file in raw mode.
const llvm::MemoryBuffer *FromFile = SM.getBuffer(FID);
Lexer RawLex(FID, FromFile, SM, PP.getLangOpts());
// Return comments as tokens, this is how we find expected diagnostics.
RawLex.SetCommentRetentionState(true);
Token Tok;
Tok.setKind(tok::comment);
while (Tok.isNot(tok::eof)) {
RawLex.Lex(Tok);
if (!Tok.is(tok::comment)) continue;
std::string Comment = PP.getSpelling(Tok);
if (Comment.empty()) continue;
// Find all expected errors/warnings/notes.
ParseDirective(Comment, ED, SM, Tok.getLocation(), PP.getDiagnostics());
};
}
void ento::createPlistMultiFileDiagnosticConsumer(AnalyzerOptions &AnalyzerOpts,
PathDiagnosticConsumers &C,
const std::string &s,
const Preprocessor &PP) {
C.push_back(new PlistDiagnostics(AnalyzerOpts, s,
PP.getLangOpts(), true));
}
/// Initializes an InclusionRewriter with a \p PP source and \p OS destination.
InclusionRewriter::InclusionRewriter(Preprocessor &PP, raw_ostream &OS,
bool ShowLineMarkers)
: PP(PP), SM(PP.getSourceManager()), OS(OS),
ShowLineMarkers(ShowLineMarkers),
LastInsertedFileChange(FileChanges.end()) {
// If we're in microsoft mode, use normal #line instead of line markers.
UseLineDirective = PP.getLangOpts().MicrosoftExt;
}
bool HandleComment(Preprocessor &PP, SourceRange Range) override {
StringRef Text =
Lexer::getSourceText(CharSourceRange::getCharRange(Range),
PP.getSourceManager(), PP.getLangOpts());
if (!Text.consume_front(IWYUPragma))
return false;
// FIXME(ioeric): resolve the header and store actual file path. For now,
// we simply assume the written header is suitable to be #included.
Includes->addMapping(PP.getSourceManager().getFilename(Range.getBegin()),
isLiteralInclude(Text) ? Text.str()
: ("\"" + Text + "\"").str());
return false;
}
static void emitPremigrationErrors(const CapturedDiagList &arcDiags,
const DiagnosticOptions &diagOpts,
Preprocessor &PP) {
TextDiagnosticPrinter printer(llvm::errs(), diagOpts);
IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
IntrusiveRefCntPtr<DiagnosticsEngine> Diags(
new DiagnosticsEngine(DiagID, &printer, /*ShouldOwnClient=*/false));
Diags->setSourceManager(&PP.getSourceManager());
printer.BeginSourceFile(PP.getLangOpts(), &PP);
arcDiags.reportDiagnostics(*Diags);
printer.EndSourceFile();
}
bool PragmaCommentHandler::HandleComment(Preprocessor &PP, SourceRange Range) {
StringRef Text =
Lexer::getSourceText(CharSourceRange::getCharRange(Range),
PP.getSourceManager(), PP.getLangOpts());
size_t Pos = Text.find(IWYUPragma);
if (Pos == StringRef::npos)
return false;
StringRef RemappingFilePath = Text.substr(Pos + std::strlen(IWYUPragma));
Collector->addHeaderMapping(
PP.getSourceManager().getFilename(Range.getBegin()),
RemappingFilePath.trim("\"<>"));
return false;
}
void clang::DoRewriteTest(Preprocessor &PP, raw_ostream* OS) {
SourceManager &SM = PP.getSourceManager();
const LangOptions &LangOpts = PP.getLangOpts();
TokenRewriter Rewriter(SM.getMainFileID(), SM, LangOpts);
// Throw <i> </i> tags around comments.
for (TokenRewriter::token_iterator I = Rewriter.token_begin(),
E = Rewriter.token_end(); I != E; ++I) {
if (I->isNot(tok::comment)) continue;
Rewriter.AddTokenBefore(I, "<i>");
Rewriter.AddTokenAfter(I, "</i>");
}
// Print out the output.
for (TokenRewriter::token_iterator I = Rewriter.token_begin(),
E = Rewriter.token_end(); I != E; ++I)
*OS << PP.getSpelling(*I);
}
bool HandleComment(Preprocessor &PP, SourceRange Range) override {
StringRef Text =
Lexer::getSourceText(CharSourceRange::getCharRange(Range),
PP.getSourceManager(), PP.getLangOpts());
SmallVector<StringRef, 4> Matches;
if (!TodoMatch.match(Text, &Matches))
return false;
StringRef Username = Matches[1];
StringRef Comment = Matches[3];
if (!Username.empty())
return false;
std::string NewText = ("// TODO(" + Twine(User) + "): " + Comment).str();
Check.diag(Range.getBegin(), "missing username/bug in TODO")
<< FixItHint::CreateReplacement(CharSourceRange::getCharRange(Range),
NewText);
return false;
}
/// HasExtension - Return true if we recognize and implement the feature
/// specified by the identifier, either as an extension or a standard language
/// feature.
static bool HasExtension(const Preprocessor &PP, const IdentifierInfo *II) {
if (HasFeature(PP, II))
return true;
// If the use of an extension results in an error diagnostic, extensions are
// effectively unavailable, so just return false here.
if (PP.getDiagnostics().getExtensionHandlingBehavior() ==
DiagnosticsEngine::Ext_Error)
return false;
const LangOptions &LangOpts = PP.getLangOpts();
StringRef Extension = II->getName();
// Normalize the extension name, __foo__ becomes foo.
if (Extension.startswith("__") && Extension.endswith("__") &&
Extension.size() >= 4)
Extension = Extension.substr(2, Extension.size() - 4);
// Because we inherit the feature list from HasFeature, this string switch
// must be less restrictive than HasFeature's.
return llvm::StringSwitch<bool>(Extension)
// C11 features supported by other languages as extensions.
.Case("c_alignas", true)
.Case("c_atomic", true)
.Case("c_generic_selections", true)
.Case("c_static_assert", true)
// C++0x features supported by other languages as extensions.
.Case("cxx_atomic", LangOpts.CPlusPlus)
.Case("cxx_deleted_functions", LangOpts.CPlusPlus)
.Case("cxx_explicit_conversions", LangOpts.CPlusPlus)
.Case("cxx_inline_namespaces", LangOpts.CPlusPlus)
.Case("cxx_local_type_template_args", LangOpts.CPlusPlus)
.Case("cxx_nonstatic_member_init", LangOpts.CPlusPlus)
.Case("cxx_override_control", LangOpts.CPlusPlus)
.Case("cxx_range_for", LangOpts.CPlusPlus)
.Case("cxx_reference_qualified_functions", LangOpts.CPlusPlus)
.Case("cxx_rvalue_references", LangOpts.CPlusPlus)
.Default(false);
}
IdentifierResolver::IdentifierResolver(Preprocessor &PP)
: LangOpt(PP.getLangOpts()), PP(PP), IdDeclInfos(new IdDeclInfoMap) {}
/// HasFeature - Return true if we recognize and implement the feature
/// specified by the identifier as a standard language feature.
static bool HasFeature(const Preprocessor &PP, const IdentifierInfo *II) {
const LangOptions &LangOpts = PP.getLangOpts();
StringRef Feature = II->getName();
// Normalize the feature name, __foo__ becomes foo.
if (Feature.startswith("__") && Feature.endswith("__") && Feature.size() >= 4)
Feature = Feature.substr(2, Feature.size() - 4);
return llvm::StringSwitch<bool>(Feature)
.Case("address_sanitizer", LangOpts.AddressSanitizer)
.Case("attribute_analyzer_noreturn", true)
.Case("attribute_availability", true)
.Case("attribute_cf_returns_not_retained", true)
.Case("attribute_cf_returns_retained", true)
.Case("attribute_deprecated_with_message", true)
.Case("attribute_ext_vector_type", true)
.Case("attribute_ns_returns_not_retained", true)
.Case("attribute_ns_returns_retained", true)
.Case("attribute_ns_consumes_self", true)
.Case("attribute_ns_consumed", true)
.Case("attribute_cf_consumed", true)
.Case("attribute_objc_ivar_unused", true)
.Case("attribute_objc_method_family", true)
.Case("attribute_overloadable", true)
.Case("attribute_unavailable_with_message", true)
.Case("attribute_unused_on_fields", true)
.Case("blocks", LangOpts.Blocks)
.Case("cxx_exceptions", LangOpts.Exceptions)
.Case("cxx_rtti", LangOpts.RTTI)
.Case("enumerator_attributes", true)
// Objective-C features
.Case("objc_arr", LangOpts.ObjCAutoRefCount) // FIXME: REMOVE?
.Case("objc_arc", LangOpts.ObjCAutoRefCount)
.Case("objc_arc_weak", LangOpts.ObjCAutoRefCount &&
LangOpts.ObjCRuntimeHasWeak)
.Case("objc_default_synthesize_properties", LangOpts.ObjC2)
.Case("objc_fixed_enum", LangOpts.ObjC2)
.Case("objc_instancetype", LangOpts.ObjC2)
.Case("objc_modules", LangOpts.ObjC2 && LangOpts.Modules)
.Case("objc_nonfragile_abi", LangOpts.ObjCRuntime.isNonFragile())
.Case("objc_weak_class", LangOpts.ObjCRuntime.hasWeakClassImport())
.Case("ownership_holds", true)
.Case("ownership_returns", true)
.Case("ownership_takes", true)
.Case("objc_bool", true)
.Case("objc_subscripting", LangOpts.ObjCRuntime.isNonFragile())
.Case("objc_array_literals", LangOpts.ObjC2)
.Case("objc_dictionary_literals", LangOpts.ObjC2)
.Case("objc_boxed_expressions", LangOpts.ObjC2)
.Case("arc_cf_code_audited", true)
// C11 features
.Case("c_alignas", LangOpts.C11)
.Case("c_atomic", LangOpts.C11)
.Case("c_generic_selections", LangOpts.C11)
.Case("c_static_assert", LangOpts.C11)
// C++11 features
.Case("cxx_access_control_sfinae", LangOpts.CPlusPlus0x)
.Case("cxx_alias_templates", LangOpts.CPlusPlus0x)
.Case("cxx_alignas", LangOpts.CPlusPlus0x)
.Case("cxx_atomic", LangOpts.CPlusPlus0x)
.Case("cxx_attributes", LangOpts.CPlusPlus0x)
.Case("cxx_auto_type", LangOpts.CPlusPlus0x)
.Case("cxx_constexpr", LangOpts.CPlusPlus0x)
.Case("cxx_decltype", LangOpts.CPlusPlus0x)
.Case("cxx_decltype_incomplete_return_types", LangOpts.CPlusPlus0x)
.Case("cxx_default_function_template_args", LangOpts.CPlusPlus0x)
.Case("cxx_defaulted_functions", LangOpts.CPlusPlus0x)
.Case("cxx_delegating_constructors", LangOpts.CPlusPlus0x)
.Case("cxx_deleted_functions", LangOpts.CPlusPlus0x)
.Case("cxx_explicit_conversions", LangOpts.CPlusPlus0x)
.Case("cxx_generalized_initializers", LangOpts.CPlusPlus0x)
.Case("cxx_implicit_moves", LangOpts.CPlusPlus0x)
//.Case("cxx_inheriting_constructors", false)
.Case("cxx_inline_namespaces", LangOpts.CPlusPlus0x)
.Case("cxx_lambdas", LangOpts.CPlusPlus0x)
.Case("cxx_local_type_template_args", LangOpts.CPlusPlus0x)
.Case("cxx_nonstatic_member_init", LangOpts.CPlusPlus0x)
.Case("cxx_noexcept", LangOpts.CPlusPlus0x)
.Case("cxx_nullptr", LangOpts.CPlusPlus0x)
.Case("cxx_override_control", LangOpts.CPlusPlus0x)
.Case("cxx_range_for", LangOpts.CPlusPlus0x)
.Case("cxx_raw_string_literals", LangOpts.CPlusPlus0x)
.Case("cxx_reference_qualified_functions", LangOpts.CPlusPlus0x)
.Case("cxx_rvalue_references", LangOpts.CPlusPlus0x)
.Case("cxx_strong_enums", LangOpts.CPlusPlus0x)
.Case("cxx_static_assert", LangOpts.CPlusPlus0x)
.Case("cxx_trailing_return", LangOpts.CPlusPlus0x)
.Case("cxx_unicode_literals", LangOpts.CPlusPlus0x)
.Case("cxx_unrestricted_unions", LangOpts.CPlusPlus0x)
.Case("cxx_user_literals", LangOpts.CPlusPlus0x)
.Case("cxx_variadic_templates", LangOpts.CPlusPlus0x)
// Type traits
.Case("has_nothrow_assign", LangOpts.CPlusPlus)
.Case("has_nothrow_copy", LangOpts.CPlusPlus)
.Case("has_nothrow_constructor", LangOpts.CPlusPlus)
.Case("has_trivial_assign", LangOpts.CPlusPlus)
.Case("has_trivial_copy", LangOpts.CPlusPlus)
.Case("has_trivial_constructor", LangOpts.CPlusPlus)
.Case("has_trivial_destructor", LangOpts.CPlusPlus)
.Case("has_virtual_destructor", LangOpts.CPlusPlus)
.Case("is_abstract", LangOpts.CPlusPlus)
.Case("is_base_of", LangOpts.CPlusPlus)
.Case("is_class", LangOpts.CPlusPlus)
.Case("is_convertible_to", LangOpts.CPlusPlus)
// __is_empty is available only if the horrible
// "struct __is_empty" parsing hack hasn't been needed in this
// translation unit. If it has, __is_empty reverts to a normal
// identifier and __has_feature(is_empty) evaluates false.
.Case("is_empty",
LangOpts.CPlusPlus &&
PP.getIdentifierInfo("__is_empty")->getTokenID()
!= tok::identifier)
.Case("is_enum", LangOpts.CPlusPlus)
.Case("is_final", LangOpts.CPlusPlus)
.Case("is_literal", LangOpts.CPlusPlus)
.Case("is_standard_layout", LangOpts.CPlusPlus)
// __is_pod is available only if the horrible
// "struct __is_pod" parsing hack hasn't been needed in this
// translation unit. If it has, __is_pod reverts to a normal
// identifier and __has_feature(is_pod) evaluates false.
.Case("is_pod",
LangOpts.CPlusPlus &&
PP.getIdentifierInfo("__is_pod")->getTokenID()
!= tok::identifier)
.Case("is_polymorphic", LangOpts.CPlusPlus)
.Case("is_trivial", LangOpts.CPlusPlus)
.Case("is_trivially_assignable", LangOpts.CPlusPlus)
.Case("is_trivially_constructible", LangOpts.CPlusPlus)
.Case("is_trivially_copyable", LangOpts.CPlusPlus)
.Case("is_union", LangOpts.CPlusPlus)
.Case("modules", LangOpts.Modules)
.Case("tls", PP.getTargetInfo().isTLSSupported())
.Case("underlying_type", LangOpts.CPlusPlus)
.Default(false);
}
// #pragma pack(...) comes in the following delicious flavors:
// pack '(' [integer] ')'
// pack '(' 'show' ')'
// pack '(' ('push' | 'pop') [',' identifier] [, integer] ')'
void PragmaPackHandler::HandlePragma(Preprocessor &PP,
PragmaIntroducerKind Introducer,
Token &PackTok) {
SourceLocation PackLoc = PackTok.getLocation();
Token Tok;
PP.Lex(Tok);
if (Tok.isNot(tok::l_paren)) {
PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_lparen) << "pack";
return;
}
Sema::PragmaPackKind Kind = Sema::PPK_Default;
IdentifierInfo *Name = 0;
Token Alignment;
Alignment.startToken();
SourceLocation LParenLoc = Tok.getLocation();
PP.Lex(Tok);
if (Tok.is(tok::numeric_constant)) {
Alignment = Tok;
PP.Lex(Tok);
// In MSVC/gcc, #pragma pack(4) sets the alignment without affecting
// the push/pop stack.
// In Apple gcc, #pragma pack(4) is equivalent to #pragma pack(push, 4)
if (PP.getLangOpts().ApplePragmaPack)
Kind = Sema::PPK_Push;
} else if (Tok.is(tok::identifier)) {
const IdentifierInfo *II = Tok.getIdentifierInfo();
if (II->isStr("show")) {
Kind = Sema::PPK_Show;
PP.Lex(Tok);
} else {
if (II->isStr("push")) {
Kind = Sema::PPK_Push;
} else if (II->isStr("pop")) {
Kind = Sema::PPK_Pop;
} else {
PP.Diag(Tok.getLocation(), diag::warn_pragma_pack_invalid_action);
return;
}
PP.Lex(Tok);
if (Tok.is(tok::comma)) {
PP.Lex(Tok);
if (Tok.is(tok::numeric_constant)) {
Alignment = Tok;
PP.Lex(Tok);
} else if (Tok.is(tok::identifier)) {
Name = Tok.getIdentifierInfo();
PP.Lex(Tok);
if (Tok.is(tok::comma)) {
PP.Lex(Tok);
if (Tok.isNot(tok::numeric_constant)) {
PP.Diag(Tok.getLocation(), diag::warn_pragma_pack_malformed);
return;
}
Alignment = Tok;
PP.Lex(Tok);
}
} else {
PP.Diag(Tok.getLocation(), diag::warn_pragma_pack_malformed);
return;
}
}
}
} else if (PP.getLangOpts().ApplePragmaPack) {
// In MSVC/gcc, #pragma pack() resets the alignment without affecting
// the push/pop stack.
// In Apple gcc #pragma pack() is equivalent to #pragma pack(pop).
Kind = Sema::PPK_Pop;
}
if (Tok.isNot(tok::r_paren)) {
PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_rparen) << "pack";
return;
}
SourceLocation RParenLoc = Tok.getLocation();
PP.Lex(Tok);
if (Tok.isNot(tok::eod)) {
PP.Diag(Tok.getLocation(), diag::warn_pragma_extra_tokens_at_eol) << "pack";
return;
}
PragmaPackInfo *Info =
(PragmaPackInfo*) PP.getPreprocessorAllocator().Allocate(
sizeof(PragmaPackInfo), llvm::alignOf<PragmaPackInfo>());
new (Info) PragmaPackInfo();
Info->Kind = Kind;
Info->Name = Name;
Info->Alignment = Alignment;
Info->LParenLoc = LParenLoc;
Info->RParenLoc = RParenLoc;
Token *Toks =
(Token*) PP.getPreprocessorAllocator().Allocate(
sizeof(Token) * 1, llvm::alignOf<Token>());
new (Toks) Token();
Toks[0].startToken();
Toks[0].setKind(tok::annot_pragma_pack);
Toks[0].setLocation(PackLoc);
Toks[0].setAnnotationValue(static_cast<void*>(Info));
PP.EnterTokenStream(Toks, 1, /*DisableMacroExpansion=*/true,
/*OwnsTokens=*/false);
}
/// integer-constant: [C99 6.4.4.1]
/// decimal-constant integer-suffix
/// octal-constant integer-suffix
/// hexadecimal-constant integer-suffix
/// binary-literal integer-suffix [GNU, C++1y]
/// user-defined-integer-literal: [C++11 lex.ext]
/// decimal-literal ud-suffix
/// octal-literal ud-suffix
/// hexadecimal-literal ud-suffix
/// binary-literal ud-suffix [GNU, C++1y]
/// decimal-constant:
/// nonzero-digit
/// decimal-constant digit
/// octal-constant:
/// 0
/// octal-constant octal-digit
/// hexadecimal-constant:
/// hexadecimal-prefix hexadecimal-digit
/// hexadecimal-constant hexadecimal-digit
/// hexadecimal-prefix: one of
/// 0x 0X
/// binary-literal:
/// 0b binary-digit
/// 0B binary-digit
/// binary-literal binary-digit
/// integer-suffix:
/// unsigned-suffix [long-suffix]
/// unsigned-suffix [long-long-suffix]
/// long-suffix [unsigned-suffix]
/// long-long-suffix [unsigned-sufix]
/// nonzero-digit:
/// 1 2 3 4 5 6 7 8 9
/// octal-digit:
/// 0 1 2 3 4 5 6 7
/// hexadecimal-digit:
/// 0 1 2 3 4 5 6 7 8 9
/// a b c d e f
/// A B C D E F
/// binary-digit:
/// 0
/// 1
/// unsigned-suffix: one of
/// u U
/// long-suffix: one of
/// l L
/// long-long-suffix: one of
/// ll LL
///
/// floating-constant: [C99 6.4.4.2]
/// TODO: add rules...
///
NumericLiteralParser::NumericLiteralParser(StringRef TokSpelling)
: ThisTokBegin(TokSpelling.begin()), ThisTokEnd(TokSpelling.end()) {
// This routine assumes that the range begin/end matches the regex for integer
// and FP constants (specifically, the 'pp-number' regex), and assumes that
// the byte at "*end" is both valid and not part of the regex. Because of
// this, it doesn't have to check for 'overscan' in various places.
assert(!isPreprocessingNumberBody(*ThisTokEnd) && "didn't maximally munch?");
s = DigitsBegin = ThisTokBegin;
saw_exponent = false;
saw_period = false;
saw_ud_suffix = false;
isLong = false;
isUnsigned = false;
isLongLong = false;
isFloat = false;
isImaginary = false;
MicrosoftInteger = 0;
hadError = false;
SourceLocation TokLoc;
if (*s == '0') { // parse radix
ParseNumberStartingWithZero(TokLoc);
if (hadError)
return;
} else { // the first digit is non-zero
radix = 10;
s = SkipDigits(s);
if (s == ThisTokEnd) {
// Done.
} else if (isHexDigit(*s) && !(*s == 'e' || *s == 'E')) {
PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s - ThisTokBegin),
diag::err_invalid_decimal_digit) << StringRef(s, 1);
hadError = true;
return;
} else if (*s == '.') {
checkSeparator(TokLoc, s, CSK_AfterDigits);
s++;
saw_period = true;
checkSeparator(TokLoc, s, CSK_BeforeDigits);
s = SkipDigits(s);
}
if ((*s == 'e' || *s == 'E')) { // exponent
checkSeparator(TokLoc, s, CSK_AfterDigits);
const char *Exponent = s;
s++;
saw_exponent = true;
if (*s == '+' || *s == '-') s++; // sign
checkSeparator(TokLoc, s, CSK_BeforeDigits);
const char *first_non_digit = SkipDigits(s);
if (first_non_digit != s) {
s = first_non_digit;
} else {
PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, Exponent - ThisTokBegin),
diag::err_exponent_has_no_digits);
hadError = true;
return;
}
}
}
SuffixBegin = s;
checkSeparator(TokLoc, s, CSK_AfterDigits);
// Parse the suffix. At this point we can classify whether we have an FP or
// integer constant.
bool isFPConstant = isFloatingLiteral();
const char *ImaginarySuffixLoc = nullptr;
// Loop over all of the characters of the suffix. If we see something bad,
// we break out of the loop.
for (; s != ThisTokEnd; ++s) {
switch (*s) {
case 'f': // FP Suffix for "float"
case 'F':
if (!isFPConstant) break; // Error for integer constant.
if (isFloat || isLong) break; // FF, LF invalid.
isFloat = true;
continue; // Success.
case 'u':
case 'U':
if (isFPConstant) break; // Error for floating constant.
if (isUnsigned) break; // Cannot be repeated.
isUnsigned = true;
continue; // Success.
case 'l':
case 'L':
if (isLong || isLongLong) break; // Cannot be repeated.
if (isFloat) break; // LF invalid.
// Check for long long. The L's need to be adjacent and the same case.
if (s[1] == s[0]) {
assert(s + 1 < ThisTokEnd && "didn't maximally munch?");
if (isFPConstant) break; // long long invalid for floats.
isLongLong = true;
++s; // Eat both of them.
} else {
isLong = true;
}
continue; // Success.
case 'i':
case 'I':
if (PP.getLangOpts().MicrosoftExt) {
if (isLong || isLongLong || MicrosoftInteger)
break;
if (!isFPConstant) {
// Allow i8, i16, i32, i64, and i128.
switch (s[1]) {
case '8':
s += 2; // i8 suffix
MicrosoftInteger = 8;
break;
case '1':
if (s[2] == '6') {
s += 3; // i16 suffix
MicrosoftInteger = 16;
} else if (s[2] == '2' && s[3] == '8') {
s += 4; // i128 suffix
MicrosoftInteger = 128;
}
break;
case '3':
if (s[2] == '2') {
s += 3; // i32 suffix
MicrosoftInteger = 32;
}
break;
case '6':
if (s[2] == '4') {
s += 3; // i64 suffix
MicrosoftInteger = 64;
}
break;
default:
break;
}
}
if (MicrosoftInteger) {
assert(s <= ThisTokEnd && "didn't maximally munch?");
break;
}
}
// "i", "if", and "il" are user-defined suffixes in C++1y.
if (*s == 'i' && PP.getLangOpts().CPlusPlus14)
break;
// fall through.
case 'j':
case 'J':
if (isImaginary) break; // Cannot be repeated.
isImaginary = true;
ImaginarySuffixLoc = s;
continue; // Success.
}
// If we reached here, there was an error or a ud-suffix.
break;
}
if (s != ThisTokEnd) {
// FIXME: Don't bother expanding UCNs if !tok.hasUCN().
expandUCNs(UDSuffixBuf, StringRef(SuffixBegin, ThisTokEnd - SuffixBegin));
if (isValidUDSuffix(PP.getLangOpts(), UDSuffixBuf)) {
// Any suffix pieces we might have parsed are actually part of the
// ud-suffix.
isLong = false;
isUnsigned = false;
isLongLong = false;
isFloat = false;
isImaginary = false;
MicrosoftInteger = 0;
saw_ud_suffix = true;
return;
}
// Report an error if there are any.
PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, SuffixBegin - ThisTokBegin),
isFPConstant ? diag::err_invalid_suffix_float_constant :
diag::err_invalid_suffix_integer_constant)
<< StringRef(SuffixBegin, ThisTokEnd-SuffixBegin);
hadError = true;
return;
}
if (isImaginary) {
PP.Diag(PP.AdvanceToTokenCharacter(TokLoc,
ImaginarySuffixLoc - ThisTokBegin),
diag::ext_imaginary_constant);
}
}
/// Finish - This does final analysis of the declspec, rejecting things like
/// "_Imaginary" (lacking an FP type). This returns a diagnostic to issue or
/// diag::NUM_DIAGNOSTICS if there is no error. After calling this method,
/// DeclSpec is guaranteed self-consistent, even if an error occurred.
void DeclSpec::Finish(DiagnosticsEngine &D, Preprocessor &PP, const PrintingPolicy &Policy) {
// Before possibly changing their values, save specs as written.
SaveWrittenBuiltinSpecs();
// Check the type specifier components first.
// If decltype(auto) is used, no other type specifiers are permitted.
if (TypeSpecType == TST_decltype_auto &&
(TypeSpecWidth != TSW_unspecified ||
TypeSpecComplex != TSC_unspecified ||
TypeSpecSign != TSS_unspecified ||
TypeAltiVecVector || TypeAltiVecPixel || TypeAltiVecBool ||
TypeQualifiers)) {
const unsigned NumLocs = 8;
SourceLocation ExtraLocs[NumLocs] = {
TSWLoc, TSCLoc, TSSLoc, AltiVecLoc,
TQ_constLoc, TQ_restrictLoc, TQ_volatileLoc, TQ_atomicLoc
};
FixItHint Hints[NumLocs];
SourceLocation FirstLoc;
for (unsigned I = 0; I != NumLocs; ++I) {
if (!ExtraLocs[I].isInvalid()) {
if (FirstLoc.isInvalid() ||
PP.getSourceManager().isBeforeInTranslationUnit(ExtraLocs[I],
FirstLoc))
FirstLoc = ExtraLocs[I];
Hints[I] = FixItHint::CreateRemoval(ExtraLocs[I]);
}
}
TypeSpecWidth = TSW_unspecified;
TypeSpecComplex = TSC_unspecified;
TypeSpecSign = TSS_unspecified;
TypeAltiVecVector = TypeAltiVecPixel = TypeAltiVecBool = false;
TypeQualifiers = 0;
Diag(D, TSTLoc, diag::err_decltype_auto_cannot_be_combined)
<< Hints[0] << Hints[1] << Hints[2] << Hints[3]
<< Hints[4] << Hints[5] << Hints[6] << Hints[7];
}
// Validate and finalize AltiVec vector declspec.
if (TypeAltiVecVector) {
if (TypeAltiVecBool) {
// Sign specifiers are not allowed with vector bool. (PIM 2.1)
if (TypeSpecSign != TSS_unspecified) {
Diag(D, TSSLoc, diag::err_invalid_vector_bool_decl_spec)
<< getSpecifierName((TSS)TypeSpecSign);
}
// Only char/int are valid with vector bool. (PIM 2.1)
if (((TypeSpecType != TST_unspecified) && (TypeSpecType != TST_char) &&
(TypeSpecType != TST_int)) || TypeAltiVecPixel) {
Diag(D, TSTLoc, diag::err_invalid_vector_bool_decl_spec)
<< (TypeAltiVecPixel ? "__pixel" :
getSpecifierName((TST)TypeSpecType, Policy));
}
// Only 'short' and 'long long' are valid with vector bool. (PIM 2.1)
if ((TypeSpecWidth != TSW_unspecified) && (TypeSpecWidth != TSW_short) &&
(TypeSpecWidth != TSW_longlong))
Diag(D, TSWLoc, diag::err_invalid_vector_bool_decl_spec)
<< getSpecifierName((TSW)TypeSpecWidth);
// vector bool long long requires VSX support.
if ((TypeSpecWidth == TSW_longlong) &&
(!PP.getTargetInfo().hasFeature("vsx")) &&
(!PP.getTargetInfo().hasFeature("power8-vector")))
Diag(D, TSTLoc, diag::err_invalid_vector_long_long_decl_spec);
// Elements of vector bool are interpreted as unsigned. (PIM 2.1)
if ((TypeSpecType == TST_char) || (TypeSpecType == TST_int) ||
(TypeSpecWidth != TSW_unspecified))
TypeSpecSign = TSS_unsigned;
} else if (TypeSpecType == TST_double) {
// vector long double and vector long long double are never allowed.
// vector double is OK for Power7 and later.
if (TypeSpecWidth == TSW_long || TypeSpecWidth == TSW_longlong)
Diag(D, TSWLoc, diag::err_invalid_vector_long_double_decl_spec);
else if (!PP.getTargetInfo().hasFeature("vsx"))
Diag(D, TSTLoc, diag::err_invalid_vector_double_decl_spec);
} else if (TypeSpecWidth == TSW_long) {
Diag(D, TSWLoc, diag::warn_vector_long_decl_spec_combination)
<< getSpecifierName((TST)TypeSpecType, Policy);
}
if (TypeAltiVecPixel) {
//TODO: perform validation
TypeSpecType = TST_int;
TypeSpecSign = TSS_unsigned;
TypeSpecWidth = TSW_short;
TypeSpecOwned = false;
}
}
// signed/unsigned are only valid with int/char/wchar_t.
if (TypeSpecSign != TSS_unspecified) {
if (TypeSpecType == TST_unspecified)
TypeSpecType = TST_int; // unsigned -> unsigned int, signed -> signed int.
else if (TypeSpecType != TST_int && TypeSpecType != TST_int128 &&
TypeSpecType != TST_char && TypeSpecType != TST_wchar) {
Diag(D, TSSLoc, diag::err_invalid_sign_spec)
<< getSpecifierName((TST)TypeSpecType, Policy);
// signed double -> double.
TypeSpecSign = TSS_unspecified;
}
}
// Validate the width of the type.
switch (TypeSpecWidth) {
case TSW_unspecified: break;
case TSW_short: // short int
case TSW_longlong: // long long int
if (TypeSpecType == TST_unspecified)
TypeSpecType = TST_int; // short -> short int, long long -> long long int.
else if (TypeSpecType != TST_int) {
Diag(D, TSWLoc,
TypeSpecWidth == TSW_short ? diag::err_invalid_short_spec
: diag::err_invalid_longlong_spec)
<< getSpecifierName((TST)TypeSpecType, Policy);
TypeSpecType = TST_int;
TypeSpecOwned = false;
}
break;
case TSW_long: // long double, long int
if (TypeSpecType == TST_unspecified)
TypeSpecType = TST_int; // long -> long int.
else if (TypeSpecType != TST_int && TypeSpecType != TST_double) {
Diag(D, TSWLoc, diag::err_invalid_long_spec)
<< getSpecifierName((TST)TypeSpecType, Policy);
TypeSpecType = TST_int;
TypeSpecOwned = false;
}
break;
}
// TODO: if the implementation does not implement _Complex or _Imaginary,
// disallow their use. Need information about the backend.
if (TypeSpecComplex != TSC_unspecified) {
if (TypeSpecType == TST_unspecified) {
Diag(D, TSCLoc, diag::ext_plain_complex)
<< FixItHint::CreateInsertion(
PP.getLocForEndOfToken(getTypeSpecComplexLoc()),
" double");
TypeSpecType = TST_double; // _Complex -> _Complex double.
} else if (TypeSpecType == TST_int || TypeSpecType == TST_char) {
// Note that this intentionally doesn't include _Complex _Bool.
if (!PP.getLangOpts().CPlusPlus)
Diag(D, TSTLoc, diag::ext_integer_complex);
} else if (TypeSpecType != TST_float && TypeSpecType != TST_double) {
Diag(D, TSCLoc, diag::err_invalid_complex_spec)
<< getSpecifierName((TST)TypeSpecType, Policy);
TypeSpecComplex = TSC_unspecified;
}
}
// C11 6.7.1/3, C++11 [dcl.stc]p1, GNU TLS: __thread, thread_local and
// _Thread_local can only appear with the 'static' and 'extern' storage class
// specifiers. We also allow __private_extern__ as an extension.
if (ThreadStorageClassSpec != TSCS_unspecified) {
switch (StorageClassSpec) {
case SCS_unspecified:
case SCS_extern:
case SCS_private_extern:
case SCS_static:
break;
default:
if (PP.getSourceManager().isBeforeInTranslationUnit(
getThreadStorageClassSpecLoc(), getStorageClassSpecLoc()))
Diag(D, getStorageClassSpecLoc(),
diag::err_invalid_decl_spec_combination)
<< DeclSpec::getSpecifierName(getThreadStorageClassSpec())
<< SourceRange(getThreadStorageClassSpecLoc());
else
Diag(D, getThreadStorageClassSpecLoc(),
diag::err_invalid_decl_spec_combination)
<< DeclSpec::getSpecifierName(getStorageClassSpec())
<< SourceRange(getStorageClassSpecLoc());
// Discard the thread storage class specifier to recover.
ThreadStorageClassSpec = TSCS_unspecified;
ThreadStorageClassSpecLoc = SourceLocation();
}
}
// If no type specifier was provided and we're parsing a language where
// the type specifier is not optional, but we got 'auto' as a storage
// class specifier, then assume this is an attempt to use C++0x's 'auto'
// type specifier.
if (PP.getLangOpts().CPlusPlus &&
TypeSpecType == TST_unspecified && StorageClassSpec == SCS_auto) {
TypeSpecType = TST_auto;
StorageClassSpec = SCS_unspecified;
TSTLoc = TSTNameLoc = StorageClassSpecLoc;
StorageClassSpecLoc = SourceLocation();
}
// Diagnose if we've recovered from an ill-formed 'auto' storage class
// specifier in a pre-C++11 dialect of C++.
if (!PP.getLangOpts().CPlusPlus11 && TypeSpecType == TST_auto)
Diag(D, TSTLoc, diag::ext_auto_type_specifier);
if (PP.getLangOpts().CPlusPlus && !PP.getLangOpts().CPlusPlus11 &&
StorageClassSpec == SCS_auto)
Diag(D, StorageClassSpecLoc, diag::warn_auto_storage_class)
<< FixItHint::CreateRemoval(StorageClassSpecLoc);
if (TypeSpecType == TST_char16 || TypeSpecType == TST_char32)
Diag(D, TSTLoc, diag::warn_cxx98_compat_unicode_type)
<< (TypeSpecType == TST_char16 ? "char16_t" : "char32_t");
if (Constexpr_specified)
Diag(D, ConstexprLoc, diag::warn_cxx98_compat_constexpr);
// C++ [class.friend]p6:
// No storage-class-specifier shall appear in the decl-specifier-seq
// of a friend declaration.
if (isFriendSpecified() &&
(getStorageClassSpec() || getThreadStorageClassSpec())) {
SmallString<32> SpecName;
SourceLocation SCLoc;
FixItHint StorageHint, ThreadHint;
if (DeclSpec::SCS SC = getStorageClassSpec()) {
SpecName = getSpecifierName(SC);
SCLoc = getStorageClassSpecLoc();
StorageHint = FixItHint::CreateRemoval(SCLoc);
}
if (DeclSpec::TSCS TSC = getThreadStorageClassSpec()) {
if (!SpecName.empty()) SpecName += " ";
SpecName += getSpecifierName(TSC);
SCLoc = getThreadStorageClassSpecLoc();
ThreadHint = FixItHint::CreateRemoval(SCLoc);
}
Diag(D, SCLoc, diag::err_friend_decl_spec)
<< SpecName << StorageHint << ThreadHint;
ClearStorageClassSpecs();
}
// C++11 [dcl.fct.spec]p5:
// The virtual specifier shall be used only in the initial
// declaration of a non-static class member function;
// C++11 [dcl.fct.spec]p6:
// The explicit specifier shall be used only in the declaration of
// a constructor or conversion function within its class
// definition;
if (isFriendSpecified() && (isVirtualSpecified() || isExplicitSpecified())) {
StringRef Keyword;
SourceLocation SCLoc;
if (isVirtualSpecified()) {
Keyword = "virtual";
SCLoc = getVirtualSpecLoc();
} else {
Keyword = "explicit";
SCLoc = getExplicitSpecLoc();
}
FixItHint Hint = FixItHint::CreateRemoval(SCLoc);
Diag(D, SCLoc, diag::err_friend_decl_spec)
<< Keyword << Hint;
FS_virtual_specified = FS_explicit_specified = false;
FS_virtualLoc = FS_explicitLoc = SourceLocation();
}
assert(!TypeSpecOwned || isDeclRep((TST) TypeSpecType));
// Okay, now we can infer the real type.
// TODO: return "auto function" and other bad things based on the real type.
// 'data definition has no type or storage class'?
}
void ento::createPlistDiagnosticConsumer(PathDiagnosticConsumers &C,
const std::string& s,
const Preprocessor &PP) {
C.push_back(new PlistDiagnostics(s, PP.getLangOpts(), false));
}
/// SyntaxHighlight - Relex the specified FileID and annotate the HTML with
/// information about keywords, macro expansions etc. This uses the macro
/// table state from the end of the file, so it won't be perfectly perfect,
/// but it will be reasonably close.
void html::SyntaxHighlight(Rewriter &R, FileID FID, const Preprocessor &PP) {
RewriteBuffer &RB = R.getEditBuffer(FID);
const SourceManager &SM = PP.getSourceManager();
const llvm::MemoryBuffer *FromFile = SM.getBuffer(FID);
Lexer L(FID, FromFile, SM, PP.getLangOpts());
const char *BufferStart = L.getBuffer().data();
// Inform the preprocessor that we want to retain comments as tokens, so we
// can highlight them.
L.SetCommentRetentionState(true);
// Lex all the tokens in raw mode, to avoid entering #includes or expanding
// macros.
Token Tok;
L.LexFromRawLexer(Tok);
while (Tok.isNot(tok::eof)) {
// Since we are lexing unexpanded tokens, all tokens are from the main
// FileID.
unsigned TokOffs = SM.getFileOffset(Tok.getLocation());
unsigned TokLen = Tok.getLength();
switch (Tok.getKind()) {
default: break;
case tok::identifier:
llvm_unreachable("tok::identifier in raw lexing mode!");
case tok::raw_identifier: {
// Fill in Result.IdentifierInfo and update the token kind,
// looking up the identifier in the identifier table.
PP.LookUpIdentifierInfo(Tok);
// If this is a pp-identifier, for a keyword, highlight it as such.
if (Tok.isNot(tok::identifier))
HighlightRange(RB, TokOffs, TokOffs+TokLen, BufferStart,
"<span class='keyword'>", "</span>");
break;
}
case tok::comment:
HighlightRange(RB, TokOffs, TokOffs+TokLen, BufferStart,
"<span class='comment'>", "</span>");
break;
case tok::utf8_string_literal:
// Chop off the u part of u8 prefix
++TokOffs;
--TokLen;
// FALL THROUGH to chop the 8
case tok::wide_string_literal:
case tok::utf16_string_literal:
case tok::utf32_string_literal:
// Chop off the L, u, U or 8 prefix
++TokOffs;
--TokLen;
// FALL THROUGH.
case tok::string_literal:
// FIXME: Exclude the optional ud-suffix from the highlighted range.
HighlightRange(RB, TokOffs, TokOffs+TokLen, BufferStart,
"<span class='string_literal'>", "</span>");
break;
case tok::hash: {
// If this is a preprocessor directive, all tokens to end of line are too.
if (!Tok.isAtStartOfLine())
break;
// Eat all of the tokens until we get to the next one at the start of
// line.
unsigned TokEnd = TokOffs+TokLen;
L.LexFromRawLexer(Tok);
while (!Tok.isAtStartOfLine() && Tok.isNot(tok::eof)) {
TokEnd = SM.getFileOffset(Tok.getLocation())+Tok.getLength();
L.LexFromRawLexer(Tok);
}
// Find end of line. This is a hack.
HighlightRange(RB, TokOffs, TokEnd, BufferStart,
"<span class='directive'>", "</span>");
// Don't skip the next token.
continue;
}
}
L.LexFromRawLexer(Tok);
}
}
/// RewriteMacrosInInput - Implement -rewrite-macros mode.
void clang::RewriteMacrosInInput(Preprocessor &PP, raw_ostream *OS) {
SourceManager &SM = PP.getSourceManager();
Rewriter Rewrite;
Rewrite.setSourceMgr(SM, PP.getLangOpts());
RewriteBuffer &RB = Rewrite.getEditBuffer(SM.getMainFileID());
std::vector<Token> RawTokens;
LexRawTokensFromMainFile(PP, RawTokens);
unsigned CurRawTok = 0;
Token RawTok = GetNextRawTok(RawTokens, CurRawTok, false);
// Get the first preprocessing token.
PP.EnterMainSourceFile();
Token PPTok;
PP.Lex(PPTok);
// Preprocess the input file in parallel with raw lexing the main file. Ignore
// all tokens that are preprocessed from a file other than the main file (e.g.
// a header). If we see tokens that are in the preprocessed file but not the
// lexed file, we have a macro expansion. If we see tokens in the lexed file
// that aren't in the preprocessed view, we have macros that expand to no
// tokens, or macro arguments etc.
while (RawTok.isNot(tok::eof) || PPTok.isNot(tok::eof)) {
SourceLocation PPLoc = SM.getExpansionLoc(PPTok.getLocation());
// If PPTok is from a different source file, ignore it.
if (!SM.isFromMainFile(PPLoc)) {
PP.Lex(PPTok);
continue;
}
// If the raw file hits a preprocessor directive, they will be extra tokens
// in the raw file that don't exist in the preprocsesed file. However, we
// choose to preserve them in the output file and otherwise handle them
// specially.
if (RawTok.is(tok::hash) && RawTok.isAtStartOfLine()) {
// If this is a #warning directive or #pragma mark (GNU extensions),
// comment the line out.
if (RawTokens[CurRawTok].is(tok::identifier)) {
const IdentifierInfo *II = RawTokens[CurRawTok].getIdentifierInfo();
if (II->getName() == "warning") {
// Comment out #warning.
RB.InsertTextAfter(SM.getFileOffset(RawTok.getLocation()), "//");
} else if (II->getName() == "pragma" &&
RawTokens[CurRawTok+1].is(tok::identifier) &&
(RawTokens[CurRawTok+1].getIdentifierInfo()->getName() ==
"mark")) {
// Comment out #pragma mark.
RB.InsertTextAfter(SM.getFileOffset(RawTok.getLocation()), "//");
}
}
// Otherwise, if this is a #include or some other directive, just leave it
// in the file by skipping over the line.
RawTok = GetNextRawTok(RawTokens, CurRawTok, false);
while (!RawTok.isAtStartOfLine() && RawTok.isNot(tok::eof))
RawTok = GetNextRawTok(RawTokens, CurRawTok, false);
continue;
}
// Okay, both tokens are from the same file. Get their offsets from the
// start of the file.
unsigned PPOffs = SM.getFileOffset(PPLoc);
unsigned RawOffs = SM.getFileOffset(RawTok.getLocation());
// If the offsets are the same and the token kind is the same, ignore them.
if (PPOffs == RawOffs && isSameToken(RawTok, PPTok)) {
RawTok = GetNextRawTok(RawTokens, CurRawTok, false);
PP.Lex(PPTok);
continue;
}
// If the PP token is farther along than the raw token, something was
// deleted. Comment out the raw token.
if (RawOffs <= PPOffs) {
// Comment out a whole run of tokens instead of bracketing each one with
// comments. Add a leading space if RawTok didn't have one.
bool HasSpace = RawTok.hasLeadingSpace();
RB.InsertTextAfter(RawOffs, &" /*"[HasSpace]);
unsigned EndPos;
do {
EndPos = RawOffs+RawTok.getLength();
RawTok = GetNextRawTok(RawTokens, CurRawTok, true);
RawOffs = SM.getFileOffset(RawTok.getLocation());
if (RawTok.is(tok::comment)) {
// Skip past the comment.
RawTok = GetNextRawTok(RawTokens, CurRawTok, false);
break;
}
} while (RawOffs <= PPOffs && !RawTok.isAtStartOfLine() &&
(PPOffs != RawOffs || !isSameToken(RawTok, PPTok)));
RB.InsertTextBefore(EndPos, "*/");
continue;
}
// Otherwise, there was a replacement an expansion. Insert the new token
// in the output buffer. Insert the whole run of new tokens at once to get
// them in the right order.
unsigned InsertPos = PPOffs;
std::string Expansion;
while (PPOffs < RawOffs) {
Expansion += ' ' + PP.getSpelling(PPTok);
PP.Lex(PPTok);
PPLoc = SM.getExpansionLoc(PPTok.getLocation());
PPOffs = SM.getFileOffset(PPLoc);
}
Expansion += ' ';
RB.InsertTextBefore(InsertPos, Expansion);
}
// Get the buffer corresponding to MainFileID. If we haven't changed it, then
// we are done.
if (const RewriteBuffer *RewriteBuf =
Rewrite.getRewriteBufferFor(SM.getMainFileID())) {
//printf("Changed:\n");
*OS << std::string(RewriteBuf->begin(), RewriteBuf->end());
} else {
fprintf(stderr, "No changes\n");
}
OS->flush();
}
/// InitializePreprocessor - Initialize the preprocessor getting it and the
/// environment ready to process a single file. This returns true on error.
///
void clang::InitializePreprocessor(
Preprocessor &PP, const PreprocessorOptions &InitOpts,
const PCHContainerReader &PCHContainerRdr,
const FrontendOptions &FEOpts) {
const LangOptions &LangOpts = PP.getLangOpts();
std::string PredefineBuffer;
PredefineBuffer.reserve(4080);
llvm::raw_string_ostream Predefines(PredefineBuffer);
MacroBuilder Builder(Predefines);
// Emit line markers for various builtin sections of the file. We don't do
// this in asm preprocessor mode, because "# 4" is not a line marker directive
// in this mode.
if (!PP.getLangOpts().AsmPreprocessor)
Builder.append("# 1 \"<built-in>\" 3");
// Install things like __POWERPC__, __GNUC__, etc into the macro table.
if (InitOpts.UsePredefines) {
if (LangOpts.CUDA && PP.getAuxTargetInfo())
InitializePredefinedMacros(*PP.getAuxTargetInfo(), LangOpts, FEOpts,
Builder);
InitializePredefinedMacros(PP.getTargetInfo(), LangOpts, FEOpts, Builder);
// Install definitions to make Objective-C++ ARC work well with various
// C++ Standard Library implementations.
if (LangOpts.ObjC1 && LangOpts.CPlusPlus &&
(LangOpts.ObjCAutoRefCount || LangOpts.ObjCWeak)) {
switch (InitOpts.ObjCXXARCStandardLibrary) {
case ARCXX_nolib:
case ARCXX_libcxx:
break;
case ARCXX_libstdcxx:
AddObjCXXARCLibstdcxxDefines(LangOpts, Builder);
break;
}
}
}
// Even with predefines off, some macros are still predefined.
// These should all be defined in the preprocessor according to the
// current language configuration.
InitializeStandardPredefinedMacros(PP.getTargetInfo(), PP.getLangOpts(),
FEOpts, Builder);
// Add on the predefines from the driver. Wrap in a #line directive to report
// that they come from the command line.
if (!PP.getLangOpts().AsmPreprocessor)
Builder.append("# 1 \"<command line>\" 1");
// Process #define's and #undef's in the order they are given.
for (unsigned i = 0, e = InitOpts.Macros.size(); i != e; ++i) {
if (InitOpts.Macros[i].second) // isUndef
Builder.undefineMacro(InitOpts.Macros[i].first);
else
DefineBuiltinMacro(Builder, InitOpts.Macros[i].first,
PP.getDiagnostics());
}
// Exit the command line and go back to <built-in> (2 is LC_LEAVE).
if (!PP.getLangOpts().AsmPreprocessor)
Builder.append("# 1 \"<built-in>\" 2");
// If -imacros are specified, include them now. These are processed before
// any -include directives.
for (unsigned i = 0, e = InitOpts.MacroIncludes.size(); i != e; ++i)
AddImplicitIncludeMacros(Builder, InitOpts.MacroIncludes[i]);
// Process -include-pch/-include-pth directives.
if (!InitOpts.ImplicitPCHInclude.empty())
AddImplicitIncludePCH(Builder, PP, PCHContainerRdr,
InitOpts.ImplicitPCHInclude);
if (!InitOpts.ImplicitPTHInclude.empty())
AddImplicitIncludePTH(Builder, PP, InitOpts.ImplicitPTHInclude);
// Process -include directives.
for (unsigned i = 0, e = InitOpts.Includes.size(); i != e; ++i) {
const std::string &Path = InitOpts.Includes[i];
AddImplicitInclude(Builder, Path);
}
// Instruct the preprocessor to skip the preamble.
PP.setSkipMainFilePreamble(InitOpts.PrecompiledPreambleBytes.first,
InitOpts.PrecompiledPreambleBytes.second);
// Copy PredefinedBuffer into the Preprocessor.
PP.setPredefines(Predefines.str());
}
/// EvaluateValue - Evaluate the token PeekTok (and any others needed) and
/// return the computed value in Result. Return true if there was an error
/// parsing. This function also returns information about the form of the
/// expression in DT. See above for information on what DT means.
///
/// If ValueLive is false, then this value is being evaluated in a context where
/// the result is not used. As such, avoid diagnostics that relate to
/// evaluation.
static bool EvaluateValue(PPValue &Result, Token &PeekTok, DefinedTracker &DT,
bool ValueLive, Preprocessor &PP) {
DT.State = DefinedTracker::Unknown;
if (PeekTok.is(tok::code_completion)) {
if (PP.getCodeCompletionHandler())
PP.getCodeCompletionHandler()->CodeCompletePreprocessorExpression();
PP.setCodeCompletionReached();
PP.LexNonComment(PeekTok);
}
// If this token's spelling is a pp-identifier, check to see if it is
// 'defined' or if it is a macro. Note that we check here because many
// keywords are pp-identifiers, so we can't check the kind.
if (IdentifierInfo *II = PeekTok.getIdentifierInfo()) {
// Handle "defined X" and "defined(X)".
if (II->isStr("defined"))
return(EvaluateDefined(Result, PeekTok, DT, ValueLive, PP));
// If this identifier isn't 'defined' or one of the special
// preprocessor keywords and it wasn't macro expanded, it turns
// into a simple 0, unless it is the C++ keyword "true", in which case it
// turns into "1".
if (ValueLive &&
II->getTokenID() != tok::kw_true &&
II->getTokenID() != tok::kw_false)
PP.Diag(PeekTok, diag::warn_pp_undef_identifier) << II;
Result.Val = II->getTokenID() == tok::kw_true;
Result.Val.setIsUnsigned(false); // "0" is signed intmax_t 0.
Result.setRange(PeekTok.getLocation());
PP.LexNonComment(PeekTok);
return false;
}
switch (PeekTok.getKind()) {
default: // Non-value token.
PP.Diag(PeekTok, diag::err_pp_expr_bad_token_start_expr);
return true;
case tok::eod:
case tok::r_paren:
// If there is no expression, report and exit.
PP.Diag(PeekTok, diag::err_pp_expected_value_in_expr);
return true;
case tok::numeric_constant: {
SmallString<64> IntegerBuffer;
bool NumberInvalid = false;
StringRef Spelling = PP.getSpelling(PeekTok, IntegerBuffer,
&NumberInvalid);
if (NumberInvalid)
return true; // a diagnostic was already reported
NumericLiteralParser Literal(Spelling, PeekTok.getLocation(), PP);
if (Literal.hadError)
return true; // a diagnostic was already reported.
if (Literal.isFloatingLiteral() || Literal.isImaginary) {
PP.Diag(PeekTok, diag::err_pp_illegal_floating_literal);
return true;
}
assert(Literal.isIntegerLiteral() && "Unknown ppnumber");
// Complain about, and drop, any ud-suffix.
if (Literal.hasUDSuffix())
PP.Diag(PeekTok, diag::err_pp_invalid_udl) << /*integer*/1;
// 'long long' is a C99 or C++11 feature.
if (!PP.getLangOpts().C99 && Literal.isLongLong) {
if (PP.getLangOpts().CPlusPlus)
PP.Diag(PeekTok,
PP.getLangOpts().CPlusPlus0x ?
diag::warn_cxx98_compat_longlong : diag::ext_cxx11_longlong);
else
PP.Diag(PeekTok, diag::ext_c99_longlong);
}
// Parse the integer literal into Result.
if (Literal.GetIntegerValue(Result.Val)) {
// Overflow parsing integer literal.
if (ValueLive) PP.Diag(PeekTok, diag::warn_integer_too_large);
Result.Val.setIsUnsigned(true);
} else {
// Set the signedness of the result to match whether there was a U suffix
// or not.
Result.Val.setIsUnsigned(Literal.isUnsigned);
// Detect overflow based on whether the value is signed. If signed
// and if the value is too large, emit a warning "integer constant is so
// large that it is unsigned" e.g. on 12345678901234567890 where intmax_t
// is 64-bits.
if (!Literal.isUnsigned && Result.Val.isNegative()) {
// Don't warn for a hex literal: 0x8000..0 shouldn't warn.
if (ValueLive && Literal.getRadix() != 16)
PP.Diag(PeekTok, diag::warn_integer_too_large_for_signed);
Result.Val.setIsUnsigned(true);
}
}
// Consume the token.
Result.setRange(PeekTok.getLocation());
PP.LexNonComment(PeekTok);
return false;
}
case tok::char_constant: // 'x'
case tok::wide_char_constant: { // L'x'
case tok::utf16_char_constant: // u'x'
case tok::utf32_char_constant: // U'x'
// Complain about, and drop, any ud-suffix.
if (PeekTok.hasUDSuffix())
PP.Diag(PeekTok, diag::err_pp_invalid_udl) << /*character*/0;
SmallString<32> CharBuffer;
bool CharInvalid = false;
StringRef ThisTok = PP.getSpelling(PeekTok, CharBuffer, &CharInvalid);
if (CharInvalid)
return true;
CharLiteralParser Literal(ThisTok.begin(), ThisTok.end(),
PeekTok.getLocation(), PP, PeekTok.getKind());
if (Literal.hadError())
return true; // A diagnostic was already emitted.
// Character literals are always int or wchar_t, expand to intmax_t.
const TargetInfo &TI = PP.getTargetInfo();
unsigned NumBits;
if (Literal.isMultiChar())
NumBits = TI.getIntWidth();
else if (Literal.isWide())
NumBits = TI.getWCharWidth();
else if (Literal.isUTF16())
NumBits = TI.getChar16Width();
else if (Literal.isUTF32())
NumBits = TI.getChar32Width();
else
NumBits = TI.getCharWidth();
// Set the width.
llvm::APSInt Val(NumBits);
// Set the value.
Val = Literal.getValue();
// Set the signedness. UTF-16 and UTF-32 are always unsigned
if (!Literal.isUTF16() && !Literal.isUTF32())
Val.setIsUnsigned(!PP.getLangOpts().CharIsSigned);
if (Result.Val.getBitWidth() > Val.getBitWidth()) {
Result.Val = Val.extend(Result.Val.getBitWidth());
} else {
assert(Result.Val.getBitWidth() == Val.getBitWidth() &&
"intmax_t smaller than char/wchar_t?");
Result.Val = Val;
}
// Consume the token.
Result.setRange(PeekTok.getLocation());
PP.LexNonComment(PeekTok);
return false;
}
case tok::l_paren: {
SourceLocation Start = PeekTok.getLocation();
PP.LexNonComment(PeekTok); // Eat the (.
// Parse the value and if there are any binary operators involved, parse
// them.
if (EvaluateValue(Result, PeekTok, DT, ValueLive, PP)) return true;
// If this is a silly value like (X), which doesn't need parens, check for
// !(defined X).
if (PeekTok.is(tok::r_paren)) {
// Just use DT unmodified as our result.
} else {
// Otherwise, we have something like (x+y), and we consumed '(x'.
if (EvaluateDirectiveSubExpr(Result, 1, PeekTok, ValueLive, PP))
return true;
if (PeekTok.isNot(tok::r_paren)) {
PP.Diag(PeekTok.getLocation(), diag::err_pp_expected_rparen)
<< Result.getRange();
PP.Diag(Start, diag::note_matching) << "(";
return true;
}
DT.State = DefinedTracker::Unknown;
}
Result.setRange(Start, PeekTok.getLocation());
PP.LexNonComment(PeekTok); // Eat the ).
return false;
}
case tok::plus: {
SourceLocation Start = PeekTok.getLocation();
// Unary plus doesn't modify the value.
PP.LexNonComment(PeekTok);
if (EvaluateValue(Result, PeekTok, DT, ValueLive, PP)) return true;
Result.setBegin(Start);
return false;
}
case tok::minus: {
SourceLocation Loc = PeekTok.getLocation();
PP.LexNonComment(PeekTok);
if (EvaluateValue(Result, PeekTok, DT, ValueLive, PP)) return true;
Result.setBegin(Loc);
// C99 6.5.3.3p3: The sign of the result matches the sign of the operand.
Result.Val = -Result.Val;
// -MININT is the only thing that overflows. Unsigned never overflows.
bool Overflow = !Result.isUnsigned() && Result.Val.isMinSignedValue();
// If this operator is live and overflowed, report the issue.
if (Overflow && ValueLive)
PP.Diag(Loc, diag::warn_pp_expr_overflow) << Result.getRange();
DT.State = DefinedTracker::Unknown;
return false;
}
case tok::tilde: {
SourceLocation Start = PeekTok.getLocation();
PP.LexNonComment(PeekTok);
if (EvaluateValue(Result, PeekTok, DT, ValueLive, PP)) return true;
Result.setBegin(Start);
// C99 6.5.3.3p4: The sign of the result matches the sign of the operand.
Result.Val = ~Result.Val;
DT.State = DefinedTracker::Unknown;
return false;
}
case tok::exclaim: {
SourceLocation Start = PeekTok.getLocation();
PP.LexNonComment(PeekTok);
if (EvaluateValue(Result, PeekTok, DT, ValueLive, PP)) return true;
Result.setBegin(Start);
Result.Val = !Result.Val;
// C99 6.5.3.3p5: The sign of the result is 'int', aka it is signed.
Result.Val.setIsUnsigned(false);
if (DT.State == DefinedTracker::DefinedMacro)
DT.State = DefinedTracker::NotDefinedMacro;
else if (DT.State == DefinedTracker::NotDefinedMacro)
DT.State = DefinedTracker::DefinedMacro;
return false;
}
// FIXME: Handle #assert
}
}
/// Finish - This does final analysis of the declspec, rejecting things like
/// "_Imaginary" (lacking an FP type). This returns a diagnostic to issue or
/// diag::NUM_DIAGNOSTICS if there is no error. After calling this method,
/// DeclSpec is guaranteed self-consistent, even if an error occurred.
void DeclSpec::Finish(DiagnosticsEngine &D, Preprocessor &PP) {
// Before possibly changing their values, save specs as written.
SaveWrittenBuiltinSpecs();
// Check the type specifier components first.
// Validate and finalize AltiVec vector declspec.
if (TypeAltiVecVector) {
if (TypeAltiVecBool) {
// Sign specifiers are not allowed with vector bool. (PIM 2.1)
if (TypeSpecSign != TSS_unspecified) {
Diag(D, TSSLoc, diag::err_invalid_vector_bool_decl_spec)
<< getSpecifierName((TSS)TypeSpecSign);
}
// Only char/int are valid with vector bool. (PIM 2.1)
if (((TypeSpecType != TST_unspecified) && (TypeSpecType != TST_char) &&
(TypeSpecType != TST_int)) || TypeAltiVecPixel) {
Diag(D, TSTLoc, diag::err_invalid_vector_bool_decl_spec)
<< (TypeAltiVecPixel ? "__pixel" :
getSpecifierName((TST)TypeSpecType));
}
// Only 'short' is valid with vector bool. (PIM 2.1)
if ((TypeSpecWidth != TSW_unspecified) && (TypeSpecWidth != TSW_short))
Diag(D, TSWLoc, diag::err_invalid_vector_bool_decl_spec)
<< getSpecifierName((TSW)TypeSpecWidth);
// Elements of vector bool are interpreted as unsigned. (PIM 2.1)
if ((TypeSpecType == TST_char) || (TypeSpecType == TST_int) ||
(TypeSpecWidth != TSW_unspecified))
TypeSpecSign = TSS_unsigned;
}
if (TypeAltiVecPixel) {
//TODO: perform validation
TypeSpecType = TST_int;
TypeSpecSign = TSS_unsigned;
TypeSpecWidth = TSW_short;
TypeSpecOwned = false;
}
}
// signed/unsigned are only valid with int/char/wchar_t.
if (TypeSpecSign != TSS_unspecified) {
if (TypeSpecType == TST_unspecified)
TypeSpecType = TST_int; // unsigned -> unsigned int, signed -> signed int.
else if (TypeSpecType != TST_int && TypeSpecType != TST_int128 &&
TypeSpecType != TST_char && TypeSpecType != TST_wchar) {
Diag(D, TSSLoc, diag::err_invalid_sign_spec)
<< getSpecifierName((TST)TypeSpecType);
// signed double -> double.
TypeSpecSign = TSS_unspecified;
}
}
// Validate the width of the type.
switch (TypeSpecWidth) {
case TSW_unspecified: break;
case TSW_short: // short int
case TSW_longlong: // long long int
if (TypeSpecType == TST_unspecified)
TypeSpecType = TST_int; // short -> short int, long long -> long long int.
else if (TypeSpecType != TST_int) {
Diag(D, TSWLoc,
TypeSpecWidth == TSW_short ? diag::err_invalid_short_spec
: diag::err_invalid_longlong_spec)
<< getSpecifierName((TST)TypeSpecType);
TypeSpecType = TST_int;
TypeSpecOwned = false;
}
break;
case TSW_long: // long double, long int
if (TypeSpecType == TST_unspecified)
TypeSpecType = TST_int; // long -> long int.
else if (TypeSpecType != TST_int && TypeSpecType != TST_double) {
Diag(D, TSWLoc, diag::err_invalid_long_spec)
<< getSpecifierName((TST)TypeSpecType);
TypeSpecType = TST_int;
TypeSpecOwned = false;
}
break;
}
// TODO: if the implementation does not implement _Complex or _Imaginary,
// disallow their use. Need information about the backend.
if (TypeSpecComplex != TSC_unspecified) {
if (TypeSpecType == TST_unspecified) {
Diag(D, TSCLoc, diag::ext_plain_complex)
<< FixItHint::CreateInsertion(
PP.getLocForEndOfToken(getTypeSpecComplexLoc()),
" double");
TypeSpecType = TST_double; // _Complex -> _Complex double.
} else if (TypeSpecType == TST_int || TypeSpecType == TST_char) {
// Note that this intentionally doesn't include _Complex _Bool.
if (!PP.getLangOpts().CPlusPlus)
Diag(D, TSTLoc, diag::ext_integer_complex);
} else if (TypeSpecType != TST_float && TypeSpecType != TST_double) {
Diag(D, TSCLoc, diag::err_invalid_complex_spec)
<< getSpecifierName((TST)TypeSpecType);
TypeSpecComplex = TSC_unspecified;
}
}
// If no type specifier was provided and we're parsing a language where
// the type specifier is not optional, but we got 'auto' as a storage
// class specifier, then assume this is an attempt to use C++0x's 'auto'
// type specifier.
// FIXME: Does Microsoft really support implicit int in C++?
if (PP.getLangOpts().CPlusPlus && !PP.getLangOpts().MicrosoftExt &&
TypeSpecType == TST_unspecified && StorageClassSpec == SCS_auto) {
TypeSpecType = TST_auto;
StorageClassSpec = SCS_unspecified;
TSTLoc = TSTNameLoc = StorageClassSpecLoc;
StorageClassSpecLoc = SourceLocation();
}
// Diagnose if we've recovered from an ill-formed 'auto' storage class
// specifier in a pre-C++0x dialect of C++.
if (!PP.getLangOpts().CPlusPlus11 && TypeSpecType == TST_auto)
Diag(D, TSTLoc, diag::ext_auto_type_specifier);
if (PP.getLangOpts().CPlusPlus && !PP.getLangOpts().CPlusPlus11 &&
StorageClassSpec == SCS_auto)
Diag(D, StorageClassSpecLoc, diag::warn_auto_storage_class)
<< FixItHint::CreateRemoval(StorageClassSpecLoc);
if (TypeSpecType == TST_char16 || TypeSpecType == TST_char32)
Diag(D, TSTLoc, diag::warn_cxx98_compat_unicode_type)
<< (TypeSpecType == TST_char16 ? "char16_t" : "char32_t");
if (Constexpr_specified)
Diag(D, ConstexprLoc, diag::warn_cxx98_compat_constexpr);
// C++ [class.friend]p6:
// No storage-class-specifier shall appear in the decl-specifier-seq
// of a friend declaration.
if (isFriendSpecified() && getStorageClassSpec()) {
DeclSpec::SCS SC = getStorageClassSpec();
const char *SpecName = getSpecifierName(SC);
SourceLocation SCLoc = getStorageClassSpecLoc();
SourceLocation SCEndLoc = SCLoc.getLocWithOffset(strlen(SpecName));
Diag(D, SCLoc, diag::err_friend_storage_spec)
<< SpecName
<< FixItHint::CreateRemoval(SourceRange(SCLoc, SCEndLoc));
ClearStorageClassSpecs();
}
assert(!TypeSpecOwned || isDeclRep((TST) TypeSpecType));
// Okay, now we can infer the real type.
// TODO: return "auto function" and other bad things based on the real type.
// 'data definition has no type or storage class'?
}
/// EvaluateDirectiveSubExpr - Evaluate the subexpression whose first token is
/// PeekTok, and whose precedence is PeekPrec. This returns the result in LHS.
///
/// If ValueLive is false, then this value is being evaluated in a context where
/// the result is not used. As such, avoid diagnostics that relate to
/// evaluation, such as division by zero warnings.
static bool EvaluateDirectiveSubExpr(PPValue &LHS, unsigned MinPrec,
Token &PeekTok, bool ValueLive,
Preprocessor &PP) {
unsigned PeekPrec = getPrecedence(PeekTok.getKind());
// If this token isn't valid, report the error.
if (PeekPrec == ~0U) {
PP.Diag(PeekTok.getLocation(), diag::err_pp_expr_bad_token_binop)
<< LHS.getRange();
return true;
}
while (1) {
// If this token has a lower precedence than we are allowed to parse, return
// it so that higher levels of the recursion can parse it.
if (PeekPrec < MinPrec)
return false;
tok::TokenKind Operator = PeekTok.getKind();
// If this is a short-circuiting operator, see if the RHS of the operator is
// dead. Note that this cannot just clobber ValueLive. Consider
// "0 && 1 ? 4 : 1 / 0", which is parsed as "(0 && 1) ? 4 : (1 / 0)". In
// this example, the RHS of the && being dead does not make the rest of the
// expr dead.
bool RHSIsLive;
if (Operator == tok::ampamp && LHS.Val == 0)
RHSIsLive = false; // RHS of "0 && x" is dead.
else if (Operator == tok::pipepipe && LHS.Val != 0)
RHSIsLive = false; // RHS of "1 || x" is dead.
else if (Operator == tok::question && LHS.Val == 0)
RHSIsLive = false; // RHS (x) of "0 ? x : y" is dead.
else
RHSIsLive = ValueLive;
// Consume the operator, remembering the operator's location for reporting.
SourceLocation OpLoc = PeekTok.getLocation();
PP.LexNonComment(PeekTok);
PPValue RHS(LHS.getBitWidth());
// Parse the RHS of the operator.
DefinedTracker DT;
if (EvaluateValue(RHS, PeekTok, DT, RHSIsLive, PP)) return true;
// Remember the precedence of this operator and get the precedence of the
// operator immediately to the right of the RHS.
unsigned ThisPrec = PeekPrec;
PeekPrec = getPrecedence(PeekTok.getKind());
// If this token isn't valid, report the error.
if (PeekPrec == ~0U) {
PP.Diag(PeekTok.getLocation(), diag::err_pp_expr_bad_token_binop)
<< RHS.getRange();
return true;
}
// Decide whether to include the next binop in this subexpression. For
// example, when parsing x+y*z and looking at '*', we want to recursively
// handle y*z as a single subexpression. We do this because the precedence
// of * is higher than that of +. The only strange case we have to handle
// here is for the ?: operator, where the precedence is actually lower than
// the LHS of the '?'. The grammar rule is:
//
// conditional-expression ::=
// logical-OR-expression ? expression : conditional-expression
// where 'expression' is actually comma-expression.
unsigned RHSPrec;
if (Operator == tok::question)
// The RHS of "?" should be maximally consumed as an expression.
RHSPrec = getPrecedence(tok::comma);
else // All others should munch while higher precedence.
RHSPrec = ThisPrec+1;
if (PeekPrec >= RHSPrec) {
if (EvaluateDirectiveSubExpr(RHS, RHSPrec, PeekTok, RHSIsLive, PP))
return true;
PeekPrec = getPrecedence(PeekTok.getKind());
}
assert(PeekPrec <= ThisPrec && "Recursion didn't work!");
// Usual arithmetic conversions (C99 6.3.1.8p1): result is unsigned if
// either operand is unsigned.
llvm::APSInt Res(LHS.getBitWidth());
switch (Operator) {
case tok::question: // No UAC for x and y in "x ? y : z".
case tok::lessless: // Shift amount doesn't UAC with shift value.
case tok::greatergreater: // Shift amount doesn't UAC with shift value.
case tok::comma: // Comma operands are not subject to UACs.
case tok::pipepipe: // Logical || does not do UACs.
case tok::ampamp: // Logical && does not do UACs.
break; // No UAC
default:
Res.setIsUnsigned(LHS.isUnsigned()|RHS.isUnsigned());
// If this just promoted something from signed to unsigned, and if the
// value was negative, warn about it.
if (ValueLive && Res.isUnsigned()) {
if (!LHS.isUnsigned() && LHS.Val.isNegative())
PP.Diag(OpLoc, diag::warn_pp_convert_lhs_to_positive)
<< LHS.Val.toString(10, true) + " to " +
LHS.Val.toString(10, false)
<< LHS.getRange() << RHS.getRange();
if (!RHS.isUnsigned() && RHS.Val.isNegative())
PP.Diag(OpLoc, diag::warn_pp_convert_rhs_to_positive)
<< RHS.Val.toString(10, true) + " to " +
RHS.Val.toString(10, false)
<< LHS.getRange() << RHS.getRange();
}
LHS.Val.setIsUnsigned(Res.isUnsigned());
RHS.Val.setIsUnsigned(Res.isUnsigned());
}
bool Overflow = false;
switch (Operator) {
default: llvm_unreachable("Unknown operator token!");
case tok::percent:
if (RHS.Val != 0)
Res = LHS.Val % RHS.Val;
else if (ValueLive) {
PP.Diag(OpLoc, diag::err_pp_remainder_by_zero)
<< LHS.getRange() << RHS.getRange();
return true;
}
break;
case tok::slash:
if (RHS.Val != 0) {
if (LHS.Val.isSigned())
Res = llvm::APSInt(LHS.Val.sdiv_ov(RHS.Val, Overflow), false);
else
Res = LHS.Val / RHS.Val;
} else if (ValueLive) {
PP.Diag(OpLoc, diag::err_pp_division_by_zero)
<< LHS.getRange() << RHS.getRange();
return true;
}
break;
case tok::star:
if (Res.isSigned())
Res = llvm::APSInt(LHS.Val.smul_ov(RHS.Val, Overflow), false);
else
Res = LHS.Val * RHS.Val;
break;
case tok::lessless: {
// Determine whether overflow is about to happen.
unsigned ShAmt = static_cast<unsigned>(RHS.Val.getLimitedValue());
if (LHS.isUnsigned()) {
Overflow = ShAmt >= LHS.Val.getBitWidth();
if (Overflow)
ShAmt = LHS.Val.getBitWidth()-1;
Res = LHS.Val << ShAmt;
} else {
Res = llvm::APSInt(LHS.Val.sshl_ov(ShAmt, Overflow), false);
}
break;
}
case tok::greatergreater: {
// Determine whether overflow is about to happen.
unsigned ShAmt = static_cast<unsigned>(RHS.Val.getLimitedValue());
if (ShAmt >= LHS.getBitWidth())
Overflow = true, ShAmt = LHS.getBitWidth()-1;
Res = LHS.Val >> ShAmt;
break;
}
case tok::plus:
if (LHS.isUnsigned())
Res = LHS.Val + RHS.Val;
else
Res = llvm::APSInt(LHS.Val.sadd_ov(RHS.Val, Overflow), false);
break;
case tok::minus:
if (LHS.isUnsigned())
Res = LHS.Val - RHS.Val;
else
Res = llvm::APSInt(LHS.Val.ssub_ov(RHS.Val, Overflow), false);
break;
case tok::lessequal:
Res = LHS.Val <= RHS.Val;
Res.setIsUnsigned(false); // C99 6.5.8p6, result is always int (signed)
break;
case tok::less:
Res = LHS.Val < RHS.Val;
Res.setIsUnsigned(false); // C99 6.5.8p6, result is always int (signed)
break;
case tok::greaterequal:
Res = LHS.Val >= RHS.Val;
Res.setIsUnsigned(false); // C99 6.5.8p6, result is always int (signed)
break;
case tok::greater:
Res = LHS.Val > RHS.Val;
Res.setIsUnsigned(false); // C99 6.5.8p6, result is always int (signed)
break;
case tok::exclaimequal:
Res = LHS.Val != RHS.Val;
Res.setIsUnsigned(false); // C99 6.5.9p3, result is always int (signed)
break;
case tok::equalequal:
Res = LHS.Val == RHS.Val;
Res.setIsUnsigned(false); // C99 6.5.9p3, result is always int (signed)
break;
case tok::amp:
Res = LHS.Val & RHS.Val;
break;
case tok::caret:
Res = LHS.Val ^ RHS.Val;
break;
case tok::pipe:
Res = LHS.Val | RHS.Val;
break;
case tok::ampamp:
Res = (LHS.Val != 0 && RHS.Val != 0);
Res.setIsUnsigned(false); // C99 6.5.13p3, result is always int (signed)
break;
case tok::pipepipe:
Res = (LHS.Val != 0 || RHS.Val != 0);
Res.setIsUnsigned(false); // C99 6.5.14p3, result is always int (signed)
break;
case tok::comma:
// Comma is invalid in pp expressions in c89/c++ mode, but is valid in C99
// if not being evaluated.
if (!PP.getLangOpts().C99 || ValueLive)
PP.Diag(OpLoc, diag::ext_pp_comma_expr)
<< LHS.getRange() << RHS.getRange();
Res = RHS.Val; // LHS = LHS,RHS -> RHS.
break;
case tok::question: {
// Parse the : part of the expression.
if (PeekTok.isNot(tok::colon)) {
PP.Diag(PeekTok.getLocation(), diag::err_expected_colon)
<< LHS.getRange(), RHS.getRange();
PP.Diag(OpLoc, diag::note_matching) << "?";
return true;
}
// Consume the :.
PP.LexNonComment(PeekTok);
// Evaluate the value after the :.
bool AfterColonLive = ValueLive && LHS.Val == 0;
PPValue AfterColonVal(LHS.getBitWidth());
DefinedTracker DT;
if (EvaluateValue(AfterColonVal, PeekTok, DT, AfterColonLive, PP))
return true;
// Parse anything after the : with the same precedence as ?. We allow
// things of equal precedence because ?: is right associative.
if (EvaluateDirectiveSubExpr(AfterColonVal, ThisPrec,
PeekTok, AfterColonLive, PP))
return true;
// Now that we have the condition, the LHS and the RHS of the :, evaluate.
Res = LHS.Val != 0 ? RHS.Val : AfterColonVal.Val;
RHS.setEnd(AfterColonVal.getRange().getEnd());
// Usual arithmetic conversions (C99 6.3.1.8p1): result is unsigned if
// either operand is unsigned.
Res.setIsUnsigned(RHS.isUnsigned() | AfterColonVal.isUnsigned());
// Figure out the precedence of the token after the : part.
PeekPrec = getPrecedence(PeekTok.getKind());
break;
}
case tok::colon:
// Don't allow :'s to float around without being part of ?: exprs.
PP.Diag(OpLoc, diag::err_pp_colon_without_question)
<< LHS.getRange() << RHS.getRange();
return true;
}
// If this operator is live and overflowed, report the issue.
if (Overflow && ValueLive)
PP.Diag(OpLoc, diag::warn_pp_expr_overflow)
<< LHS.getRange() << RHS.getRange();
// Put the result back into 'LHS' for our next iteration.
LHS.Val = Res;
LHS.setEnd(RHS.getRange().getEnd());
}
}
/// HighlightMacros - This uses the macro table state from the end of the
/// file, to re-expand macros and insert (into the HTML) information about the
/// macro expansions. This won't be perfectly perfect, but it will be
/// reasonably close.
void html::HighlightMacros(Rewriter &R, FileID FID, const Preprocessor& PP) {
// Re-lex the raw token stream into a token buffer.
const SourceManager &SM = PP.getSourceManager();
std::vector<Token> TokenStream;
const llvm::MemoryBuffer *FromFile = SM.getBuffer(FID);
Lexer L(FID, FromFile, SM, PP.getLangOpts());
// Lex all the tokens in raw mode, to avoid entering #includes or expanding
// macros.
while (1) {
Token Tok;
L.LexFromRawLexer(Tok);
// If this is a # at the start of a line, discard it from the token stream.
// We don't want the re-preprocess step to see #defines, #includes or other
// preprocessor directives.
if (Tok.is(tok::hash) && Tok.isAtStartOfLine())
continue;
// If this is a ## token, change its kind to unknown so that repreprocessing
// it will not produce an error.
if (Tok.is(tok::hashhash))
Tok.setKind(tok::unknown);
// If this raw token is an identifier, the raw lexer won't have looked up
// the corresponding identifier info for it. Do this now so that it will be
// macro expanded when we re-preprocess it.
if (Tok.is(tok::raw_identifier))
PP.LookUpIdentifierInfo(Tok);
TokenStream.push_back(Tok);
if (Tok.is(tok::eof)) break;
}
// Temporarily change the diagnostics object so that we ignore any generated
// diagnostics from this pass.
DiagnosticsEngine TmpDiags(PP.getDiagnostics().getDiagnosticIDs(),
&PP.getDiagnostics().getDiagnosticOptions(),
new IgnoringDiagConsumer);
// FIXME: This is a huge hack; we reuse the input preprocessor because we want
// its state, but we aren't actually changing it (we hope). This should really
// construct a copy of the preprocessor.
Preprocessor &TmpPP = const_cast<Preprocessor&>(PP);
DiagnosticsEngine *OldDiags = &TmpPP.getDiagnostics();
TmpPP.setDiagnostics(TmpDiags);
// Inform the preprocessor that we don't want comments.
TmpPP.SetCommentRetentionState(false, false);
// We don't want pragmas either. Although we filtered out #pragma, removing
// _Pragma and __pragma is much harder.
bool PragmasPreviouslyEnabled = TmpPP.getPragmasEnabled();
TmpPP.setPragmasEnabled(false);
// Enter the tokens we just lexed. This will cause them to be macro expanded
// but won't enter sub-files (because we removed #'s).
TmpPP.EnterTokenStream(TokenStream, false);
TokenConcatenation ConcatInfo(TmpPP);
// Lex all the tokens.
Token Tok;
TmpPP.Lex(Tok);
while (Tok.isNot(tok::eof)) {
// Ignore non-macro tokens.
if (!Tok.getLocation().isMacroID()) {
TmpPP.Lex(Tok);
continue;
}
// Okay, we have the first token of a macro expansion: highlight the
// expansion by inserting a start tag before the macro expansion and
// end tag after it.
std::pair<SourceLocation, SourceLocation> LLoc =
SM.getExpansionRange(Tok.getLocation());
// Ignore tokens whose instantiation location was not the main file.
if (SM.getFileID(LLoc.first) != FID) {
TmpPP.Lex(Tok);
continue;
}
assert(SM.getFileID(LLoc.second) == FID &&
"Start and end of expansion must be in the same ultimate file!");
std::string Expansion = EscapeText(TmpPP.getSpelling(Tok));
unsigned LineLen = Expansion.size();
Token PrevPrevTok;
Token PrevTok = Tok;
// Okay, eat this token, getting the next one.
TmpPP.Lex(Tok);
// Skip all the rest of the tokens that are part of this macro
// instantiation. It would be really nice to pop up a window with all the
// spelling of the tokens or something.
while (!Tok.is(tok::eof) &&
SM.getExpansionLoc(Tok.getLocation()) == LLoc.first) {
// Insert a newline if the macro expansion is getting large.
if (LineLen > 60) {
Expansion += "<br>";
LineLen = 0;
}
LineLen -= Expansion.size();
// If the tokens were already space separated, or if they must be to avoid
// them being implicitly pasted, add a space between them.
if (Tok.hasLeadingSpace() ||
ConcatInfo.AvoidConcat(PrevPrevTok, PrevTok, Tok))
Expansion += ' ';
// Escape any special characters in the token text.
Expansion += EscapeText(TmpPP.getSpelling(Tok));
LineLen += Expansion.size();
PrevPrevTok = PrevTok;
PrevTok = Tok;
TmpPP.Lex(Tok);
}
// Insert the expansion as the end tag, so that multi-line macros all get
// highlighted.
Expansion = "<span class='expansion'>" + Expansion + "</span></span>";
HighlightRange(R, LLoc.first, LLoc.second,
"<span class='macro'>", Expansion.c_str());
}
// Restore the preprocessor's old state.
TmpPP.setDiagnostics(*OldDiags);
TmpPP.setPragmasEnabled(PragmasPreviouslyEnabled);
}
/// ParseNumberStartingWithZero - This method is called when the first character
/// of the number is found to be a zero. This means it is either an octal
/// number (like '04') or a hex number ('0x123a') a binary number ('0b1010') or
/// a floating point number (01239.123e4). Eat the prefix, determining the
/// radix etc.
void NumericLiteralParser::ParseNumberStartingWithZero(SourceLocation TokLoc) {
assert(s[0] == '0' && "Invalid method call");
s++;
int c1 = s[0];
// Handle a hex number like 0x1234.
if ((c1 == 'x' || c1 == 'X') && (isHexDigit(s[1]) || s[1] == '.')) {
s++;
assert(s < ThisTokEnd && "didn't maximally munch?");
radix = 16;
DigitsBegin = s;
s = SkipHexDigits(s);
bool noSignificand = (s == DigitsBegin);
if (s == ThisTokEnd) {
// Done.
} else if (*s == '.') {
s++;
saw_period = true;
const char *floatDigitsBegin = s;
checkSeparator(TokLoc, s, CSK_BeforeDigits);
s = SkipHexDigits(s);
noSignificand &= (floatDigitsBegin == s);
}
if (noSignificand) {
PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s - ThisTokBegin),
diag::err_hexconstant_requires_digits);
hadError = true;
return;
}
// A binary exponent can appear with or with a '.'. If dotted, the
// binary exponent is required.
if (*s == 'p' || *s == 'P') {
checkSeparator(TokLoc, s, CSK_AfterDigits);
const char *Exponent = s;
s++;
saw_exponent = true;
if (*s == '+' || *s == '-') s++; // sign
const char *first_non_digit = SkipDigits(s);
if (first_non_digit == s) {
PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, Exponent-ThisTokBegin),
diag::err_exponent_has_no_digits);
hadError = true;
return;
}
checkSeparator(TokLoc, s, CSK_BeforeDigits);
s = first_non_digit;
if (!PP.getLangOpts().HexFloats)
PP.Diag(TokLoc, diag::ext_hexconstant_invalid);
} else if (saw_period) {
PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-ThisTokBegin),
diag::err_hexconstant_requires_exponent);
hadError = true;
}
return;
}
// Handle simple binary numbers 0b01010
if ((c1 == 'b' || c1 == 'B') && (s[1] == '0' || s[1] == '1')) {
// 0b101010 is a C++1y / GCC extension.
PP.Diag(TokLoc,
PP.getLangOpts().CPlusPlus14
? diag::warn_cxx11_compat_binary_literal
: PP.getLangOpts().CPlusPlus
? diag::ext_binary_literal_cxx14
: diag::ext_binary_literal);
++s;
assert(s < ThisTokEnd && "didn't maximally munch?");
radix = 2;
DigitsBegin = s;
s = SkipBinaryDigits(s);
if (s == ThisTokEnd) {
// Done.
} else if (isHexDigit(*s)) {
PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-ThisTokBegin),
diag::err_invalid_binary_digit) << StringRef(s, 1);
hadError = true;
}
// Other suffixes will be diagnosed by the caller.
return;
}
// For now, the radix is set to 8. If we discover that we have a
// floating point constant, the radix will change to 10. Octal floating
// point constants are not permitted (only decimal and hexadecimal).
radix = 8;
DigitsBegin = s;
s = SkipOctalDigits(s);
if (s == ThisTokEnd)
return; // Done, simple octal number like 01234
// If we have some other non-octal digit that *is* a decimal digit, see if
// this is part of a floating point number like 094.123 or 09e1.
if (isDigit(*s)) {
const char *EndDecimal = SkipDigits(s);
if (EndDecimal[0] == '.' || EndDecimal[0] == 'e' || EndDecimal[0] == 'E') {
s = EndDecimal;
radix = 10;
}
}
// If we have a hex digit other than 'e' (which denotes a FP exponent) then
// the code is using an incorrect base.
if (isHexDigit(*s) && *s != 'e' && *s != 'E') {
PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-ThisTokBegin),
diag::err_invalid_octal_digit) << StringRef(s, 1);
hadError = true;
return;
}
if (*s == '.') {
s++;
radix = 10;
saw_period = true;
checkSeparator(TokLoc, s, CSK_BeforeDigits);
s = SkipDigits(s); // Skip suffix.
}
if (*s == 'e' || *s == 'E') { // exponent
checkSeparator(TokLoc, s, CSK_AfterDigits);
const char *Exponent = s;
s++;
radix = 10;
saw_exponent = true;
if (*s == '+' || *s == '-') s++; // sign
const char *first_non_digit = SkipDigits(s);
if (first_non_digit != s) {
checkSeparator(TokLoc, s, CSK_BeforeDigits);
s = first_non_digit;
} else {
PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, Exponent-ThisTokBegin),
diag::err_exponent_has_no_digits);
hadError = true;
return;
}
}
}