bool TokenLexer::MaybeRemoveCommaBeforeVaArgs( SmallVectorImpl<Token> &ResultToks, bool HasPasteOperator, MacroInfo *Macro, unsigned MacroArgNo, Preprocessor &PP) { // Is the macro argument __VA_ARGS__? if (!Macro->isVariadic() || MacroArgNo != Macro->getNumArgs()-1) return false; // In Microsoft-compatibility mode, a comma is removed in the expansion // of " ... , __VA_ARGS__ " if __VA_ARGS__ is empty. This extension is // not supported by gcc. if (!HasPasteOperator && !PP.getLangOpts().MSVCCompat) return false; // GCC removes the comma in the expansion of " ... , ## __VA_ARGS__ " if // __VA_ARGS__ is empty, but not in strict C99 mode where there are no // named arguments, where it remains. In all other modes, including C99 // with GNU extensions, it is removed regardless of named arguments. // Microsoft also appears to support this extension, unofficially. if (PP.getLangOpts().C99 && !PP.getLangOpts().GNUMode && Macro->getNumArgs() < 2) return false; // Is a comma available to be removed? if (ResultToks.empty() || !ResultToks.back().is(tok::comma)) return false; // Issue an extension diagnostic for the paste operator. if (HasPasteOperator) PP.Diag(ResultToks.back().getLocation(), diag::ext_paste_comma); // Remove the comma. ResultToks.pop_back(); if (!ResultToks.empty()) { // If the comma was right after another paste (e.g. "X##,##__VA_ARGS__"), // then removal of the comma should produce a placemarker token (in C99 // terms) which we model by popping off the previous ##, giving us a plain // "X" when __VA_ARGS__ is empty. if (ResultToks.back().is(tok::hashhash)) ResultToks.pop_back(); // Remember that this comma was elided. ResultToks.back().setFlag(Token::CommaAfterElided); } // Never add a space, even if the comma, ##, or arg had a space. NextTokGetsSpace = false; return true; }
Sema::Sema(Preprocessor &pp, ASTContext &ctxt, ASTConsumer &consumer, TranslationUnitKind TUKind, CodeCompleteConsumer *CodeCompleter) : TheTargetAttributesSema(0), ExternalSource(0), isMultiplexExternalSource(false), FPFeatures(pp.getLangOpts()), LangOpts(pp.getLangOpts()), PP(pp), Context(ctxt), Consumer(consumer), Diags(PP.getDiagnostics()), SourceMgr(PP.getSourceManager()), CollectStats(false), CodeCompleter(CodeCompleter), CurContext(0), OriginalLexicalContext(0), PackContext(0), MSStructPragmaOn(false), VisContext(0), IsBuildingRecoveryCallExpr(false), ExprNeedsCleanups(false), LateTemplateParser(0), OpaqueParser(0), IdResolver(pp), StdInitializerList(0), CXXTypeInfoDecl(0), MSVCGuidDecl(0), NSNumberDecl(0), NSStringDecl(0), StringWithUTF8StringMethod(0), NSArrayDecl(0), ArrayWithObjectsMethod(0), NSDictionaryDecl(0), DictionaryWithObjectsMethod(0), GlobalNewDeleteDeclared(false), TUKind(TUKind), NumSFINAEErrors(0), IsTransformingLambdaCallOperatorProtoType(false), InFunctionDeclarator(0), AccessCheckingSFINAE(false), InNonInstantiationSFINAEContext(false), NonInstantiationEntries(0), ArgumentPackSubstitutionIndex(-1), CurrentInstantiationScope(0), TyposCorrected(0), AnalysisWarnings(*this) { TUScope = 0; LoadedExternalKnownNamespaces = false; for (unsigned I = 0; I != NSAPI::NumNSNumberLiteralMethods; ++I) NSNumberLiteralMethods[I] = 0; if (getLangOpts().ObjC1) NSAPIObj.reset(new NSAPI(Context)); if (getLangOpts().CPlusPlus) FieldCollector.reset(new CXXFieldCollector()); // Tell diagnostics how to render things from the AST library. PP.getDiagnostics().SetArgToStringFn(&FormatASTNodeDiagnosticArgument, &Context); ExprEvalContexts.push_back( ExpressionEvaluationContextRecord(PotentiallyEvaluated, 0, false, 0, false)); FunctionScopes.push_back(new FunctionScopeInfo(Diags)); }
/// LexRawTokensFromMainFile - Lets all the raw tokens from the main file into /// the specified vector. static void LexRawTokensFromMainFile(Preprocessor &PP, std::vector<Token> &RawTokens) { SourceManager &SM = PP.getSourceManager(); // Create a lexer to lex all the tokens of the main file in raw mode. Even // though it is in raw mode, it will not return comments. const llvm::MemoryBuffer *FromFile = SM.getBuffer(SM.getMainFileID()); Lexer RawLex(SM.getMainFileID(), FromFile, SM, PP.getLangOpts()); // Switch on comment lexing because we really do want them. RawLex.SetCommentRetentionState(true); Token RawTok; do { RawLex.LexFromRawLexer(RawTok); // If we have an identifier with no identifier info for our raw token, look // up the indentifier info. This is important for equality comparison of // identifier tokens. if (RawTok.is(tok::raw_identifier)) PP.LookUpIdentifierInfo(RawTok); RawTokens.push_back(RawTok); } while (RawTok.isNot(tok::eof)); }
/// Returns true if the statement is expanded from a configuration macro. static bool isExpandedFromConfigurationMacro(const Stmt *S, Preprocessor &PP, bool IgnoreYES_NO = false) { // FIXME: This is not very precise. Here we just check to see if the // value comes from a macro, but we can do much better. This is likely // to be over conservative. This logic is factored into a separate function // so that we can refine it later. SourceLocation L = S->getBeginLoc(); if (L.isMacroID()) { SourceManager &SM = PP.getSourceManager(); if (IgnoreYES_NO) { // The Objective-C constant 'YES' and 'NO' // are defined as macros. Do not treat them // as configuration values. SourceLocation TopL = getTopMostMacro(L, SM); StringRef MacroName = PP.getImmediateMacroName(TopL); if (MacroName == "YES" || MacroName == "NO") return false; } else if (!PP.getLangOpts().CPlusPlus) { // Do not treat C 'false' and 'true' macros as configuration values. SourceLocation TopL = getTopMostMacro(L, SM); StringRef MacroName = PP.getImmediateMacroName(TopL); if (MacroName == "false" || MacroName == "true") return false; } return true; } return false; }
/// FindExpectedDiags - Lex the main source file to find all of the // expected errors and warnings. static void FindExpectedDiags(const Preprocessor &PP, ExpectedData &ED, FileID FID) { // Create a raw lexer to pull all the comments out of FID. if (FID.isInvalid()) return; SourceManager& SM = PP.getSourceManager(); // Create a lexer to lex all the tokens of the main file in raw mode. const llvm::MemoryBuffer *FromFile = SM.getBuffer(FID); Lexer RawLex(FID, FromFile, SM, PP.getLangOpts()); // Return comments as tokens, this is how we find expected diagnostics. RawLex.SetCommentRetentionState(true); Token Tok; Tok.setKind(tok::comment); while (Tok.isNot(tok::eof)) { RawLex.Lex(Tok); if (!Tok.is(tok::comment)) continue; std::string Comment = PP.getSpelling(Tok); if (Comment.empty()) continue; // Find all expected errors/warnings/notes. ParseDirective(Comment, ED, SM, Tok.getLocation(), PP.getDiagnostics()); }; }
void ento::createPlistMultiFileDiagnosticConsumer(AnalyzerOptions &AnalyzerOpts, PathDiagnosticConsumers &C, const std::string &s, const Preprocessor &PP) { C.push_back(new PlistDiagnostics(AnalyzerOpts, s, PP.getLangOpts(), true)); }
/// Initializes an InclusionRewriter with a \p PP source and \p OS destination. InclusionRewriter::InclusionRewriter(Preprocessor &PP, raw_ostream &OS, bool ShowLineMarkers) : PP(PP), SM(PP.getSourceManager()), OS(OS), ShowLineMarkers(ShowLineMarkers), LastInsertedFileChange(FileChanges.end()) { // If we're in microsoft mode, use normal #line instead of line markers. UseLineDirective = PP.getLangOpts().MicrosoftExt; }
bool HandleComment(Preprocessor &PP, SourceRange Range) override { StringRef Text = Lexer::getSourceText(CharSourceRange::getCharRange(Range), PP.getSourceManager(), PP.getLangOpts()); if (!Text.consume_front(IWYUPragma)) return false; // FIXME(ioeric): resolve the header and store actual file path. For now, // we simply assume the written header is suitable to be #included. Includes->addMapping(PP.getSourceManager().getFilename(Range.getBegin()), isLiteralInclude(Text) ? Text.str() : ("\"" + Text + "\"").str()); return false; }
static void emitPremigrationErrors(const CapturedDiagList &arcDiags, const DiagnosticOptions &diagOpts, Preprocessor &PP) { TextDiagnosticPrinter printer(llvm::errs(), diagOpts); IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs()); IntrusiveRefCntPtr<DiagnosticsEngine> Diags( new DiagnosticsEngine(DiagID, &printer, /*ShouldOwnClient=*/false)); Diags->setSourceManager(&PP.getSourceManager()); printer.BeginSourceFile(PP.getLangOpts(), &PP); arcDiags.reportDiagnostics(*Diags); printer.EndSourceFile(); }
bool PragmaCommentHandler::HandleComment(Preprocessor &PP, SourceRange Range) { StringRef Text = Lexer::getSourceText(CharSourceRange::getCharRange(Range), PP.getSourceManager(), PP.getLangOpts()); size_t Pos = Text.find(IWYUPragma); if (Pos == StringRef::npos) return false; StringRef RemappingFilePath = Text.substr(Pos + std::strlen(IWYUPragma)); Collector->addHeaderMapping( PP.getSourceManager().getFilename(Range.getBegin()), RemappingFilePath.trim("\"<>")); return false; }
void clang::DoRewriteTest(Preprocessor &PP, raw_ostream* OS) { SourceManager &SM = PP.getSourceManager(); const LangOptions &LangOpts = PP.getLangOpts(); TokenRewriter Rewriter(SM.getMainFileID(), SM, LangOpts); // Throw <i> </i> tags around comments. for (TokenRewriter::token_iterator I = Rewriter.token_begin(), E = Rewriter.token_end(); I != E; ++I) { if (I->isNot(tok::comment)) continue; Rewriter.AddTokenBefore(I, "<i>"); Rewriter.AddTokenAfter(I, "</i>"); } // Print out the output. for (TokenRewriter::token_iterator I = Rewriter.token_begin(), E = Rewriter.token_end(); I != E; ++I) *OS << PP.getSpelling(*I); }
bool HandleComment(Preprocessor &PP, SourceRange Range) override { StringRef Text = Lexer::getSourceText(CharSourceRange::getCharRange(Range), PP.getSourceManager(), PP.getLangOpts()); SmallVector<StringRef, 4> Matches; if (!TodoMatch.match(Text, &Matches)) return false; StringRef Username = Matches[1]; StringRef Comment = Matches[3]; if (!Username.empty()) return false; std::string NewText = ("// TODO(" + Twine(User) + "): " + Comment).str(); Check.diag(Range.getBegin(), "missing username/bug in TODO") << FixItHint::CreateReplacement(CharSourceRange::getCharRange(Range), NewText); return false; }
/// HasExtension - Return true if we recognize and implement the feature /// specified by the identifier, either as an extension or a standard language /// feature. static bool HasExtension(const Preprocessor &PP, const IdentifierInfo *II) { if (HasFeature(PP, II)) return true; // If the use of an extension results in an error diagnostic, extensions are // effectively unavailable, so just return false here. if (PP.getDiagnostics().getExtensionHandlingBehavior() == DiagnosticsEngine::Ext_Error) return false; const LangOptions &LangOpts = PP.getLangOpts(); StringRef Extension = II->getName(); // Normalize the extension name, __foo__ becomes foo. if (Extension.startswith("__") && Extension.endswith("__") && Extension.size() >= 4) Extension = Extension.substr(2, Extension.size() - 4); // Because we inherit the feature list from HasFeature, this string switch // must be less restrictive than HasFeature's. return llvm::StringSwitch<bool>(Extension) // C11 features supported by other languages as extensions. .Case("c_alignas", true) .Case("c_atomic", true) .Case("c_generic_selections", true) .Case("c_static_assert", true) // C++0x features supported by other languages as extensions. .Case("cxx_atomic", LangOpts.CPlusPlus) .Case("cxx_deleted_functions", LangOpts.CPlusPlus) .Case("cxx_explicit_conversions", LangOpts.CPlusPlus) .Case("cxx_inline_namespaces", LangOpts.CPlusPlus) .Case("cxx_local_type_template_args", LangOpts.CPlusPlus) .Case("cxx_nonstatic_member_init", LangOpts.CPlusPlus) .Case("cxx_override_control", LangOpts.CPlusPlus) .Case("cxx_range_for", LangOpts.CPlusPlus) .Case("cxx_reference_qualified_functions", LangOpts.CPlusPlus) .Case("cxx_rvalue_references", LangOpts.CPlusPlus) .Default(false); }
IdentifierResolver::IdentifierResolver(Preprocessor &PP) : LangOpt(PP.getLangOpts()), PP(PP), IdDeclInfos(new IdDeclInfoMap) {}
/// HasFeature - Return true if we recognize and implement the feature /// specified by the identifier as a standard language feature. static bool HasFeature(const Preprocessor &PP, const IdentifierInfo *II) { const LangOptions &LangOpts = PP.getLangOpts(); StringRef Feature = II->getName(); // Normalize the feature name, __foo__ becomes foo. if (Feature.startswith("__") && Feature.endswith("__") && Feature.size() >= 4) Feature = Feature.substr(2, Feature.size() - 4); return llvm::StringSwitch<bool>(Feature) .Case("address_sanitizer", LangOpts.AddressSanitizer) .Case("attribute_analyzer_noreturn", true) .Case("attribute_availability", true) .Case("attribute_cf_returns_not_retained", true) .Case("attribute_cf_returns_retained", true) .Case("attribute_deprecated_with_message", true) .Case("attribute_ext_vector_type", true) .Case("attribute_ns_returns_not_retained", true) .Case("attribute_ns_returns_retained", true) .Case("attribute_ns_consumes_self", true) .Case("attribute_ns_consumed", true) .Case("attribute_cf_consumed", true) .Case("attribute_objc_ivar_unused", true) .Case("attribute_objc_method_family", true) .Case("attribute_overloadable", true) .Case("attribute_unavailable_with_message", true) .Case("attribute_unused_on_fields", true) .Case("blocks", LangOpts.Blocks) .Case("cxx_exceptions", LangOpts.Exceptions) .Case("cxx_rtti", LangOpts.RTTI) .Case("enumerator_attributes", true) // Objective-C features .Case("objc_arr", LangOpts.ObjCAutoRefCount) // FIXME: REMOVE? .Case("objc_arc", LangOpts.ObjCAutoRefCount) .Case("objc_arc_weak", LangOpts.ObjCAutoRefCount && LangOpts.ObjCRuntimeHasWeak) .Case("objc_default_synthesize_properties", LangOpts.ObjC2) .Case("objc_fixed_enum", LangOpts.ObjC2) .Case("objc_instancetype", LangOpts.ObjC2) .Case("objc_modules", LangOpts.ObjC2 && LangOpts.Modules) .Case("objc_nonfragile_abi", LangOpts.ObjCRuntime.isNonFragile()) .Case("objc_weak_class", LangOpts.ObjCRuntime.hasWeakClassImport()) .Case("ownership_holds", true) .Case("ownership_returns", true) .Case("ownership_takes", true) .Case("objc_bool", true) .Case("objc_subscripting", LangOpts.ObjCRuntime.isNonFragile()) .Case("objc_array_literals", LangOpts.ObjC2) .Case("objc_dictionary_literals", LangOpts.ObjC2) .Case("objc_boxed_expressions", LangOpts.ObjC2) .Case("arc_cf_code_audited", true) // C11 features .Case("c_alignas", LangOpts.C11) .Case("c_atomic", LangOpts.C11) .Case("c_generic_selections", LangOpts.C11) .Case("c_static_assert", LangOpts.C11) // C++11 features .Case("cxx_access_control_sfinae", LangOpts.CPlusPlus0x) .Case("cxx_alias_templates", LangOpts.CPlusPlus0x) .Case("cxx_alignas", LangOpts.CPlusPlus0x) .Case("cxx_atomic", LangOpts.CPlusPlus0x) .Case("cxx_attributes", LangOpts.CPlusPlus0x) .Case("cxx_auto_type", LangOpts.CPlusPlus0x) .Case("cxx_constexpr", LangOpts.CPlusPlus0x) .Case("cxx_decltype", LangOpts.CPlusPlus0x) .Case("cxx_decltype_incomplete_return_types", LangOpts.CPlusPlus0x) .Case("cxx_default_function_template_args", LangOpts.CPlusPlus0x) .Case("cxx_defaulted_functions", LangOpts.CPlusPlus0x) .Case("cxx_delegating_constructors", LangOpts.CPlusPlus0x) .Case("cxx_deleted_functions", LangOpts.CPlusPlus0x) .Case("cxx_explicit_conversions", LangOpts.CPlusPlus0x) .Case("cxx_generalized_initializers", LangOpts.CPlusPlus0x) .Case("cxx_implicit_moves", LangOpts.CPlusPlus0x) //.Case("cxx_inheriting_constructors", false) .Case("cxx_inline_namespaces", LangOpts.CPlusPlus0x) .Case("cxx_lambdas", LangOpts.CPlusPlus0x) .Case("cxx_local_type_template_args", LangOpts.CPlusPlus0x) .Case("cxx_nonstatic_member_init", LangOpts.CPlusPlus0x) .Case("cxx_noexcept", LangOpts.CPlusPlus0x) .Case("cxx_nullptr", LangOpts.CPlusPlus0x) .Case("cxx_override_control", LangOpts.CPlusPlus0x) .Case("cxx_range_for", LangOpts.CPlusPlus0x) .Case("cxx_raw_string_literals", LangOpts.CPlusPlus0x) .Case("cxx_reference_qualified_functions", LangOpts.CPlusPlus0x) .Case("cxx_rvalue_references", LangOpts.CPlusPlus0x) .Case("cxx_strong_enums", LangOpts.CPlusPlus0x) .Case("cxx_static_assert", LangOpts.CPlusPlus0x) .Case("cxx_trailing_return", LangOpts.CPlusPlus0x) .Case("cxx_unicode_literals", LangOpts.CPlusPlus0x) .Case("cxx_unrestricted_unions", LangOpts.CPlusPlus0x) .Case("cxx_user_literals", LangOpts.CPlusPlus0x) .Case("cxx_variadic_templates", LangOpts.CPlusPlus0x) // Type traits .Case("has_nothrow_assign", LangOpts.CPlusPlus) .Case("has_nothrow_copy", LangOpts.CPlusPlus) .Case("has_nothrow_constructor", LangOpts.CPlusPlus) .Case("has_trivial_assign", LangOpts.CPlusPlus) .Case("has_trivial_copy", LangOpts.CPlusPlus) .Case("has_trivial_constructor", LangOpts.CPlusPlus) .Case("has_trivial_destructor", LangOpts.CPlusPlus) .Case("has_virtual_destructor", LangOpts.CPlusPlus) .Case("is_abstract", LangOpts.CPlusPlus) .Case("is_base_of", LangOpts.CPlusPlus) .Case("is_class", LangOpts.CPlusPlus) .Case("is_convertible_to", LangOpts.CPlusPlus) // __is_empty is available only if the horrible // "struct __is_empty" parsing hack hasn't been needed in this // translation unit. If it has, __is_empty reverts to a normal // identifier and __has_feature(is_empty) evaluates false. .Case("is_empty", LangOpts.CPlusPlus && PP.getIdentifierInfo("__is_empty")->getTokenID() != tok::identifier) .Case("is_enum", LangOpts.CPlusPlus) .Case("is_final", LangOpts.CPlusPlus) .Case("is_literal", LangOpts.CPlusPlus) .Case("is_standard_layout", LangOpts.CPlusPlus) // __is_pod is available only if the horrible // "struct __is_pod" parsing hack hasn't been needed in this // translation unit. If it has, __is_pod reverts to a normal // identifier and __has_feature(is_pod) evaluates false. .Case("is_pod", LangOpts.CPlusPlus && PP.getIdentifierInfo("__is_pod")->getTokenID() != tok::identifier) .Case("is_polymorphic", LangOpts.CPlusPlus) .Case("is_trivial", LangOpts.CPlusPlus) .Case("is_trivially_assignable", LangOpts.CPlusPlus) .Case("is_trivially_constructible", LangOpts.CPlusPlus) .Case("is_trivially_copyable", LangOpts.CPlusPlus) .Case("is_union", LangOpts.CPlusPlus) .Case("modules", LangOpts.Modules) .Case("tls", PP.getTargetInfo().isTLSSupported()) .Case("underlying_type", LangOpts.CPlusPlus) .Default(false); }
// #pragma pack(...) comes in the following delicious flavors: // pack '(' [integer] ')' // pack '(' 'show' ')' // pack '(' ('push' | 'pop') [',' identifier] [, integer] ')' void PragmaPackHandler::HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer, Token &PackTok) { SourceLocation PackLoc = PackTok.getLocation(); Token Tok; PP.Lex(Tok); if (Tok.isNot(tok::l_paren)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_lparen) << "pack"; return; } Sema::PragmaPackKind Kind = Sema::PPK_Default; IdentifierInfo *Name = 0; Token Alignment; Alignment.startToken(); SourceLocation LParenLoc = Tok.getLocation(); PP.Lex(Tok); if (Tok.is(tok::numeric_constant)) { Alignment = Tok; PP.Lex(Tok); // In MSVC/gcc, #pragma pack(4) sets the alignment without affecting // the push/pop stack. // In Apple gcc, #pragma pack(4) is equivalent to #pragma pack(push, 4) if (PP.getLangOpts().ApplePragmaPack) Kind = Sema::PPK_Push; } else if (Tok.is(tok::identifier)) { const IdentifierInfo *II = Tok.getIdentifierInfo(); if (II->isStr("show")) { Kind = Sema::PPK_Show; PP.Lex(Tok); } else { if (II->isStr("push")) { Kind = Sema::PPK_Push; } else if (II->isStr("pop")) { Kind = Sema::PPK_Pop; } else { PP.Diag(Tok.getLocation(), diag::warn_pragma_pack_invalid_action); return; } PP.Lex(Tok); if (Tok.is(tok::comma)) { PP.Lex(Tok); if (Tok.is(tok::numeric_constant)) { Alignment = Tok; PP.Lex(Tok); } else if (Tok.is(tok::identifier)) { Name = Tok.getIdentifierInfo(); PP.Lex(Tok); if (Tok.is(tok::comma)) { PP.Lex(Tok); if (Tok.isNot(tok::numeric_constant)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_pack_malformed); return; } Alignment = Tok; PP.Lex(Tok); } } else { PP.Diag(Tok.getLocation(), diag::warn_pragma_pack_malformed); return; } } } } else if (PP.getLangOpts().ApplePragmaPack) { // In MSVC/gcc, #pragma pack() resets the alignment without affecting // the push/pop stack. // In Apple gcc #pragma pack() is equivalent to #pragma pack(pop). Kind = Sema::PPK_Pop; } if (Tok.isNot(tok::r_paren)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_rparen) << "pack"; return; } SourceLocation RParenLoc = Tok.getLocation(); PP.Lex(Tok); if (Tok.isNot(tok::eod)) { PP.Diag(Tok.getLocation(), diag::warn_pragma_extra_tokens_at_eol) << "pack"; return; } PragmaPackInfo *Info = (PragmaPackInfo*) PP.getPreprocessorAllocator().Allocate( sizeof(PragmaPackInfo), llvm::alignOf<PragmaPackInfo>()); new (Info) PragmaPackInfo(); Info->Kind = Kind; Info->Name = Name; Info->Alignment = Alignment; Info->LParenLoc = LParenLoc; Info->RParenLoc = RParenLoc; Token *Toks = (Token*) PP.getPreprocessorAllocator().Allocate( sizeof(Token) * 1, llvm::alignOf<Token>()); new (Toks) Token(); Toks[0].startToken(); Toks[0].setKind(tok::annot_pragma_pack); Toks[0].setLocation(PackLoc); Toks[0].setAnnotationValue(static_cast<void*>(Info)); PP.EnterTokenStream(Toks, 1, /*DisableMacroExpansion=*/true, /*OwnsTokens=*/false); }
/// integer-constant: [C99 6.4.4.1] /// decimal-constant integer-suffix /// octal-constant integer-suffix /// hexadecimal-constant integer-suffix /// binary-literal integer-suffix [GNU, C++1y] /// user-defined-integer-literal: [C++11 lex.ext] /// decimal-literal ud-suffix /// octal-literal ud-suffix /// hexadecimal-literal ud-suffix /// binary-literal ud-suffix [GNU, C++1y] /// decimal-constant: /// nonzero-digit /// decimal-constant digit /// octal-constant: /// 0 /// octal-constant octal-digit /// hexadecimal-constant: /// hexadecimal-prefix hexadecimal-digit /// hexadecimal-constant hexadecimal-digit /// hexadecimal-prefix: one of /// 0x 0X /// binary-literal: /// 0b binary-digit /// 0B binary-digit /// binary-literal binary-digit /// integer-suffix: /// unsigned-suffix [long-suffix] /// unsigned-suffix [long-long-suffix] /// long-suffix [unsigned-suffix] /// long-long-suffix [unsigned-sufix] /// nonzero-digit: /// 1 2 3 4 5 6 7 8 9 /// octal-digit: /// 0 1 2 3 4 5 6 7 /// hexadecimal-digit: /// 0 1 2 3 4 5 6 7 8 9 /// a b c d e f /// A B C D E F /// binary-digit: /// 0 /// 1 /// unsigned-suffix: one of /// u U /// long-suffix: one of /// l L /// long-long-suffix: one of /// ll LL /// /// floating-constant: [C99 6.4.4.2] /// TODO: add rules... /// NumericLiteralParser::NumericLiteralParser(StringRef TokSpelling) : ThisTokBegin(TokSpelling.begin()), ThisTokEnd(TokSpelling.end()) { // This routine assumes that the range begin/end matches the regex for integer // and FP constants (specifically, the 'pp-number' regex), and assumes that // the byte at "*end" is both valid and not part of the regex. Because of // this, it doesn't have to check for 'overscan' in various places. assert(!isPreprocessingNumberBody(*ThisTokEnd) && "didn't maximally munch?"); s = DigitsBegin = ThisTokBegin; saw_exponent = false; saw_period = false; saw_ud_suffix = false; isLong = false; isUnsigned = false; isLongLong = false; isFloat = false; isImaginary = false; MicrosoftInteger = 0; hadError = false; SourceLocation TokLoc; if (*s == '0') { // parse radix ParseNumberStartingWithZero(TokLoc); if (hadError) return; } else { // the first digit is non-zero radix = 10; s = SkipDigits(s); if (s == ThisTokEnd) { // Done. } else if (isHexDigit(*s) && !(*s == 'e' || *s == 'E')) { PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s - ThisTokBegin), diag::err_invalid_decimal_digit) << StringRef(s, 1); hadError = true; return; } else if (*s == '.') { checkSeparator(TokLoc, s, CSK_AfterDigits); s++; saw_period = true; checkSeparator(TokLoc, s, CSK_BeforeDigits); s = SkipDigits(s); } if ((*s == 'e' || *s == 'E')) { // exponent checkSeparator(TokLoc, s, CSK_AfterDigits); const char *Exponent = s; s++; saw_exponent = true; if (*s == '+' || *s == '-') s++; // sign checkSeparator(TokLoc, s, CSK_BeforeDigits); const char *first_non_digit = SkipDigits(s); if (first_non_digit != s) { s = first_non_digit; } else { PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, Exponent - ThisTokBegin), diag::err_exponent_has_no_digits); hadError = true; return; } } } SuffixBegin = s; checkSeparator(TokLoc, s, CSK_AfterDigits); // Parse the suffix. At this point we can classify whether we have an FP or // integer constant. bool isFPConstant = isFloatingLiteral(); const char *ImaginarySuffixLoc = nullptr; // Loop over all of the characters of the suffix. If we see something bad, // we break out of the loop. for (; s != ThisTokEnd; ++s) { switch (*s) { case 'f': // FP Suffix for "float" case 'F': if (!isFPConstant) break; // Error for integer constant. if (isFloat || isLong) break; // FF, LF invalid. isFloat = true; continue; // Success. case 'u': case 'U': if (isFPConstant) break; // Error for floating constant. if (isUnsigned) break; // Cannot be repeated. isUnsigned = true; continue; // Success. case 'l': case 'L': if (isLong || isLongLong) break; // Cannot be repeated. if (isFloat) break; // LF invalid. // Check for long long. The L's need to be adjacent and the same case. if (s[1] == s[0]) { assert(s + 1 < ThisTokEnd && "didn't maximally munch?"); if (isFPConstant) break; // long long invalid for floats. isLongLong = true; ++s; // Eat both of them. } else { isLong = true; } continue; // Success. case 'i': case 'I': if (PP.getLangOpts().MicrosoftExt) { if (isLong || isLongLong || MicrosoftInteger) break; if (!isFPConstant) { // Allow i8, i16, i32, i64, and i128. switch (s[1]) { case '8': s += 2; // i8 suffix MicrosoftInteger = 8; break; case '1': if (s[2] == '6') { s += 3; // i16 suffix MicrosoftInteger = 16; } else if (s[2] == '2' && s[3] == '8') { s += 4; // i128 suffix MicrosoftInteger = 128; } break; case '3': if (s[2] == '2') { s += 3; // i32 suffix MicrosoftInteger = 32; } break; case '6': if (s[2] == '4') { s += 3; // i64 suffix MicrosoftInteger = 64; } break; default: break; } } if (MicrosoftInteger) { assert(s <= ThisTokEnd && "didn't maximally munch?"); break; } } // "i", "if", and "il" are user-defined suffixes in C++1y. if (*s == 'i' && PP.getLangOpts().CPlusPlus14) break; // fall through. case 'j': case 'J': if (isImaginary) break; // Cannot be repeated. isImaginary = true; ImaginarySuffixLoc = s; continue; // Success. } // If we reached here, there was an error or a ud-suffix. break; } if (s != ThisTokEnd) { // FIXME: Don't bother expanding UCNs if !tok.hasUCN(). expandUCNs(UDSuffixBuf, StringRef(SuffixBegin, ThisTokEnd - SuffixBegin)); if (isValidUDSuffix(PP.getLangOpts(), UDSuffixBuf)) { // Any suffix pieces we might have parsed are actually part of the // ud-suffix. isLong = false; isUnsigned = false; isLongLong = false; isFloat = false; isImaginary = false; MicrosoftInteger = 0; saw_ud_suffix = true; return; } // Report an error if there are any. PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, SuffixBegin - ThisTokBegin), isFPConstant ? diag::err_invalid_suffix_float_constant : diag::err_invalid_suffix_integer_constant) << StringRef(SuffixBegin, ThisTokEnd-SuffixBegin); hadError = true; return; } if (isImaginary) { PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, ImaginarySuffixLoc - ThisTokBegin), diag::ext_imaginary_constant); } }
/// Finish - This does final analysis of the declspec, rejecting things like /// "_Imaginary" (lacking an FP type). This returns a diagnostic to issue or /// diag::NUM_DIAGNOSTICS if there is no error. After calling this method, /// DeclSpec is guaranteed self-consistent, even if an error occurred. void DeclSpec::Finish(DiagnosticsEngine &D, Preprocessor &PP, const PrintingPolicy &Policy) { // Before possibly changing their values, save specs as written. SaveWrittenBuiltinSpecs(); // Check the type specifier components first. // If decltype(auto) is used, no other type specifiers are permitted. if (TypeSpecType == TST_decltype_auto && (TypeSpecWidth != TSW_unspecified || TypeSpecComplex != TSC_unspecified || TypeSpecSign != TSS_unspecified || TypeAltiVecVector || TypeAltiVecPixel || TypeAltiVecBool || TypeQualifiers)) { const unsigned NumLocs = 8; SourceLocation ExtraLocs[NumLocs] = { TSWLoc, TSCLoc, TSSLoc, AltiVecLoc, TQ_constLoc, TQ_restrictLoc, TQ_volatileLoc, TQ_atomicLoc }; FixItHint Hints[NumLocs]; SourceLocation FirstLoc; for (unsigned I = 0; I != NumLocs; ++I) { if (!ExtraLocs[I].isInvalid()) { if (FirstLoc.isInvalid() || PP.getSourceManager().isBeforeInTranslationUnit(ExtraLocs[I], FirstLoc)) FirstLoc = ExtraLocs[I]; Hints[I] = FixItHint::CreateRemoval(ExtraLocs[I]); } } TypeSpecWidth = TSW_unspecified; TypeSpecComplex = TSC_unspecified; TypeSpecSign = TSS_unspecified; TypeAltiVecVector = TypeAltiVecPixel = TypeAltiVecBool = false; TypeQualifiers = 0; Diag(D, TSTLoc, diag::err_decltype_auto_cannot_be_combined) << Hints[0] << Hints[1] << Hints[2] << Hints[3] << Hints[4] << Hints[5] << Hints[6] << Hints[7]; } // Validate and finalize AltiVec vector declspec. if (TypeAltiVecVector) { if (TypeAltiVecBool) { // Sign specifiers are not allowed with vector bool. (PIM 2.1) if (TypeSpecSign != TSS_unspecified) { Diag(D, TSSLoc, diag::err_invalid_vector_bool_decl_spec) << getSpecifierName((TSS)TypeSpecSign); } // Only char/int are valid with vector bool. (PIM 2.1) if (((TypeSpecType != TST_unspecified) && (TypeSpecType != TST_char) && (TypeSpecType != TST_int)) || TypeAltiVecPixel) { Diag(D, TSTLoc, diag::err_invalid_vector_bool_decl_spec) << (TypeAltiVecPixel ? "__pixel" : getSpecifierName((TST)TypeSpecType, Policy)); } // Only 'short' and 'long long' are valid with vector bool. (PIM 2.1) if ((TypeSpecWidth != TSW_unspecified) && (TypeSpecWidth != TSW_short) && (TypeSpecWidth != TSW_longlong)) Diag(D, TSWLoc, diag::err_invalid_vector_bool_decl_spec) << getSpecifierName((TSW)TypeSpecWidth); // vector bool long long requires VSX support. if ((TypeSpecWidth == TSW_longlong) && (!PP.getTargetInfo().hasFeature("vsx")) && (!PP.getTargetInfo().hasFeature("power8-vector"))) Diag(D, TSTLoc, diag::err_invalid_vector_long_long_decl_spec); // Elements of vector bool are interpreted as unsigned. (PIM 2.1) if ((TypeSpecType == TST_char) || (TypeSpecType == TST_int) || (TypeSpecWidth != TSW_unspecified)) TypeSpecSign = TSS_unsigned; } else if (TypeSpecType == TST_double) { // vector long double and vector long long double are never allowed. // vector double is OK for Power7 and later. if (TypeSpecWidth == TSW_long || TypeSpecWidth == TSW_longlong) Diag(D, TSWLoc, diag::err_invalid_vector_long_double_decl_spec); else if (!PP.getTargetInfo().hasFeature("vsx")) Diag(D, TSTLoc, diag::err_invalid_vector_double_decl_spec); } else if (TypeSpecWidth == TSW_long) { Diag(D, TSWLoc, diag::warn_vector_long_decl_spec_combination) << getSpecifierName((TST)TypeSpecType, Policy); } if (TypeAltiVecPixel) { //TODO: perform validation TypeSpecType = TST_int; TypeSpecSign = TSS_unsigned; TypeSpecWidth = TSW_short; TypeSpecOwned = false; } } // signed/unsigned are only valid with int/char/wchar_t. if (TypeSpecSign != TSS_unspecified) { if (TypeSpecType == TST_unspecified) TypeSpecType = TST_int; // unsigned -> unsigned int, signed -> signed int. else if (TypeSpecType != TST_int && TypeSpecType != TST_int128 && TypeSpecType != TST_char && TypeSpecType != TST_wchar) { Diag(D, TSSLoc, diag::err_invalid_sign_spec) << getSpecifierName((TST)TypeSpecType, Policy); // signed double -> double. TypeSpecSign = TSS_unspecified; } } // Validate the width of the type. switch (TypeSpecWidth) { case TSW_unspecified: break; case TSW_short: // short int case TSW_longlong: // long long int if (TypeSpecType == TST_unspecified) TypeSpecType = TST_int; // short -> short int, long long -> long long int. else if (TypeSpecType != TST_int) { Diag(D, TSWLoc, TypeSpecWidth == TSW_short ? diag::err_invalid_short_spec : diag::err_invalid_longlong_spec) << getSpecifierName((TST)TypeSpecType, Policy); TypeSpecType = TST_int; TypeSpecOwned = false; } break; case TSW_long: // long double, long int if (TypeSpecType == TST_unspecified) TypeSpecType = TST_int; // long -> long int. else if (TypeSpecType != TST_int && TypeSpecType != TST_double) { Diag(D, TSWLoc, diag::err_invalid_long_spec) << getSpecifierName((TST)TypeSpecType, Policy); TypeSpecType = TST_int; TypeSpecOwned = false; } break; } // TODO: if the implementation does not implement _Complex or _Imaginary, // disallow their use. Need information about the backend. if (TypeSpecComplex != TSC_unspecified) { if (TypeSpecType == TST_unspecified) { Diag(D, TSCLoc, diag::ext_plain_complex) << FixItHint::CreateInsertion( PP.getLocForEndOfToken(getTypeSpecComplexLoc()), " double"); TypeSpecType = TST_double; // _Complex -> _Complex double. } else if (TypeSpecType == TST_int || TypeSpecType == TST_char) { // Note that this intentionally doesn't include _Complex _Bool. if (!PP.getLangOpts().CPlusPlus) Diag(D, TSTLoc, diag::ext_integer_complex); } else if (TypeSpecType != TST_float && TypeSpecType != TST_double) { Diag(D, TSCLoc, diag::err_invalid_complex_spec) << getSpecifierName((TST)TypeSpecType, Policy); TypeSpecComplex = TSC_unspecified; } } // C11 6.7.1/3, C++11 [dcl.stc]p1, GNU TLS: __thread, thread_local and // _Thread_local can only appear with the 'static' and 'extern' storage class // specifiers. We also allow __private_extern__ as an extension. if (ThreadStorageClassSpec != TSCS_unspecified) { switch (StorageClassSpec) { case SCS_unspecified: case SCS_extern: case SCS_private_extern: case SCS_static: break; default: if (PP.getSourceManager().isBeforeInTranslationUnit( getThreadStorageClassSpecLoc(), getStorageClassSpecLoc())) Diag(D, getStorageClassSpecLoc(), diag::err_invalid_decl_spec_combination) << DeclSpec::getSpecifierName(getThreadStorageClassSpec()) << SourceRange(getThreadStorageClassSpecLoc()); else Diag(D, getThreadStorageClassSpecLoc(), diag::err_invalid_decl_spec_combination) << DeclSpec::getSpecifierName(getStorageClassSpec()) << SourceRange(getStorageClassSpecLoc()); // Discard the thread storage class specifier to recover. ThreadStorageClassSpec = TSCS_unspecified; ThreadStorageClassSpecLoc = SourceLocation(); } } // If no type specifier was provided and we're parsing a language where // the type specifier is not optional, but we got 'auto' as a storage // class specifier, then assume this is an attempt to use C++0x's 'auto' // type specifier. if (PP.getLangOpts().CPlusPlus && TypeSpecType == TST_unspecified && StorageClassSpec == SCS_auto) { TypeSpecType = TST_auto; StorageClassSpec = SCS_unspecified; TSTLoc = TSTNameLoc = StorageClassSpecLoc; StorageClassSpecLoc = SourceLocation(); } // Diagnose if we've recovered from an ill-formed 'auto' storage class // specifier in a pre-C++11 dialect of C++. if (!PP.getLangOpts().CPlusPlus11 && TypeSpecType == TST_auto) Diag(D, TSTLoc, diag::ext_auto_type_specifier); if (PP.getLangOpts().CPlusPlus && !PP.getLangOpts().CPlusPlus11 && StorageClassSpec == SCS_auto) Diag(D, StorageClassSpecLoc, diag::warn_auto_storage_class) << FixItHint::CreateRemoval(StorageClassSpecLoc); if (TypeSpecType == TST_char16 || TypeSpecType == TST_char32) Diag(D, TSTLoc, diag::warn_cxx98_compat_unicode_type) << (TypeSpecType == TST_char16 ? "char16_t" : "char32_t"); if (Constexpr_specified) Diag(D, ConstexprLoc, diag::warn_cxx98_compat_constexpr); // C++ [class.friend]p6: // No storage-class-specifier shall appear in the decl-specifier-seq // of a friend declaration. if (isFriendSpecified() && (getStorageClassSpec() || getThreadStorageClassSpec())) { SmallString<32> SpecName; SourceLocation SCLoc; FixItHint StorageHint, ThreadHint; if (DeclSpec::SCS SC = getStorageClassSpec()) { SpecName = getSpecifierName(SC); SCLoc = getStorageClassSpecLoc(); StorageHint = FixItHint::CreateRemoval(SCLoc); } if (DeclSpec::TSCS TSC = getThreadStorageClassSpec()) { if (!SpecName.empty()) SpecName += " "; SpecName += getSpecifierName(TSC); SCLoc = getThreadStorageClassSpecLoc(); ThreadHint = FixItHint::CreateRemoval(SCLoc); } Diag(D, SCLoc, diag::err_friend_decl_spec) << SpecName << StorageHint << ThreadHint; ClearStorageClassSpecs(); } // C++11 [dcl.fct.spec]p5: // The virtual specifier shall be used only in the initial // declaration of a non-static class member function; // C++11 [dcl.fct.spec]p6: // The explicit specifier shall be used only in the declaration of // a constructor or conversion function within its class // definition; if (isFriendSpecified() && (isVirtualSpecified() || isExplicitSpecified())) { StringRef Keyword; SourceLocation SCLoc; if (isVirtualSpecified()) { Keyword = "virtual"; SCLoc = getVirtualSpecLoc(); } else { Keyword = "explicit"; SCLoc = getExplicitSpecLoc(); } FixItHint Hint = FixItHint::CreateRemoval(SCLoc); Diag(D, SCLoc, diag::err_friend_decl_spec) << Keyword << Hint; FS_virtual_specified = FS_explicit_specified = false; FS_virtualLoc = FS_explicitLoc = SourceLocation(); } assert(!TypeSpecOwned || isDeclRep((TST) TypeSpecType)); // Okay, now we can infer the real type. // TODO: return "auto function" and other bad things based on the real type. // 'data definition has no type or storage class'? }
void ento::createPlistDiagnosticConsumer(PathDiagnosticConsumers &C, const std::string& s, const Preprocessor &PP) { C.push_back(new PlistDiagnostics(s, PP.getLangOpts(), false)); }
/// SyntaxHighlight - Relex the specified FileID and annotate the HTML with /// information about keywords, macro expansions etc. This uses the macro /// table state from the end of the file, so it won't be perfectly perfect, /// but it will be reasonably close. void html::SyntaxHighlight(Rewriter &R, FileID FID, const Preprocessor &PP) { RewriteBuffer &RB = R.getEditBuffer(FID); const SourceManager &SM = PP.getSourceManager(); const llvm::MemoryBuffer *FromFile = SM.getBuffer(FID); Lexer L(FID, FromFile, SM, PP.getLangOpts()); const char *BufferStart = L.getBuffer().data(); // Inform the preprocessor that we want to retain comments as tokens, so we // can highlight them. L.SetCommentRetentionState(true); // Lex all the tokens in raw mode, to avoid entering #includes or expanding // macros. Token Tok; L.LexFromRawLexer(Tok); while (Tok.isNot(tok::eof)) { // Since we are lexing unexpanded tokens, all tokens are from the main // FileID. unsigned TokOffs = SM.getFileOffset(Tok.getLocation()); unsigned TokLen = Tok.getLength(); switch (Tok.getKind()) { default: break; case tok::identifier: llvm_unreachable("tok::identifier in raw lexing mode!"); case tok::raw_identifier: { // Fill in Result.IdentifierInfo and update the token kind, // looking up the identifier in the identifier table. PP.LookUpIdentifierInfo(Tok); // If this is a pp-identifier, for a keyword, highlight it as such. if (Tok.isNot(tok::identifier)) HighlightRange(RB, TokOffs, TokOffs+TokLen, BufferStart, "<span class='keyword'>", "</span>"); break; } case tok::comment: HighlightRange(RB, TokOffs, TokOffs+TokLen, BufferStart, "<span class='comment'>", "</span>"); break; case tok::utf8_string_literal: // Chop off the u part of u8 prefix ++TokOffs; --TokLen; // FALL THROUGH to chop the 8 case tok::wide_string_literal: case tok::utf16_string_literal: case tok::utf32_string_literal: // Chop off the L, u, U or 8 prefix ++TokOffs; --TokLen; // FALL THROUGH. case tok::string_literal: // FIXME: Exclude the optional ud-suffix from the highlighted range. HighlightRange(RB, TokOffs, TokOffs+TokLen, BufferStart, "<span class='string_literal'>", "</span>"); break; case tok::hash: { // If this is a preprocessor directive, all tokens to end of line are too. if (!Tok.isAtStartOfLine()) break; // Eat all of the tokens until we get to the next one at the start of // line. unsigned TokEnd = TokOffs+TokLen; L.LexFromRawLexer(Tok); while (!Tok.isAtStartOfLine() && Tok.isNot(tok::eof)) { TokEnd = SM.getFileOffset(Tok.getLocation())+Tok.getLength(); L.LexFromRawLexer(Tok); } // Find end of line. This is a hack. HighlightRange(RB, TokOffs, TokEnd, BufferStart, "<span class='directive'>", "</span>"); // Don't skip the next token. continue; } } L.LexFromRawLexer(Tok); } }
/// RewriteMacrosInInput - Implement -rewrite-macros mode. void clang::RewriteMacrosInInput(Preprocessor &PP, raw_ostream *OS) { SourceManager &SM = PP.getSourceManager(); Rewriter Rewrite; Rewrite.setSourceMgr(SM, PP.getLangOpts()); RewriteBuffer &RB = Rewrite.getEditBuffer(SM.getMainFileID()); std::vector<Token> RawTokens; LexRawTokensFromMainFile(PP, RawTokens); unsigned CurRawTok = 0; Token RawTok = GetNextRawTok(RawTokens, CurRawTok, false); // Get the first preprocessing token. PP.EnterMainSourceFile(); Token PPTok; PP.Lex(PPTok); // Preprocess the input file in parallel with raw lexing the main file. Ignore // all tokens that are preprocessed from a file other than the main file (e.g. // a header). If we see tokens that are in the preprocessed file but not the // lexed file, we have a macro expansion. If we see tokens in the lexed file // that aren't in the preprocessed view, we have macros that expand to no // tokens, or macro arguments etc. while (RawTok.isNot(tok::eof) || PPTok.isNot(tok::eof)) { SourceLocation PPLoc = SM.getExpansionLoc(PPTok.getLocation()); // If PPTok is from a different source file, ignore it. if (!SM.isFromMainFile(PPLoc)) { PP.Lex(PPTok); continue; } // If the raw file hits a preprocessor directive, they will be extra tokens // in the raw file that don't exist in the preprocsesed file. However, we // choose to preserve them in the output file and otherwise handle them // specially. if (RawTok.is(tok::hash) && RawTok.isAtStartOfLine()) { // If this is a #warning directive or #pragma mark (GNU extensions), // comment the line out. if (RawTokens[CurRawTok].is(tok::identifier)) { const IdentifierInfo *II = RawTokens[CurRawTok].getIdentifierInfo(); if (II->getName() == "warning") { // Comment out #warning. RB.InsertTextAfter(SM.getFileOffset(RawTok.getLocation()), "//"); } else if (II->getName() == "pragma" && RawTokens[CurRawTok+1].is(tok::identifier) && (RawTokens[CurRawTok+1].getIdentifierInfo()->getName() == "mark")) { // Comment out #pragma mark. RB.InsertTextAfter(SM.getFileOffset(RawTok.getLocation()), "//"); } } // Otherwise, if this is a #include or some other directive, just leave it // in the file by skipping over the line. RawTok = GetNextRawTok(RawTokens, CurRawTok, false); while (!RawTok.isAtStartOfLine() && RawTok.isNot(tok::eof)) RawTok = GetNextRawTok(RawTokens, CurRawTok, false); continue; } // Okay, both tokens are from the same file. Get their offsets from the // start of the file. unsigned PPOffs = SM.getFileOffset(PPLoc); unsigned RawOffs = SM.getFileOffset(RawTok.getLocation()); // If the offsets are the same and the token kind is the same, ignore them. if (PPOffs == RawOffs && isSameToken(RawTok, PPTok)) { RawTok = GetNextRawTok(RawTokens, CurRawTok, false); PP.Lex(PPTok); continue; } // If the PP token is farther along than the raw token, something was // deleted. Comment out the raw token. if (RawOffs <= PPOffs) { // Comment out a whole run of tokens instead of bracketing each one with // comments. Add a leading space if RawTok didn't have one. bool HasSpace = RawTok.hasLeadingSpace(); RB.InsertTextAfter(RawOffs, &" /*"[HasSpace]); unsigned EndPos; do { EndPos = RawOffs+RawTok.getLength(); RawTok = GetNextRawTok(RawTokens, CurRawTok, true); RawOffs = SM.getFileOffset(RawTok.getLocation()); if (RawTok.is(tok::comment)) { // Skip past the comment. RawTok = GetNextRawTok(RawTokens, CurRawTok, false); break; } } while (RawOffs <= PPOffs && !RawTok.isAtStartOfLine() && (PPOffs != RawOffs || !isSameToken(RawTok, PPTok))); RB.InsertTextBefore(EndPos, "*/"); continue; } // Otherwise, there was a replacement an expansion. Insert the new token // in the output buffer. Insert the whole run of new tokens at once to get // them in the right order. unsigned InsertPos = PPOffs; std::string Expansion; while (PPOffs < RawOffs) { Expansion += ' ' + PP.getSpelling(PPTok); PP.Lex(PPTok); PPLoc = SM.getExpansionLoc(PPTok.getLocation()); PPOffs = SM.getFileOffset(PPLoc); } Expansion += ' '; RB.InsertTextBefore(InsertPos, Expansion); } // Get the buffer corresponding to MainFileID. If we haven't changed it, then // we are done. if (const RewriteBuffer *RewriteBuf = Rewrite.getRewriteBufferFor(SM.getMainFileID())) { //printf("Changed:\n"); *OS << std::string(RewriteBuf->begin(), RewriteBuf->end()); } else { fprintf(stderr, "No changes\n"); } OS->flush(); }
/// InitializePreprocessor - Initialize the preprocessor getting it and the /// environment ready to process a single file. This returns true on error. /// void clang::InitializePreprocessor( Preprocessor &PP, const PreprocessorOptions &InitOpts, const PCHContainerReader &PCHContainerRdr, const FrontendOptions &FEOpts) { const LangOptions &LangOpts = PP.getLangOpts(); std::string PredefineBuffer; PredefineBuffer.reserve(4080); llvm::raw_string_ostream Predefines(PredefineBuffer); MacroBuilder Builder(Predefines); // Emit line markers for various builtin sections of the file. We don't do // this in asm preprocessor mode, because "# 4" is not a line marker directive // in this mode. if (!PP.getLangOpts().AsmPreprocessor) Builder.append("# 1 \"<built-in>\" 3"); // Install things like __POWERPC__, __GNUC__, etc into the macro table. if (InitOpts.UsePredefines) { if (LangOpts.CUDA && PP.getAuxTargetInfo()) InitializePredefinedMacros(*PP.getAuxTargetInfo(), LangOpts, FEOpts, Builder); InitializePredefinedMacros(PP.getTargetInfo(), LangOpts, FEOpts, Builder); // Install definitions to make Objective-C++ ARC work well with various // C++ Standard Library implementations. if (LangOpts.ObjC1 && LangOpts.CPlusPlus && (LangOpts.ObjCAutoRefCount || LangOpts.ObjCWeak)) { switch (InitOpts.ObjCXXARCStandardLibrary) { case ARCXX_nolib: case ARCXX_libcxx: break; case ARCXX_libstdcxx: AddObjCXXARCLibstdcxxDefines(LangOpts, Builder); break; } } } // Even with predefines off, some macros are still predefined. // These should all be defined in the preprocessor according to the // current language configuration. InitializeStandardPredefinedMacros(PP.getTargetInfo(), PP.getLangOpts(), FEOpts, Builder); // Add on the predefines from the driver. Wrap in a #line directive to report // that they come from the command line. if (!PP.getLangOpts().AsmPreprocessor) Builder.append("# 1 \"<command line>\" 1"); // Process #define's and #undef's in the order they are given. for (unsigned i = 0, e = InitOpts.Macros.size(); i != e; ++i) { if (InitOpts.Macros[i].second) // isUndef Builder.undefineMacro(InitOpts.Macros[i].first); else DefineBuiltinMacro(Builder, InitOpts.Macros[i].first, PP.getDiagnostics()); } // Exit the command line and go back to <built-in> (2 is LC_LEAVE). if (!PP.getLangOpts().AsmPreprocessor) Builder.append("# 1 \"<built-in>\" 2"); // If -imacros are specified, include them now. These are processed before // any -include directives. for (unsigned i = 0, e = InitOpts.MacroIncludes.size(); i != e; ++i) AddImplicitIncludeMacros(Builder, InitOpts.MacroIncludes[i]); // Process -include-pch/-include-pth directives. if (!InitOpts.ImplicitPCHInclude.empty()) AddImplicitIncludePCH(Builder, PP, PCHContainerRdr, InitOpts.ImplicitPCHInclude); if (!InitOpts.ImplicitPTHInclude.empty()) AddImplicitIncludePTH(Builder, PP, InitOpts.ImplicitPTHInclude); // Process -include directives. for (unsigned i = 0, e = InitOpts.Includes.size(); i != e; ++i) { const std::string &Path = InitOpts.Includes[i]; AddImplicitInclude(Builder, Path); } // Instruct the preprocessor to skip the preamble. PP.setSkipMainFilePreamble(InitOpts.PrecompiledPreambleBytes.first, InitOpts.PrecompiledPreambleBytes.second); // Copy PredefinedBuffer into the Preprocessor. PP.setPredefines(Predefines.str()); }
/// EvaluateValue - Evaluate the token PeekTok (and any others needed) and /// return the computed value in Result. Return true if there was an error /// parsing. This function also returns information about the form of the /// expression in DT. See above for information on what DT means. /// /// If ValueLive is false, then this value is being evaluated in a context where /// the result is not used. As such, avoid diagnostics that relate to /// evaluation. static bool EvaluateValue(PPValue &Result, Token &PeekTok, DefinedTracker &DT, bool ValueLive, Preprocessor &PP) { DT.State = DefinedTracker::Unknown; if (PeekTok.is(tok::code_completion)) { if (PP.getCodeCompletionHandler()) PP.getCodeCompletionHandler()->CodeCompletePreprocessorExpression(); PP.setCodeCompletionReached(); PP.LexNonComment(PeekTok); } // If this token's spelling is a pp-identifier, check to see if it is // 'defined' or if it is a macro. Note that we check here because many // keywords are pp-identifiers, so we can't check the kind. if (IdentifierInfo *II = PeekTok.getIdentifierInfo()) { // Handle "defined X" and "defined(X)". if (II->isStr("defined")) return(EvaluateDefined(Result, PeekTok, DT, ValueLive, PP)); // If this identifier isn't 'defined' or one of the special // preprocessor keywords and it wasn't macro expanded, it turns // into a simple 0, unless it is the C++ keyword "true", in which case it // turns into "1". if (ValueLive && II->getTokenID() != tok::kw_true && II->getTokenID() != tok::kw_false) PP.Diag(PeekTok, diag::warn_pp_undef_identifier) << II; Result.Val = II->getTokenID() == tok::kw_true; Result.Val.setIsUnsigned(false); // "0" is signed intmax_t 0. Result.setRange(PeekTok.getLocation()); PP.LexNonComment(PeekTok); return false; } switch (PeekTok.getKind()) { default: // Non-value token. PP.Diag(PeekTok, diag::err_pp_expr_bad_token_start_expr); return true; case tok::eod: case tok::r_paren: // If there is no expression, report and exit. PP.Diag(PeekTok, diag::err_pp_expected_value_in_expr); return true; case tok::numeric_constant: { SmallString<64> IntegerBuffer; bool NumberInvalid = false; StringRef Spelling = PP.getSpelling(PeekTok, IntegerBuffer, &NumberInvalid); if (NumberInvalid) return true; // a diagnostic was already reported NumericLiteralParser Literal(Spelling, PeekTok.getLocation(), PP); if (Literal.hadError) return true; // a diagnostic was already reported. if (Literal.isFloatingLiteral() || Literal.isImaginary) { PP.Diag(PeekTok, diag::err_pp_illegal_floating_literal); return true; } assert(Literal.isIntegerLiteral() && "Unknown ppnumber"); // Complain about, and drop, any ud-suffix. if (Literal.hasUDSuffix()) PP.Diag(PeekTok, diag::err_pp_invalid_udl) << /*integer*/1; // 'long long' is a C99 or C++11 feature. if (!PP.getLangOpts().C99 && Literal.isLongLong) { if (PP.getLangOpts().CPlusPlus) PP.Diag(PeekTok, PP.getLangOpts().CPlusPlus0x ? diag::warn_cxx98_compat_longlong : diag::ext_cxx11_longlong); else PP.Diag(PeekTok, diag::ext_c99_longlong); } // Parse the integer literal into Result. if (Literal.GetIntegerValue(Result.Val)) { // Overflow parsing integer literal. if (ValueLive) PP.Diag(PeekTok, diag::warn_integer_too_large); Result.Val.setIsUnsigned(true); } else { // Set the signedness of the result to match whether there was a U suffix // or not. Result.Val.setIsUnsigned(Literal.isUnsigned); // Detect overflow based on whether the value is signed. If signed // and if the value is too large, emit a warning "integer constant is so // large that it is unsigned" e.g. on 12345678901234567890 where intmax_t // is 64-bits. if (!Literal.isUnsigned && Result.Val.isNegative()) { // Don't warn for a hex literal: 0x8000..0 shouldn't warn. if (ValueLive && Literal.getRadix() != 16) PP.Diag(PeekTok, diag::warn_integer_too_large_for_signed); Result.Val.setIsUnsigned(true); } } // Consume the token. Result.setRange(PeekTok.getLocation()); PP.LexNonComment(PeekTok); return false; } case tok::char_constant: // 'x' case tok::wide_char_constant: { // L'x' case tok::utf16_char_constant: // u'x' case tok::utf32_char_constant: // U'x' // Complain about, and drop, any ud-suffix. if (PeekTok.hasUDSuffix()) PP.Diag(PeekTok, diag::err_pp_invalid_udl) << /*character*/0; SmallString<32> CharBuffer; bool CharInvalid = false; StringRef ThisTok = PP.getSpelling(PeekTok, CharBuffer, &CharInvalid); if (CharInvalid) return true; CharLiteralParser Literal(ThisTok.begin(), ThisTok.end(), PeekTok.getLocation(), PP, PeekTok.getKind()); if (Literal.hadError()) return true; // A diagnostic was already emitted. // Character literals are always int or wchar_t, expand to intmax_t. const TargetInfo &TI = PP.getTargetInfo(); unsigned NumBits; if (Literal.isMultiChar()) NumBits = TI.getIntWidth(); else if (Literal.isWide()) NumBits = TI.getWCharWidth(); else if (Literal.isUTF16()) NumBits = TI.getChar16Width(); else if (Literal.isUTF32()) NumBits = TI.getChar32Width(); else NumBits = TI.getCharWidth(); // Set the width. llvm::APSInt Val(NumBits); // Set the value. Val = Literal.getValue(); // Set the signedness. UTF-16 and UTF-32 are always unsigned if (!Literal.isUTF16() && !Literal.isUTF32()) Val.setIsUnsigned(!PP.getLangOpts().CharIsSigned); if (Result.Val.getBitWidth() > Val.getBitWidth()) { Result.Val = Val.extend(Result.Val.getBitWidth()); } else { assert(Result.Val.getBitWidth() == Val.getBitWidth() && "intmax_t smaller than char/wchar_t?"); Result.Val = Val; } // Consume the token. Result.setRange(PeekTok.getLocation()); PP.LexNonComment(PeekTok); return false; } case tok::l_paren: { SourceLocation Start = PeekTok.getLocation(); PP.LexNonComment(PeekTok); // Eat the (. // Parse the value and if there are any binary operators involved, parse // them. if (EvaluateValue(Result, PeekTok, DT, ValueLive, PP)) return true; // If this is a silly value like (X), which doesn't need parens, check for // !(defined X). if (PeekTok.is(tok::r_paren)) { // Just use DT unmodified as our result. } else { // Otherwise, we have something like (x+y), and we consumed '(x'. if (EvaluateDirectiveSubExpr(Result, 1, PeekTok, ValueLive, PP)) return true; if (PeekTok.isNot(tok::r_paren)) { PP.Diag(PeekTok.getLocation(), diag::err_pp_expected_rparen) << Result.getRange(); PP.Diag(Start, diag::note_matching) << "("; return true; } DT.State = DefinedTracker::Unknown; } Result.setRange(Start, PeekTok.getLocation()); PP.LexNonComment(PeekTok); // Eat the ). return false; } case tok::plus: { SourceLocation Start = PeekTok.getLocation(); // Unary plus doesn't modify the value. PP.LexNonComment(PeekTok); if (EvaluateValue(Result, PeekTok, DT, ValueLive, PP)) return true; Result.setBegin(Start); return false; } case tok::minus: { SourceLocation Loc = PeekTok.getLocation(); PP.LexNonComment(PeekTok); if (EvaluateValue(Result, PeekTok, DT, ValueLive, PP)) return true; Result.setBegin(Loc); // C99 6.5.3.3p3: The sign of the result matches the sign of the operand. Result.Val = -Result.Val; // -MININT is the only thing that overflows. Unsigned never overflows. bool Overflow = !Result.isUnsigned() && Result.Val.isMinSignedValue(); // If this operator is live and overflowed, report the issue. if (Overflow && ValueLive) PP.Diag(Loc, diag::warn_pp_expr_overflow) << Result.getRange(); DT.State = DefinedTracker::Unknown; return false; } case tok::tilde: { SourceLocation Start = PeekTok.getLocation(); PP.LexNonComment(PeekTok); if (EvaluateValue(Result, PeekTok, DT, ValueLive, PP)) return true; Result.setBegin(Start); // C99 6.5.3.3p4: The sign of the result matches the sign of the operand. Result.Val = ~Result.Val; DT.State = DefinedTracker::Unknown; return false; } case tok::exclaim: { SourceLocation Start = PeekTok.getLocation(); PP.LexNonComment(PeekTok); if (EvaluateValue(Result, PeekTok, DT, ValueLive, PP)) return true; Result.setBegin(Start); Result.Val = !Result.Val; // C99 6.5.3.3p5: The sign of the result is 'int', aka it is signed. Result.Val.setIsUnsigned(false); if (DT.State == DefinedTracker::DefinedMacro) DT.State = DefinedTracker::NotDefinedMacro; else if (DT.State == DefinedTracker::NotDefinedMacro) DT.State = DefinedTracker::DefinedMacro; return false; } // FIXME: Handle #assert } }
/// Finish - This does final analysis of the declspec, rejecting things like /// "_Imaginary" (lacking an FP type). This returns a diagnostic to issue or /// diag::NUM_DIAGNOSTICS if there is no error. After calling this method, /// DeclSpec is guaranteed self-consistent, even if an error occurred. void DeclSpec::Finish(DiagnosticsEngine &D, Preprocessor &PP) { // Before possibly changing their values, save specs as written. SaveWrittenBuiltinSpecs(); // Check the type specifier components first. // Validate and finalize AltiVec vector declspec. if (TypeAltiVecVector) { if (TypeAltiVecBool) { // Sign specifiers are not allowed with vector bool. (PIM 2.1) if (TypeSpecSign != TSS_unspecified) { Diag(D, TSSLoc, diag::err_invalid_vector_bool_decl_spec) << getSpecifierName((TSS)TypeSpecSign); } // Only char/int are valid with vector bool. (PIM 2.1) if (((TypeSpecType != TST_unspecified) && (TypeSpecType != TST_char) && (TypeSpecType != TST_int)) || TypeAltiVecPixel) { Diag(D, TSTLoc, diag::err_invalid_vector_bool_decl_spec) << (TypeAltiVecPixel ? "__pixel" : getSpecifierName((TST)TypeSpecType)); } // Only 'short' is valid with vector bool. (PIM 2.1) if ((TypeSpecWidth != TSW_unspecified) && (TypeSpecWidth != TSW_short)) Diag(D, TSWLoc, diag::err_invalid_vector_bool_decl_spec) << getSpecifierName((TSW)TypeSpecWidth); // Elements of vector bool are interpreted as unsigned. (PIM 2.1) if ((TypeSpecType == TST_char) || (TypeSpecType == TST_int) || (TypeSpecWidth != TSW_unspecified)) TypeSpecSign = TSS_unsigned; } if (TypeAltiVecPixel) { //TODO: perform validation TypeSpecType = TST_int; TypeSpecSign = TSS_unsigned; TypeSpecWidth = TSW_short; TypeSpecOwned = false; } } // signed/unsigned are only valid with int/char/wchar_t. if (TypeSpecSign != TSS_unspecified) { if (TypeSpecType == TST_unspecified) TypeSpecType = TST_int; // unsigned -> unsigned int, signed -> signed int. else if (TypeSpecType != TST_int && TypeSpecType != TST_int128 && TypeSpecType != TST_char && TypeSpecType != TST_wchar) { Diag(D, TSSLoc, diag::err_invalid_sign_spec) << getSpecifierName((TST)TypeSpecType); // signed double -> double. TypeSpecSign = TSS_unspecified; } } // Validate the width of the type. switch (TypeSpecWidth) { case TSW_unspecified: break; case TSW_short: // short int case TSW_longlong: // long long int if (TypeSpecType == TST_unspecified) TypeSpecType = TST_int; // short -> short int, long long -> long long int. else if (TypeSpecType != TST_int) { Diag(D, TSWLoc, TypeSpecWidth == TSW_short ? diag::err_invalid_short_spec : diag::err_invalid_longlong_spec) << getSpecifierName((TST)TypeSpecType); TypeSpecType = TST_int; TypeSpecOwned = false; } break; case TSW_long: // long double, long int if (TypeSpecType == TST_unspecified) TypeSpecType = TST_int; // long -> long int. else if (TypeSpecType != TST_int && TypeSpecType != TST_double) { Diag(D, TSWLoc, diag::err_invalid_long_spec) << getSpecifierName((TST)TypeSpecType); TypeSpecType = TST_int; TypeSpecOwned = false; } break; } // TODO: if the implementation does not implement _Complex or _Imaginary, // disallow their use. Need information about the backend. if (TypeSpecComplex != TSC_unspecified) { if (TypeSpecType == TST_unspecified) { Diag(D, TSCLoc, diag::ext_plain_complex) << FixItHint::CreateInsertion( PP.getLocForEndOfToken(getTypeSpecComplexLoc()), " double"); TypeSpecType = TST_double; // _Complex -> _Complex double. } else if (TypeSpecType == TST_int || TypeSpecType == TST_char) { // Note that this intentionally doesn't include _Complex _Bool. if (!PP.getLangOpts().CPlusPlus) Diag(D, TSTLoc, diag::ext_integer_complex); } else if (TypeSpecType != TST_float && TypeSpecType != TST_double) { Diag(D, TSCLoc, diag::err_invalid_complex_spec) << getSpecifierName((TST)TypeSpecType); TypeSpecComplex = TSC_unspecified; } } // If no type specifier was provided and we're parsing a language where // the type specifier is not optional, but we got 'auto' as a storage // class specifier, then assume this is an attempt to use C++0x's 'auto' // type specifier. // FIXME: Does Microsoft really support implicit int in C++? if (PP.getLangOpts().CPlusPlus && !PP.getLangOpts().MicrosoftExt && TypeSpecType == TST_unspecified && StorageClassSpec == SCS_auto) { TypeSpecType = TST_auto; StorageClassSpec = SCS_unspecified; TSTLoc = TSTNameLoc = StorageClassSpecLoc; StorageClassSpecLoc = SourceLocation(); } // Diagnose if we've recovered from an ill-formed 'auto' storage class // specifier in a pre-C++0x dialect of C++. if (!PP.getLangOpts().CPlusPlus11 && TypeSpecType == TST_auto) Diag(D, TSTLoc, diag::ext_auto_type_specifier); if (PP.getLangOpts().CPlusPlus && !PP.getLangOpts().CPlusPlus11 && StorageClassSpec == SCS_auto) Diag(D, StorageClassSpecLoc, diag::warn_auto_storage_class) << FixItHint::CreateRemoval(StorageClassSpecLoc); if (TypeSpecType == TST_char16 || TypeSpecType == TST_char32) Diag(D, TSTLoc, diag::warn_cxx98_compat_unicode_type) << (TypeSpecType == TST_char16 ? "char16_t" : "char32_t"); if (Constexpr_specified) Diag(D, ConstexprLoc, diag::warn_cxx98_compat_constexpr); // C++ [class.friend]p6: // No storage-class-specifier shall appear in the decl-specifier-seq // of a friend declaration. if (isFriendSpecified() && getStorageClassSpec()) { DeclSpec::SCS SC = getStorageClassSpec(); const char *SpecName = getSpecifierName(SC); SourceLocation SCLoc = getStorageClassSpecLoc(); SourceLocation SCEndLoc = SCLoc.getLocWithOffset(strlen(SpecName)); Diag(D, SCLoc, diag::err_friend_storage_spec) << SpecName << FixItHint::CreateRemoval(SourceRange(SCLoc, SCEndLoc)); ClearStorageClassSpecs(); } assert(!TypeSpecOwned || isDeclRep((TST) TypeSpecType)); // Okay, now we can infer the real type. // TODO: return "auto function" and other bad things based on the real type. // 'data definition has no type or storage class'? }
/// EvaluateDirectiveSubExpr - Evaluate the subexpression whose first token is /// PeekTok, and whose precedence is PeekPrec. This returns the result in LHS. /// /// If ValueLive is false, then this value is being evaluated in a context where /// the result is not used. As such, avoid diagnostics that relate to /// evaluation, such as division by zero warnings. static bool EvaluateDirectiveSubExpr(PPValue &LHS, unsigned MinPrec, Token &PeekTok, bool ValueLive, Preprocessor &PP) { unsigned PeekPrec = getPrecedence(PeekTok.getKind()); // If this token isn't valid, report the error. if (PeekPrec == ~0U) { PP.Diag(PeekTok.getLocation(), diag::err_pp_expr_bad_token_binop) << LHS.getRange(); return true; } while (1) { // If this token has a lower precedence than we are allowed to parse, return // it so that higher levels of the recursion can parse it. if (PeekPrec < MinPrec) return false; tok::TokenKind Operator = PeekTok.getKind(); // If this is a short-circuiting operator, see if the RHS of the operator is // dead. Note that this cannot just clobber ValueLive. Consider // "0 && 1 ? 4 : 1 / 0", which is parsed as "(0 && 1) ? 4 : (1 / 0)". In // this example, the RHS of the && being dead does not make the rest of the // expr dead. bool RHSIsLive; if (Operator == tok::ampamp && LHS.Val == 0) RHSIsLive = false; // RHS of "0 && x" is dead. else if (Operator == tok::pipepipe && LHS.Val != 0) RHSIsLive = false; // RHS of "1 || x" is dead. else if (Operator == tok::question && LHS.Val == 0) RHSIsLive = false; // RHS (x) of "0 ? x : y" is dead. else RHSIsLive = ValueLive; // Consume the operator, remembering the operator's location for reporting. SourceLocation OpLoc = PeekTok.getLocation(); PP.LexNonComment(PeekTok); PPValue RHS(LHS.getBitWidth()); // Parse the RHS of the operator. DefinedTracker DT; if (EvaluateValue(RHS, PeekTok, DT, RHSIsLive, PP)) return true; // Remember the precedence of this operator and get the precedence of the // operator immediately to the right of the RHS. unsigned ThisPrec = PeekPrec; PeekPrec = getPrecedence(PeekTok.getKind()); // If this token isn't valid, report the error. if (PeekPrec == ~0U) { PP.Diag(PeekTok.getLocation(), diag::err_pp_expr_bad_token_binop) << RHS.getRange(); return true; } // Decide whether to include the next binop in this subexpression. For // example, when parsing x+y*z and looking at '*', we want to recursively // handle y*z as a single subexpression. We do this because the precedence // of * is higher than that of +. The only strange case we have to handle // here is for the ?: operator, where the precedence is actually lower than // the LHS of the '?'. The grammar rule is: // // conditional-expression ::= // logical-OR-expression ? expression : conditional-expression // where 'expression' is actually comma-expression. unsigned RHSPrec; if (Operator == tok::question) // The RHS of "?" should be maximally consumed as an expression. RHSPrec = getPrecedence(tok::comma); else // All others should munch while higher precedence. RHSPrec = ThisPrec+1; if (PeekPrec >= RHSPrec) { if (EvaluateDirectiveSubExpr(RHS, RHSPrec, PeekTok, RHSIsLive, PP)) return true; PeekPrec = getPrecedence(PeekTok.getKind()); } assert(PeekPrec <= ThisPrec && "Recursion didn't work!"); // Usual arithmetic conversions (C99 6.3.1.8p1): result is unsigned if // either operand is unsigned. llvm::APSInt Res(LHS.getBitWidth()); switch (Operator) { case tok::question: // No UAC for x and y in "x ? y : z". case tok::lessless: // Shift amount doesn't UAC with shift value. case tok::greatergreater: // Shift amount doesn't UAC with shift value. case tok::comma: // Comma operands are not subject to UACs. case tok::pipepipe: // Logical || does not do UACs. case tok::ampamp: // Logical && does not do UACs. break; // No UAC default: Res.setIsUnsigned(LHS.isUnsigned()|RHS.isUnsigned()); // If this just promoted something from signed to unsigned, and if the // value was negative, warn about it. if (ValueLive && Res.isUnsigned()) { if (!LHS.isUnsigned() && LHS.Val.isNegative()) PP.Diag(OpLoc, diag::warn_pp_convert_lhs_to_positive) << LHS.Val.toString(10, true) + " to " + LHS.Val.toString(10, false) << LHS.getRange() << RHS.getRange(); if (!RHS.isUnsigned() && RHS.Val.isNegative()) PP.Diag(OpLoc, diag::warn_pp_convert_rhs_to_positive) << RHS.Val.toString(10, true) + " to " + RHS.Val.toString(10, false) << LHS.getRange() << RHS.getRange(); } LHS.Val.setIsUnsigned(Res.isUnsigned()); RHS.Val.setIsUnsigned(Res.isUnsigned()); } bool Overflow = false; switch (Operator) { default: llvm_unreachable("Unknown operator token!"); case tok::percent: if (RHS.Val != 0) Res = LHS.Val % RHS.Val; else if (ValueLive) { PP.Diag(OpLoc, diag::err_pp_remainder_by_zero) << LHS.getRange() << RHS.getRange(); return true; } break; case tok::slash: if (RHS.Val != 0) { if (LHS.Val.isSigned()) Res = llvm::APSInt(LHS.Val.sdiv_ov(RHS.Val, Overflow), false); else Res = LHS.Val / RHS.Val; } else if (ValueLive) { PP.Diag(OpLoc, diag::err_pp_division_by_zero) << LHS.getRange() << RHS.getRange(); return true; } break; case tok::star: if (Res.isSigned()) Res = llvm::APSInt(LHS.Val.smul_ov(RHS.Val, Overflow), false); else Res = LHS.Val * RHS.Val; break; case tok::lessless: { // Determine whether overflow is about to happen. unsigned ShAmt = static_cast<unsigned>(RHS.Val.getLimitedValue()); if (LHS.isUnsigned()) { Overflow = ShAmt >= LHS.Val.getBitWidth(); if (Overflow) ShAmt = LHS.Val.getBitWidth()-1; Res = LHS.Val << ShAmt; } else { Res = llvm::APSInt(LHS.Val.sshl_ov(ShAmt, Overflow), false); } break; } case tok::greatergreater: { // Determine whether overflow is about to happen. unsigned ShAmt = static_cast<unsigned>(RHS.Val.getLimitedValue()); if (ShAmt >= LHS.getBitWidth()) Overflow = true, ShAmt = LHS.getBitWidth()-1; Res = LHS.Val >> ShAmt; break; } case tok::plus: if (LHS.isUnsigned()) Res = LHS.Val + RHS.Val; else Res = llvm::APSInt(LHS.Val.sadd_ov(RHS.Val, Overflow), false); break; case tok::minus: if (LHS.isUnsigned()) Res = LHS.Val - RHS.Val; else Res = llvm::APSInt(LHS.Val.ssub_ov(RHS.Val, Overflow), false); break; case tok::lessequal: Res = LHS.Val <= RHS.Val; Res.setIsUnsigned(false); // C99 6.5.8p6, result is always int (signed) break; case tok::less: Res = LHS.Val < RHS.Val; Res.setIsUnsigned(false); // C99 6.5.8p6, result is always int (signed) break; case tok::greaterequal: Res = LHS.Val >= RHS.Val; Res.setIsUnsigned(false); // C99 6.5.8p6, result is always int (signed) break; case tok::greater: Res = LHS.Val > RHS.Val; Res.setIsUnsigned(false); // C99 6.5.8p6, result is always int (signed) break; case tok::exclaimequal: Res = LHS.Val != RHS.Val; Res.setIsUnsigned(false); // C99 6.5.9p3, result is always int (signed) break; case tok::equalequal: Res = LHS.Val == RHS.Val; Res.setIsUnsigned(false); // C99 6.5.9p3, result is always int (signed) break; case tok::amp: Res = LHS.Val & RHS.Val; break; case tok::caret: Res = LHS.Val ^ RHS.Val; break; case tok::pipe: Res = LHS.Val | RHS.Val; break; case tok::ampamp: Res = (LHS.Val != 0 && RHS.Val != 0); Res.setIsUnsigned(false); // C99 6.5.13p3, result is always int (signed) break; case tok::pipepipe: Res = (LHS.Val != 0 || RHS.Val != 0); Res.setIsUnsigned(false); // C99 6.5.14p3, result is always int (signed) break; case tok::comma: // Comma is invalid in pp expressions in c89/c++ mode, but is valid in C99 // if not being evaluated. if (!PP.getLangOpts().C99 || ValueLive) PP.Diag(OpLoc, diag::ext_pp_comma_expr) << LHS.getRange() << RHS.getRange(); Res = RHS.Val; // LHS = LHS,RHS -> RHS. break; case tok::question: { // Parse the : part of the expression. if (PeekTok.isNot(tok::colon)) { PP.Diag(PeekTok.getLocation(), diag::err_expected_colon) << LHS.getRange(), RHS.getRange(); PP.Diag(OpLoc, diag::note_matching) << "?"; return true; } // Consume the :. PP.LexNonComment(PeekTok); // Evaluate the value after the :. bool AfterColonLive = ValueLive && LHS.Val == 0; PPValue AfterColonVal(LHS.getBitWidth()); DefinedTracker DT; if (EvaluateValue(AfterColonVal, PeekTok, DT, AfterColonLive, PP)) return true; // Parse anything after the : with the same precedence as ?. We allow // things of equal precedence because ?: is right associative. if (EvaluateDirectiveSubExpr(AfterColonVal, ThisPrec, PeekTok, AfterColonLive, PP)) return true; // Now that we have the condition, the LHS and the RHS of the :, evaluate. Res = LHS.Val != 0 ? RHS.Val : AfterColonVal.Val; RHS.setEnd(AfterColonVal.getRange().getEnd()); // Usual arithmetic conversions (C99 6.3.1.8p1): result is unsigned if // either operand is unsigned. Res.setIsUnsigned(RHS.isUnsigned() | AfterColonVal.isUnsigned()); // Figure out the precedence of the token after the : part. PeekPrec = getPrecedence(PeekTok.getKind()); break; } case tok::colon: // Don't allow :'s to float around without being part of ?: exprs. PP.Diag(OpLoc, diag::err_pp_colon_without_question) << LHS.getRange() << RHS.getRange(); return true; } // If this operator is live and overflowed, report the issue. if (Overflow && ValueLive) PP.Diag(OpLoc, diag::warn_pp_expr_overflow) << LHS.getRange() << RHS.getRange(); // Put the result back into 'LHS' for our next iteration. LHS.Val = Res; LHS.setEnd(RHS.getRange().getEnd()); } }
/// HighlightMacros - This uses the macro table state from the end of the /// file, to re-expand macros and insert (into the HTML) information about the /// macro expansions. This won't be perfectly perfect, but it will be /// reasonably close. void html::HighlightMacros(Rewriter &R, FileID FID, const Preprocessor& PP) { // Re-lex the raw token stream into a token buffer. const SourceManager &SM = PP.getSourceManager(); std::vector<Token> TokenStream; const llvm::MemoryBuffer *FromFile = SM.getBuffer(FID); Lexer L(FID, FromFile, SM, PP.getLangOpts()); // Lex all the tokens in raw mode, to avoid entering #includes or expanding // macros. while (1) { Token Tok; L.LexFromRawLexer(Tok); // If this is a # at the start of a line, discard it from the token stream. // We don't want the re-preprocess step to see #defines, #includes or other // preprocessor directives. if (Tok.is(tok::hash) && Tok.isAtStartOfLine()) continue; // If this is a ## token, change its kind to unknown so that repreprocessing // it will not produce an error. if (Tok.is(tok::hashhash)) Tok.setKind(tok::unknown); // If this raw token is an identifier, the raw lexer won't have looked up // the corresponding identifier info for it. Do this now so that it will be // macro expanded when we re-preprocess it. if (Tok.is(tok::raw_identifier)) PP.LookUpIdentifierInfo(Tok); TokenStream.push_back(Tok); if (Tok.is(tok::eof)) break; } // Temporarily change the diagnostics object so that we ignore any generated // diagnostics from this pass. DiagnosticsEngine TmpDiags(PP.getDiagnostics().getDiagnosticIDs(), &PP.getDiagnostics().getDiagnosticOptions(), new IgnoringDiagConsumer); // FIXME: This is a huge hack; we reuse the input preprocessor because we want // its state, but we aren't actually changing it (we hope). This should really // construct a copy of the preprocessor. Preprocessor &TmpPP = const_cast<Preprocessor&>(PP); DiagnosticsEngine *OldDiags = &TmpPP.getDiagnostics(); TmpPP.setDiagnostics(TmpDiags); // Inform the preprocessor that we don't want comments. TmpPP.SetCommentRetentionState(false, false); // We don't want pragmas either. Although we filtered out #pragma, removing // _Pragma and __pragma is much harder. bool PragmasPreviouslyEnabled = TmpPP.getPragmasEnabled(); TmpPP.setPragmasEnabled(false); // Enter the tokens we just lexed. This will cause them to be macro expanded // but won't enter sub-files (because we removed #'s). TmpPP.EnterTokenStream(TokenStream, false); TokenConcatenation ConcatInfo(TmpPP); // Lex all the tokens. Token Tok; TmpPP.Lex(Tok); while (Tok.isNot(tok::eof)) { // Ignore non-macro tokens. if (!Tok.getLocation().isMacroID()) { TmpPP.Lex(Tok); continue; } // Okay, we have the first token of a macro expansion: highlight the // expansion by inserting a start tag before the macro expansion and // end tag after it. std::pair<SourceLocation, SourceLocation> LLoc = SM.getExpansionRange(Tok.getLocation()); // Ignore tokens whose instantiation location was not the main file. if (SM.getFileID(LLoc.first) != FID) { TmpPP.Lex(Tok); continue; } assert(SM.getFileID(LLoc.second) == FID && "Start and end of expansion must be in the same ultimate file!"); std::string Expansion = EscapeText(TmpPP.getSpelling(Tok)); unsigned LineLen = Expansion.size(); Token PrevPrevTok; Token PrevTok = Tok; // Okay, eat this token, getting the next one. TmpPP.Lex(Tok); // Skip all the rest of the tokens that are part of this macro // instantiation. It would be really nice to pop up a window with all the // spelling of the tokens or something. while (!Tok.is(tok::eof) && SM.getExpansionLoc(Tok.getLocation()) == LLoc.first) { // Insert a newline if the macro expansion is getting large. if (LineLen > 60) { Expansion += "<br>"; LineLen = 0; } LineLen -= Expansion.size(); // If the tokens were already space separated, or if they must be to avoid // them being implicitly pasted, add a space between them. if (Tok.hasLeadingSpace() || ConcatInfo.AvoidConcat(PrevPrevTok, PrevTok, Tok)) Expansion += ' '; // Escape any special characters in the token text. Expansion += EscapeText(TmpPP.getSpelling(Tok)); LineLen += Expansion.size(); PrevPrevTok = PrevTok; PrevTok = Tok; TmpPP.Lex(Tok); } // Insert the expansion as the end tag, so that multi-line macros all get // highlighted. Expansion = "<span class='expansion'>" + Expansion + "</span></span>"; HighlightRange(R, LLoc.first, LLoc.second, "<span class='macro'>", Expansion.c_str()); } // Restore the preprocessor's old state. TmpPP.setDiagnostics(*OldDiags); TmpPP.setPragmasEnabled(PragmasPreviouslyEnabled); }
/// ParseNumberStartingWithZero - This method is called when the first character /// of the number is found to be a zero. This means it is either an octal /// number (like '04') or a hex number ('0x123a') a binary number ('0b1010') or /// a floating point number (01239.123e4). Eat the prefix, determining the /// radix etc. void NumericLiteralParser::ParseNumberStartingWithZero(SourceLocation TokLoc) { assert(s[0] == '0' && "Invalid method call"); s++; int c1 = s[0]; // Handle a hex number like 0x1234. if ((c1 == 'x' || c1 == 'X') && (isHexDigit(s[1]) || s[1] == '.')) { s++; assert(s < ThisTokEnd && "didn't maximally munch?"); radix = 16; DigitsBegin = s; s = SkipHexDigits(s); bool noSignificand = (s == DigitsBegin); if (s == ThisTokEnd) { // Done. } else if (*s == '.') { s++; saw_period = true; const char *floatDigitsBegin = s; checkSeparator(TokLoc, s, CSK_BeforeDigits); s = SkipHexDigits(s); noSignificand &= (floatDigitsBegin == s); } if (noSignificand) { PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s - ThisTokBegin), diag::err_hexconstant_requires_digits); hadError = true; return; } // A binary exponent can appear with or with a '.'. If dotted, the // binary exponent is required. if (*s == 'p' || *s == 'P') { checkSeparator(TokLoc, s, CSK_AfterDigits); const char *Exponent = s; s++; saw_exponent = true; if (*s == '+' || *s == '-') s++; // sign const char *first_non_digit = SkipDigits(s); if (first_non_digit == s) { PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, Exponent-ThisTokBegin), diag::err_exponent_has_no_digits); hadError = true; return; } checkSeparator(TokLoc, s, CSK_BeforeDigits); s = first_non_digit; if (!PP.getLangOpts().HexFloats) PP.Diag(TokLoc, diag::ext_hexconstant_invalid); } else if (saw_period) { PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-ThisTokBegin), diag::err_hexconstant_requires_exponent); hadError = true; } return; } // Handle simple binary numbers 0b01010 if ((c1 == 'b' || c1 == 'B') && (s[1] == '0' || s[1] == '1')) { // 0b101010 is a C++1y / GCC extension. PP.Diag(TokLoc, PP.getLangOpts().CPlusPlus14 ? diag::warn_cxx11_compat_binary_literal : PP.getLangOpts().CPlusPlus ? diag::ext_binary_literal_cxx14 : diag::ext_binary_literal); ++s; assert(s < ThisTokEnd && "didn't maximally munch?"); radix = 2; DigitsBegin = s; s = SkipBinaryDigits(s); if (s == ThisTokEnd) { // Done. } else if (isHexDigit(*s)) { PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-ThisTokBegin), diag::err_invalid_binary_digit) << StringRef(s, 1); hadError = true; } // Other suffixes will be diagnosed by the caller. return; } // For now, the radix is set to 8. If we discover that we have a // floating point constant, the radix will change to 10. Octal floating // point constants are not permitted (only decimal and hexadecimal). radix = 8; DigitsBegin = s; s = SkipOctalDigits(s); if (s == ThisTokEnd) return; // Done, simple octal number like 01234 // If we have some other non-octal digit that *is* a decimal digit, see if // this is part of a floating point number like 094.123 or 09e1. if (isDigit(*s)) { const char *EndDecimal = SkipDigits(s); if (EndDecimal[0] == '.' || EndDecimal[0] == 'e' || EndDecimal[0] == 'E') { s = EndDecimal; radix = 10; } } // If we have a hex digit other than 'e' (which denotes a FP exponent) then // the code is using an incorrect base. if (isHexDigit(*s) && *s != 'e' && *s != 'E') { PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-ThisTokBegin), diag::err_invalid_octal_digit) << StringRef(s, 1); hadError = true; return; } if (*s == '.') { s++; radix = 10; saw_period = true; checkSeparator(TokLoc, s, CSK_BeforeDigits); s = SkipDigits(s); // Skip suffix. } if (*s == 'e' || *s == 'E') { // exponent checkSeparator(TokLoc, s, CSK_AfterDigits); const char *Exponent = s; s++; radix = 10; saw_exponent = true; if (*s == '+' || *s == '-') s++; // sign const char *first_non_digit = SkipDigits(s); if (first_non_digit != s) { checkSeparator(TokLoc, s, CSK_BeforeDigits); s = first_non_digit; } else { PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, Exponent-ThisTokBegin), diag::err_exponent_has_no_digits); hadError = true; return; } } }