TokenRewriter::TokenRewriter(FileID FID, SourceManager &SM, const LangOptions &LangOpts) { ScratchBuf.reset(new ScratchBuffer(SM)); // Create a lexer to lex all the tokens of the main file in raw mode. const llvm::MemoryBuffer *FromFile = SM.getBuffer(FID); Lexer RawLex(FID, FromFile, SM, LangOpts); // Return all comments and whitespace as tokens. RawLex.SetKeepWhitespaceMode(true); // Lex the file, populating our datastructures. Token RawTok; RawLex.LexFromRawLexer(RawTok); while (RawTok.isNot(tok::eof)) { #if 0 if (Tok.is(tok::raw_identifier)) { // Look up the identifier info for the token. This should use // IdentifierTable directly instead of PP. PP.LookUpIdentifierInfo(Tok); } #endif AddToken(RawTok, TokenList.end()); RawLex.LexFromRawLexer(RawTok); } }
/// \brief Lex the specified source file to determine whether it contains /// any expected-* directives. As a Lexer is used rather than a full-blown /// Preprocessor, directives inside skipped #if blocks will still be found. /// /// \return true if any directives were found. static bool findDirectives(SourceManager &SM, FileID FID, const LangOptions &LangOpts) { // Create a raw lexer to pull all the comments out of FID. if (FID.isInvalid()) return false; // Create a lexer to lex all the tokens of the main file in raw mode. const llvm::MemoryBuffer *FromFile = SM.getBuffer(FID); Lexer RawLex(FID, FromFile, SM, LangOpts); // Return comments as tokens, this is how we find expected diagnostics. RawLex.SetCommentRetentionState(true); Token Tok; Tok.setKind(tok::comment); VerifyDiagnosticConsumer::DirectiveStatus Status = VerifyDiagnosticConsumer::HasNoDirectives; while (Tok.isNot(tok::eof)) { RawLex.Lex(Tok); if (!Tok.is(tok::comment)) continue; std::string Comment = RawLex.getSpelling(Tok, SM, LangOpts); if (Comment.empty()) continue; // Find first directive. if (ParseDirective(Comment, 0, SM, Tok.getLocation(), SM.getDiagnostics(), Status)) return true; } return false; }
static void PrintSourceForLocation(const SourceLocation &Loc, SourceManager &SM) { const char *LocData = SM.getCharacterData(Loc, /*Invalid=*/nullptr); unsigned LocColumn = SM.getSpellingColumnNumber(Loc, /*Invalid=*/nullptr) - 1; FileID FID = SM.getFileID(Loc); llvm::MemoryBuffer *Buffer = SM.getBuffer(FID, Loc, /*Invalid=*/nullptr); assert(LocData >= Buffer->getBufferStart() && LocData < Buffer->getBufferEnd()); const char *LineBegin = LocData - LocColumn; assert(LineBegin >= Buffer->getBufferStart()); const char *LineEnd = nullptr; for (LineEnd = LineBegin; *LineEnd != '\n' && *LineEnd != '\r' && LineEnd < Buffer->getBufferEnd(); ++LineEnd) ; llvm::StringRef LineString(LineBegin, LineEnd - LineBegin); llvm::errs() << LineString << '\n'; std::string Space(LocColumn, ' '); llvm::errs() << Space.c_str() << '\n'; }
// Checks if 'typedef' keyword can be removed - we do it only if // it is the only declaration in a declaration chain. static bool CheckRemoval(SourceManager &SM, const SourceLocation &LocStart, const SourceLocation &LocEnd, ASTContext &Context, SourceRange &ResultRange) { FileID FID = SM.getFileID(LocEnd); llvm::MemoryBuffer *Buffer = SM.getBuffer(FID, LocEnd); Lexer DeclLexer(SM.getLocForStartOfFile(FID), Context.getLangOpts(), Buffer->getBufferStart(), SM.getCharacterData(LocStart), Buffer->getBufferEnd()); Token DeclToken; bool result = false; int parenthesisLevel = 0; while (!DeclLexer.LexFromRawLexer(DeclToken)) { if (DeclToken.getKind() == tok::TokenKind::l_paren) parenthesisLevel++; if (DeclToken.getKind() == tok::TokenKind::r_paren) parenthesisLevel--; if (DeclToken.getKind() == tok::TokenKind::semi) break; // if there is comma and we are not between open parenthesis then it is // two or more declatarions in this chain if (parenthesisLevel == 0 && DeclToken.getKind() == tok::TokenKind::comma) return false; if (DeclToken.isOneOf(tok::TokenKind::identifier, tok::TokenKind::raw_identifier)) { auto TokenStr = DeclToken.getRawIdentifier().str(); if (TokenStr == "typedef") { ResultRange = SourceRange(DeclToken.getLocation(), DeclToken.getEndLoc()); result = true; } } } // assert if there was keyword 'typedef' in declaration assert(result && "No typedef found"); return result; }
void TextDiagnosticPrinter::EmitCaretDiagnostic(SourceLocation Loc, SourceRange *Ranges, unsigned NumRanges, SourceManager &SM, const CodeModificationHint *Hints, unsigned NumHints, unsigned Columns) { assert(!Loc.isInvalid() && "must have a valid source location here"); // If this is a macro ID, first emit information about where this was // instantiated (recursively) then emit information about where. the token was // spelled from. if (!Loc.isFileID()) { SourceLocation OneLevelUp = SM.getImmediateInstantiationRange(Loc).first; // FIXME: Map ranges? EmitCaretDiagnostic(OneLevelUp, Ranges, NumRanges, SM, 0, 0, Columns); Loc = SM.getImmediateSpellingLoc(Loc); // Map the ranges. for (unsigned i = 0; i != NumRanges; ++i) { SourceLocation S = Ranges[i].getBegin(), E = Ranges[i].getEnd(); if (S.isMacroID()) S = SM.getImmediateSpellingLoc(S); if (E.isMacroID()) E = SM.getImmediateSpellingLoc(E); Ranges[i] = SourceRange(S, E); } if (ShowLocation) { std::pair<FileID, unsigned> IInfo = SM.getDecomposedInstantiationLoc(Loc); // Emit the file/line/column that this expansion came from. OS << SM.getBuffer(IInfo.first)->getBufferIdentifier() << ':' << SM.getLineNumber(IInfo.first, IInfo.second) << ':'; if (ShowColumn) OS << SM.getColumnNumber(IInfo.first, IInfo.second) << ':'; OS << ' '; } OS << "note: instantiated from:\n"; EmitCaretDiagnostic(Loc, Ranges, NumRanges, SM, Hints, NumHints, Columns); return; } // Decompose the location into a FID/Offset pair. std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc); FileID FID = LocInfo.first; unsigned FileOffset = LocInfo.second; // Get information about the buffer it points into. std::pair<const char*, const char*> BufferInfo = SM.getBufferData(FID); const char *BufStart = BufferInfo.first; unsigned ColNo = SM.getColumnNumber(FID, FileOffset); unsigned CaretEndColNo = ColNo + Lexer::MeasureTokenLength(Loc, SM, *LangOpts); // Rewind from the current position to the start of the line. const char *TokPtr = BufStart+FileOffset; const char *LineStart = TokPtr-ColNo+1; // Column # is 1-based. // Compute the line end. Scan forward from the error position to the end of // the line. const char *LineEnd = TokPtr; while (*LineEnd != '\n' && *LineEnd != '\r' && *LineEnd != '\0') ++LineEnd; // Copy the line of code into an std::string for ease of manipulation. std::string SourceLine(LineStart, LineEnd); // Create a line for the caret that is filled with spaces that is the same // length as the line of source code. std::string CaretLine(LineEnd-LineStart, ' '); // Highlight all of the characters covered by Ranges with ~ characters. if (NumRanges) { unsigned LineNo = SM.getLineNumber(FID, FileOffset); for (unsigned i = 0, e = NumRanges; i != e; ++i) HighlightRange(Ranges[i], SM, LineNo, FID, CaretLine, SourceLine); } // Next, insert the caret itself. if (ColNo-1 < CaretLine.size()) CaretLine[ColNo-1] = '^'; else CaretLine.push_back('^'); // Scan the source line, looking for tabs. If we find any, manually expand // them to 8 characters and update the CaretLine to match. for (unsigned i = 0; i != SourceLine.size(); ++i) { if (SourceLine[i] != '\t') continue; // Replace this tab with at least one space. SourceLine[i] = ' '; // Compute the number of spaces we need to insert. unsigned NumSpaces = ((i+8)&~7) - (i+1); assert(NumSpaces < 8 && "Invalid computation of space amt"); // Insert spaces into the SourceLine. SourceLine.insert(i+1, NumSpaces, ' '); // Insert spaces or ~'s into CaretLine. CaretLine.insert(i+1, NumSpaces, CaretLine[i] == '~' ? '~' : ' '); } // If we are in -fdiagnostics-print-source-range-info mode, we are trying to // produce easily machine parsable output. Add a space before the source line // and the caret to make it trivial to tell the main diagnostic line from what // the user is intended to see. if (PrintRangeInfo) { SourceLine = ' ' + SourceLine; CaretLine = ' ' + CaretLine; } std::string FixItInsertionLine; if (NumHints && PrintFixItInfo) { for (const CodeModificationHint *Hint = Hints, *LastHint = Hints + NumHints; Hint != LastHint; ++Hint) { if (Hint->InsertionLoc.isValid()) { // We have an insertion hint. Determine whether the inserted // code is on the same line as the caret. std::pair<FileID, unsigned> HintLocInfo = SM.getDecomposedInstantiationLoc(Hint->InsertionLoc); if (SM.getLineNumber(HintLocInfo.first, HintLocInfo.second) == SM.getLineNumber(FID, FileOffset)) { // Insert the new code into the line just below the code // that the user wrote. unsigned HintColNo = SM.getColumnNumber(HintLocInfo.first, HintLocInfo.second); unsigned LastColumnModified = HintColNo - 1 + Hint->CodeToInsert.size(); if (LastColumnModified > FixItInsertionLine.size()) FixItInsertionLine.resize(LastColumnModified, ' '); std::copy(Hint->CodeToInsert.begin(), Hint->CodeToInsert.end(), FixItInsertionLine.begin() + HintColNo - 1); } else { FixItInsertionLine.clear(); break; } } } } // If the source line is too long for our terminal, select only the // "interesting" source region within that line. if (Columns && SourceLine.size() > Columns) SelectInterestingSourceRegion(SourceLine, CaretLine, FixItInsertionLine, CaretEndColNo, Columns); // Finally, remove any blank spaces from the end of CaretLine. while (CaretLine[CaretLine.size()-1] == ' ') CaretLine.erase(CaretLine.end()-1); // Emit what we have computed. OS << SourceLine << '\n'; if (UseColors) OS.changeColor(caretColor, true); OS << CaretLine << '\n'; if (UseColors) OS.resetColor(); if (!FixItInsertionLine.empty()) { if (UseColors) // Print fixit line in color OS.changeColor(fixitColor, false); if (PrintRangeInfo) OS << ' '; OS << FixItInsertionLine << '\n'; if (UseColors) OS.resetColor(); } }
bool XdfObject::Read(SourceManager& sm, Diagnostic& diags) { const llvm::MemoryBuffer& in = *sm.getBuffer(sm.getMainFileID()); InputBuffer inbuf(in); inbuf.setLittleEndian(); // Read object header if (inbuf.getReadableSize() < FILEHEAD_SIZE) { diags.Report(SourceLocation(), diag::err_object_header_unreadable); return false; } unsigned long magic = ReadU32(inbuf); if (magic != XDF_MAGIC) { diags.Report(SourceLocation(), diag::err_not_file_type) << "XDF"; return false; } unsigned long scnum = ReadU32(inbuf); unsigned long symnum = ReadU32(inbuf); unsigned long headers_len = ReadU32(inbuf); if (inbuf.getReadableSize() < headers_len) { diags.Report(SourceLocation(), diag::err_xdf_headers_unreadable); return false; } unsigned long section_offset = FILEHEAD_SIZE; unsigned long symtab_offset = section_offset + SECTHEAD_SIZE*scnum; unsigned long strtab_offset = symtab_offset + SYMBOL_SIZE*symnum; ReadString read_string(in, strtab_offset, FILEHEAD_SIZE+headers_len-strtab_offset, diags); // Storage for nrelocs, indexed by section number std::vector<unsigned long> sects_nrelocs; sects_nrelocs.reserve(scnum); // Create sections for (unsigned long i=0; i<scnum; ++i) { // Start with symbol=0 as it's not created yet; updated later. std::auto_ptr<XdfSection> xsect(new XdfSection(SymbolRef(0))); unsigned long name_sym_index; IntNum lma, vma; unsigned long align; bool bss; unsigned long filepos; unsigned long nrelocs; xsect->Read(in, section_offset+SECTHEAD_SIZE*i, &name_sym_index, &lma, &vma, &align, &bss, &filepos, &nrelocs); xsect->scnum = i; // get section name from section symbol entry inbuf.setPosition(symtab_offset+name_sym_index*SYMBOL_SIZE+8); llvm::StringRef sectname = read_string(ReadU32(inbuf)); std::auto_ptr<Section> section( new Section(sectname, xsect->bits != 0, bss, SourceLocation())); section->setFilePos(filepos); section->setVMA(vma); section->setLMA(lma); if (bss) { Bytecode& gap = section->AppendGap(xsect->size, SourceLocation()); Diagnostic nodiags(0); gap.CalcLen(0, nodiags); // force length calculation of gap } else { // Read section data inbuf.setPosition(filepos); if (inbuf.getReadableSize() < xsect->size) { diags.Report(SourceLocation(), diag::err_section_data_unreadable) << sectname; return false; } section->bytecodes_front().getFixed().Write(inbuf.Read(xsect->size), xsect->size); } // Associate section data with section section->AddAssocData(xsect); // Add section to object m_object.AppendSection(section); sects_nrelocs.push_back(nrelocs); } // Create symbols inbuf.setPosition(symtab_offset); for (unsigned long i=0; i<symnum; ++i) { unsigned long sym_scnum = ReadU32(inbuf); // section number unsigned long value = ReadU32(inbuf); // value llvm::StringRef symname = read_string(ReadU32(inbuf));// name unsigned long flags = ReadU32(inbuf); // flags SymbolRef sym = m_object.getSymbol(symname); if ((flags & XdfSymbol::XDF_GLOBAL) != 0) sym->Declare(Symbol::GLOBAL); else if ((flags & XdfSymbol::XDF_EXTERN) != 0) sym->Declare(Symbol::EXTERN); if ((flags & XdfSymbol::XDF_EQU) != 0) sym->DefineEqu(Expr(value)); else if (sym_scnum < scnum) { Section& sect = m_object.getSection(sym_scnum); Location loc = {§.bytecodes_front(), value}; sym->DefineLabel(loc); } // Save index in symrec data sym->AddAssocData(std::auto_ptr<XdfSymbol>(new XdfSymbol(i))); } // Update section symbol info, and create section relocations std::vector<unsigned long>::iterator nrelocsi = sects_nrelocs.begin(); for (Object::section_iterator sect=m_object.sections_begin(), end=m_object.sections_end(); sect != end; ++sect, ++nrelocsi) { XdfSection* xsect = sect->getAssocData<XdfSection>(); assert(xsect != 0); // Read relocations inbuf.setPosition(xsect->relptr); if (inbuf.getReadableSize() < (*nrelocsi) * RELOC_SIZE) { diags.Report(SourceLocation(), diag::err_section_relocs_unreadable) << sect->getName(); return false; } for (unsigned long i=0; i<(*nrelocsi); ++i) { unsigned long addr = ReadU32(inbuf); unsigned long sym_index = ReadU32(inbuf); unsigned long basesym_index = ReadU32(inbuf); XdfReloc::Type type = static_cast<XdfReloc::Type>(ReadU8(inbuf)); XdfReloc::Size size = static_cast<XdfReloc::Size>(ReadU8(inbuf)); unsigned char shift = ReadU8(inbuf); ReadU8(inbuf); // flags; ignored SymbolRef sym = m_object.getSymbol(sym_index); SymbolRef basesym(0); if (type == XdfReloc::XDF_WRT) basesym = m_object.getSymbol(basesym_index); sect->AddReloc(std::auto_ptr<Reloc>( new XdfReloc(addr, sym, basesym, type, size, shift))); } } if (diags.hasErrorOccurred()) return false; return true; }