/** * @brief Highlights rest of line as import directive */ void PythonHighlighter::highlightImport(Scanner &scanner) { FormatToken tk; while ((tk = scanner.read()).format() != Format_EndOfBlock) { Format format = tk.format(); if (tk.format() == Format_Identifier) format = Format_ImportedModule; setFormat(tk.begin(), tk.length(), formatForCategory(format)); } }
int GoHighlighter::highlightLine(const QString &text, int initialState) { GoScanner scanner(text.constData(), text.size()); scanner.setState(initialState); FormatToken tk; while ((tk = scanner.read()).format() != Format_EndOfBlock) setFormat(tk.begin(), tk.length(), formatForCategory(tk.format())); return scanner.state(); }
void Highlighter::highlightImport(Scanner &scanner) { FormatToken tk; while ((tk = scanner.read()).format() != Format_EndOfBlock) { Format format = tk.format(); if (tk.format() == Format_IDENTIFIER) format = Format_IMPORTED_MODULE; setFormat(tk.begin(), tk.length(), _formats[format]); } }
int PythonIndenter::getIndentDiff(const QString &previousLine, const TextEditor::TabSettings &tabSettings) const { PythonScanner sc(previousLine.constData(), previousLine.length()); forever { FormatToken tk = sc.read(); if ((tk.format() == Internal::Format_Keyword) && m_jumpKeywords.contains(sc.value(tk))) return -tabSettings.m_indentSize; if (tk.format() != Internal::Format_Whitespace) break; } return 0; }
void WhitespaceManager::breakToken(const FormatToken &Tok, unsigned Offset, unsigned ReplaceChars, StringRef PreviousPostfix, StringRef CurrentPrefix, bool InPPDirective, unsigned Spaces) { Changes.push_back(Change( true, SourceRange(Tok.getStartOfNonWhitespace().getLocWithOffset(Offset), Tok.getStartOfNonWhitespace().getLocWithOffset( Offset + ReplaceChars)), Spaces, Spaces, 1, PreviousPostfix, CurrentPrefix, // FIXME: Unify token adjustment, so we don't split it between // BreakableToken and the WhitespaceManager. That would also allow us to // correctly store a tok::TokenKind instead of rolling our own enum. tok::unknown, InPPDirective && !Tok.IsFirst)); }
void WhitespaceManager::replaceWhitespaceInToken( const FormatToken &Tok, unsigned Offset, unsigned ReplaceChars, StringRef PreviousPostfix, StringRef CurrentPrefix, bool InPPDirective, unsigned Newlines, unsigned Spaces) { Changes.push_back(Change( true, SourceRange(Tok.getStartOfNonWhitespace().getLocWithOffset(Offset), Tok.getStartOfNonWhitespace().getLocWithOffset( Offset + ReplaceChars)), Spaces, Spaces, Newlines, PreviousPostfix, CurrentPrefix, // If we don't add a newline this change doesn't start a comment. Thus, // when we align line comments, we don't need to treat this change as one. // FIXME: We still need to take this change in account to properly // calculate the new length of the comment and to calculate the changes // for which to do the alignment when aligning comments. Tok.Type == TT_LineComment && Newlines > 0 ? tok::comment : tok::unknown, InPPDirective && !Tok.IsFirst)); }
bool AffectedRangeManager::affectsTokenRange(const FormatToken &First, const FormatToken &Last, bool IncludeLeadingNewlines) { SourceLocation Start = First.WhitespaceRange.getBegin(); if (!IncludeLeadingNewlines) Start = Start.getLocWithOffset(First.LastNewlineOffset); SourceLocation End = Last.getStartOfNonWhitespace(); End = End.getLocWithOffset(Last.TokenText.size()); CharSourceRange Range = CharSourceRange::getCharRange(Start, End); return affectsCharSourceRange(Range); }
int LuaHighlighter::highlightLine(QString const& text, int initialState) { Scanner scanner(text.constData(),text.length()); scanner.setState(initialState); FormatToken tk; bool hasOnlyWhitespace = true; while((tk = scanner.read()).format() != Format_EndOfBlock) { Format format = tk.format(); if(format == Format_Keyword) { if(isImportKeyword(scanner.value(tk)) && hasOnlyWhitespace) { setFormat(tk.begin(), tk.length(), formatForCategory(format)); highlightImport(scanner); break; } } setFormat(tk.begin(), tk.length(), formatForCategory(format)); if(format != Format_Whitespace) hasOnlyWhitespace = false; } return scanner.state(); }
void UnwrappedLineFormatter::formatFirstToken(FormatToken &RootToken, const AnnotatedLine *PreviousLine, unsigned IndentLevel, unsigned Indent, bool InPPDirective) { unsigned Newlines = std::min(RootToken.NewlinesBefore, Style.MaxEmptyLinesToKeep + 1); // Remove empty lines before "}" where applicable. if (RootToken.is(tok::r_brace) && (!RootToken.Next || (RootToken.Next->is(tok::semi) && !RootToken.Next->Next))) Newlines = std::min(Newlines, 1u); if (Newlines == 0 && !RootToken.IsFirst) Newlines = 1; if (RootToken.IsFirst && !RootToken.HasUnescapedNewline) Newlines = 0; // Remove empty lines after "{". if (!Style.KeepEmptyLinesAtTheStartOfBlocks && PreviousLine && PreviousLine->Last->is(tok::l_brace) && PreviousLine->First->isNot(tok::kw_namespace) && !startsExternCBlock(*PreviousLine)) Newlines = 1; // Insert extra new line before access specifiers. if (PreviousLine && PreviousLine->Last->isOneOf(tok::semi, tok::r_brace) && RootToken.isAccessSpecifier() && RootToken.NewlinesBefore == 1) ++Newlines; // Remove empty lines after access specifiers. if (PreviousLine && PreviousLine->First->isAccessSpecifier() && (!PreviousLine->InPPDirective || !RootToken.HasUnescapedNewline)) Newlines = std::min(1u, Newlines); Whitespaces->replaceWhitespace(RootToken, Newlines, IndentLevel, Indent, Indent, InPPDirective && !RootToken.HasUnescapedNewline); }
void UnwrappedLineParser::calculateBraceTypes() { // We'll parse forward through the tokens until we hit // a closing brace or eof - note that getNextToken() will // parse macros, so this will magically work inside macro // definitions, too. unsigned StoredPosition = Tokens->getPosition(); unsigned Position = StoredPosition; FormatToken *Tok = FormatTok; // Keep a stack of positions of lbrace tokens. We will // update information about whether an lbrace starts a // braced init list or a different block during the loop. SmallVector<FormatToken *, 8> LBraceStack; assert(Tok->Tok.is(tok::l_brace)); do { // Get next none-comment token. FormatToken *NextTok; unsigned ReadTokens = 0; do { NextTok = Tokens->getNextToken(); ++ReadTokens; } while (NextTok->is(tok::comment)); switch (Tok->Tok.getKind()) { case tok::l_brace: LBraceStack.push_back(Tok); break; case tok::r_brace: if (!LBraceStack.empty()) { if (LBraceStack.back()->BlockKind == BK_Unknown) { bool ProbablyBracedList = false; if (Style.Language == FormatStyle::LK_Proto) { ProbablyBracedList = NextTok->isOneOf(tok::comma, tok::r_square); } else { // Using OriginalColumn to distinguish between ObjC methods and // binary operators is a bit hacky. bool NextIsObjCMethod = NextTok->isOneOf(tok::plus, tok::minus) && NextTok->OriginalColumn == 0; // If there is a comma, semicolon or right paren after the closing // brace, we assume this is a braced initializer list. Note that // regardless how we mark inner braces here, we will overwrite the // BlockKind later if we parse a braced list (where all blocks // inside are by default braced lists), or when we explicitly detect // blocks (for example while parsing lambdas). // // We exclude + and - as they can be ObjC visibility modifiers. ProbablyBracedList = NextTok->isOneOf(tok::comma, tok::semi, tok::period, tok::colon, tok::r_paren, tok::r_square, tok::l_brace, tok::l_paren) || (NextTok->isBinaryOperator() && !NextIsObjCMethod); } if (ProbablyBracedList) { Tok->BlockKind = BK_BracedInit; LBraceStack.back()->BlockKind = BK_BracedInit; } else { Tok->BlockKind = BK_Block; LBraceStack.back()->BlockKind = BK_Block; } } LBraceStack.pop_back(); } break; case tok::at: case tok::semi: case tok::kw_if: case tok::kw_while: case tok::kw_for: case tok::kw_switch: case tok::kw_try: if (!LBraceStack.empty()) LBraceStack.back()->BlockKind = BK_Block; break; default: break; } Tok = NextTok; Position += ReadTokens; } while (Tok->Tok.isNot(tok::eof) && !LBraceStack.empty()); // Assume other blocks for all unclosed opening braces. for (unsigned i = 0, e = LBraceStack.size(); i != e; ++i) { if (LBraceStack[i]->BlockKind == BK_Unknown) LBraceStack[i]->BlockKind = BK_Block; } FormatTok = Tokens->setPosition(StoredPosition); }
void CommaSeparatedList::precomputeFormattingInfos(const FormatToken *Token) { // FIXME: At some point we might want to do this for other lists, too. if (!Token->MatchingParen || Token->isNot(tok::l_brace)) return; FormatToken *ItemBegin = Token->Next; SmallVector<bool, 8> MustBreakBeforeItem; // The lengths of an item if it is put at the end of the line. This includes // trailing comments which are otherwise ignored for column alignment. SmallVector<unsigned, 8> EndOfLineItemLength; for (unsigned i = 0, e = Commas.size() + 1; i != e; ++i) { // Skip comments on their own line. while (ItemBegin->HasUnescapedNewline && ItemBegin->isTrailingComment()) ItemBegin = ItemBegin->Next; MustBreakBeforeItem.push_back(ItemBegin->MustBreakBefore); const FormatToken *ItemEnd = NULL; if (i == Commas.size()) { ItemEnd = Token->MatchingParen; const FormatToken *NonCommentEnd = ItemEnd->getPreviousNonComment(); ItemLengths.push_back(CodePointsBetween(ItemBegin, NonCommentEnd)); if (Style.Cpp11BracedListStyle) { // In Cpp11 braced list style, the } and possibly other subsequent // tokens will need to stay on a line with the last element. while (ItemEnd->Next && !ItemEnd->Next->CanBreakBefore) ItemEnd = ItemEnd->Next; } else { // In other braced lists styles, the "}" can be wrapped to the new line. ItemEnd = Token->MatchingParen->Previous; } } else { ItemEnd = Commas[i]; // The comma is counted as part of the item when calculating the length. ItemLengths.push_back(CodePointsBetween(ItemBegin, ItemEnd)); // Consume trailing comments so the are included in EndOfLineItemLength. if (ItemEnd->Next && !ItemEnd->Next->HasUnescapedNewline && ItemEnd->Next->isTrailingComment()) ItemEnd = ItemEnd->Next; } EndOfLineItemLength.push_back(CodePointsBetween(ItemBegin, ItemEnd)); // If there is a trailing comma in the list, the next item will start at the // closing brace. Don't create an extra item for this. if (ItemEnd->getNextNonComment() == Token->MatchingParen) break; ItemBegin = ItemEnd->Next; } // We can never place more than ColumnLimit / 3 items in a row (because of the // spaces and the comma). for (unsigned Columns = 1; Columns <= Style.ColumnLimit / 3; ++Columns) { ColumnFormat Format; Format.Columns = Columns; Format.ColumnSizes.resize(Columns); Format.LineCount = 0; bool HasRowWithSufficientColumns = false; unsigned Column = 0; for (unsigned i = 0, e = ItemLengths.size(); i != e; ++i) { if (MustBreakBeforeItem[i] || Column == Columns) { ++Format.LineCount; Column = 0; } if (Column == Columns - 1) HasRowWithSufficientColumns = true; unsigned length = (Column == Columns - 1) ? EndOfLineItemLength[i] : ItemLengths[i]; Format.ColumnSizes[Column] = std::max(Format.ColumnSizes[Column], length); ++Column; } // If all rows are terminated early (e.g. by trailing comments), we don't // need to look further. if (!HasRowWithSufficientColumns) break; Format.TotalWidth = Columns - 1; // Width of the N-1 spaces. for (unsigned i = 0; i < Columns; ++i) { Format.TotalWidth += Format.ColumnSizes[i]; } // Ignore layouts that are bound to violate the column limit. if (Format.TotalWidth > Style.ColumnLimit) continue; Formats.push_back(Format); } }
DebugCommand* PrintCommand::Build(std::vector<Token*> args) { if (args.size() < 2) { return new InvalidCommand(args, -1, "too few arguments"); } Token* target = args[1]; if (!target->IsRegister() && !target->IsFPRegister() && !target->IsIdentifier()) { return new InvalidCommand(args, 1, "expects reg or identifier"); } FormatToken* format = NULL; int target_size = 0; if (target->IsRegister()) { Register reg = RegisterToken::Cast(target)->value(); target_size = reg.SizeInBytes(); } else if (target->IsFPRegister()) { FPRegister fpreg = FPRegisterToken::Cast(target)->value(); target_size = fpreg.SizeInBytes(); } // If the target is an identifier there must be no format. This is checked // in the switch statement below. switch (args.size()) { case 2: { if (target->IsRegister()) { switch (target_size) { case 4: format = new Format<uint32_t>("%08" PRIx32, 'x'); break; case 8: format = new Format<uint64_t>("%016" PRIx64, 'x'); break; default: VIXL_UNREACHABLE(); } } else if (target->IsFPRegister()) { switch (target_size) { case 4: format = new Format<float>("%8g", 'f'); break; case 8: format = new Format<double>("%8g", 'f'); break; default: VIXL_UNREACHABLE(); } } break; } case 3: { if (target->IsIdentifier()) { return new InvalidCommand(args, 2, "format is only allowed with registers"); } Token* second = args[2]; if (!second->IsFormat()) { return new InvalidCommand(args, 2, "expects format"); } format = FormatToken::Cast(second); if (format->SizeOf() > target_size) { return new InvalidCommand(args, 2, "format too wide"); } break; } default: return new InvalidCommand(args, -1, "too many arguments"); } return new PrintCommand(args[0], target, format); }
void CommaSeparatedList::precomputeFormattingInfos(const FormatToken *Token) { // FIXME: At some point we might want to do this for other lists, too. if (!Token->MatchingParen || Token->isNot(tok::l_brace)) return; // In C++11 braced list style, we should not format in columns unless they // have many items (20 or more) or we allow bin-packing of function call // arguments. if (Style.Cpp11BracedListStyle && !Style.BinPackArguments && Commas.size() < 19) return; // Column format doesn't really make sense if we don't align after brackets. if (!Style.AlignAfterOpenBracket) return; FormatToken *ItemBegin = Token->Next; while (ItemBegin->isTrailingComment()) ItemBegin = ItemBegin->Next; SmallVector<bool, 8> MustBreakBeforeItem; // The lengths of an item if it is put at the end of the line. This includes // trailing comments which are otherwise ignored for column alignment. SmallVector<unsigned, 8> EndOfLineItemLength; bool HasSeparatingComment = false; for (unsigned i = 0, e = Commas.size() + 1; i != e; ++i) { // Skip comments on their own line. while (ItemBegin->HasUnescapedNewline && ItemBegin->isTrailingComment()) { ItemBegin = ItemBegin->Next; HasSeparatingComment = i > 0; } MustBreakBeforeItem.push_back(ItemBegin->MustBreakBefore); if (ItemBegin->is(tok::l_brace)) HasNestedBracedList = true; const FormatToken *ItemEnd = nullptr; if (i == Commas.size()) { ItemEnd = Token->MatchingParen; const FormatToken *NonCommentEnd = ItemEnd->getPreviousNonComment(); ItemLengths.push_back(CodePointsBetween(ItemBegin, NonCommentEnd)); if (Style.Cpp11BracedListStyle) { // In Cpp11 braced list style, the } and possibly other subsequent // tokens will need to stay on a line with the last element. while (ItemEnd->Next && !ItemEnd->Next->CanBreakBefore) ItemEnd = ItemEnd->Next; } else { // In other braced lists styles, the "}" can be wrapped to the new line. ItemEnd = Token->MatchingParen->Previous; } } else { ItemEnd = Commas[i]; // The comma is counted as part of the item when calculating the length. ItemLengths.push_back(CodePointsBetween(ItemBegin, ItemEnd)); // Consume trailing comments so the are included in EndOfLineItemLength. if (ItemEnd->Next && !ItemEnd->Next->HasUnescapedNewline && ItemEnd->Next->isTrailingComment()) ItemEnd = ItemEnd->Next; } EndOfLineItemLength.push_back(CodePointsBetween(ItemBegin, ItemEnd)); // If there is a trailing comma in the list, the next item will start at the // closing brace. Don't create an extra item for this. if (ItemEnd->getNextNonComment() == Token->MatchingParen) break; ItemBegin = ItemEnd->Next; } // Don't use column layout for nested lists, lists with few elements and in // presence of separating comments. if (Token->NestingLevel != 0 || Commas.size() < 5 || HasSeparatingComment) return; // We can never place more than ColumnLimit / 3 items in a row (because of the // spaces and the comma). unsigned MaxItems = Style.ColumnLimit / 3; std::vector<unsigned> MinSizeInColumn; MinSizeInColumn.reserve(MaxItems); for (unsigned Columns = 1; Columns <= MaxItems; ++Columns) { ColumnFormat Format; Format.Columns = Columns; Format.ColumnSizes.resize(Columns); MinSizeInColumn.assign(Columns, UINT_MAX); Format.LineCount = 1; bool HasRowWithSufficientColumns = false; unsigned Column = 0; for (unsigned i = 0, e = ItemLengths.size(); i != e; ++i) { assert(i < MustBreakBeforeItem.size()); if (MustBreakBeforeItem[i] || Column == Columns) { ++Format.LineCount; Column = 0; } if (Column == Columns - 1) HasRowWithSufficientColumns = true; unsigned Length = (Column == Columns - 1) ? EndOfLineItemLength[i] : ItemLengths[i]; Format.ColumnSizes[Column] = std::max(Format.ColumnSizes[Column], Length); MinSizeInColumn[Column] = std::min(MinSizeInColumn[Column], Length); ++Column; } // If all rows are terminated early (e.g. by trailing comments), we don't // need to look further. if (!HasRowWithSufficientColumns) break; Format.TotalWidth = Columns - 1; // Width of the N-1 spaces. for (unsigned i = 0; i < Columns; ++i) Format.TotalWidth += Format.ColumnSizes[i]; // Don't use this Format, if the difference between the longest and shortest // element in a column exceeds a threshold to avoid excessive spaces. if ([&] { for (unsigned i = 0; i < Columns - 1; ++i) if (Format.ColumnSizes[i] - MinSizeInColumn[i] > 10) return true; return false; }()) continue; // Ignore layouts that are bound to violate the column limit. if (Format.TotalWidth > Style.ColumnLimit) continue; Formats.push_back(Format); } }
void UnwrappedLineParser::calculateBraceTypes() { // We'll parse forward through the tokens until we hit // a closing brace or eof - note that getNextToken() will // parse macros, so this will magically work inside macro // definitions, too. unsigned StoredPosition = Tokens->getPosition(); unsigned Position = StoredPosition; FormatToken *Tok = FormatTok; // Keep a stack of positions of lbrace tokens. We will // update information about whether an lbrace starts a // braced init list or a different block during the loop. SmallVector<FormatToken *, 8> LBraceStack; assert(Tok->Tok.is(tok::l_brace)); do { // Get next none-comment token. FormatToken *NextTok; unsigned ReadTokens = 0; do { NextTok = Tokens->getNextToken(); ++ReadTokens; } while (NextTok->is(tok::comment)); switch (Tok->Tok.getKind()) { case tok::l_brace: LBraceStack.push_back(Tok); break; case tok::r_brace: if (!LBraceStack.empty()) { if (LBraceStack.back()->BlockKind == BK_Unknown) { // If there is a comma, semicolon or right paren after the closing // brace, we assume this is a braced initializer list. // FIXME: Note that this currently works only because we do not // use the brace information while inside a braced init list. // Thus, if the parent is a braced init list, we consider all // brace blocks inside it braced init list. That works good enough // for now, but we will need to fix it to correctly handle lambdas. if (NextTok->isOneOf(tok::comma, tok::semi, tok::r_paren, tok::l_brace, tok::colon)) { Tok->BlockKind = BK_BracedInit; LBraceStack.back()->BlockKind = BK_BracedInit; } else { Tok->BlockKind = BK_Block; LBraceStack.back()->BlockKind = BK_Block; } } LBraceStack.pop_back(); } break; case tok::semi: case tok::kw_if: case tok::kw_while: case tok::kw_for: case tok::kw_switch: case tok::kw_try: if (!LBraceStack.empty()) LBraceStack.back()->BlockKind = BK_Block; break; default: break; } Tok = NextTok; Position += ReadTokens; } while (Tok->Tok.isNot(tok::eof)); // Assume other blocks for all unclosed opening braces. for (unsigned i = 0, e = LBraceStack.size(); i != e; ++i) { if (LBraceStack[i]->BlockKind == BK_Unknown) LBraceStack[i]->BlockKind = BK_Block; } FormatTok = Tokens->setPosition(StoredPosition); }
/** * @brief Highlighter::highlight_impl * @param text - source code to highlight * @param initialState - initial state of scanner, retrieved from previous block * @return final state of scanner, should be saved with current block */ int Highlighter::highlightLine(const QString &text, int initialState) { Scanner scanner(text.constData(), text.size()); scanner.setState(initialState); HState hstate = HState_NORMAL; int blockNo = currentBlock().blockNumber(); QMap<int, QString>::iterator it = _userTypes.find(blockNo); if (it != _userTypes.end()) _userTypes.erase(it); FormatToken tk; bool hasOnlyWhitespace = true; while ((tk = scanner.read()).format() != Format_EndOfBlock) { Format format = tk.format(); switch (hstate) { default: // HState_NORMAL if (format == Format_KEYWORD) { QString value = scanner.value(tk); if (value == "class") { hstate = HState_ON_CLASS; } else if (isImportKeyword(value) && hasOnlyWhitespace) { setFormat(tk.begin(), tk.length(), _formats[format]); highlightImport(scanner); return scanner.getState(); } } if (format == Format_IDENTIFIER) { QString value = scanner.value(tk); QMap<int, QString>::const_iterator it; for (it = _userTypes.begin(); it != _userTypes.end(); ++it) { if (value == it.value()) { format = Format_TYPE; break; } } } break; case HState_ON_CLASS: if (format == Format_WHITESPACE) hstate = HState_AFTER_CLASS; else hstate = HState_NORMAL; break; case HState_AFTER_CLASS: hstate = HState_NORMAL; if (format == Format_IDENTIFIER) { _userTypes[blockNo] = scanner.value(tk); format = Format_TYPE; } break; } setFormat(tk.begin(), tk.length(), _formats[format]); if (format != Format_WHITESPACE) hasOnlyWhitespace = false; } return scanner.getState(); }