/**
* @brief Highlights rest of line as import directive
*/
void PythonHighlighter::highlightImport(Scanner &scanner)
{
FormatToken tk;
while ((tk = scanner.read()).format() != Format_EndOfBlock) {
Format format = tk.format();
if (tk.format() == Format_Identifier)
format = Format_ImportedModule;
setFormat(tk.begin(), tk.length(), formatForCategory(format));
}
}
int GoHighlighter::highlightLine(const QString &text, int initialState)
{
GoScanner scanner(text.constData(), text.size());
scanner.setState(initialState);
FormatToken tk;
while ((tk = scanner.read()).format() != Format_EndOfBlock)
setFormat(tk.begin(), tk.length(), formatForCategory(tk.format()));
return scanner.state();
}
void Highlighter::highlightImport(Scanner &scanner)
{
FormatToken tk;
while ((tk = scanner.read()).format() != Format_EndOfBlock)
{
Format format = tk.format();
if (tk.format() == Format_IDENTIFIER)
format = Format_IMPORTED_MODULE;
setFormat(tk.begin(), tk.length(), _formats[format]);
}
}
int PythonIndenter::getIndentDiff(const QString &previousLine, const TextEditor::TabSettings &tabSettings) const
{
PythonScanner sc(previousLine.constData(), previousLine.length());
forever {
FormatToken tk = sc.read();
if ((tk.format() == Internal::Format_Keyword) && m_jumpKeywords.contains(sc.value(tk)))
return -tabSettings.m_indentSize;
if (tk.format() != Internal::Format_Whitespace)
break;
}
return 0;
}
void WhitespaceManager::breakToken(const FormatToken &Tok, unsigned Offset,
unsigned ReplaceChars,
StringRef PreviousPostfix,
StringRef CurrentPrefix, bool InPPDirective,
unsigned Spaces) {
Changes.push_back(Change(
true, SourceRange(Tok.getStartOfNonWhitespace().getLocWithOffset(Offset),
Tok.getStartOfNonWhitespace().getLocWithOffset(
Offset + ReplaceChars)),
Spaces, Spaces, 1, PreviousPostfix, CurrentPrefix,
// FIXME: Unify token adjustment, so we don't split it between
// BreakableToken and the WhitespaceManager. That would also allow us to
// correctly store a tok::TokenKind instead of rolling our own enum.
tok::unknown, InPPDirective && !Tok.IsFirst));
}
void WhitespaceManager::replaceWhitespaceInToken(
const FormatToken &Tok, unsigned Offset, unsigned ReplaceChars,
StringRef PreviousPostfix, StringRef CurrentPrefix, bool InPPDirective,
unsigned Newlines, unsigned Spaces) {
Changes.push_back(Change(
true, SourceRange(Tok.getStartOfNonWhitespace().getLocWithOffset(Offset),
Tok.getStartOfNonWhitespace().getLocWithOffset(
Offset + ReplaceChars)),
Spaces, Spaces, Newlines, PreviousPostfix, CurrentPrefix,
// If we don't add a newline this change doesn't start a comment. Thus,
// when we align line comments, we don't need to treat this change as one.
// FIXME: We still need to take this change in account to properly
// calculate the new length of the comment and to calculate the changes
// for which to do the alignment when aligning comments.
Tok.Type == TT_LineComment && Newlines > 0 ? tok::comment : tok::unknown,
InPPDirective && !Tok.IsFirst));
}
bool AffectedRangeManager::affectsTokenRange(const FormatToken &First,
const FormatToken &Last,
bool IncludeLeadingNewlines) {
SourceLocation Start = First.WhitespaceRange.getBegin();
if (!IncludeLeadingNewlines)
Start = Start.getLocWithOffset(First.LastNewlineOffset);
SourceLocation End = Last.getStartOfNonWhitespace();
End = End.getLocWithOffset(Last.TokenText.size());
CharSourceRange Range = CharSourceRange::getCharRange(Start, End);
return affectsCharSourceRange(Range);
}
int LuaHighlighter::highlightLine(QString const& text, int initialState)
{
Scanner scanner(text.constData(),text.length());
scanner.setState(initialState);
FormatToken tk;
bool hasOnlyWhitespace = true;
while((tk = scanner.read()).format() != Format_EndOfBlock)
{
Format format = tk.format();
if(format == Format_Keyword) {
if(isImportKeyword(scanner.value(tk)) && hasOnlyWhitespace)
{
setFormat(tk.begin(), tk.length(), formatForCategory(format));
highlightImport(scanner);
break;
}
}
setFormat(tk.begin(), tk.length(), formatForCategory(format));
if(format != Format_Whitespace)
hasOnlyWhitespace = false;
}
return scanner.state();
}
void UnwrappedLineFormatter::formatFirstToken(FormatToken &RootToken,
const AnnotatedLine *PreviousLine,
unsigned IndentLevel,
unsigned Indent,
bool InPPDirective) {
unsigned Newlines =
std::min(RootToken.NewlinesBefore, Style.MaxEmptyLinesToKeep + 1);
// Remove empty lines before "}" where applicable.
if (RootToken.is(tok::r_brace) &&
(!RootToken.Next ||
(RootToken.Next->is(tok::semi) && !RootToken.Next->Next)))
Newlines = std::min(Newlines, 1u);
if (Newlines == 0 && !RootToken.IsFirst)
Newlines = 1;
if (RootToken.IsFirst && !RootToken.HasUnescapedNewline)
Newlines = 0;
// Remove empty lines after "{".
if (!Style.KeepEmptyLinesAtTheStartOfBlocks && PreviousLine &&
PreviousLine->Last->is(tok::l_brace) &&
PreviousLine->First->isNot(tok::kw_namespace) &&
!startsExternCBlock(*PreviousLine))
Newlines = 1;
// Insert extra new line before access specifiers.
if (PreviousLine && PreviousLine->Last->isOneOf(tok::semi, tok::r_brace) &&
RootToken.isAccessSpecifier() && RootToken.NewlinesBefore == 1)
++Newlines;
// Remove empty lines after access specifiers.
if (PreviousLine && PreviousLine->First->isAccessSpecifier() &&
(!PreviousLine->InPPDirective || !RootToken.HasUnescapedNewline))
Newlines = std::min(1u, Newlines);
Whitespaces->replaceWhitespace(RootToken, Newlines, IndentLevel, Indent,
Indent, InPPDirective &&
!RootToken.HasUnescapedNewline);
}
void UnwrappedLineParser::calculateBraceTypes() {
// We'll parse forward through the tokens until we hit
// a closing brace or eof - note that getNextToken() will
// parse macros, so this will magically work inside macro
// definitions, too.
unsigned StoredPosition = Tokens->getPosition();
unsigned Position = StoredPosition;
FormatToken *Tok = FormatTok;
// Keep a stack of positions of lbrace tokens. We will
// update information about whether an lbrace starts a
// braced init list or a different block during the loop.
SmallVector<FormatToken *, 8> LBraceStack;
assert(Tok->Tok.is(tok::l_brace));
do {
// Get next none-comment token.
FormatToken *NextTok;
unsigned ReadTokens = 0;
do {
NextTok = Tokens->getNextToken();
++ReadTokens;
} while (NextTok->is(tok::comment));
switch (Tok->Tok.getKind()) {
case tok::l_brace:
LBraceStack.push_back(Tok);
break;
case tok::r_brace:
if (!LBraceStack.empty()) {
if (LBraceStack.back()->BlockKind == BK_Unknown) {
bool ProbablyBracedList = false;
if (Style.Language == FormatStyle::LK_Proto) {
ProbablyBracedList = NextTok->isOneOf(tok::comma, tok::r_square);
} else {
// Using OriginalColumn to distinguish between ObjC methods and
// binary operators is a bit hacky.
bool NextIsObjCMethod = NextTok->isOneOf(tok::plus, tok::minus) &&
NextTok->OriginalColumn == 0;
// If there is a comma, semicolon or right paren after the closing
// brace, we assume this is a braced initializer list. Note that
// regardless how we mark inner braces here, we will overwrite the
// BlockKind later if we parse a braced list (where all blocks
// inside are by default braced lists), or when we explicitly detect
// blocks (for example while parsing lambdas).
//
// We exclude + and - as they can be ObjC visibility modifiers.
ProbablyBracedList =
NextTok->isOneOf(tok::comma, tok::semi, tok::period, tok::colon,
tok::r_paren, tok::r_square, tok::l_brace,
tok::l_paren) ||
(NextTok->isBinaryOperator() && !NextIsObjCMethod);
}
if (ProbablyBracedList) {
Tok->BlockKind = BK_BracedInit;
LBraceStack.back()->BlockKind = BK_BracedInit;
} else {
Tok->BlockKind = BK_Block;
LBraceStack.back()->BlockKind = BK_Block;
}
}
LBraceStack.pop_back();
}
break;
case tok::at:
case tok::semi:
case tok::kw_if:
case tok::kw_while:
case tok::kw_for:
case tok::kw_switch:
case tok::kw_try:
if (!LBraceStack.empty())
LBraceStack.back()->BlockKind = BK_Block;
break;
default:
break;
}
Tok = NextTok;
Position += ReadTokens;
} while (Tok->Tok.isNot(tok::eof) && !LBraceStack.empty());
// Assume other blocks for all unclosed opening braces.
for (unsigned i = 0, e = LBraceStack.size(); i != e; ++i) {
if (LBraceStack[i]->BlockKind == BK_Unknown)
LBraceStack[i]->BlockKind = BK_Block;
}
FormatTok = Tokens->setPosition(StoredPosition);
}
void CommaSeparatedList::precomputeFormattingInfos(const FormatToken *Token) {
// FIXME: At some point we might want to do this for other lists, too.
if (!Token->MatchingParen || Token->isNot(tok::l_brace))
return;
FormatToken *ItemBegin = Token->Next;
SmallVector<bool, 8> MustBreakBeforeItem;
// The lengths of an item if it is put at the end of the line. This includes
// trailing comments which are otherwise ignored for column alignment.
SmallVector<unsigned, 8> EndOfLineItemLength;
for (unsigned i = 0, e = Commas.size() + 1; i != e; ++i) {
// Skip comments on their own line.
while (ItemBegin->HasUnescapedNewline && ItemBegin->isTrailingComment())
ItemBegin = ItemBegin->Next;
MustBreakBeforeItem.push_back(ItemBegin->MustBreakBefore);
const FormatToken *ItemEnd = NULL;
if (i == Commas.size()) {
ItemEnd = Token->MatchingParen;
const FormatToken *NonCommentEnd = ItemEnd->getPreviousNonComment();
ItemLengths.push_back(CodePointsBetween(ItemBegin, NonCommentEnd));
if (Style.Cpp11BracedListStyle) {
// In Cpp11 braced list style, the } and possibly other subsequent
// tokens will need to stay on a line with the last element.
while (ItemEnd->Next && !ItemEnd->Next->CanBreakBefore)
ItemEnd = ItemEnd->Next;
} else {
// In other braced lists styles, the "}" can be wrapped to the new line.
ItemEnd = Token->MatchingParen->Previous;
}
} else {
ItemEnd = Commas[i];
// The comma is counted as part of the item when calculating the length.
ItemLengths.push_back(CodePointsBetween(ItemBegin, ItemEnd));
// Consume trailing comments so the are included in EndOfLineItemLength.
if (ItemEnd->Next && !ItemEnd->Next->HasUnescapedNewline &&
ItemEnd->Next->isTrailingComment())
ItemEnd = ItemEnd->Next;
}
EndOfLineItemLength.push_back(CodePointsBetween(ItemBegin, ItemEnd));
// If there is a trailing comma in the list, the next item will start at the
// closing brace. Don't create an extra item for this.
if (ItemEnd->getNextNonComment() == Token->MatchingParen)
break;
ItemBegin = ItemEnd->Next;
}
// We can never place more than ColumnLimit / 3 items in a row (because of the
// spaces and the comma).
for (unsigned Columns = 1; Columns <= Style.ColumnLimit / 3; ++Columns) {
ColumnFormat Format;
Format.Columns = Columns;
Format.ColumnSizes.resize(Columns);
Format.LineCount = 0;
bool HasRowWithSufficientColumns = false;
unsigned Column = 0;
for (unsigned i = 0, e = ItemLengths.size(); i != e; ++i) {
if (MustBreakBeforeItem[i] || Column == Columns) {
++Format.LineCount;
Column = 0;
}
if (Column == Columns - 1)
HasRowWithSufficientColumns = true;
unsigned length =
(Column == Columns - 1) ? EndOfLineItemLength[i] : ItemLengths[i];
Format.ColumnSizes[Column] =
std::max(Format.ColumnSizes[Column], length);
++Column;
}
// If all rows are terminated early (e.g. by trailing comments), we don't
// need to look further.
if (!HasRowWithSufficientColumns)
break;
Format.TotalWidth = Columns - 1; // Width of the N-1 spaces.
for (unsigned i = 0; i < Columns; ++i) {
Format.TotalWidth += Format.ColumnSizes[i];
}
// Ignore layouts that are bound to violate the column limit.
if (Format.TotalWidth > Style.ColumnLimit)
continue;
Formats.push_back(Format);
}
}
DebugCommand* PrintCommand::Build(std::vector<Token*> args) {
if (args.size() < 2) {
return new InvalidCommand(args, -1, "too few arguments");
}
Token* target = args[1];
if (!target->IsRegister() &&
!target->IsFPRegister() &&
!target->IsIdentifier()) {
return new InvalidCommand(args, 1, "expects reg or identifier");
}
FormatToken* format = NULL;
int target_size = 0;
if (target->IsRegister()) {
Register reg = RegisterToken::Cast(target)->value();
target_size = reg.SizeInBytes();
} else if (target->IsFPRegister()) {
FPRegister fpreg = FPRegisterToken::Cast(target)->value();
target_size = fpreg.SizeInBytes();
}
// If the target is an identifier there must be no format. This is checked
// in the switch statement below.
switch (args.size()) {
case 2: {
if (target->IsRegister()) {
switch (target_size) {
case 4: format = new Format<uint32_t>("%08" PRIx32, 'x'); break;
case 8: format = new Format<uint64_t>("%016" PRIx64, 'x'); break;
default: VIXL_UNREACHABLE();
}
} else if (target->IsFPRegister()) {
switch (target_size) {
case 4: format = new Format<float>("%8g", 'f'); break;
case 8: format = new Format<double>("%8g", 'f'); break;
default: VIXL_UNREACHABLE();
}
}
break;
}
case 3: {
if (target->IsIdentifier()) {
return new InvalidCommand(args, 2,
"format is only allowed with registers");
}
Token* second = args[2];
if (!second->IsFormat()) {
return new InvalidCommand(args, 2, "expects format");
}
format = FormatToken::Cast(second);
if (format->SizeOf() > target_size) {
return new InvalidCommand(args, 2, "format too wide");
}
break;
}
default:
return new InvalidCommand(args, -1, "too many arguments");
}
return new PrintCommand(args[0], target, format);
}
void CommaSeparatedList::precomputeFormattingInfos(const FormatToken *Token) {
// FIXME: At some point we might want to do this for other lists, too.
if (!Token->MatchingParen || Token->isNot(tok::l_brace))
return;
// In C++11 braced list style, we should not format in columns unless they
// have many items (20 or more) or we allow bin-packing of function call
// arguments.
if (Style.Cpp11BracedListStyle && !Style.BinPackArguments &&
Commas.size() < 19)
return;
// Column format doesn't really make sense if we don't align after brackets.
if (!Style.AlignAfterOpenBracket)
return;
FormatToken *ItemBegin = Token->Next;
while (ItemBegin->isTrailingComment())
ItemBegin = ItemBegin->Next;
SmallVector<bool, 8> MustBreakBeforeItem;
// The lengths of an item if it is put at the end of the line. This includes
// trailing comments which are otherwise ignored for column alignment.
SmallVector<unsigned, 8> EndOfLineItemLength;
bool HasSeparatingComment = false;
for (unsigned i = 0, e = Commas.size() + 1; i != e; ++i) {
// Skip comments on their own line.
while (ItemBegin->HasUnescapedNewline && ItemBegin->isTrailingComment()) {
ItemBegin = ItemBegin->Next;
HasSeparatingComment = i > 0;
}
MustBreakBeforeItem.push_back(ItemBegin->MustBreakBefore);
if (ItemBegin->is(tok::l_brace))
HasNestedBracedList = true;
const FormatToken *ItemEnd = nullptr;
if (i == Commas.size()) {
ItemEnd = Token->MatchingParen;
const FormatToken *NonCommentEnd = ItemEnd->getPreviousNonComment();
ItemLengths.push_back(CodePointsBetween(ItemBegin, NonCommentEnd));
if (Style.Cpp11BracedListStyle) {
// In Cpp11 braced list style, the } and possibly other subsequent
// tokens will need to stay on a line with the last element.
while (ItemEnd->Next && !ItemEnd->Next->CanBreakBefore)
ItemEnd = ItemEnd->Next;
} else {
// In other braced lists styles, the "}" can be wrapped to the new line.
ItemEnd = Token->MatchingParen->Previous;
}
} else {
ItemEnd = Commas[i];
// The comma is counted as part of the item when calculating the length.
ItemLengths.push_back(CodePointsBetween(ItemBegin, ItemEnd));
// Consume trailing comments so the are included in EndOfLineItemLength.
if (ItemEnd->Next && !ItemEnd->Next->HasUnescapedNewline &&
ItemEnd->Next->isTrailingComment())
ItemEnd = ItemEnd->Next;
}
EndOfLineItemLength.push_back(CodePointsBetween(ItemBegin, ItemEnd));
// If there is a trailing comma in the list, the next item will start at the
// closing brace. Don't create an extra item for this.
if (ItemEnd->getNextNonComment() == Token->MatchingParen)
break;
ItemBegin = ItemEnd->Next;
}
// Don't use column layout for nested lists, lists with few elements and in
// presence of separating comments.
if (Token->NestingLevel != 0 || Commas.size() < 5 || HasSeparatingComment)
return;
// We can never place more than ColumnLimit / 3 items in a row (because of the
// spaces and the comma).
unsigned MaxItems = Style.ColumnLimit / 3;
std::vector<unsigned> MinSizeInColumn;
MinSizeInColumn.reserve(MaxItems);
for (unsigned Columns = 1; Columns <= MaxItems; ++Columns) {
ColumnFormat Format;
Format.Columns = Columns;
Format.ColumnSizes.resize(Columns);
MinSizeInColumn.assign(Columns, UINT_MAX);
Format.LineCount = 1;
bool HasRowWithSufficientColumns = false;
unsigned Column = 0;
for (unsigned i = 0, e = ItemLengths.size(); i != e; ++i) {
assert(i < MustBreakBeforeItem.size());
if (MustBreakBeforeItem[i] || Column == Columns) {
++Format.LineCount;
Column = 0;
}
if (Column == Columns - 1)
HasRowWithSufficientColumns = true;
unsigned Length =
(Column == Columns - 1) ? EndOfLineItemLength[i] : ItemLengths[i];
Format.ColumnSizes[Column] = std::max(Format.ColumnSizes[Column], Length);
MinSizeInColumn[Column] = std::min(MinSizeInColumn[Column], Length);
++Column;
}
// If all rows are terminated early (e.g. by trailing comments), we don't
// need to look further.
if (!HasRowWithSufficientColumns)
break;
Format.TotalWidth = Columns - 1; // Width of the N-1 spaces.
for (unsigned i = 0; i < Columns; ++i)
Format.TotalWidth += Format.ColumnSizes[i];
// Don't use this Format, if the difference between the longest and shortest
// element in a column exceeds a threshold to avoid excessive spaces.
if ([&] {
for (unsigned i = 0; i < Columns - 1; ++i)
if (Format.ColumnSizes[i] - MinSizeInColumn[i] > 10)
return true;
return false;
}())
continue;
// Ignore layouts that are bound to violate the column limit.
if (Format.TotalWidth > Style.ColumnLimit)
continue;
Formats.push_back(Format);
}
}
void UnwrappedLineParser::calculateBraceTypes() {
// We'll parse forward through the tokens until we hit
// a closing brace or eof - note that getNextToken() will
// parse macros, so this will magically work inside macro
// definitions, too.
unsigned StoredPosition = Tokens->getPosition();
unsigned Position = StoredPosition;
FormatToken *Tok = FormatTok;
// Keep a stack of positions of lbrace tokens. We will
// update information about whether an lbrace starts a
// braced init list or a different block during the loop.
SmallVector<FormatToken *, 8> LBraceStack;
assert(Tok->Tok.is(tok::l_brace));
do {
// Get next none-comment token.
FormatToken *NextTok;
unsigned ReadTokens = 0;
do {
NextTok = Tokens->getNextToken();
++ReadTokens;
} while (NextTok->is(tok::comment));
switch (Tok->Tok.getKind()) {
case tok::l_brace:
LBraceStack.push_back(Tok);
break;
case tok::r_brace:
if (!LBraceStack.empty()) {
if (LBraceStack.back()->BlockKind == BK_Unknown) {
// If there is a comma, semicolon or right paren after the closing
// brace, we assume this is a braced initializer list.
// FIXME: Note that this currently works only because we do not
// use the brace information while inside a braced init list.
// Thus, if the parent is a braced init list, we consider all
// brace blocks inside it braced init list. That works good enough
// for now, but we will need to fix it to correctly handle lambdas.
if (NextTok->isOneOf(tok::comma, tok::semi, tok::r_paren,
tok::l_brace, tok::colon)) {
Tok->BlockKind = BK_BracedInit;
LBraceStack.back()->BlockKind = BK_BracedInit;
} else {
Tok->BlockKind = BK_Block;
LBraceStack.back()->BlockKind = BK_Block;
}
}
LBraceStack.pop_back();
}
break;
case tok::semi:
case tok::kw_if:
case tok::kw_while:
case tok::kw_for:
case tok::kw_switch:
case tok::kw_try:
if (!LBraceStack.empty())
LBraceStack.back()->BlockKind = BK_Block;
break;
default:
break;
}
Tok = NextTok;
Position += ReadTokens;
} while (Tok->Tok.isNot(tok::eof));
// Assume other blocks for all unclosed opening braces.
for (unsigned i = 0, e = LBraceStack.size(); i != e; ++i) {
if (LBraceStack[i]->BlockKind == BK_Unknown)
LBraceStack[i]->BlockKind = BK_Block;
}
FormatTok = Tokens->setPosition(StoredPosition);
}
/**
* @brief Highlighter::highlight_impl
* @param text - source code to highlight
* @param initialState - initial state of scanner, retrieved from previous block
* @return final state of scanner, should be saved with current block
*/
int Highlighter::highlightLine(const QString &text, int initialState)
{
Scanner scanner(text.constData(), text.size());
scanner.setState(initialState);
HState hstate = HState_NORMAL;
int blockNo = currentBlock().blockNumber();
QMap<int, QString>::iterator it = _userTypes.find(blockNo);
if (it != _userTypes.end())
_userTypes.erase(it);
FormatToken tk;
bool hasOnlyWhitespace = true;
while ((tk = scanner.read()).format() != Format_EndOfBlock)
{
Format format = tk.format();
switch (hstate)
{
default: // HState_NORMAL
if (format == Format_KEYWORD) {
QString value = scanner.value(tk);
if (value == "class") {
hstate = HState_ON_CLASS;
} else if (isImportKeyword(value) && hasOnlyWhitespace) {
setFormat(tk.begin(), tk.length(), _formats[format]);
highlightImport(scanner);
return scanner.getState();
}
}
if (format == Format_IDENTIFIER) {
QString value = scanner.value(tk);
QMap<int, QString>::const_iterator it;
for (it = _userTypes.begin(); it != _userTypes.end(); ++it) {
if (value == it.value()) {
format = Format_TYPE;
break;
}
}
}
break;
case HState_ON_CLASS:
if (format == Format_WHITESPACE)
hstate = HState_AFTER_CLASS;
else
hstate = HState_NORMAL;
break;
case HState_AFTER_CLASS:
hstate = HState_NORMAL;
if (format == Format_IDENTIFIER) {
_userTypes[blockNo] = scanner.value(tk);
format = Format_TYPE;
}
break;
}
setFormat(tk.begin(), tk.length(), _formats[format]);
if (format != Format_WHITESPACE)
hasOnlyWhitespace = false;
}
return scanner.getState();
}
