static void createTokens (int startPosition, const String& lineText,
CodeDocument::Iterator& source,
CodeTokeniser* analyser,
Array <SyntaxToken>& newTokens)
{
CodeDocument::Iterator lastIterator (source);
const int lineLength = lineText.length();
for (;;)
{
int tokenType = analyser->readNextToken (source);
int tokenStart = lastIterator.getPosition();
int tokenEnd = source.getPosition();
if (tokenEnd <= tokenStart)
break;
tokenEnd -= startPosition;
if (tokenEnd > 0)
{
tokenStart -= startPosition;
newTokens.add (SyntaxToken (lineText.substring (jmax (0, tokenStart), tokenEnd),
tokenType));
if (tokenEnd >= lineLength)
break;
}
lastIterator = source;
}
source = lastIterator;
}
static void addToken (Array<SyntaxToken>& dest, const String& text,
const int length, const int type)
{
if (length > 1000)
{
// subdivide very long tokens to avoid unwieldy glyph sequences
addToken (dest, text.substring (0, length / 2), length / 2, type);
addToken (dest, text.substring (length / 2), length - length / 2, type);
}
else
{
dest.add (SyntaxToken (text, length, type));
}
}
bool update (CodeDocument& document, int lineNum,
CodeDocument::Iterator& source,
CodeTokeniser* analyser, const int spacesPerTab,
const CodeDocument::Position& selectionStart,
const CodeDocument::Position& selectionEnd)
{
Array <SyntaxToken> newTokens;
newTokens.ensureStorageAllocated (8);
if (analyser == nullptr)
{
newTokens.add (SyntaxToken (document.getLine (lineNum), -1));
}
else if (lineNum < document.getNumLines())
{
const CodeDocument::Position pos (&document, lineNum, 0);
createTokens (pos.getPosition(), pos.getLineText(),
source, analyser, newTokens);
}
replaceTabsWithSpaces (newTokens, spacesPerTab);
int newHighlightStart = 0;
int newHighlightEnd = 0;
if (selectionStart.getLineNumber() <= lineNum && selectionEnd.getLineNumber() >= lineNum)
{
const String line (document.getLine (lineNum));
CodeDocument::Position lineStart (&document, lineNum, 0), lineEnd (&document, lineNum + 1, 0);
newHighlightStart = indexToColumn (jmax (0, selectionStart.getPosition() - lineStart.getPosition()),
line, spacesPerTab);
newHighlightEnd = indexToColumn (jmin (lineEnd.getPosition() - lineStart.getPosition(), selectionEnd.getPosition() - lineStart.getPosition()),
line, spacesPerTab);
}
if (newHighlightStart != highlightColumnStart || newHighlightEnd != highlightColumnEnd)
{
highlightColumnStart = newHighlightStart;
highlightColumnEnd = newHighlightEnd;
}
else
{
if (tokens.size() == newTokens.size())
{
bool allTheSame = true;
for (int i = newTokens.size(); --i >= 0;)
{
if (tokens.getReference(i) != newTokens.getReference(i))
{
allTheSame = false;
break;
}
}
if (allTheSame)
return false;
}
}
tokens.swapWithArray (newTokens);
return true;
}