QVector<Type> RppLexer::lex(const TokenContainer &tokenContainer)
{
QVector<Type> tokenTypes;
const int numTokens = tokenContainer.count();
tokenTypes.reserve(numTokens);
QByteArray text = tokenContainer.fullText();
m_buffer = text.constData();
for(int t=0; t<numTokens; ++t) {
TokenEngine::Token token = tokenContainer.token(t);
tokenTypes.append(indentify(token.start, token.length));
}
return tokenTypes;
}
void RppTreeEvaluator::evaluateText(const Text *textLine)
{
const int numTokens = textLine->count();
const TokenContainer tokenContainer = textLine->text().tokenContainer(0);
int t = 0;
int startTokenRun = 0;
while(t < numTokens) {
const Token *currentToken = textLine->token(t);
int currentContainerIndex = currentToken->index();
//handle macro replacements
if(currentToken->toIdToken()) {
const int tokenIndex = currentToken->index();
const QByteArray tokenText = tokenContainer.tempText(tokenIndex);
if(m_activeDefinitions->contains(tokenText)) {
//crate section
TokenSection section(tokenContainer, textLine->token(startTokenRun)->index(), t - startTokenRun);
m_tokenSections.append(section);
//evaluate macro
const int oldContainerIndex = currentContainerIndex;
TokenContainer evaluatedText = evaluateMacro(tokenContainer, currentContainerIndex);
TokenSection evalSection(evaluatedText, 0, evaluatedText.count());
m_tokenSections.append(evalSection);
t += currentContainerIndex - oldContainerIndex;
startTokenRun = t;
}
++t;
continue;
}
//handle comments
if(currentToken->toLineComment() || currentToken->toMultiLineComment()) {
//create section
TokenSection section(tokenContainer, textLine->token(startTokenRun)->index(), t - startTokenRun );
m_tokenSections.append(section);
t++; //skip comment
startTokenRun = t;
t++;
continue;
}
// handle escaped newlines
if (currentContainerIndex + 1 < numTokens) {
const TokenTempRef tokenRef1 = tokenContainer.tokenTempRef(currentContainerIndex);
const TokenTempRef tokenRef2 = tokenContainer.tokenTempRef(currentContainerIndex + 1);
// This is i slight hack. We want to check if the next token is a newline token,
// but since we don't have any lexical info at this point we just check if it starts
// with \r or \n
if (tokenRef1.at(0) == '\\' && (tokenRef2.at(0) == '\n' || tokenRef2.at(0) == '\r')) {
//create section
TokenSection section(tokenContainer, textLine->token(startTokenRun)->index(), t - startTokenRun );
m_tokenSections.append(section);
t += 2;
startTokenRun = t;
t++;
continue;
}
}
t++;
}
//round up any tokens at the end and put them in a section
if(t - startTokenRun > 1) {
TokenSection section(tokenContainer, textLine->token(startTokenRun)->index(), t - startTokenRun );
m_tokenSections.append(section);
}
m_tokenSections.append(*newlineSection);
}