// # undef identifier newline bool Preprocessor::parseUndefDirective(Item *group) { Q_ASSERT(group->toItemComposite()); const TokenSection tokenSection = readLine(); const QVector<int> cleanedLine = cleanTokenRange(tokenSection); if(cleanedLine.count() < 3) return false; UndefDirective *undefDirective = createNode<UndefDirective>(m_memoryPool, group); group->toItemComposite()->add(undefDirective); undefDirective->setIdentifier(TokenList(m_tokenContainer, QVector<int>() << cleanedLine.at(2))); undefDirective->setTokenSection(tokenSection); return true; }
//parses an "ifdef-like" directive, like #ifdef and #ifndef :) //# ifdef identifier newline group[opt] bool Preprocessor::parseIfdefLikeDirective(IfdefLikeDirective *node) { Q_ASSERT(node->toItemComposite()); const TokenSection tokenSection = readLine(); const QVector<int> cleanedLine = cleanTokenRange(tokenSection); if(cleanedLine.count() < 3) return false; node->setTokenSection(tokenSection); node->setIdentifier(TokenList(m_tokenContainer, QVector<int>() << cleanedLine.at(2))); parseGroup(node); return true; }
/* Creates a tokenList containing one token */ TokenList ExpressionBuilder::createTokenList(int tokenIndex) const { return TokenList(m_tokenList.tokenContainer(tokenIndex), QVector<int>() << m_tokenList.containerIndex(tokenIndex)); }
/* # define identifier replacement-list new-line # define identifier lparen identifier-list[opt] ) replacement-list new-line # define identifier lparen ... ) replacement-list new-line # define identifier lparen identifier-list, ... ) replacement-list new-line */ bool Preprocessor::parseDefineDirective(Item *group) { Q_ASSERT(group->toItemComposite()); const TokenSection line = readLine(); const QVector<int> cleanedLine = cleanTokenRange(line); if(cleanedLine.count() < 3) return false; // get identifier const int identifier = cleanedLine.at(2); //skip "#" and "define" DefineDirective *defineDirective = 0; int replacementListStart; // check if this is a macro function if (cleanedLine.count() >= 4 && m_tokenContainer.text(cleanedLine.at(3)) == "(" && !isWhiteSpace(cleanedLine.at(3) - 1)) { MacroFunctionDefinition *macro; macro = createNode<MacroFunctionDefinition>(m_memoryPool, group); int tokenIndex = 4; //point to first argument or ')' QVector<int> macroParameterList; while(tokenIndex < cleanedLine.count()) { QByteArray currentText = m_tokenContainer.text(cleanedLine.at(tokenIndex)); ++tokenIndex; if(currentText == ")") break; if(currentText == ",") continue; macroParameterList.append(cleanedLine.at(tokenIndex - 1)); } macro->setParameters(TokenList(m_tokenContainer, macroParameterList)); defineDirective = macro; replacementListStart = tokenIndex; } else { MacroDefinition *macro; macro = createNode<MacroDefinition>(m_memoryPool, group); defineDirective = macro; replacementListStart = 3; } Q_ASSERT(defineDirective); // This is a bit hackish.. we want the replacement list with whitepspace // tokens, but cleanedLine() has already removed those. And we can't use // the original line, because that may contain escaped newline tokens. // So we remove the esacped newlines and search for the token number // given by cleanedLine.at(replacementListStart) QVector<int> replacementList; const QVector<int> noEscNewline = cleanEscapedNewLines(line); if (replacementListStart < cleanedLine.count()) { const int cleanedLineReplacementListStart = cleanedLine.at(replacementListStart); const int rListStart = noEscNewline.indexOf(cleanedLineReplacementListStart); if (rListStart != -1) { const int skipNewLineToken = 1; for (int i = rListStart; i < noEscNewline.count() - skipNewLineToken; ++i) { const int tokenContainerIndex = noEscNewline.at(i); const Type type = m_tokenTypeList.at(tokenContainerIndex); // Don't append comment tokens. if (type != Token_line_comment && type != Token_multiline_comment) { replacementList.append(tokenContainerIndex); } } } } defineDirective->setTokenSection(line); defineDirective->setIdentifier(TokenList(m_tokenContainer, QVector<int>() << identifier)); defineDirective->setReplacementList(TokenList(m_tokenContainer, replacementList)); group->toItemComposite()->add(defineDirective); return true; }
std::string serialize(TokenList tokenList) { return tokenList.empty()? "" : (tokenList.front().text() + serialize(TokenList(tokenList.begin()+1,tokenList.end()))); }
TokenList BracketMatcher::ReadCombination() { // TODO: split into smaller functions enum EMode { eOther , eOperator , eFirstOperand }; TokenList ret; unsigned int bracket_level = 0; EMode mode = eOther; while( true ) { Token token = lexer_.NextToken(); // If the stream ended before a combo was closed, return an empty // list - the user pressed Ctrl-D and doesn't want any output. if( token.IsEndOfStream() ) { return TokenList(); } token.AddToColumn( newline_processor_.GetIndent() ); // If we have got to the first operand of a combination // store the indent level so that we can indent correctly // in the repl when the user presses return. switch( mode ) { case eOperator: { mode = eFirstOperand; break; } case eFirstOperand: { newline_processor_.PushIndent( bracket_level, token.Column() ); mode = eOther; break; } default: { break; } } ret.AddToken( token ); // If we've finished this combination, return now if( token.Name() == ")" && bracket_level == 1 ) { break; } // Otherwise, adjust our bracket level if there is a ( or ) if( token.Name() == "(" ) { ++bracket_level; mode = eOperator; // The next token will be the operator } else if( token.Name() == ")" ) { newline_processor_.PopIndent( bracket_level ); --bracket_level; mode = eOther; } // If we got a bare symbol (no brackets) we exit here // (except that we skip straight past quotes) if( bracket_level == 0 && token.Name() != "'" ) { break; } // If the lexer's token ended with newline, it couldn't call // newline (because we hadn't processed the token yet). We // have now processed it, so we can call NewLine now. if( print_continuation_ && lexer_.EndedWithNewLine() ) { newline_processor_.NewLine(); } } // When the combination (or bare symbol) is finished, we don't // need to track indentation any more, so reset the newline processor, // and we don't care about any newlines before the next token, so // we skip whitespace in the lexer. newline_processor_.Reset(); return ret; }