PTHEntry PTHWriter::LexTokens(Lexer& L) { // Pad 0's so that we emit tokens to a 4-byte alignment. // This speed up reading them back in. Pad(Out, 4); Offset TokenOff = (Offset) Out.tell(); // Keep track of matching '#if' ... '#endif'. typedef std::vector<std::pair<Offset, unsigned> > PPCondTable; PPCondTable PPCond; std::vector<unsigned> PPStartCond; bool ParsingPreprocessorDirective = false; Token Tok; do { L.LexFromRawLexer(Tok); NextToken: if ((Tok.isAtStartOfLine() || Tok.is(tok::eof)) && ParsingPreprocessorDirective) { // Insert an eod token into the token cache. It has the same // position as the next token that is not on the same line as the // preprocessor directive. Observe that we continue processing // 'Tok' when we exit this branch. Token Tmp = Tok; Tmp.setKind(tok::eod); Tmp.clearFlag(Token::StartOfLine); Tmp.setIdentifierInfo(0); EmitToken(Tmp); ParsingPreprocessorDirective = false; } if (Tok.is(tok::raw_identifier)) { PP.LookUpIdentifierInfo(Tok); EmitToken(Tok); continue; } if (Tok.is(tok::hash) && Tok.isAtStartOfLine()) { // Special processing for #include. Store the '#' token and lex // the next token. assert(!ParsingPreprocessorDirective); Offset HashOff = (Offset) Out.tell(); // Get the next token. Token NextTok; L.LexFromRawLexer(NextTok); // If we see the start of line, then we had a null directive "#". In // this case, discard both tokens. if (NextTok.isAtStartOfLine()) goto NextToken; // The token is the start of a directive. Emit it. EmitToken(Tok); Tok = NextTok; // Did we see 'include'/'import'/'include_next'? if (Tok.isNot(tok::raw_identifier)) { EmitToken(Tok); continue; } IdentifierInfo* II = PP.LookUpIdentifierInfo(Tok); tok::PPKeywordKind K = II->getPPKeywordID(); ParsingPreprocessorDirective = true; switch (K) { case tok::pp_not_keyword: // Invalid directives "#foo" can occur in #if 0 blocks etc, just pass // them through. default: break; case tok::pp_include: case tok::pp_import: case tok::pp_include_next: { // Save the 'include' token. EmitToken(Tok); // Lex the next token as an include string. L.setParsingPreprocessorDirective(true); L.LexIncludeFilename(Tok); L.setParsingPreprocessorDirective(false); assert(!Tok.isAtStartOfLine()); if (Tok.is(tok::raw_identifier)) PP.LookUpIdentifierInfo(Tok); break; } case tok::pp_if: case tok::pp_ifdef: case tok::pp_ifndef: { // Add an entry for '#if' and friends. We initially set the target // index to 0. This will get backpatched when we hit #endif. PPStartCond.push_back(PPCond.size()); PPCond.push_back(std::make_pair(HashOff, 0U)); break; } case tok::pp_endif: { // Add an entry for '#endif'. We set the target table index to itself. // This will later be set to zero when emitting to the PTH file. We // use 0 for uninitialized indices because that is easier to debug. unsigned index = PPCond.size(); // Backpatch the opening '#if' entry. assert(!PPStartCond.empty()); assert(PPCond.size() > PPStartCond.back()); assert(PPCond[PPStartCond.back()].second == 0); PPCond[PPStartCond.back()].second = index; PPStartCond.pop_back(); // Add the new entry to PPCond. PPCond.push_back(std::make_pair(HashOff, index)); EmitToken(Tok); // Some files have gibberish on the same line as '#endif'. // Discard these tokens. do L.LexFromRawLexer(Tok); while (Tok.isNot(tok::eof) && !Tok.isAtStartOfLine()); // We have the next token in hand. // Don't immediately lex the next one. goto NextToken; } case tok::pp_elif: case tok::pp_else: { // Add an entry for #elif or #else. // This serves as both a closing and opening of a conditional block. // This means that its entry will get backpatched later. unsigned index = PPCond.size(); // Backpatch the previous '#if' entry. assert(!PPStartCond.empty()); assert(PPCond.size() > PPStartCond.back()); assert(PPCond[PPStartCond.back()].second == 0); PPCond[PPStartCond.back()].second = index; PPStartCond.pop_back(); // Now add '#elif' as a new block opening. PPCond.push_back(std::make_pair(HashOff, 0U)); PPStartCond.push_back(index); break; } } } EmitToken(Tok); } while (Tok.isNot(tok::eof)); assert(PPStartCond.empty() && "Error: imblanced preprocessor conditionals."); // Next write out PPCond. Offset PPCondOff = (Offset) Out.tell(); // Write out the size of PPCond so that clients can identifer empty tables. Emit32(PPCond.size()); for (unsigned i = 0, e = PPCond.size(); i!=e; ++i) { Emit32(PPCond[i].first - TokenOff); uint32_t x = PPCond[i].second; assert(x != 0 && "PPCond entry not backpatched."); // Emit zero for #endifs. This allows us to do checking when // we read the PTH file back in. Emit32(x == i ? 0 : x); } return PTHEntry(TokenOff, PPCondOff); }