void tokenize(const LangOptions &LangOpts, const SourceManager &SM,
unsigned BufferID, unsigned Offset, unsigned EndOffset,
CommentRetentionMode RetainComments,
TriviaRetentionMode TriviaRetention,
bool TokenizeInterpolatedString, ArrayRef<Token> SplitTokens,
DF &&DestFunc) {
assert((TriviaRetention != TriviaRetentionMode::WithTrivia ||
!TokenizeInterpolatedString) &&
"string interpolation with trivia is not implemented yet");
if (Offset == 0 && EndOffset == 0)
EndOffset = SM.getRangeForBuffer(BufferID).getByteLength();
Lexer L(LangOpts, SM, BufferID, /*Diags=*/nullptr, /*InSILMode=*/false,
RetainComments, TriviaRetention, Offset, EndOffset);
auto TokComp = [&](const Token &A, const Token &B) {
return SM.isBeforeInBuffer(A.getLoc(), B.getLoc());
};
std::set<Token, decltype(TokComp)> ResetTokens(TokComp);
for (auto C = SplitTokens.begin(), E = SplitTokens.end(); C != E; ++C) {
ResetTokens.insert(*C);
}
Token Tok;
syntax::Trivia LeadingTrivia, TrailingTrivia;
do {
L.lex(Tok, LeadingTrivia, TrailingTrivia);
// If the token has the same location as a reset location,
// reset the token stream
auto F = ResetTokens.find(Tok);
if (F != ResetTokens.end()) {
assert(F->isNot(tok::string_literal));
DestFunc(*F, syntax::Trivia(), syntax::Trivia());
auto NewState = L.getStateForBeginningOfTokenLoc(
F->getLoc().getAdvancedLoc(F->getLength()));
L.restoreState(NewState);
continue;
}
if (Tok.is(tok::string_literal) && TokenizeInterpolatedString) {
std::vector<Token> StrTokens;
getStringPartTokens(Tok, LangOpts, SM, BufferID, StrTokens);
for (auto &StrTok : StrTokens) {
DestFunc(StrTok, syntax::Trivia(), syntax::Trivia());
}
} else {
DestFunc(Tok, LeadingTrivia, TrailingTrivia);
}
} while (Tok.getKind() != tok::eof);
}
void StackDestroy(stk_stack * theStack,void DestFunc(void * a)) {
if(theStack) {
stk_stack_node * x=theStack->top;
stk_stack_node * y;
while(x) {
y=x->next;
DestFunc(x->info);
free(x);
x=y;
}
free(theStack);
}
}
void StackDestroy(STKSTACK theStack,void DestFunc(void * a))
{
STKNODE x=theStack->top;
STKNODE y=0;
if(theStack) {
while(x) {
y=x->next;
DestFunc(x->info);
free(x);
x=y;
}
free(theStack);
}
}
