//
// The main functional entry-point into the preprocessor, which will
// scan the source strings to figure out and return the next processing token.
//
// Return string pointer to next token.
// Return 0 when no more tokens.
//
const char* TPpContext::tokenize(TPpToken* ppToken)
{
int token = '\n';
for(;;) {
const char* tokenString = nullptr;
token = scanToken(ppToken);
ppToken->token = token;
if (token == EOF) {
missingEndifCheck();
return nullptr;
}
if (token == '#') {
if (previous_token == '\n') {
token = readCPPline(ppToken);
if (token == EOF) {
missingEndifCheck();
return nullptr;
}
continue;
} else {
parseContext.error(ppToken->loc, "preprocessor directive cannot be preceded by another token", "#", "");
return nullptr;
}
}
previous_token = token;
if (token == '\n')
continue;
// expand macros
if (token == CPP_IDENTIFIER && MacroExpand(ppToken->atom, ppToken, false, true) != 0)
continue;
if (token == CPP_IDENTIFIER)
tokenString = GetAtomString(ppToken->atom);
else if (token == CPP_INTCONSTANT || token == CPP_UINTCONSTANT ||
token == CPP_FLOATCONSTANT || token == CPP_DOUBLECONSTANT)
tokenString = ppToken->name;
else if (token == CPP_STRCONSTANT) {
parseContext.error(ppToken->loc, "string literals not supported", "\"\"", "");
tokenString = nullptr;
} else if (token == '\'') {
parseContext.error(ppToken->loc, "character literals not supported", "\'", "");
tokenString = nullptr;
} else
tokenString = GetAtomString(token);
if (tokenString) {
if (tokenString[0] != 0)
parseContext.tokensBeforeEOF = 1;
return tokenString;
}
}
}
//
// The main functional entry point into the preprocessor, which will
// scan the source strings to figure out and return the next processing token.
//
// Return the token, or EndOfInput when no more tokens.
//
int TPpContext::tokenize(TPpToken& ppToken)
{
for(;;) {
int token = scanToken(&ppToken);
// Handle token-pasting logic
token = tokenPaste(token, ppToken);
if (token == EndOfInput) {
missingEndifCheck();
return EndOfInput;
}
if (token == '#') {
if (previous_token == '\n') {
token = readCPPline(&ppToken);
if (token == EndOfInput) {
missingEndifCheck();
return EndOfInput;
}
continue;
} else {
_parseContext.ppError(ppToken.loc, "preprocessor directive cannot be preceded by another token", "#", "");
return EndOfInput;
}
}
previous_token = token;
if (token == '\n')
continue;
// expand macros
if (token == PpAtomIdentifier && MacroExpand(&ppToken, false, true) != 0)
continue;
switch (token) {
case PpAtomIdentifier:
case PpAtomConstInt:
case PpAtomConstUint:
case PpAtomConstFloat:
case PpAtomConstInt64:
case PpAtomConstUint64:
case PpAtomConstInt16:
case PpAtomConstUint16:
case PpAtomConstDouble:
case PpAtomConstFloat16:
if (ppToken.name[0] == '\0')
continue;
break;
case PpAtomConstString:
if (ifdepth == 0 && _parseContext.intermediate.getSource() != EShSourceHlsl) {
// HLSL allows string literals.
_parseContext.ppError(ppToken.loc, "string literals not supported", "\"\"", "");
continue;
}
break;
case '\'':
_parseContext.ppError(ppToken.loc, "character literals not supported", "\'", "");
continue;
default:
strcpy(ppToken.name, atomStrings.getString(token));
break;
}
return token;
}
}
