// Handle #pragma
int TPpContext::CPPpragma(TPpToken* ppToken)
{
char SrcStrName[2];
TVector<TString> tokens;
TSourceLoc loc = ppToken->loc; // because we go to the next line before processing
int token = scanToken(ppToken);
while (token != '\n' && token != EndOfInput) {
switch (token) {
case PpAtomIdentifier:
case PpAtomConstInt:
case PpAtomConstUint:
case PpAtomConstFloat:
case PpAtomConstDouble:
tokens.push_back(ppToken->name);
break;
default:
SrcStrName[0] = (char)token;
SrcStrName[1] = '\0';
tokens.push_back(SrcStrName);
}
token = scanToken(ppToken);
}
if (token == EndOfInput)
parseContext.ppError(loc, "directive must end with a newline", "#pragma", "");
else
parseContext.handlePragma(loc, tokens);
return token;
}
// Handle #error
int TPpContext::CPPerror(TPpToken* ppToken)
{
int token = scanToken(ppToken);
std::string message;
TSourceLoc loc = ppToken->loc;
while (token != '\n' && token != EndOfInput) {
if (token == PpAtomConstInt || token == PpAtomConstUint ||
token == PpAtomConstInt64 || token == PpAtomConstUint64 ||
#ifdef AMD_EXTENSIONS
token == PpAtomConstFloat16 ||
#endif
token == PpAtomConstFloat || token == PpAtomConstDouble) {
message.append(ppToken->name);
} else if (token == PpAtomIdentifier || token == PpAtomConstString) {
message.append(ppToken->name);
} else {
message.append(GetAtomString(token));
}
message.append(" ");
token = scanToken(ppToken);
}
parseContext.notifyErrorDirective(loc.line, message.c_str());
//store this msg into the shader's information log..set the Compile Error flag!!!!
parseContext.ppError(loc, message.c_str(), "#error", "");
return '\n';
}
// Handle #ifdef
int TPpContext::CPPifdef(int defined, TPpToken* ppToken)
{
int token = scanToken(ppToken);
if (++ifdepth > maxIfNesting) {
parseContext.ppError(ppToken->loc, "maximum nesting depth exceeded", "#ifdef", "");
return 0;
}
elsetracker++;
if (token != PpAtomIdentifier) {
if (defined)
parseContext.ppError(ppToken->loc, "must be followed by macro name", "#ifdef", "");
else
parseContext.ppError(ppToken->loc, "must be followed by macro name", "#ifndef", "");
} else {
MacroSymbol* macro = lookupMacroDef(atomStrings.getAtom(ppToken->name));
token = scanToken(ppToken);
if (token != '\n') {
parseContext.ppError(ppToken->loc, "unexpected tokens following #ifdef directive - expected a newline", "#ifdef", "");
while (token != '\n' && token != EndOfInput)
token = scanToken(ppToken);
}
if (((macro != nullptr && !macro->undef) ? 1 : 0) != defined)
token = CPPelse(1, ppToken);
}
return token;
}
// Handle #ifdef
int TPpContext::CPPifdef(int defined, TPpToken* ppToken)
{
int token = scanToken(ppToken);
int name = ppToken->atom;
if (++ifdepth > maxIfNesting) {
parseContext.ppError(ppToken->loc, "maximum nesting depth exceeded", "#ifdef", "");
return 0;
}
elsetracker++;
if (token != PpAtomIdentifier) {
if (defined)
parseContext.ppError(ppToken->loc, "must be followed by macro name", "#ifdef", "");
else
parseContext.ppError(ppToken->loc, "must be followed by macro name", "#ifndef", "");
} else {
Symbol *s = LookUpSymbol(name);
token = scanToken(ppToken);
if (token != '\n') {
parseContext.ppError(ppToken->loc, "unexpected tokens following #ifdef directive - expected a newline", "#ifdef", "");
while (token != '\n' && token != EndOfInput)
token = scanToken(ppToken);
}
if (((s && !s->mac.undef) ? 1 : 0) != defined)
token = CPPelse(1, ppToken);
}
return token;
}
// Expand macros, skipping empty expansions, to get to the first real token in those expansions.
int TPpContext::evalToToken(int token, bool shortCircuit, int& res, bool& err, TPpToken* ppToken)
{
while (token == PpAtomIdentifier && ppToken->atom != PpAtomDefined) {
int macroReturn = MacroExpand(ppToken->atom, ppToken, true, false);
if (macroReturn == 0) {
parseContext.ppError(ppToken->loc, "can't evaluate expression", "preprocessor evaluation", "");
err = true;
res = 0;
token = scanToken(ppToken);
break;
}
if (macroReturn == -1) {
if (! shortCircuit && parseContext.profile == EEsProfile) {
const char* message = "undefined macro in expression not allowed in es profile";
if (parseContext.relaxedErrors())
parseContext.ppWarn(ppToken->loc, message, "preprocessor evaluation", ppToken->name);
else
parseContext.ppError(ppToken->loc, message, "preprocessor evaluation", ppToken->name);
}
}
token = scanToken(ppToken);
}
return token;
}
// Handle #line
int TPpContext::CPPline(TPpToken* ppToken)
{
// "#line must have, after macro substitution, one of the following forms:
// "#line line
// "#line line source-string-number"
int token = scanToken(ppToken);
const TSourceLoc directiveLoc = ppToken->loc;
if (token == '\n') {
parseContext.ppError(ppToken->loc, "must by followed by an integral literal", "#line", "");
return token;
}
int lineRes = 0; // Line number after macro expansion.
int lineToken = 0;
bool hasFile = false;
int fileRes = 0; // Source file number after macro expansion.
const char* sourceName = nullptr; // Optional source file name.
bool lineErr = false;
bool fileErr = false;
token = eval(token, MIN_PRECEDENCE, false, lineRes, lineErr, ppToken);
if (! lineErr) {
lineToken = lineRes;
if (token == '\n')
++lineRes;
if (parseContext.lineDirectiveShouldSetNextLine())
--lineRes;
parseContext.setCurrentLine(lineRes);
if (token != '\n') {
if (token == PpAtomConstString) {
parseContext.ppRequireExtensions(directiveLoc, 1, &E_GL_GOOGLE_cpp_style_line_directive, "filename-based #line");
// We need to save a copy of the string instead of pointing
// to the name field of the token since the name field
// will likely be overwritten by the next token scan.
sourceName = GetAtomString(LookUpAddString(ppToken->name));
parseContext.setCurrentSourceName(sourceName);
hasFile = true;
token = scanToken(ppToken);
} else {
token = eval(token, MIN_PRECEDENCE, false, fileRes, fileErr, ppToken);
if (! fileErr) {
parseContext.setCurrentString(fileRes);
hasFile = true;
}
}
}
}
if (!fileErr && !lineErr) {
parseContext.notifyLineDirective(directiveLoc.line, lineToken, hasFile, fileRes, sourceName);
}
token = extraTokenCheck(PpAtomLine, ppToken, token);
return token;
}
TPpContext::TokenStream* TPpContext::PrescanMacroArg(TokenStream* a, TPpToken* ppToken, bool newLineOkay)
{
int token;
TokenStream *n;
RewindTokenStream(a);
do {
token = ReadToken(a, ppToken);
if (token == PpAtomIdentifier && LookUpSymbol(ppToken->atom))
break;
} while (token != EndOfInput);
if (token == EndOfInput)
return a;
n = new TokenStream;
pushInput(new tMarkerInput(this));
pushTokenStreamInput(a);
while ((token = scanToken(ppToken)) != tMarkerInput::marker) {
if (token == PpAtomIdentifier && MacroExpand(ppToken->atom, ppToken, false, newLineOkay) != 0)
continue;
RecordToken(n, token, ppToken);
}
popInput();
delete a;
return n;
}
// Macro-expand a macro argument 'arg' to create 'expandedArg'.
// Does not replace 'arg'.
// Returns nullptr if no expanded argument is created.
TPpContext::TokenStream* TPpContext::PrescanMacroArg(TokenStream& arg, TPpToken* ppToken, bool newLineOkay)
{
// expand the argument
TokenStream* expandedArg = new TokenStream;
pushInput(new tMarkerInput(this));
pushTokenStreamInput(arg);
int token;
while ((token = scanToken(ppToken)) != tMarkerInput::marker && token != EndOfInput) {
token = tokenPaste(token, *ppToken);
if (token == tMarkerInput::marker || token == EndOfInput)
break;
if (token == PpAtomIdentifier && MacroExpand(ppToken, false, newLineOkay) != 0)
continue;
expandedArg->putToken(token, ppToken);
}
if (token == EndOfInput) {
// MacroExpand ate the marker, so had bad input, recover
delete expandedArg;
expandedArg = nullptr;
} else {
// remove the marker
popInput();
}
return expandedArg;
}
// Call when there should be no more tokens left on a line.
int TPpContext::extraTokenCheck(int atom, TPpToken* ppToken, int token)
{
if (token != '\n' && token != EndOfInput) {
static const char* message = "unexpected tokens following directive";
const char* label;
if (atom == PpAtomElse)
label = "#else";
else if (atom == PpAtomElif)
label = "#elif";
else if (atom == PpAtomEndif)
label = "#endif";
else if (atom == PpAtomIf)
label = "#if";
else if (atom == PpAtomLine)
label = "#line";
else
label = "";
if (parseContext.relaxedErrors())
parseContext.ppWarn(ppToken->loc, message, label, "");
else
parseContext.ppError(ppToken->loc, message, label, "");
while (token != '\n' && token != EndOfInput)
token = scanToken(ppToken);
}
return token;
}
// Expand macros, skipping empty expansions, to get to the first real token in those expansions.
int TPpContext::evalToToken(int token, bool shortCircuit, int& res, bool& err, TPpToken* ppToken)
{
while (token == PpAtomIdentifier && strcmp("defined", ppToken->name) != 0) {
switch (MacroExpand(ppToken, true, false)) {
case MacroExpandNotStarted:
case MacroExpandError:
parseContext.ppError(ppToken->loc, "can't evaluate expression", "preprocessor evaluation", "");
err = true;
res = 0;
break;
case MacroExpandStarted:
break;
case MacroExpandUndef:
if (! shortCircuit && parseContext.profile == EEsProfile) {
const char* message = "undefined macro in expression not allowed in es profile";
if (parseContext.relaxedErrors())
parseContext.ppWarn(ppToken->loc, message, "preprocessor evaluation", ppToken->name);
else
parseContext.ppError(ppToken->loc, message, "preprocessor evaluation", ppToken->name);
}
break;
}
token = scanToken(ppToken);
if (err)
break;
}
return token;
}
// #version: This is just for error checking: the version and profile are decided before preprocessing starts
int TPpContext::CPPversion(TPpToken* ppToken)
{
int token = scanToken(ppToken);
if (errorOnVersion || versionSeen) {
if (parseContext.isReadingHLSL())
parseContext.ppError(ppToken->loc, "invalid preprocessor command", "#version", "");
else
parseContext.ppError(ppToken->loc, "must occur first in shader", "#version", "");
}
versionSeen = true;
if (token == '\n') {
parseContext.ppError(ppToken->loc, "must be followed by version number", "#version", "");
return token;
}
if (token != PpAtomConstInt)
parseContext.ppError(ppToken->loc, "must be followed by version number", "#version", "");
ppToken->ival = atoi(ppToken->name);
int versionNumber = ppToken->ival;
int line = ppToken->loc.line;
token = scanToken(ppToken);
if (token == '\n') {
parseContext.notifyVersion(line, versionNumber, nullptr);
return token;
} else {
int profileAtom = atomStrings.getAtom(ppToken->name);
if (profileAtom != PpAtomCore &&
profileAtom != PpAtomCompatibility &&
profileAtom != PpAtomEs)
parseContext.ppError(ppToken->loc, "bad profile name; use es, core, or compatibility", "#version", "");
parseContext.notifyVersion(line, versionNumber, ppToken->name);
token = scanToken(ppToken);
if (token == '\n')
return token;
else
parseContext.ppError(ppToken->loc, "bad tokens following profile -- expected newline", "#version", "");
}
return token;
}
//
// The main functional entry-point into the preprocessor, which will
// scan the source strings to figure out and return the next processing token.
//
// Return string pointer to next token.
// Return 0 when no more tokens.
//
const char* TPpContext::tokenize(TPpToken* ppToken)
{
int token = '\n';
for(;;) {
const char* tokenString = nullptr;
token = scanToken(ppToken);
ppToken->token = token;
if (token == EOF) {
missingEndifCheck();
return nullptr;
}
if (token == '#') {
if (previous_token == '\n') {
token = readCPPline(ppToken);
if (token == EOF) {
missingEndifCheck();
return nullptr;
}
continue;
} else {
parseContext.error(ppToken->loc, "preprocessor directive cannot be preceded by another token", "#", "");
return nullptr;
}
}
previous_token = token;
if (token == '\n')
continue;
// expand macros
if (token == CPP_IDENTIFIER && MacroExpand(ppToken->atom, ppToken, false, true) != 0)
continue;
if (token == CPP_IDENTIFIER)
tokenString = GetAtomString(ppToken->atom);
else if (token == CPP_INTCONSTANT || token == CPP_UINTCONSTANT ||
token == CPP_FLOATCONSTANT || token == CPP_DOUBLECONSTANT)
tokenString = ppToken->name;
else if (token == CPP_STRCONSTANT) {
parseContext.error(ppToken->loc, "string literals not supported", "\"\"", "");
tokenString = nullptr;
} else if (token == '\'') {
parseContext.error(ppToken->loc, "character literals not supported", "\'", "");
tokenString = nullptr;
} else
tokenString = GetAtomString(token);
if (tokenString) {
if (tokenString[0] != 0)
parseContext.tokensBeforeEOF = 1;
return tokenString;
}
}
}
std::vector<Token> Scanner::scanTokens()
{
while (!isAtEnd())
{
start_ = current_;
scanToken();
}
tokens_.push_back(Token(EOF_T, "", "", line_));
return tokens_;
}
// Handle #undef
int TPpContext::CPPundef(TPpToken* ppToken)
{
int token = scanToken(ppToken);
if (token != PpAtomIdentifier) {
parseContext.ppError(ppToken->loc, "must be followed by macro name", "#undef", "");
return token;
}
parseContext.reservedPpErrorCheck(ppToken->loc, ppToken->name, "#undef");
MacroSymbol* macro = lookupMacroDef(atomStrings.getAtom(ppToken->name));
if (macro != nullptr)
macro->undef = 1;
token = scanToken(ppToken);
if (token != '\n')
parseContext.ppError(ppToken->loc, "can only be followed by a single macro name", "#undef", "");
return token;
}
// Handle #include
int TPpContext::CPPinclude(TPpToken* ppToken)
{
const TSourceLoc directiveLoc = ppToken->loc;
int token = scanToken(ppToken);
if (token != PpAtomConstString) {
// TODO: handle angle brackets.
parseContext.ppError(directiveLoc, "must be followed by a file designation", "#include", "");
} else {
// Make a copy of the name because it will be overwritten by the next token scan.
const std::string filename = ppToken->name;
token = scanToken(ppToken);
if (token != '\n' && token != EndOfInput) {
parseContext.ppError(ppToken->loc, "extra content after file designation", "#include", "");
} else {
TShader::Includer::IncludeResult* res = includer.include(filename.c_str(), TShader::Includer::EIncludeRelative, currentSourceFile.c_str(), includeStack.size() + 1);
if (res && !res->file_name.empty()) {
if (res->file_data && res->file_length) {
const bool forNextLine = parseContext.lineDirectiveShouldSetNextLine();
std::ostringstream prologue;
std::ostringstream epilogue;
prologue << "#line " << forNextLine << " " << "\"" << res->file_name << "\"\n";
epilogue << (res->file_data[res->file_length - 1] == '\n'? "" : "\n") << "#line " << directiveLoc.line + forNextLine << " " << directiveLoc.getStringNameOrNum() << "\n";
pushInput(new TokenizableIncludeFile(directiveLoc, prologue.str(), res, epilogue.str(), this));
}
// At EOF, there's no "current" location anymore.
if (token != EndOfInput) parseContext.setCurrentColumn(0);
// Don't accidentally return EndOfInput, which will end all preprocessing.
return '\n';
} else {
std::string message =
res ? std::string(res->file_data, res->file_length)
: std::string("Could not process include directive");
parseContext.ppError(directiveLoc, message.c_str(), "#include", "");
if (res) {
includer.releaseInclude(res);
}
}
}
}
return token;
}
// Handle #extension
int TPpContext::CPPextension(TPpToken* ppToken)
{
int line = ppToken->loc.line;
int token = scanToken(ppToken);
char extensionName[MaxTokenLength + 1];
if (token=='\n') {
parseContext.ppError(ppToken->loc, "extension name not specified", "#extension", "");
return token;
}
if (token != PpAtomIdentifier)
parseContext.ppError(ppToken->loc, "extension name expected", "#extension", "");
assert(strlen(ppToken->name) <= MaxTokenLength);
strcpy(extensionName, ppToken->name);
token = scanToken(ppToken);
if (token != ':') {
parseContext.ppError(ppToken->loc, "':' missing after extension name", "#extension", "");
return token;
}
token = scanToken(ppToken);
if (token != PpAtomIdentifier) {
parseContext.ppError(ppToken->loc, "behavior for extension not specified", "#extension", "");
return token;
}
parseContext.updateExtensionBehavior(line, extensionName, ppToken->name);
parseContext.notifyExtensionDirective(line, extensionName, ppToken->name);
token = scanToken(ppToken);
if (token == '\n')
return token;
else
parseContext.ppError(ppToken->loc, "extra tokens -- expected newline", "#extension","");
return token;
}
// Handle #undef
int TPpContext::CPPundef(TPpToken* ppToken)
{
int token = scanToken(ppToken);
Symbol *symb;
if (token != PpAtomIdentifier) {
parseContext.ppError(ppToken->loc, "must be followed by macro name", "#undef", "");
return token;
}
parseContext.reservedPpErrorCheck(ppToken->loc, ppToken->name, "#undef");
symb = LookUpSymbol(ppToken->atom);
if (symb) {
symb->mac.undef = 1;
}
token = scanToken(ppToken);
if (token != '\n')
parseContext.ppError(ppToken->loc, "can only be followed by a single macro name", "#undef", "");
return token;
}
// Handle #include
int TPpContext::CPPinclude(TPpToken* ppToken)
{
const TSourceLoc directiveLoc = ppToken->loc;
int token = scanToken(ppToken);
if (token != PpAtomConstString) {
// TODO: handle angle brackets.
parseContext.ppError(directiveLoc, "must be followed by a file designation", "#include", "");
} else {
// Make a copy of the name because it will be overwritten by the next token scan.
const std::string filename = ppToken->name;
token = scanToken(ppToken);
if (token != '\n' && token != EndOfInput) {
parseContext.ppError(ppToken->loc, "extra content after file designation", "#include", "");
} else {
std::string sourceName;
std::string replacement;
std::tie(sourceName, replacement) = includer.include(filename.c_str());
if (!sourceName.empty()) {
if (!replacement.empty()) {
const bool forNextLine = parseContext.lineDirectiveShouldSetNextLine();
std::ostringstream content;
content << "#line " << forNextLine << " " << "\"" << sourceName << "\"\n";
content << replacement << (replacement.back() == '\n' ? "" : "\n");
content << "#line " << directiveLoc.line + forNextLine << " " << directiveLoc.getStringNameOrNum() << "\n";
pushInput(new TokenizableString(directiveLoc, content.str(), this));
}
// At EOF, there's no "current" location anymore.
if (token != EndOfInput) parseContext.setCurrentColumn(0);
// Don't accidentally return EndOfInput, which will end all preprocessing.
return '\n';
} else {
parseContext.ppError(directiveLoc, replacement.c_str(), "#include", "");
}
}
}
return token;
}
// Macro-expand a macro argument 'arg' to create 'expandedArg'.
// Does not replace 'arg'.
// Returns nullptr if no expanded argument is created.
TPpContext::TokenStream* TPpContext::PrescanMacroArg(TokenStream& arg, TPpToken* ppToken, bool newLineOkay)
{
// expand the argument
TokenStream* expandedArg = new TokenStream;
pushInput(new tMarkerInput(this));
pushTokenStreamInput(arg);
int token;
while ((token = scanToken(ppToken)) != tMarkerInput::marker && token != EndOfInput) {
token = tokenPaste(token, *ppToken);
if (token == PpAtomIdentifier) {
switch (MacroExpand(ppToken, false, newLineOkay)) {
case MacroExpandNotStarted:
break;
case MacroExpandError:
// toss the rest of the pushed-input argument by scanning until tMarkerInput
while ((token = scanToken(ppToken)) != tMarkerInput::marker && token != EndOfInput)
;
break;
case MacroExpandStarted:
case MacroExpandUndef:
continue;
}
}
if (token == tMarkerInput::marker || token == EndOfInput)
break;
expandedArg->putToken(token, ppToken);
}
if (token != tMarkerInput::marker) {
// Error, or MacroExpand ate the marker, so had bad input, recover
delete expandedArg;
expandedArg = nullptr;
}
return expandedArg;
}
// fetchToken() will return the first token in the lookahead store (if the
// lookahead store has tokens) or it will get a new token from the frontier
int Scanner::fetchToken(ScannerToken &t, Location &l) {
m_token = &t;
m_loc = &l;
int tokid;
if (!m_lookahead.empty()) {
// If there is a lookahead token, return that. No need to perform
// special logic for "ReturnAllTokens", we already accounted for
// that when the tokens were inserted into m_lookahead
TokenStore::iterator it = m_lookahead.begin();
tokid = it->t;
*m_token = it->token;
*m_loc = it->loc;
return tokid;
}
return scanToken(t,l);
}
// Handle #if
int TPpContext::CPPif(TPpToken* ppToken)
{
int token = scanToken(ppToken);
elsetracker++;
ifdepth++;
if (ifdepth > maxIfNesting) {
parseContext.ppError(ppToken->loc, "maximum nesting depth exceeded", "#if", "");
return 0;
}
int res = 0;
bool err = false;
token = eval(token, MIN_PRECEDENCE, false, res, err, ppToken);
token = extraTokenCheck(PpAtomIf, ppToken, token);
if (!res && !err)
token = CPPelse(1, ppToken);
return token;
}
// nextLookahead() advances an iterator forward in the lookahead store.
// If the end of the store is reached, a new token will be scanned from
// the frontier. nextLookahead skips over whitespace and comments.
void Scanner::nextLookahead(TokenStore::iterator& pos) {
for (;;) {
++pos;
if (pos == m_lookahead.end()) {
pos = m_lookahead.appendNew();
pos->loc = *m_loc;
pos->t = scanToken(pos->token, pos->loc);
}
switch (pos->t) {
case T_DOC_COMMENT:
case T_COMMENT:
case T_OPEN_TAG:
case T_WHITESPACE:
break;
default:
return;
}
}
}
void scanLine(int line_no, char *line, FILE *result) {
strtrim(line);
if (strlen(line) == 0) return;
int token_starts[128] = {0};
int token_count = getTokenCountByStartIndexs(line, token_starts);
int i;
char token[32];
int start, len;
for (i = 0; i < token_count; i++) {
start = token_starts[i];
if (line[start] == ' ') continue;
len = token_starts[i + 1] - start;
substr(line, token, start, len);
scanToken(token, line_no);
}
}
void Scanner::scan (Parser& parser)
{
while (scanToken (parser));
}
//
// Do all token-pasting related combining of two pasted tokens when getting a
// stream of tokens from a replacement list. Degenerates to no processing if a
// replacement list is not the source of the token stream.
//
int TPpContext::tokenPaste(int token, TPpToken& ppToken)
{
// starting with ## is illegal, skip to next token
if (token == PpAtomPaste) {
_parseContext.ppError(ppToken.loc, "unexpected location", "##", "");
return scanToken(&ppToken);
}
int resultToken = token; // "foo" pasted with "35" is an identifier, not a number
// ## can be chained, process all in the chain at once
while (peekPasting()) {
TPpToken pastedPpToken;
// next token has to be ##
token = scanToken(&pastedPpToken);
assert(token == PpAtomPaste);
// This covers end of macro expansion
if (endOfReplacementList()) {
_parseContext.ppError(ppToken.loc, "unexpected location; end of replacement list", "##", "");
break;
}
// get the token after the ##
token = scanToken(&pastedPpToken);
// This covers end of argument expansion
if (token == tMarkerInput::marker) {
_parseContext.ppError(ppToken.loc, "unexpected location; end of argument", "##", "");
break;
}
// get the token text
switch (resultToken) {
case PpAtomIdentifier:
// already have the correct text in token.names
break;
case '=':
case '!':
case '-':
case '~':
case '+':
case '*':
case '/':
case '%':
case '<':
case '>':
case '|':
case '^':
case '&':
case PpAtomRight:
case PpAtomLeft:
case PpAtomAnd:
case PpAtomOr:
case PpAtomXor:
strcpy(ppToken.name, atomStrings.getString(resultToken));
strcpy(pastedPpToken.name, atomStrings.getString(token));
break;
default:
_parseContext.ppError(ppToken.loc, "not supported for these tokens", "##", "");
return resultToken;
}
// combine the tokens
if (strlen(ppToken.name) + strlen(pastedPpToken.name) > MaxTokenLength) {
_parseContext.ppError(ppToken.loc, "combined tokens are too long", "##", "");
return resultToken;
}
strncat(ppToken.name, pastedPpToken.name, MaxTokenLength - strlen(ppToken.name));
// correct the kind of token we are making, if needed (identifiers stay identifiers)
if (resultToken != PpAtomIdentifier) {
int newToken = atomStrings.getAtom(ppToken.name);
if (newToken > 0)
resultToken = newToken;
else
_parseContext.ppError(ppToken.loc, "combined token is invalid", "##", "");
}
}
return resultToken;
}
int TPpContext::readCPPline(TPpToken* ppToken)
{
int token = scanToken(ppToken);
if (token == PpAtomIdentifier) {
switch (ppToken->atom) {
case PpAtomDefine:
token = CPPdefine(ppToken);
break;
case PpAtomElse:
if (elsetracker[elseSeen])
parseContext.ppError(ppToken->loc, "#else after #else", "#else", "");
elsetracker[elseSeen] = true;
if (! ifdepth)
parseContext.ppError(ppToken->loc, "mismatched statements", "#else", "");
token = extraTokenCheck(PpAtomElse, ppToken, scanToken(ppToken));
token = CPPelse(0, ppToken);
break;
case PpAtomElif:
if (! ifdepth)
parseContext.ppError(ppToken->loc, "mismatched statements", "#elif", "");
if (elseSeen[elsetracker])
parseContext.ppError(ppToken->loc, "#elif after #else", "#elif", "");
// this token is really a dont care, but we still need to eat the tokens
token = scanToken(ppToken);
while (token != '\n' && token != EndOfInput)
token = scanToken(ppToken);
token = CPPelse(0, ppToken);
break;
case PpAtomEndif:
elseSeen[elsetracker] = false;
--elsetracker;
if (! ifdepth)
parseContext.ppError(ppToken->loc, "mismatched statements", "#endif", "");
else
--ifdepth;
token = extraTokenCheck(PpAtomEndif, ppToken, scanToken(ppToken));
break;
case PpAtomIf:
token = CPPif(ppToken);
break;
case PpAtomIfdef:
token = CPPifdef(1, ppToken);
break;
case PpAtomIfndef:
token = CPPifdef(0, ppToken);
break;
case PpAtomInclude:
parseContext.ppRequireExtensions(ppToken->loc, 1, &E_GL_GOOGLE_include_directive, "#include");
token = CPPinclude(ppToken);
break;
case PpAtomLine:
token = CPPline(ppToken);
break;
case PpAtomPragma:
token = CPPpragma(ppToken);
break;
case PpAtomUndef:
token = CPPundef(ppToken);
break;
case PpAtomError:
token = CPPerror(ppToken);
break;
case PpAtomVersion:
token = CPPversion(ppToken);
break;
case PpAtomExtension:
token = CPPextension(ppToken);
break;
default:
parseContext.ppError(ppToken->loc, "invalid directive:", "#", ppToken->name);
break;
}
} else if (token != '\n' && token != EndOfInput)
parseContext.ppError(ppToken->loc, "invalid directive", "#", "");
while (token != '\n' && token != EndOfInput)
token = scanToken(ppToken);
return token;
}
//
// Check an identifier (atom) to see if it is a macro that should be expanded.
// If it is, and defined, push a tInput that will produce the appropriate expansion
// and return 1.
// If it is, but undefined, and expandUndef is requested, push a tInput that will
// expand to 0 and return -1.
// Otherwise, return 0 to indicate no expansion, which is not necessarily an error.
//
int TPpContext::MacroExpand(int atom, TPpToken* ppToken, bool expandUndef, bool newLineOkay)
{
ppToken->space = false;
switch (atom) {
case PpAtomLineMacro:
ppToken->ival = parseContext.getCurrentLoc().line;
sprintf(ppToken->name, "%d", ppToken->ival);
UngetToken(PpAtomConstInt, ppToken);
return 1;
case PpAtomFileMacro: {
if (parseContext.getCurrentLoc().name)
parseContext.ppRequireExtensions(ppToken->loc, 1, &E_GL_GOOGLE_cpp_style_line_directive, "filename-based __FILE__");
ppToken->ival = parseContext.getCurrentLoc().string;
sprintf(ppToken->name, "%s", ppToken->loc.getStringNameOrNum().c_str());
UngetToken(PpAtomConstInt, ppToken);
return 1;
}
case PpAtomVersionMacro:
ppToken->ival = parseContext.version;
sprintf(ppToken->name, "%d", ppToken->ival);
UngetToken(PpAtomConstInt, ppToken);
return 1;
default:
break;
}
Symbol *sym = LookUpSymbol(atom);
int token;
int depth = 0;
// no recursive expansions
if (sym && sym->mac.busy)
return 0;
// not expanding undefined macros
if ((! sym || sym->mac.undef) && ! expandUndef)
return 0;
// 0 is the value of an undefined macro
if ((! sym || sym->mac.undef) && expandUndef) {
pushInput(new tZeroInput(this));
return -1;
}
tMacroInput *in = new tMacroInput(this);
TSourceLoc loc = ppToken->loc; // in case we go to the next line before discovering the error
in->mac = &sym->mac;
if (sym->mac.args) {
token = scanToken(ppToken);
if (newLineOkay) {
while (token == '\n')
token = scanToken(ppToken);
}
if (token != '(') {
parseContext.ppError(loc, "expected '(' following", "macro expansion", GetAtomString(atom));
UngetToken(token, ppToken);
ppToken->atom = atom;
delete in;
return 0;
}
in->args.resize(in->mac->argc);
for (int i = 0; i < in->mac->argc; i++)
in->args[i] = new TokenStream;
int arg = 0;
bool tokenRecorded = false;
do {
depth = 0;
while (1) {
token = scanToken(ppToken);
if (token == EndOfInput) {
parseContext.ppError(loc, "End of input in macro", "macro expansion", GetAtomString(atom));
delete in;
return 0;
}
if (token == '\n') {
if (! newLineOkay) {
parseContext.ppError(loc, "End of line in macro substitution:", "macro expansion", GetAtomString(atom));
delete in;
return 0;
}
continue;
}
if (token == '#') {
parseContext.ppError(ppToken->loc, "unexpected '#'", "macro expansion", GetAtomString(atom));
delete in;
return 0;
}
if (in->mac->argc == 0 && token != ')')
break;
if (depth == 0 && (token == ',' || token == ')'))
break;
if (token == '(')
depth++;
if (token == ')')
depth--;
RecordToken(in->args[arg], token, ppToken);
tokenRecorded = true;
}
if (token == ')') {
if (in->mac->argc == 1 && tokenRecorded == 0)
break;
arg++;
break;
}
arg++;
} while (arg < in->mac->argc);
if (arg < in->mac->argc)
parseContext.ppError(loc, "Too few args in Macro", "macro expansion", GetAtomString(atom));
else if (token != ')') {
depth=0;
while (token != EndOfInput && (depth > 0 || token != ')')) {
if (token == ')')
depth--;
token = scanToken(ppToken);
if (token == '(')
depth++;
}
if (token == EndOfInput) {
parseContext.ppError(loc, "End of input in macro", "macro expansion", GetAtomString(atom));
delete in;
return 0;
}
parseContext.ppError(loc, "Too many args in macro", "macro expansion", GetAtomString(atom));
}
for (int i = 0; i < in->mac->argc; i++)
in->args[i] = PrescanMacroArg(in->args[i], ppToken, newLineOkay);
}
pushInput(in);
sym->mac.busy = 1;
RewindTokenStream(sym->mac.body);
return 1;
}
// Handle #define
int TPpContext::CPPdefine(TPpToken* ppToken)
{
MacroSymbol mac;
Symbol *symb;
// get macro name
int token = scanToken(ppToken);
if (token != PpAtomIdentifier) {
parseContext.ppError(ppToken->loc, "must be followed by macro name", "#define", "");
return token;
}
if (ppToken->loc.string >= 0) {
// We are in user code; check for reserved name use:
parseContext.reservedPpErrorCheck(ppToken->loc, ppToken->name, "#define");
}
// save the original atom
const int defAtom = ppToken->atom;
// gather parameters to the macro, between (...)
token = scanToken(ppToken);
if (token == '(' && ! ppToken->space) {
int argc = 0;
int args[maxMacroArgs];
do {
token = scanToken(ppToken);
if (argc == 0 && token == ')')
break;
if (token != PpAtomIdentifier) {
parseContext.ppError(ppToken->loc, "bad argument", "#define", "");
return token;
}
// check for duplication of parameter name
bool duplicate = false;
for (int a = 0; a < argc; ++a) {
if (args[a] == ppToken->atom) {
parseContext.ppError(ppToken->loc, "duplicate macro parameter", "#define", "");
duplicate = true;
break;
}
}
if (! duplicate) {
if (argc < maxMacroArgs)
args[argc++] = ppToken->atom;
else
parseContext.ppError(ppToken->loc, "too many macro parameters", "#define", "");
}
token = scanToken(ppToken);
} while (token == ',');
if (token != ')') {
parseContext.ppError(ppToken->loc, "missing parenthesis", "#define", "");
return token;
}
mac.argc = argc;
mac.args = (int*)mem_Alloc(pool, argc * sizeof(int));
memcpy(mac.args, args, argc * sizeof(int));
token = scanToken(ppToken);
}
// record the definition of the macro
TSourceLoc defineLoc = ppToken->loc; // because ppToken is going to go to the next line before we report errors
mac.body = new TokenStream;
while (token != '\n' && token != EndOfInput) {
RecordToken(mac.body, token, ppToken);
token = scanToken(ppToken);
if (token != '\n' && ppToken->space)
RecordToken(mac.body, ' ', ppToken);
}
// check for duplicate definition
symb = LookUpSymbol(defAtom);
if (symb) {
if (! symb->mac.undef) {
// Already defined -- need to make sure they are identical:
// "Two replacement lists are identical if and only if the preprocessing tokens in both have the same number,
// ordering, spelling, and white-space separation, where all white-space separations are considered identical."
if (symb->mac.argc != mac.argc)
parseContext.ppError(defineLoc, "Macro redefined; different number of arguments:", "#define", GetAtomString(defAtom));
else {
for (int argc = 0; argc < mac.argc; argc++) {
if (symb->mac.args[argc] != mac.args[argc])
parseContext.ppError(defineLoc, "Macro redefined; different argument names:", "#define", GetAtomString(defAtom));
}
RewindTokenStream(symb->mac.body);
RewindTokenStream(mac.body);
int newToken;
do {
int oldToken;
TPpToken oldPpToken;
TPpToken newPpToken;
oldToken = ReadToken(symb->mac.body, &oldPpToken);
newToken = ReadToken(mac.body, &newPpToken);
if (oldToken != newToken || oldPpToken != newPpToken) {
parseContext.ppError(defineLoc, "Macro redefined; different substitutions:", "#define", GetAtomString(defAtom));
break;
}
} while (newToken > 0);
}
}
} else
symb = AddSymbol(defAtom);
delete symb->mac.body;
symb->mac = mac;
return '\n';
}
/* Skip forward to appropriate spot. This is used both
** to skip to a #endif after seeing an #else, AND to skip to a #else,
** #elif, or #endif after a #if/#ifdef/#ifndef/#elif test was false.
*/
int TPpContext::CPPelse(int matchelse, TPpToken* ppToken)
{
int atom;
int depth = 0;
int token = scanToken(ppToken);
while (token != EndOfInput) {
if (token != '#') {
while (token != '\n' && token != EndOfInput)
token = scanToken(ppToken);
if (token == EndOfInput)
return token;
token = scanToken(ppToken);
continue;
}
if ((token = scanToken(ppToken)) != PpAtomIdentifier)
continue;
atom = ppToken->atom;
if (atom == PpAtomIf || atom == PpAtomIfdef || atom == PpAtomIfndef) {
depth++;
ifdepth++;
elsetracker++;
} else if (atom == PpAtomEndif) {
token = extraTokenCheck(atom, ppToken, scanToken(ppToken));
elseSeen[elsetracker] = false;
--elsetracker;
if (depth == 0) {
// found the #endif we are looking for
if (ifdepth)
--ifdepth;
break;
}
--depth;
--ifdepth;
} else if (matchelse && depth == 0) {
if (atom == PpAtomElse) {
elseSeen[elsetracker] = true;
token = extraTokenCheck(atom, ppToken, scanToken(ppToken));
// found the #else we are looking for
break;
} else if (atom == PpAtomElif) {
if (elseSeen[elsetracker])
parseContext.ppError(ppToken->loc, "#elif after #else", "#elif", "");
/* we decrement ifdepth here, because CPPif will increment
* it and we really want to leave it alone */
if (ifdepth) {
--ifdepth;
elseSeen[elsetracker] = false;
--elsetracker;
}
return CPPif(ppToken);
}
} else if (atom == PpAtomElse) {
if (elseSeen[elsetracker])
parseContext.ppError(ppToken->loc, "#else after #else", "#else", "");
else
elseSeen[elsetracker] = true;
token = extraTokenCheck(atom, ppToken, scanToken(ppToken));
} else if (atom == PpAtomElif) {
if (elseSeen[elsetracker])
parseContext.ppError(ppToken->loc, "#elif after #else", "#elif", "");
}
}
return token;
}
int TPpContext::eval(int token, int precedence, bool shortCircuit, int& res, bool& err, TPpToken* ppToken)
{
TSourceLoc loc = ppToken->loc; // because we sometimes read the newline before reporting the error
if (token == PpAtomIdentifier) {
if (ppToken->atom == PpAtomDefined) {
bool needclose = 0;
token = scanToken(ppToken);
if (token == '(') {
needclose = true;
token = scanToken(ppToken);
}
if (token != PpAtomIdentifier) {
parseContext.ppError(loc, "incorrect directive, expected identifier", "preprocessor evaluation", "");
err = true;
res = 0;
return token;
}
Symbol* s = LookUpSymbol(ppToken->atom);
res = s ? ! s->mac.undef : 0;
token = scanToken(ppToken);
if (needclose) {
if (token != ')') {
parseContext.ppError(loc, "expected ')'", "preprocessor evaluation", "");
err = true;
res = 0;
return token;
}
token = scanToken(ppToken);
}
} else {
token = evalToToken(token, shortCircuit, res, err, ppToken);
return eval(token, precedence, shortCircuit, res, err, ppToken);
}
} else if (token == PpAtomConstInt) {
res = ppToken->ival;
token = scanToken(ppToken);
} else if (token == '(') {
token = scanToken(ppToken);
token = eval(token, MIN_PRECEDENCE, shortCircuit, res, err, ppToken);
if (! err) {
if (token != ')') {
parseContext.ppError(loc, "expected ')'", "preprocessor evaluation", "");
err = true;
res = 0;
return token;
}
token = scanToken(ppToken);
}
} else {
int op = NUM_ELEMENTS(unop) - 1;
for (; op >= 0; op--) {
if (unop[op].token == token)
break;
}
if (op >= 0) {
token = scanToken(ppToken);
token = eval(token, UNARY, shortCircuit, res, err, ppToken);
res = unop[op].op(res);
} else {
parseContext.ppError(loc, "bad expression", "preprocessor evaluation", "");
err = true;
res = 0;
return token;
}
}
token = evalToToken(token, shortCircuit, res, err, ppToken);
// Perform evaluation of binary operation, if there is one, otherwise we are done.
while (! err) {
if (token == ')' || token == '\n')
break;
int op;
for (op = NUM_ELEMENTS(binop) - 1; op >= 0; op--) {
if (binop[op].token == token)
break;
}
if (op < 0 || binop[op].precedence <= precedence)
break;
int leftSide = res;
// Setup short-circuiting, needed for ES, unless already in a short circuit.
// (Once in a short-circuit, can't turn off again, until that whole subexpression is done.
if (! shortCircuit) {
if ((token == PpAtomOr && leftSide == 1) ||
(token == PpAtomAnd && leftSide == 0))
shortCircuit = true;
}
token = scanToken(ppToken);
token = eval(token, binop[op].precedence, shortCircuit, res, err, ppToken);
if (binop[op].op == op_div || binop[op].op == op_mod) {
if (res == 0) {
parseContext.ppError(loc, "division by 0", "preprocessor evaluation", "");
res = 1;
}
}
res = binop[op].op(leftSide, res);
}
return token;
}
