// Handle #pragma int TPpContext::CPPpragma(TPpToken* ppToken) { char SrcStrName[2]; TVector<TString> tokens; TSourceLoc loc = ppToken->loc; // because we go to the next line before processing int token = scanToken(ppToken); while (token != '\n' && token != EndOfInput) { switch (token) { case PpAtomIdentifier: case PpAtomConstInt: case PpAtomConstUint: case PpAtomConstFloat: case PpAtomConstDouble: tokens.push_back(ppToken->name); break; default: SrcStrName[0] = (char)token; SrcStrName[1] = '\0'; tokens.push_back(SrcStrName); } token = scanToken(ppToken); } if (token == EndOfInput) parseContext.ppError(loc, "directive must end with a newline", "#pragma", ""); else parseContext.handlePragma(loc, tokens); return token; }
// Handle #error int TPpContext::CPPerror(TPpToken* ppToken) { int token = scanToken(ppToken); std::string message; TSourceLoc loc = ppToken->loc; while (token != '\n' && token != EndOfInput) { if (token == PpAtomConstInt || token == PpAtomConstUint || token == PpAtomConstInt64 || token == PpAtomConstUint64 || #ifdef AMD_EXTENSIONS token == PpAtomConstFloat16 || #endif token == PpAtomConstFloat || token == PpAtomConstDouble) { message.append(ppToken->name); } else if (token == PpAtomIdentifier || token == PpAtomConstString) { message.append(ppToken->name); } else { message.append(GetAtomString(token)); } message.append(" "); token = scanToken(ppToken); } parseContext.notifyErrorDirective(loc.line, message.c_str()); //store this msg into the shader's information log..set the Compile Error flag!!!! parseContext.ppError(loc, message.c_str(), "#error", ""); return '\n'; }
// Handle #ifdef int TPpContext::CPPifdef(int defined, TPpToken* ppToken) { int token = scanToken(ppToken); if (++ifdepth > maxIfNesting) { parseContext.ppError(ppToken->loc, "maximum nesting depth exceeded", "#ifdef", ""); return 0; } elsetracker++; if (token != PpAtomIdentifier) { if (defined) parseContext.ppError(ppToken->loc, "must be followed by macro name", "#ifdef", ""); else parseContext.ppError(ppToken->loc, "must be followed by macro name", "#ifndef", ""); } else { MacroSymbol* macro = lookupMacroDef(atomStrings.getAtom(ppToken->name)); token = scanToken(ppToken); if (token != '\n') { parseContext.ppError(ppToken->loc, "unexpected tokens following #ifdef directive - expected a newline", "#ifdef", ""); while (token != '\n' && token != EndOfInput) token = scanToken(ppToken); } if (((macro != nullptr && !macro->undef) ? 1 : 0) != defined) token = CPPelse(1, ppToken); } return token; }
// Handle #ifdef int TPpContext::CPPifdef(int defined, TPpToken* ppToken) { int token = scanToken(ppToken); int name = ppToken->atom; if (++ifdepth > maxIfNesting) { parseContext.ppError(ppToken->loc, "maximum nesting depth exceeded", "#ifdef", ""); return 0; } elsetracker++; if (token != PpAtomIdentifier) { if (defined) parseContext.ppError(ppToken->loc, "must be followed by macro name", "#ifdef", ""); else parseContext.ppError(ppToken->loc, "must be followed by macro name", "#ifndef", ""); } else { Symbol *s = LookUpSymbol(name); token = scanToken(ppToken); if (token != '\n') { parseContext.ppError(ppToken->loc, "unexpected tokens following #ifdef directive - expected a newline", "#ifdef", ""); while (token != '\n' && token != EndOfInput) token = scanToken(ppToken); } if (((s && !s->mac.undef) ? 1 : 0) != defined) token = CPPelse(1, ppToken); } return token; }
// Expand macros, skipping empty expansions, to get to the first real token in those expansions. int TPpContext::evalToToken(int token, bool shortCircuit, int& res, bool& err, TPpToken* ppToken) { while (token == PpAtomIdentifier && ppToken->atom != PpAtomDefined) { int macroReturn = MacroExpand(ppToken->atom, ppToken, true, false); if (macroReturn == 0) { parseContext.ppError(ppToken->loc, "can't evaluate expression", "preprocessor evaluation", ""); err = true; res = 0; token = scanToken(ppToken); break; } if (macroReturn == -1) { if (! shortCircuit && parseContext.profile == EEsProfile) { const char* message = "undefined macro in expression not allowed in es profile"; if (parseContext.relaxedErrors()) parseContext.ppWarn(ppToken->loc, message, "preprocessor evaluation", ppToken->name); else parseContext.ppError(ppToken->loc, message, "preprocessor evaluation", ppToken->name); } } token = scanToken(ppToken); } return token; }
// Handle #line int TPpContext::CPPline(TPpToken* ppToken) { // "#line must have, after macro substitution, one of the following forms: // "#line line // "#line line source-string-number" int token = scanToken(ppToken); const TSourceLoc directiveLoc = ppToken->loc; if (token == '\n') { parseContext.ppError(ppToken->loc, "must by followed by an integral literal", "#line", ""); return token; } int lineRes = 0; // Line number after macro expansion. int lineToken = 0; bool hasFile = false; int fileRes = 0; // Source file number after macro expansion. const char* sourceName = nullptr; // Optional source file name. bool lineErr = false; bool fileErr = false; token = eval(token, MIN_PRECEDENCE, false, lineRes, lineErr, ppToken); if (! lineErr) { lineToken = lineRes; if (token == '\n') ++lineRes; if (parseContext.lineDirectiveShouldSetNextLine()) --lineRes; parseContext.setCurrentLine(lineRes); if (token != '\n') { if (token == PpAtomConstString) { parseContext.ppRequireExtensions(directiveLoc, 1, &E_GL_GOOGLE_cpp_style_line_directive, "filename-based #line"); // We need to save a copy of the string instead of pointing // to the name field of the token since the name field // will likely be overwritten by the next token scan. sourceName = GetAtomString(LookUpAddString(ppToken->name)); parseContext.setCurrentSourceName(sourceName); hasFile = true; token = scanToken(ppToken); } else { token = eval(token, MIN_PRECEDENCE, false, fileRes, fileErr, ppToken); if (! fileErr) { parseContext.setCurrentString(fileRes); hasFile = true; } } } } if (!fileErr && !lineErr) { parseContext.notifyLineDirective(directiveLoc.line, lineToken, hasFile, fileRes, sourceName); } token = extraTokenCheck(PpAtomLine, ppToken, token); return token; }
TPpContext::TokenStream* TPpContext::PrescanMacroArg(TokenStream* a, TPpToken* ppToken, bool newLineOkay) { int token; TokenStream *n; RewindTokenStream(a); do { token = ReadToken(a, ppToken); if (token == PpAtomIdentifier && LookUpSymbol(ppToken->atom)) break; } while (token != EndOfInput); if (token == EndOfInput) return a; n = new TokenStream; pushInput(new tMarkerInput(this)); pushTokenStreamInput(a); while ((token = scanToken(ppToken)) != tMarkerInput::marker) { if (token == PpAtomIdentifier && MacroExpand(ppToken->atom, ppToken, false, newLineOkay) != 0) continue; RecordToken(n, token, ppToken); } popInput(); delete a; return n; }
// Macro-expand a macro argument 'arg' to create 'expandedArg'. // Does not replace 'arg'. // Returns nullptr if no expanded argument is created. TPpContext::TokenStream* TPpContext::PrescanMacroArg(TokenStream& arg, TPpToken* ppToken, bool newLineOkay) { // expand the argument TokenStream* expandedArg = new TokenStream; pushInput(new tMarkerInput(this)); pushTokenStreamInput(arg); int token; while ((token = scanToken(ppToken)) != tMarkerInput::marker && token != EndOfInput) { token = tokenPaste(token, *ppToken); if (token == tMarkerInput::marker || token == EndOfInput) break; if (token == PpAtomIdentifier && MacroExpand(ppToken, false, newLineOkay) != 0) continue; expandedArg->putToken(token, ppToken); } if (token == EndOfInput) { // MacroExpand ate the marker, so had bad input, recover delete expandedArg; expandedArg = nullptr; } else { // remove the marker popInput(); } return expandedArg; }
// Call when there should be no more tokens left on a line. int TPpContext::extraTokenCheck(int atom, TPpToken* ppToken, int token) { if (token != '\n' && token != EndOfInput) { static const char* message = "unexpected tokens following directive"; const char* label; if (atom == PpAtomElse) label = "#else"; else if (atom == PpAtomElif) label = "#elif"; else if (atom == PpAtomEndif) label = "#endif"; else if (atom == PpAtomIf) label = "#if"; else if (atom == PpAtomLine) label = "#line"; else label = ""; if (parseContext.relaxedErrors()) parseContext.ppWarn(ppToken->loc, message, label, ""); else parseContext.ppError(ppToken->loc, message, label, ""); while (token != '\n' && token != EndOfInput) token = scanToken(ppToken); } return token; }
// Expand macros, skipping empty expansions, to get to the first real token in those expansions. int TPpContext::evalToToken(int token, bool shortCircuit, int& res, bool& err, TPpToken* ppToken) { while (token == PpAtomIdentifier && strcmp("defined", ppToken->name) != 0) { switch (MacroExpand(ppToken, true, false)) { case MacroExpandNotStarted: case MacroExpandError: parseContext.ppError(ppToken->loc, "can't evaluate expression", "preprocessor evaluation", ""); err = true; res = 0; break; case MacroExpandStarted: break; case MacroExpandUndef: if (! shortCircuit && parseContext.profile == EEsProfile) { const char* message = "undefined macro in expression not allowed in es profile"; if (parseContext.relaxedErrors()) parseContext.ppWarn(ppToken->loc, message, "preprocessor evaluation", ppToken->name); else parseContext.ppError(ppToken->loc, message, "preprocessor evaluation", ppToken->name); } break; } token = scanToken(ppToken); if (err) break; } return token; }
// #version: This is just for error checking: the version and profile are decided before preprocessing starts int TPpContext::CPPversion(TPpToken* ppToken) { int token = scanToken(ppToken); if (errorOnVersion || versionSeen) { if (parseContext.isReadingHLSL()) parseContext.ppError(ppToken->loc, "invalid preprocessor command", "#version", ""); else parseContext.ppError(ppToken->loc, "must occur first in shader", "#version", ""); } versionSeen = true; if (token == '\n') { parseContext.ppError(ppToken->loc, "must be followed by version number", "#version", ""); return token; } if (token != PpAtomConstInt) parseContext.ppError(ppToken->loc, "must be followed by version number", "#version", ""); ppToken->ival = atoi(ppToken->name); int versionNumber = ppToken->ival; int line = ppToken->loc.line; token = scanToken(ppToken); if (token == '\n') { parseContext.notifyVersion(line, versionNumber, nullptr); return token; } else { int profileAtom = atomStrings.getAtom(ppToken->name); if (profileAtom != PpAtomCore && profileAtom != PpAtomCompatibility && profileAtom != PpAtomEs) parseContext.ppError(ppToken->loc, "bad profile name; use es, core, or compatibility", "#version", ""); parseContext.notifyVersion(line, versionNumber, ppToken->name); token = scanToken(ppToken); if (token == '\n') return token; else parseContext.ppError(ppToken->loc, "bad tokens following profile -- expected newline", "#version", ""); } return token; }
// // The main functional entry-point into the preprocessor, which will // scan the source strings to figure out and return the next processing token. // // Return string pointer to next token. // Return 0 when no more tokens. // const char* TPpContext::tokenize(TPpToken* ppToken) { int token = '\n'; for(;;) { const char* tokenString = nullptr; token = scanToken(ppToken); ppToken->token = token; if (token == EOF) { missingEndifCheck(); return nullptr; } if (token == '#') { if (previous_token == '\n') { token = readCPPline(ppToken); if (token == EOF) { missingEndifCheck(); return nullptr; } continue; } else { parseContext.error(ppToken->loc, "preprocessor directive cannot be preceded by another token", "#", ""); return nullptr; } } previous_token = token; if (token == '\n') continue; // expand macros if (token == CPP_IDENTIFIER && MacroExpand(ppToken->atom, ppToken, false, true) != 0) continue; if (token == CPP_IDENTIFIER) tokenString = GetAtomString(ppToken->atom); else if (token == CPP_INTCONSTANT || token == CPP_UINTCONSTANT || token == CPP_FLOATCONSTANT || token == CPP_DOUBLECONSTANT) tokenString = ppToken->name; else if (token == CPP_STRCONSTANT) { parseContext.error(ppToken->loc, "string literals not supported", "\"\"", ""); tokenString = nullptr; } else if (token == '\'') { parseContext.error(ppToken->loc, "character literals not supported", "\'", ""); tokenString = nullptr; } else tokenString = GetAtomString(token); if (tokenString) { if (tokenString[0] != 0) parseContext.tokensBeforeEOF = 1; return tokenString; } } }
std::vector<Token> Scanner::scanTokens() { while (!isAtEnd()) { start_ = current_; scanToken(); } tokens_.push_back(Token(EOF_T, "", "", line_)); return tokens_; }
// Handle #undef int TPpContext::CPPundef(TPpToken* ppToken) { int token = scanToken(ppToken); if (token != PpAtomIdentifier) { parseContext.ppError(ppToken->loc, "must be followed by macro name", "#undef", ""); return token; } parseContext.reservedPpErrorCheck(ppToken->loc, ppToken->name, "#undef"); MacroSymbol* macro = lookupMacroDef(atomStrings.getAtom(ppToken->name)); if (macro != nullptr) macro->undef = 1; token = scanToken(ppToken); if (token != '\n') parseContext.ppError(ppToken->loc, "can only be followed by a single macro name", "#undef", ""); return token; }
// Handle #include int TPpContext::CPPinclude(TPpToken* ppToken) { const TSourceLoc directiveLoc = ppToken->loc; int token = scanToken(ppToken); if (token != PpAtomConstString) { // TODO: handle angle brackets. parseContext.ppError(directiveLoc, "must be followed by a file designation", "#include", ""); } else { // Make a copy of the name because it will be overwritten by the next token scan. const std::string filename = ppToken->name; token = scanToken(ppToken); if (token != '\n' && token != EndOfInput) { parseContext.ppError(ppToken->loc, "extra content after file designation", "#include", ""); } else { TShader::Includer::IncludeResult* res = includer.include(filename.c_str(), TShader::Includer::EIncludeRelative, currentSourceFile.c_str(), includeStack.size() + 1); if (res && !res->file_name.empty()) { if (res->file_data && res->file_length) { const bool forNextLine = parseContext.lineDirectiveShouldSetNextLine(); std::ostringstream prologue; std::ostringstream epilogue; prologue << "#line " << forNextLine << " " << "\"" << res->file_name << "\"\n"; epilogue << (res->file_data[res->file_length - 1] == '\n'? "" : "\n") << "#line " << directiveLoc.line + forNextLine << " " << directiveLoc.getStringNameOrNum() << "\n"; pushInput(new TokenizableIncludeFile(directiveLoc, prologue.str(), res, epilogue.str(), this)); } // At EOF, there's no "current" location anymore. if (token != EndOfInput) parseContext.setCurrentColumn(0); // Don't accidentally return EndOfInput, which will end all preprocessing. return '\n'; } else { std::string message = res ? std::string(res->file_data, res->file_length) : std::string("Could not process include directive"); parseContext.ppError(directiveLoc, message.c_str(), "#include", ""); if (res) { includer.releaseInclude(res); } } } } return token; }
// Handle #extension int TPpContext::CPPextension(TPpToken* ppToken) { int line = ppToken->loc.line; int token = scanToken(ppToken); char extensionName[MaxTokenLength + 1]; if (token=='\n') { parseContext.ppError(ppToken->loc, "extension name not specified", "#extension", ""); return token; } if (token != PpAtomIdentifier) parseContext.ppError(ppToken->loc, "extension name expected", "#extension", ""); assert(strlen(ppToken->name) <= MaxTokenLength); strcpy(extensionName, ppToken->name); token = scanToken(ppToken); if (token != ':') { parseContext.ppError(ppToken->loc, "':' missing after extension name", "#extension", ""); return token; } token = scanToken(ppToken); if (token != PpAtomIdentifier) { parseContext.ppError(ppToken->loc, "behavior for extension not specified", "#extension", ""); return token; } parseContext.updateExtensionBehavior(line, extensionName, ppToken->name); parseContext.notifyExtensionDirective(line, extensionName, ppToken->name); token = scanToken(ppToken); if (token == '\n') return token; else parseContext.ppError(ppToken->loc, "extra tokens -- expected newline", "#extension",""); return token; }
// Handle #undef int TPpContext::CPPundef(TPpToken* ppToken) { int token = scanToken(ppToken); Symbol *symb; if (token != PpAtomIdentifier) { parseContext.ppError(ppToken->loc, "must be followed by macro name", "#undef", ""); return token; } parseContext.reservedPpErrorCheck(ppToken->loc, ppToken->name, "#undef"); symb = LookUpSymbol(ppToken->atom); if (symb) { symb->mac.undef = 1; } token = scanToken(ppToken); if (token != '\n') parseContext.ppError(ppToken->loc, "can only be followed by a single macro name", "#undef", ""); return token; }
// Handle #include int TPpContext::CPPinclude(TPpToken* ppToken) { const TSourceLoc directiveLoc = ppToken->loc; int token = scanToken(ppToken); if (token != PpAtomConstString) { // TODO: handle angle brackets. parseContext.ppError(directiveLoc, "must be followed by a file designation", "#include", ""); } else { // Make a copy of the name because it will be overwritten by the next token scan. const std::string filename = ppToken->name; token = scanToken(ppToken); if (token != '\n' && token != EndOfInput) { parseContext.ppError(ppToken->loc, "extra content after file designation", "#include", ""); } else { std::string sourceName; std::string replacement; std::tie(sourceName, replacement) = includer.include(filename.c_str()); if (!sourceName.empty()) { if (!replacement.empty()) { const bool forNextLine = parseContext.lineDirectiveShouldSetNextLine(); std::ostringstream content; content << "#line " << forNextLine << " " << "\"" << sourceName << "\"\n"; content << replacement << (replacement.back() == '\n' ? "" : "\n"); content << "#line " << directiveLoc.line + forNextLine << " " << directiveLoc.getStringNameOrNum() << "\n"; pushInput(new TokenizableString(directiveLoc, content.str(), this)); } // At EOF, there's no "current" location anymore. if (token != EndOfInput) parseContext.setCurrentColumn(0); // Don't accidentally return EndOfInput, which will end all preprocessing. return '\n'; } else { parseContext.ppError(directiveLoc, replacement.c_str(), "#include", ""); } } } return token; }
// Macro-expand a macro argument 'arg' to create 'expandedArg'. // Does not replace 'arg'. // Returns nullptr if no expanded argument is created. TPpContext::TokenStream* TPpContext::PrescanMacroArg(TokenStream& arg, TPpToken* ppToken, bool newLineOkay) { // expand the argument TokenStream* expandedArg = new TokenStream; pushInput(new tMarkerInput(this)); pushTokenStreamInput(arg); int token; while ((token = scanToken(ppToken)) != tMarkerInput::marker && token != EndOfInput) { token = tokenPaste(token, *ppToken); if (token == PpAtomIdentifier) { switch (MacroExpand(ppToken, false, newLineOkay)) { case MacroExpandNotStarted: break; case MacroExpandError: // toss the rest of the pushed-input argument by scanning until tMarkerInput while ((token = scanToken(ppToken)) != tMarkerInput::marker && token != EndOfInput) ; break; case MacroExpandStarted: case MacroExpandUndef: continue; } } if (token == tMarkerInput::marker || token == EndOfInput) break; expandedArg->putToken(token, ppToken); } if (token != tMarkerInput::marker) { // Error, or MacroExpand ate the marker, so had bad input, recover delete expandedArg; expandedArg = nullptr; } return expandedArg; }
// fetchToken() will return the first token in the lookahead store (if the // lookahead store has tokens) or it will get a new token from the frontier int Scanner::fetchToken(ScannerToken &t, Location &l) { m_token = &t; m_loc = &l; int tokid; if (!m_lookahead.empty()) { // If there is a lookahead token, return that. No need to perform // special logic for "ReturnAllTokens", we already accounted for // that when the tokens were inserted into m_lookahead TokenStore::iterator it = m_lookahead.begin(); tokid = it->t; *m_token = it->token; *m_loc = it->loc; return tokid; } return scanToken(t,l); }
// Handle #if int TPpContext::CPPif(TPpToken* ppToken) { int token = scanToken(ppToken); elsetracker++; ifdepth++; if (ifdepth > maxIfNesting) { parseContext.ppError(ppToken->loc, "maximum nesting depth exceeded", "#if", ""); return 0; } int res = 0; bool err = false; token = eval(token, MIN_PRECEDENCE, false, res, err, ppToken); token = extraTokenCheck(PpAtomIf, ppToken, token); if (!res && !err) token = CPPelse(1, ppToken); return token; }
// nextLookahead() advances an iterator forward in the lookahead store. // If the end of the store is reached, a new token will be scanned from // the frontier. nextLookahead skips over whitespace and comments. void Scanner::nextLookahead(TokenStore::iterator& pos) { for (;;) { ++pos; if (pos == m_lookahead.end()) { pos = m_lookahead.appendNew(); pos->loc = *m_loc; pos->t = scanToken(pos->token, pos->loc); } switch (pos->t) { case T_DOC_COMMENT: case T_COMMENT: case T_OPEN_TAG: case T_WHITESPACE: break; default: return; } } }
void scanLine(int line_no, char *line, FILE *result) { strtrim(line); if (strlen(line) == 0) return; int token_starts[128] = {0}; int token_count = getTokenCountByStartIndexs(line, token_starts); int i; char token[32]; int start, len; for (i = 0; i < token_count; i++) { start = token_starts[i]; if (line[start] == ' ') continue; len = token_starts[i + 1] - start; substr(line, token, start, len); scanToken(token, line_no); } }
void Scanner::scan (Parser& parser) { while (scanToken (parser)); }
// // Do all token-pasting related combining of two pasted tokens when getting a // stream of tokens from a replacement list. Degenerates to no processing if a // replacement list is not the source of the token stream. // int TPpContext::tokenPaste(int token, TPpToken& ppToken) { // starting with ## is illegal, skip to next token if (token == PpAtomPaste) { _parseContext.ppError(ppToken.loc, "unexpected location", "##", ""); return scanToken(&ppToken); } int resultToken = token; // "foo" pasted with "35" is an identifier, not a number // ## can be chained, process all in the chain at once while (peekPasting()) { TPpToken pastedPpToken; // next token has to be ## token = scanToken(&pastedPpToken); assert(token == PpAtomPaste); // This covers end of macro expansion if (endOfReplacementList()) { _parseContext.ppError(ppToken.loc, "unexpected location; end of replacement list", "##", ""); break; } // get the token after the ## token = scanToken(&pastedPpToken); // This covers end of argument expansion if (token == tMarkerInput::marker) { _parseContext.ppError(ppToken.loc, "unexpected location; end of argument", "##", ""); break; } // get the token text switch (resultToken) { case PpAtomIdentifier: // already have the correct text in token.names break; case '=': case '!': case '-': case '~': case '+': case '*': case '/': case '%': case '<': case '>': case '|': case '^': case '&': case PpAtomRight: case PpAtomLeft: case PpAtomAnd: case PpAtomOr: case PpAtomXor: strcpy(ppToken.name, atomStrings.getString(resultToken)); strcpy(pastedPpToken.name, atomStrings.getString(token)); break; default: _parseContext.ppError(ppToken.loc, "not supported for these tokens", "##", ""); return resultToken; } // combine the tokens if (strlen(ppToken.name) + strlen(pastedPpToken.name) > MaxTokenLength) { _parseContext.ppError(ppToken.loc, "combined tokens are too long", "##", ""); return resultToken; } strncat(ppToken.name, pastedPpToken.name, MaxTokenLength - strlen(ppToken.name)); // correct the kind of token we are making, if needed (identifiers stay identifiers) if (resultToken != PpAtomIdentifier) { int newToken = atomStrings.getAtom(ppToken.name); if (newToken > 0) resultToken = newToken; else _parseContext.ppError(ppToken.loc, "combined token is invalid", "##", ""); } } return resultToken; }
int TPpContext::readCPPline(TPpToken* ppToken) { int token = scanToken(ppToken); if (token == PpAtomIdentifier) { switch (ppToken->atom) { case PpAtomDefine: token = CPPdefine(ppToken); break; case PpAtomElse: if (elsetracker[elseSeen]) parseContext.ppError(ppToken->loc, "#else after #else", "#else", ""); elsetracker[elseSeen] = true; if (! ifdepth) parseContext.ppError(ppToken->loc, "mismatched statements", "#else", ""); token = extraTokenCheck(PpAtomElse, ppToken, scanToken(ppToken)); token = CPPelse(0, ppToken); break; case PpAtomElif: if (! ifdepth) parseContext.ppError(ppToken->loc, "mismatched statements", "#elif", ""); if (elseSeen[elsetracker]) parseContext.ppError(ppToken->loc, "#elif after #else", "#elif", ""); // this token is really a dont care, but we still need to eat the tokens token = scanToken(ppToken); while (token != '\n' && token != EndOfInput) token = scanToken(ppToken); token = CPPelse(0, ppToken); break; case PpAtomEndif: elseSeen[elsetracker] = false; --elsetracker; if (! ifdepth) parseContext.ppError(ppToken->loc, "mismatched statements", "#endif", ""); else --ifdepth; token = extraTokenCheck(PpAtomEndif, ppToken, scanToken(ppToken)); break; case PpAtomIf: token = CPPif(ppToken); break; case PpAtomIfdef: token = CPPifdef(1, ppToken); break; case PpAtomIfndef: token = CPPifdef(0, ppToken); break; case PpAtomInclude: parseContext.ppRequireExtensions(ppToken->loc, 1, &E_GL_GOOGLE_include_directive, "#include"); token = CPPinclude(ppToken); break; case PpAtomLine: token = CPPline(ppToken); break; case PpAtomPragma: token = CPPpragma(ppToken); break; case PpAtomUndef: token = CPPundef(ppToken); break; case PpAtomError: token = CPPerror(ppToken); break; case PpAtomVersion: token = CPPversion(ppToken); break; case PpAtomExtension: token = CPPextension(ppToken); break; default: parseContext.ppError(ppToken->loc, "invalid directive:", "#", ppToken->name); break; } } else if (token != '\n' && token != EndOfInput) parseContext.ppError(ppToken->loc, "invalid directive", "#", ""); while (token != '\n' && token != EndOfInput) token = scanToken(ppToken); return token; }
// // Check an identifier (atom) to see if it is a macro that should be expanded. // If it is, and defined, push a tInput that will produce the appropriate expansion // and return 1. // If it is, but undefined, and expandUndef is requested, push a tInput that will // expand to 0 and return -1. // Otherwise, return 0 to indicate no expansion, which is not necessarily an error. // int TPpContext::MacroExpand(int atom, TPpToken* ppToken, bool expandUndef, bool newLineOkay) { ppToken->space = false; switch (atom) { case PpAtomLineMacro: ppToken->ival = parseContext.getCurrentLoc().line; sprintf(ppToken->name, "%d", ppToken->ival); UngetToken(PpAtomConstInt, ppToken); return 1; case PpAtomFileMacro: { if (parseContext.getCurrentLoc().name) parseContext.ppRequireExtensions(ppToken->loc, 1, &E_GL_GOOGLE_cpp_style_line_directive, "filename-based __FILE__"); ppToken->ival = parseContext.getCurrentLoc().string; sprintf(ppToken->name, "%s", ppToken->loc.getStringNameOrNum().c_str()); UngetToken(PpAtomConstInt, ppToken); return 1; } case PpAtomVersionMacro: ppToken->ival = parseContext.version; sprintf(ppToken->name, "%d", ppToken->ival); UngetToken(PpAtomConstInt, ppToken); return 1; default: break; } Symbol *sym = LookUpSymbol(atom); int token; int depth = 0; // no recursive expansions if (sym && sym->mac.busy) return 0; // not expanding undefined macros if ((! sym || sym->mac.undef) && ! expandUndef) return 0; // 0 is the value of an undefined macro if ((! sym || sym->mac.undef) && expandUndef) { pushInput(new tZeroInput(this)); return -1; } tMacroInput *in = new tMacroInput(this); TSourceLoc loc = ppToken->loc; // in case we go to the next line before discovering the error in->mac = &sym->mac; if (sym->mac.args) { token = scanToken(ppToken); if (newLineOkay) { while (token == '\n') token = scanToken(ppToken); } if (token != '(') { parseContext.ppError(loc, "expected '(' following", "macro expansion", GetAtomString(atom)); UngetToken(token, ppToken); ppToken->atom = atom; delete in; return 0; } in->args.resize(in->mac->argc); for (int i = 0; i < in->mac->argc; i++) in->args[i] = new TokenStream; int arg = 0; bool tokenRecorded = false; do { depth = 0; while (1) { token = scanToken(ppToken); if (token == EndOfInput) { parseContext.ppError(loc, "End of input in macro", "macro expansion", GetAtomString(atom)); delete in; return 0; } if (token == '\n') { if (! newLineOkay) { parseContext.ppError(loc, "End of line in macro substitution:", "macro expansion", GetAtomString(atom)); delete in; return 0; } continue; } if (token == '#') { parseContext.ppError(ppToken->loc, "unexpected '#'", "macro expansion", GetAtomString(atom)); delete in; return 0; } if (in->mac->argc == 0 && token != ')') break; if (depth == 0 && (token == ',' || token == ')')) break; if (token == '(') depth++; if (token == ')') depth--; RecordToken(in->args[arg], token, ppToken); tokenRecorded = true; } if (token == ')') { if (in->mac->argc == 1 && tokenRecorded == 0) break; arg++; break; } arg++; } while (arg < in->mac->argc); if (arg < in->mac->argc) parseContext.ppError(loc, "Too few args in Macro", "macro expansion", GetAtomString(atom)); else if (token != ')') { depth=0; while (token != EndOfInput && (depth > 0 || token != ')')) { if (token == ')') depth--; token = scanToken(ppToken); if (token == '(') depth++; } if (token == EndOfInput) { parseContext.ppError(loc, "End of input in macro", "macro expansion", GetAtomString(atom)); delete in; return 0; } parseContext.ppError(loc, "Too many args in macro", "macro expansion", GetAtomString(atom)); } for (int i = 0; i < in->mac->argc; i++) in->args[i] = PrescanMacroArg(in->args[i], ppToken, newLineOkay); } pushInput(in); sym->mac.busy = 1; RewindTokenStream(sym->mac.body); return 1; }
// Handle #define int TPpContext::CPPdefine(TPpToken* ppToken) { MacroSymbol mac; Symbol *symb; // get macro name int token = scanToken(ppToken); if (token != PpAtomIdentifier) { parseContext.ppError(ppToken->loc, "must be followed by macro name", "#define", ""); return token; } if (ppToken->loc.string >= 0) { // We are in user code; check for reserved name use: parseContext.reservedPpErrorCheck(ppToken->loc, ppToken->name, "#define"); } // save the original atom const int defAtom = ppToken->atom; // gather parameters to the macro, between (...) token = scanToken(ppToken); if (token == '(' && ! ppToken->space) { int argc = 0; int args[maxMacroArgs]; do { token = scanToken(ppToken); if (argc == 0 && token == ')') break; if (token != PpAtomIdentifier) { parseContext.ppError(ppToken->loc, "bad argument", "#define", ""); return token; } // check for duplication of parameter name bool duplicate = false; for (int a = 0; a < argc; ++a) { if (args[a] == ppToken->atom) { parseContext.ppError(ppToken->loc, "duplicate macro parameter", "#define", ""); duplicate = true; break; } } if (! duplicate) { if (argc < maxMacroArgs) args[argc++] = ppToken->atom; else parseContext.ppError(ppToken->loc, "too many macro parameters", "#define", ""); } token = scanToken(ppToken); } while (token == ','); if (token != ')') { parseContext.ppError(ppToken->loc, "missing parenthesis", "#define", ""); return token; } mac.argc = argc; mac.args = (int*)mem_Alloc(pool, argc * sizeof(int)); memcpy(mac.args, args, argc * sizeof(int)); token = scanToken(ppToken); } // record the definition of the macro TSourceLoc defineLoc = ppToken->loc; // because ppToken is going to go to the next line before we report errors mac.body = new TokenStream; while (token != '\n' && token != EndOfInput) { RecordToken(mac.body, token, ppToken); token = scanToken(ppToken); if (token != '\n' && ppToken->space) RecordToken(mac.body, ' ', ppToken); } // check for duplicate definition symb = LookUpSymbol(defAtom); if (symb) { if (! symb->mac.undef) { // Already defined -- need to make sure they are identical: // "Two replacement lists are identical if and only if the preprocessing tokens in both have the same number, // ordering, spelling, and white-space separation, where all white-space separations are considered identical." if (symb->mac.argc != mac.argc) parseContext.ppError(defineLoc, "Macro redefined; different number of arguments:", "#define", GetAtomString(defAtom)); else { for (int argc = 0; argc < mac.argc; argc++) { if (symb->mac.args[argc] != mac.args[argc]) parseContext.ppError(defineLoc, "Macro redefined; different argument names:", "#define", GetAtomString(defAtom)); } RewindTokenStream(symb->mac.body); RewindTokenStream(mac.body); int newToken; do { int oldToken; TPpToken oldPpToken; TPpToken newPpToken; oldToken = ReadToken(symb->mac.body, &oldPpToken); newToken = ReadToken(mac.body, &newPpToken); if (oldToken != newToken || oldPpToken != newPpToken) { parseContext.ppError(defineLoc, "Macro redefined; different substitutions:", "#define", GetAtomString(defAtom)); break; } } while (newToken > 0); } } } else symb = AddSymbol(defAtom); delete symb->mac.body; symb->mac = mac; return '\n'; }
/* Skip forward to appropriate spot. This is used both ** to skip to a #endif after seeing an #else, AND to skip to a #else, ** #elif, or #endif after a #if/#ifdef/#ifndef/#elif test was false. */ int TPpContext::CPPelse(int matchelse, TPpToken* ppToken) { int atom; int depth = 0; int token = scanToken(ppToken); while (token != EndOfInput) { if (token != '#') { while (token != '\n' && token != EndOfInput) token = scanToken(ppToken); if (token == EndOfInput) return token; token = scanToken(ppToken); continue; } if ((token = scanToken(ppToken)) != PpAtomIdentifier) continue; atom = ppToken->atom; if (atom == PpAtomIf || atom == PpAtomIfdef || atom == PpAtomIfndef) { depth++; ifdepth++; elsetracker++; } else if (atom == PpAtomEndif) { token = extraTokenCheck(atom, ppToken, scanToken(ppToken)); elseSeen[elsetracker] = false; --elsetracker; if (depth == 0) { // found the #endif we are looking for if (ifdepth) --ifdepth; break; } --depth; --ifdepth; } else if (matchelse && depth == 0) { if (atom == PpAtomElse) { elseSeen[elsetracker] = true; token = extraTokenCheck(atom, ppToken, scanToken(ppToken)); // found the #else we are looking for break; } else if (atom == PpAtomElif) { if (elseSeen[elsetracker]) parseContext.ppError(ppToken->loc, "#elif after #else", "#elif", ""); /* we decrement ifdepth here, because CPPif will increment * it and we really want to leave it alone */ if (ifdepth) { --ifdepth; elseSeen[elsetracker] = false; --elsetracker; } return CPPif(ppToken); } } else if (atom == PpAtomElse) { if (elseSeen[elsetracker]) parseContext.ppError(ppToken->loc, "#else after #else", "#else", ""); else elseSeen[elsetracker] = true; token = extraTokenCheck(atom, ppToken, scanToken(ppToken)); } else if (atom == PpAtomElif) { if (elseSeen[elsetracker]) parseContext.ppError(ppToken->loc, "#elif after #else", "#elif", ""); } } return token; }
int TPpContext::eval(int token, int precedence, bool shortCircuit, int& res, bool& err, TPpToken* ppToken) { TSourceLoc loc = ppToken->loc; // because we sometimes read the newline before reporting the error if (token == PpAtomIdentifier) { if (ppToken->atom == PpAtomDefined) { bool needclose = 0; token = scanToken(ppToken); if (token == '(') { needclose = true; token = scanToken(ppToken); } if (token != PpAtomIdentifier) { parseContext.ppError(loc, "incorrect directive, expected identifier", "preprocessor evaluation", ""); err = true; res = 0; return token; } Symbol* s = LookUpSymbol(ppToken->atom); res = s ? ! s->mac.undef : 0; token = scanToken(ppToken); if (needclose) { if (token != ')') { parseContext.ppError(loc, "expected ')'", "preprocessor evaluation", ""); err = true; res = 0; return token; } token = scanToken(ppToken); } } else { token = evalToToken(token, shortCircuit, res, err, ppToken); return eval(token, precedence, shortCircuit, res, err, ppToken); } } else if (token == PpAtomConstInt) { res = ppToken->ival; token = scanToken(ppToken); } else if (token == '(') { token = scanToken(ppToken); token = eval(token, MIN_PRECEDENCE, shortCircuit, res, err, ppToken); if (! err) { if (token != ')') { parseContext.ppError(loc, "expected ')'", "preprocessor evaluation", ""); err = true; res = 0; return token; } token = scanToken(ppToken); } } else { int op = NUM_ELEMENTS(unop) - 1; for (; op >= 0; op--) { if (unop[op].token == token) break; } if (op >= 0) { token = scanToken(ppToken); token = eval(token, UNARY, shortCircuit, res, err, ppToken); res = unop[op].op(res); } else { parseContext.ppError(loc, "bad expression", "preprocessor evaluation", ""); err = true; res = 0; return token; } } token = evalToToken(token, shortCircuit, res, err, ppToken); // Perform evaluation of binary operation, if there is one, otherwise we are done. while (! err) { if (token == ')' || token == '\n') break; int op; for (op = NUM_ELEMENTS(binop) - 1; op >= 0; op--) { if (binop[op].token == token) break; } if (op < 0 || binop[op].precedence <= precedence) break; int leftSide = res; // Setup short-circuiting, needed for ES, unless already in a short circuit. // (Once in a short-circuit, can't turn off again, until that whole subexpression is done. if (! shortCircuit) { if ((token == PpAtomOr && leftSide == 1) || (token == PpAtomAnd && leftSide == 0)) shortCircuit = true; } token = scanToken(ppToken); token = eval(token, binop[op].precedence, shortCircuit, res, err, ppToken); if (binop[op].op == op_div || binop[op].op == op_mod) { if (res == 0) { parseContext.ppError(loc, "division by 0", "preprocessor evaluation", ""); res = 1; } } res = binop[op].op(leftSide, res); } return token; }