int yylex_CPP(char* buf, int maxSize) { yystypepp yylvalpp; int token = '\n'; for(;;) { char* tokenString = 0; token = cpp->currentInput->scan(cpp->currentInput, &yylvalpp); if(check_EOF(token)) return 0; if (token == '#') { if (cpp->previous_token == '\n'|| cpp->previous_token == 0) { token = readCPPline(&yylvalpp); if(check_EOF(token)) return 0; continue; } else { CPPErrorToInfoLog("preprocessor command must not be preceded by any other statement in that line"); return 0; } } cpp->previous_token = token; // expand macros if (token == CPP_IDENTIFIER && MacroExpand(yylvalpp.sc_ident, &yylvalpp)) { cpp->pastFirstStatement = 1; continue; } if (token == '\n') continue; cpp->pastFirstStatement = 1; if (token == CPP_IDENTIFIER) { tokenString = GetStringOfAtom(atable,yylvalpp.sc_ident); } else if (token == CPP_FLOATCONSTANT || token == CPP_INTCONSTANT){ tokenString = yylvalpp.symbol_name; } else { tokenString = GetStringOfAtom(atable,token); } if (tokenString) { int len = strlen(tokenString); cpp->tokensBeforeEOF = 1; if (len >= maxSize) { return maxSize; } else if (len > 0) { strcpy(buf, tokenString); return len; } return 0; } } return 0; } // yylex
// // The main functional entry-point into the preprocessor, which will // scan the source strings to figure out and return the next processing token. // // Return string pointer to next token. // Return 0 when no more tokens. // const char* TPpContext::tokenize(TPpToken* ppToken) { int token = '\n'; for(;;) { const char* tokenString = nullptr; token = scanToken(ppToken); ppToken->token = token; if (token == EOF) { missingEndifCheck(); return nullptr; } if (token == '#') { if (previous_token == '\n') { token = readCPPline(ppToken); if (token == EOF) { missingEndifCheck(); return nullptr; } continue; } else { parseContext.error(ppToken->loc, "preprocessor directive cannot be preceded by another token", "#", ""); return nullptr; } } previous_token = token; if (token == '\n') continue; // expand macros if (token == CPP_IDENTIFIER && MacroExpand(ppToken->atom, ppToken, false, true) != 0) continue; if (token == CPP_IDENTIFIER) tokenString = GetAtomString(ppToken->atom); else if (token == CPP_INTCONSTANT || token == CPP_UINTCONSTANT || token == CPP_FLOATCONSTANT || token == CPP_DOUBLECONSTANT) tokenString = ppToken->name; else if (token == CPP_STRCONSTANT) { parseContext.error(ppToken->loc, "string literals not supported", "\"\"", ""); tokenString = nullptr; } else if (token == '\'') { parseContext.error(ppToken->loc, "character literals not supported", "\'", ""); tokenString = nullptr; } else tokenString = GetAtomString(token); if (tokenString) { if (tokenString[0] != 0) parseContext.tokensBeforeEOF = 1; return tokenString; } } }
// // The main functional entry point into the preprocessor, which will // scan the source strings to figure out and return the next processing token. // // Return the token, or EndOfInput when no more tokens. // int TPpContext::tokenize(TPpToken& ppToken) { for(;;) { int token = scanToken(&ppToken); // Handle token-pasting logic token = tokenPaste(token, ppToken); if (token == EndOfInput) { missingEndifCheck(); return EndOfInput; } if (token == '#') { if (previous_token == '\n') { token = readCPPline(&ppToken); if (token == EndOfInput) { missingEndifCheck(); return EndOfInput; } continue; } else { _parseContext.ppError(ppToken.loc, "preprocessor directive cannot be preceded by another token", "#", ""); return EndOfInput; } } previous_token = token; if (token == '\n') continue; // expand macros if (token == PpAtomIdentifier && MacroExpand(&ppToken, false, true) != 0) continue; switch (token) { case PpAtomIdentifier: case PpAtomConstInt: case PpAtomConstUint: case PpAtomConstFloat: case PpAtomConstInt64: case PpAtomConstUint64: case PpAtomConstInt16: case PpAtomConstUint16: case PpAtomConstDouble: case PpAtomConstFloat16: if (ppToken.name[0] == '\0') continue; break; case PpAtomConstString: if (ifdepth == 0 && _parseContext.intermediate.getSource() != EShSourceHlsl) { // HLSL allows string literals. _parseContext.ppError(ppToken.loc, "string literals not supported", "\"\"", ""); continue; } break; case '\'': _parseContext.ppError(ppToken.loc, "character literals not supported", "\'", ""); continue; default: strcpy(ppToken.name, atomStrings.getString(token)); break; } return token; } }