asCScriptNode *asCParser::ParseInterfaceMethod() { asCScriptNode *node = new asCScriptNode(snFunction); node->AddChildLast(ParseType(true)); if( isSyntaxError ) return node; node->AddChildLast(ParseTypeMod(false)); if( isSyntaxError ) return node; node->AddChildLast(ParseIdentifier()); if( isSyntaxError ) return node; node->AddChildLast(ParseParameterList()); if( isSyntaxError ) return node; // Parse an optional const after the method definition sToken t1; GetToken(&t1); RewindTo(&t1); if( t1.type == ttConst ) node->AddChildLast(ParseToken(ttConst)); GetToken(&t1); if( t1.type != ttEndStatement ) { Error(ExpectedToken(";").AddressOf(), &t1); return node; } node->UpdateSourcePos(t1.pos, t1.length); return node; }
bool Read() { bool result = false; EatSpace(); if (index < length) { result = true; char c = Current(); if (IsAlpha(c) || c == '_') ParseIdentifier(); else if (IsDigit(c)) ParseIntegerLiteral(); else if (c == '"') ParseStringLiteral(); else if (c == '\'') ParseCodePointLiteral(); else if (IsSymbol(c)) ParseSymbols(); else result = false; } return result; }
asCScriptNode *asCParser::ParseGlobalVar() { asCScriptNode *node = new asCScriptNode(snGlobalVar); // Parse data type node->AddChildLast(ParseType(true)); if( isSyntaxError ) return node; sToken t; for(;;) { // Parse identifier node->AddChildLast(ParseIdentifier()); if( isSyntaxError ) return node; // If next token is assignment, parse expression GetToken(&t); if( t.type == ttAssignment ) { GetToken(&t); RewindTo(&t); if( t.type == ttStartStatementBlock ) { node->AddChildLast(ParseInitList()); if( isSyntaxError ) return node; } else { node->AddChildLast(ParseAssignment()); if( isSyntaxError ) return node; } } else if( t.type == ttOpenParanthesis ) { RewindTo(&t); node->AddChildLast(ParseArgList()); if( isSyntaxError ) return node; } else RewindTo(&t); // continue if list separator, else terminate with end statement GetToken(&t); if( t.type == ttListSeparator ) continue; else if( t.type == ttEndStatement ) { node->UpdateSourcePos(t.pos, t.length); return node; } else { Error(ExpectedTokens(",", ";").AddressOf(), &t); return node; } } return 0; }
BOOL CXMLAttribute::ParseString(LPCTSTR& strXML) { if ( ! ParseIdentifier( strXML, m_sName ) ) return FALSE; if ( ! ParseMatch( strXML, _T("=") ) ) return FALSE; if ( ParseMatch( strXML, _T("\"") ) ) { LPCTSTR pszQuote = _tcschr( strXML, '\"' ); if ( ! pszQuote || *pszQuote != '\"' ) return FALSE; m_sValue = Unescape( strXML, (int)( pszQuote - strXML ) ); strXML = pszQuote; return ParseMatch( strXML, _T("\"") ); } if ( ParseMatch( strXML, _T("'") ) ) { LPCTSTR pszQuote = _tcschr( strXML, '\'' ); if ( ! pszQuote || *pszQuote != '\'' ) return FALSE; m_sValue = Unescape( strXML, (int)( pszQuote - strXML ) ); strXML = pszQuote; return ParseMatch( strXML, _T("\'") ); } return FALSE; }
bool CScanner::NextToken() { SkipWhiteSpace(); if (ErrorString != "") return false; // if (Pos >= SourceLength) // return false; _off_t OldPos = Pos; char c = cget(); if (isdigit(c) || // Thinks a little, while reading this condition, it might be a bit unreadable. ( ( c == '.' && ( (IncPos(), isdigit(cget())) || (DecPos(), false) ) ) && (DecPos(), true) ) ) return ParseNumber() && OldPos != Pos; else if (isalpha(c) || c == '_') return ParseIdentifier() && OldPos != Pos; else if (MatchPrefix("'")) return ParseCharacterLiteral() && OldPos != Pos; else if (MatchPrefix("\"")) return ParseStringLiteral() && OldPos != Pos; else if (SymbolicTokenMapping.find(string() += c)) return ParseSymbol() && OldPos != Pos; else if (c == 0) return true; else ErrorString = "Unrecognized character at line: ";// + itoa(CurrentLine) + ", Column: " + itoa(CurrentColumn); return false; }
int DoSet (Parser *p) { Value v; int r; DynamicBuffer buf; DBufInit(&buf); r = ParseIdentifier(p, &buf); if (r) return r; /* Allow optional equals-sign: SET var = value */ if (ParseNonSpaceChar(p, &r, 1) == '=') { ParseNonSpaceChar(p, &r, 0); } r = EvaluateExpr(p, &v); if (r) { DBufFree(&buf); return r; } if (*DBufValue(&buf) == '$') r = SetSysVar(DBufValue(&buf)+1, &v); else r = SetVar(DBufValue(&buf), &v); DBufFree(&buf); return r; }
asCScriptNode *asCParser::ParseFunctionDefinition() { asCScriptNode *node = new asCScriptNode(snFunction); node->AddChildLast(ParseType(true)); if( isSyntaxError ) return node; node->AddChildLast(ParseTypeMod(false)); if( isSyntaxError ) return node; node->AddChildLast(ParseIdentifier()); if( isSyntaxError ) return node; node->AddChildLast(ParseParameterList()); if( isSyntaxError ) return node; // Parse an optional const after the function definition (used for object methods) sToken t1; GetToken(&t1); RewindTo(&t1); if( t1.type == ttConst ) node->AddChildLast(ParseToken(ttConst)); return node; }
asCScriptNode *asCParser::ParseFunction(bool isMethod) { asCScriptNode *node = new asCScriptNode(snFunction); // If this is a class constructor then no return type will be declared sToken t1,t2; GetToken(&t1); GetToken(&t2); RewindTo(&t1); if( !isMethod || t2.type != ttOpenParanthesis ) { node->AddChildLast(ParseType(false)); if( isSyntaxError ) return node; node->AddChildLast(ParseTypeMod(false)); if( isSyntaxError ) return node; } node->AddChildLast(ParseIdentifier()); if( isSyntaxError ) return node; node->AddChildLast(ParseParameterList()); if( isSyntaxError ) return node; node->AddChildLast(ParseStatementBlock()); return node; }
// <source-name> ::= <positive length number> <identifier> static bool ParseSourceName(State *state) { State copy = *state; if (ParseNumber(state) && ParseIdentifier(state)) { return true; } *state = copy; return false; }
internal bool KwmParseRule(std::string RuleSym, window_rule *Rule) { tokenizer Tokenizer = {}; Tokenizer.At = const_cast<char*>(RuleSym.c_str()); bool Result = true; bool Parsing = true; while(Parsing) { token Token = GetToken(&Tokenizer); switch(Token.Type) { case Token_EndOfStream: { Parsing = false; } break; case Token_Unknown: { } break; case Token_Identifier: { if(TokenEquals(Token, "owner")) Result = Result && ParseIdentifier(&Tokenizer, &Rule->Owner); else if(TokenEquals(Token, "name")) Result = Result && ParseIdentifier(&Tokenizer, &Rule->Name); else if(TokenEquals(Token, "role")) Result = Result && ParseIdentifier(&Tokenizer, &Rule->Role); else if(TokenEquals(Token, "crole")) Result = Result && ParseIdentifier(&Tokenizer, &Rule->CustomRole); else if(TokenEquals(Token, "properties")) Result = Result && ParseProperties(&Tokenizer, &Rule->Properties); else if(TokenEquals(Token, "except")) Result = Result && ParseIdentifier(&Tokenizer, &Rule->Except); } break; default: { } break; } } return Result; }
static int GetToken(EvalState *c, PVAL *pval) { int tkn; /* check for a saved token */ if ((tkn = c->savedToken) != TKN_NONE) { c->savedToken = TKN_NONE; return tkn; } /* skip leading spaces */ while (*c->linePtr != '\0' && isspace(*c->linePtr)) ++c->linePtr; /* check for end of file (string) */ if (*c->linePtr == '\0') return TKN_EOF; /* check for a number */ if (*c->linePtr == '.' || isdigit(*c->linePtr)) { ParseNumber(c, pval); tkn = TKN_NUMBER; } /* check for an identifier */ else if (isalpha(*c->linePtr)) tkn = ParseIdentifier(c, pval); /* handle operators */ else { switch (tkn = *c->linePtr) { case '<': if (c->linePtr[1] == '<') { tkn = TKN_SHL; ++c->linePtr; } break; case '>': if (c->linePtr[1] == '>') { tkn = TKN_SHR; ++c->linePtr; } break; default: /* nothing to do */ break; } ++c->linePtr; } return tkn; }
JsonValue * JsonParser::ReadValue(JsonValue * parent) { if (*cur_ == '"') { return ReadString(parent); } else if (IsDigit(*cur_) || *cur_ == '-') { return ReadNumber(parent); } else if (IsAlpha(*cur_)) { std::string keyword; ParseIdentifier(keyword); if (keyword == "null") { return 0; } if (keyword == "true") { return new JsonBooleanValue(parent, true); } else if (keyword == "false") { return new JsonBooleanValue(parent, false); } else { throw Error( row_, column_ - keyword.length(), FormatString("Invalid bareword \"%s\", while value expected", keyword.c_str()) ); } } else if (*cur_ == '[') { return ReadArray(parent); } else if (*cur_ == '{') { return ReadObject(parent); } else { RaiseError("Invalid symbol, while value expected"); return 0; } }
int asCParser::ParsePropertyDeclaration(asCScriptCode *script) { Reset(); this->script = script; scriptNode = new asCScriptNode(snDeclaration); scriptNode->AddChildLast(ParseType(true)); if( isSyntaxError ) return -1; scriptNode->AddChildLast(ParseIdentifier()); if( isSyntaxError ) return -1; return 0; }
asCScriptNode *asCParser::ParseInterface() { asCScriptNode *node = new asCScriptNode(snInterface); sToken t; GetToken(&t); if( t.type != ttInterface ) { Error(ExpectedToken("interface").AddressOf(), &t); return node; } node->SetToken(&t); node->AddChildLast(ParseIdentifier()); GetToken(&t); if( t.type != ttStartStatementBlock ) { Error(ExpectedToken("{").AddressOf(), &t); return node; } // Parse interface methods GetToken(&t); RewindTo(&t); while( t.type != ttEndStatementBlock && t.type != ttEnd ) { // Parse the method signature node->AddChildLast(ParseInterfaceMethod()); if( isSyntaxError ) return node; GetToken(&t); RewindTo(&t); } GetToken(&t); if( t.type != ttEndStatementBlock ) { Error(ExpectedToken("}").AddressOf(), &t); return node; } node->UpdateSourcePos(t.pos, t.length); return node; }
asCScriptNode *asCParser::ParseExprPostOp() { asCScriptNode *node = new asCScriptNode(snExprPostOp); sToken t; GetToken(&t); if( !IsPostOperator(t.type) ) { Error(TXT_EXPECTED_POST_OPERATOR, &t); return node; } node->SetToken(&t); node->UpdateSourcePos(t.pos, t.length); if( t.type == ttDot ) { sToken t1, t2; GetToken(&t1); GetToken(&t2); RewindTo(&t1); if( t2.type == ttOpenParanthesis ) node->AddChildLast(ParseFunctionCall()); else node->AddChildLast(ParseIdentifier()); } else if( t.type == ttOpenBracket ) { node->AddChildLast(ParseAssignment()); GetToken(&t); if( t.type != ttCloseBracket ) { ExpectedToken("]"); return node; } node->UpdateSourcePos(t.pos, t.length); } return node; }
asCScriptNode *asCParser::ParseFunction() { asCScriptNode *node = new asCScriptNode(snFunction); node->AddChildLast(ParseType(false)); if( isSyntaxError ) return node; node->AddChildLast(ParseTypeMod(false)); if( isSyntaxError ) return node; node->AddChildLast(ParseIdentifier()); if( isSyntaxError ) return node; node->AddChildLast(ParseParameterList()); if( isSyntaxError ) return node; node->AddChildLast(ParseStatementBlock()); return node; }
bool MaterialParser::Parse(Lexer* lexer, const boost::filesystem::path& dir) { m_rootDir = dir; Token token = lexer->NextToken(); while(true) { switch(token.type) { case Token::token_misc: if(token.str != ";") { Error(token.line, "unexpected token: '" + token.str + "'"); return false; } token = lexer->SkipToken(token); break; case Token::token_id: if(false == ParseIdentifier(lexer)) { return false; } token = lexer->CurToken(); break; case Token::token_eof: return true; default: Error(token.line, "unexpected token: '" + token.str + "'"); token = lexer->NextToken(); break; } } Error(-1, "====== failed. ======"); return false; }
asCScriptNode *asCParser::ParseExprValue() { asCScriptNode *node = new asCScriptNode(snExprValue); sToken t1; GetToken(&t1); RewindTo(&t1); if( t1.type == ttIdentifier || IsRealType(t1.type) ) { if( IsFunctionCall() ) node->AddChildLast(ParseFunctionCall()); else node->AddChildLast(ParseIdentifier()); } else if( t1.type == ttCast ) node->AddChildLast(ParseCast()); else if( IsConstant(t1.type) ) node->AddChildLast(ParseConstant()); else if( t1.type == ttOpenParanthesis ) { GetToken(&t1); node->UpdateSourcePos(t1.pos, t1.length); node->AddChildLast(ParseAssignment()); if( isSyntaxError ) return node; GetToken(&t1); if( t1.type != ttCloseParanthesis ) Error(ExpectedToken(")").AddressOf(), &t1); node->UpdateSourcePos(t1.pos, t1.length); } else Error(TXT_EXPECTED_EXPRESSION_VALUE, &t1); return node; }
asCScriptNode *asCParser::ParseClass() { asCScriptNode *node = new asCScriptNode(snClass); sToken t; GetToken(&t); if( t.type != ttClass ) { Error(ExpectedToken("class").AddressOf(), &t); return node; } node->SetToken(&t); node->AddChildLast(ParseIdentifier()); GetToken(&t); // Optional list of interfaces that are being implemented and classes that are being inherited if( t.type == ttColon ) { node->AddChildLast(ParseIdentifier()); GetToken(&t); while( t.type == ttListSeparator ) { node->AddChildLast(ParseIdentifier()); GetToken(&t); } } if( t.type != ttStartStatementBlock ) { Error(ExpectedToken("{").AddressOf(), &t); return node; } // Parse properties GetToken(&t); RewindTo(&t); while( t.type != ttEndStatementBlock && t.type != ttEnd ) { // Is it a property or a method? if( IsFuncDecl(true) ) { // Parse the method node->AddChildLast(ParseFunction(true)); } else if( IsVarDecl() ) { // Parse a property declaration asCScriptNode *prop = new asCScriptNode(snDeclaration); node->AddChildLast(prop); prop->AddChildLast(ParseType(true)); if( isSyntaxError ) return node; prop->AddChildLast(ParseIdentifier()); if( isSyntaxError ) return node; GetToken(&t); if( t.type != ttEndStatement ) { Error(ExpectedToken(";").AddressOf(), &t); return node; } prop->UpdateSourcePos(t.pos, t.length); } else { Error(TXT_EXPECTED_METHOD_OR_PROPERTY, &t); return node; } GetToken(&t); RewindTo(&t); } GetToken(&t); if( t.type != ttEndStatementBlock ) { Error(ExpectedToken("}").AddressOf(), &t); return node; } node->UpdateSourcePos(t.pos, t.length); return node; }
ExprNode* Parser :: ParseDeclaration(SymbolType* sym_type){ if (isEq(scan.Get(), _SEPARATION, ";")){ scan.Next(); return nullptr; } SymbolType* t_symbol; if (!sym_type) { t_symbol = ParseType(); }else t_symbol = sym_type; VarSymbol* var = nullptr; BinOpNode *node = nullptr; while(true){ Token *token; token = scan.Get(); if(isEq(token, _SEPARATION, ";" )&& (t_symbol->isStruct() || dynamic_cast<TypedefSymbol*>(t_symbol))) break; if (isEq(scan.Get(), _SEPARATION,"(")) var = ParseComplexDeclaration(t_symbol); else var = ParseIdentifier(t_symbol); if(symStack->tables.back()->find_symbol(var->name) == 0) symStack->add_symbol(var); if(isEq(scan.Get(), _OPERATION, "=")){ Token *asgn = scan.Get(); scan.Next(); ExprNode* Assing_operand = ParseExpr(priorityTable[","] + 1); node = new BinOpNode(asgn, new IdentifierNode(token, var), Assing_operand); if(dynamic_cast<ConstSymbolType*> (var->getType())) errorIf(!var->getType()->getType()->canConvertTo(node->right->getType()),"Cannot perform conversion",scan.Get() ); else node->getType(); blocks.top()->AddStatement(node); } if(isEq(scan.Get() ,_SEPARATION, ";") || isEq(scan.Get() ,_SEPARATION, "{")) break; errorIf(!isEq(scan.Get(), _OPERATION, ","), "Comma Expected"); scan.Next(); } if(isEq (scan.Get(),_SEPARATION,"{")){ FuncSymbol *func = dynamic_cast<FuncSymbol*>(var->type); errorIf(!func, "Unexpected brace", scan.Get()); errorIf((symStack->tables.size() != 1), "Can not define the function in the block", scan.Get()); parsingFunc = func; func->body = ParseBlock(); parsingFunc = 0; CheckReturn(func); if(func->name == "main") main_func = func; scan.Next(); } else scan.Next(); return node; }
/** * Construct the final microcode from the compiled and verified shader source. * @param ShaderOutput - Where to store the microcode and parameter map. * @param InShaderSource - Metal source with input/output signature. * @param SourceLen - The length of the Metal source code. */ static void BuildMetalShaderOutput( FShaderCompilerOutput& ShaderOutput, const FShaderCompilerInput& ShaderInput, const ANSICHAR* InShaderSource, int32 SourceLen, TArray<FShaderCompilerError>& OutErrors ) { FMetalCodeHeader Header = {0}; const ANSICHAR* ShaderSource = InShaderSource; FShaderParameterMap& ParameterMap = ShaderOutput.ParameterMap; EShaderFrequency Frequency = (EShaderFrequency)ShaderOutput.Target.Frequency; TBitArray<> UsedUniformBufferSlots; UsedUniformBufferSlots.Init(false,32); // Write out the magic markers. Header.Frequency = Frequency; #define DEF_PREFIX_STR(Str) \ const ANSICHAR* Str##Prefix = "// @" #Str ": "; \ const int32 Str##PrefixLen = FCStringAnsi::Strlen(Str##Prefix) DEF_PREFIX_STR(Inputs); DEF_PREFIX_STR(Outputs); DEF_PREFIX_STR(UniformBlocks); DEF_PREFIX_STR(Uniforms); DEF_PREFIX_STR(PackedGlobals); DEF_PREFIX_STR(PackedUB); DEF_PREFIX_STR(PackedUBCopies); DEF_PREFIX_STR(PackedUBGlobalCopies); DEF_PREFIX_STR(Samplers); DEF_PREFIX_STR(UAVs); DEF_PREFIX_STR(SamplerStates); DEF_PREFIX_STR(NumThreads); #undef DEF_PREFIX_STR // Skip any comments that come before the signature. while ( FCStringAnsi::Strncmp(ShaderSource, "//", 2) == 0 && FCStringAnsi::Strncmp(ShaderSource, "// @", 4) != 0 ) { while (*ShaderSource && *ShaderSource++ != '\n') {} } // HLSLCC first prints the list of inputs. if (FCStringAnsi::Strncmp(ShaderSource, InputsPrefix, InputsPrefixLen) == 0) { ShaderSource += InputsPrefixLen; // Only inputs for vertex shaders must be tracked. if (Frequency == SF_Vertex) { const ANSICHAR* AttributePrefix = "in_ATTRIBUTE"; const int32 AttributePrefixLen = FCStringAnsi::Strlen(AttributePrefix); while (*ShaderSource && *ShaderSource != '\n') { // Skip the type. while (*ShaderSource && *ShaderSource++ != ':') {} // Only process attributes. if (FCStringAnsi::Strncmp(ShaderSource, AttributePrefix, AttributePrefixLen) == 0) { ShaderSource += AttributePrefixLen; uint8 AttributeIndex = ParseNumber(ShaderSource); Header.Bindings.InOutMask |= (1 << AttributeIndex); } // Skip to the next. while (*ShaderSource && *ShaderSource != ',' && *ShaderSource != '\n') { ShaderSource++; } if (Match(ShaderSource, '\n')) { break; } verify(Match(ShaderSource, ',')); } } else { // Skip to the next line. while (*ShaderSource && *ShaderSource++ != '\n') {} } } // Then the list of outputs. if (FCStringAnsi::Strncmp(ShaderSource, OutputsPrefix, OutputsPrefixLen) == 0) { ShaderSource += OutputsPrefixLen; // Only outputs for pixel shaders must be tracked. if (Frequency == SF_Pixel) { const ANSICHAR* TargetPrefix = "out_Target"; const int32 TargetPrefixLen = FCStringAnsi::Strlen(TargetPrefix); while (*ShaderSource && *ShaderSource != '\n') { // Skip the type. while (*ShaderSource && *ShaderSource++ != ':') {} // Handle targets. if (FCStringAnsi::Strncmp(ShaderSource, TargetPrefix, TargetPrefixLen) == 0) { ShaderSource += TargetPrefixLen; uint8 TargetIndex = ParseNumber(ShaderSource); Header.Bindings.InOutMask |= (1 << TargetIndex); } // Handle depth writes. else if (FCStringAnsi::Strcmp(ShaderSource, "gl_FragDepth") == 0) { Header.Bindings.InOutMask |= 0x8000; } // Skip to the next. while (*ShaderSource && *ShaderSource != ',' && *ShaderSource != '\n') { ShaderSource++; } if (Match(ShaderSource, '\n')) { break; } verify(Match(ShaderSource, ',')); } } else { // Skip to the next line. while (*ShaderSource && *ShaderSource++ != '\n') {} } } bool bHasRegularUniformBuffers = false; // Then 'normal' uniform buffers. if (FCStringAnsi::Strncmp(ShaderSource, UniformBlocksPrefix, UniformBlocksPrefixLen) == 0) { ShaderSource += UniformBlocksPrefixLen; while (*ShaderSource && *ShaderSource != '\n') { FString BufferName = ParseIdentifier(ShaderSource); verify(BufferName.Len() > 0); verify(Match(ShaderSource, '(')); uint16 UBIndex = ParseNumber(ShaderSource); if (UBIndex >= Header.Bindings.NumUniformBuffers) { Header.Bindings.NumUniformBuffers = UBIndex + 1; } UsedUniformBufferSlots[UBIndex] = true; verify(Match(ShaderSource, ')')); ParameterMap.AddParameterAllocation(*BufferName, UBIndex, 0, 0); bHasRegularUniformBuffers = true; // Skip the comma. if (Match(ShaderSource, '\n')) { break; } verify(Match(ShaderSource, ',')); } Match(ShaderSource, '\n'); } // Then uniforms. const uint16 BytesPerComponent = 4; /* uint16 PackedUniformSize[OGL_NUM_PACKED_UNIFORM_ARRAYS] = {0}; FMemory::Memzero(&PackedUniformSize, sizeof(PackedUniformSize)); */ if (FCStringAnsi::Strncmp(ShaderSource, UniformsPrefix, UniformsPrefixLen) == 0) { // @todo-mobile: Will we ever need to support this code path? check(0); /* ShaderSource += UniformsPrefixLen; while (*ShaderSource && *ShaderSource != '\n') { uint16 ArrayIndex = 0; uint16 Offset = 0; uint16 NumComponents = 0; FString ParameterName = ParseIdentifier(ShaderSource); verify(ParameterName.Len() > 0); verify(Match(ShaderSource, '(')); ArrayIndex = ParseNumber(ShaderSource); verify(Match(ShaderSource, ':')); Offset = ParseNumber(ShaderSource); verify(Match(ShaderSource, ':')); NumComponents = ParseNumber(ShaderSource); verify(Match(ShaderSource, ')')); ParameterMap.AddParameterAllocation( *ParameterName, ArrayIndex, Offset * BytesPerComponent, NumComponents * BytesPerComponent ); if (ArrayIndex < OGL_NUM_PACKED_UNIFORM_ARRAYS) { PackedUniformSize[ArrayIndex] = FMath::Max<uint16>( PackedUniformSize[ArrayIndex], BytesPerComponent * (Offset + NumComponents) ); } // Skip the comma. if (Match(ShaderSource, '\n')) { break; } verify(Match(ShaderSource, ',')); } Match(ShaderSource, '\n'); */ } // Packed global uniforms TMap<ANSICHAR, uint16> PackedGlobalArraySize; if (FCStringAnsi::Strncmp(ShaderSource, PackedGlobalsPrefix, PackedGlobalsPrefixLen) == 0) { ShaderSource += PackedGlobalsPrefixLen; while (*ShaderSource && *ShaderSource != '\n') { ANSICHAR ArrayIndex = 0; uint16 Offset = 0; uint16 NumComponents = 0; FString ParameterName = ParseIdentifier(ShaderSource); verify(ParameterName.Len() > 0); verify(Match(ShaderSource, '(')); ArrayIndex = *ShaderSource++; verify(Match(ShaderSource, ':')); Offset = ParseNumber(ShaderSource); verify(Match(ShaderSource, ',')); NumComponents = ParseNumber(ShaderSource); verify(Match(ShaderSource, ')')); ParameterMap.AddParameterAllocation( *ParameterName, ArrayIndex, Offset * BytesPerComponent, NumComponents * BytesPerComponent ); uint16& Size = PackedGlobalArraySize.FindOrAdd(ArrayIndex); Size = FMath::Max<uint16>(BytesPerComponent * (Offset + NumComponents), Size); if (Match(ShaderSource, '\n')) { break; } // Skip the comma. verify(Match(ShaderSource, ',')); } Match(ShaderSource, '\n'); } // Packed Uniform Buffers TMap<int, TMap<ANSICHAR, uint16> > PackedUniformBuffersSize; while (FCStringAnsi::Strncmp(ShaderSource, PackedUBPrefix, PackedUBPrefixLen) == 0) { ShaderSource += PackedUBPrefixLen; FString BufferName = ParseIdentifier(ShaderSource); verify(BufferName.Len() > 0); verify(Match(ShaderSource, '(')); uint16 BufferIndex = ParseNumber(ShaderSource); check(BufferIndex == Header.Bindings.NumUniformBuffers); verify(Match(ShaderSource, ')')); ParameterMap.AddParameterAllocation(*BufferName, Header.Bindings.NumUniformBuffers++, 0, 0); verify(Match(ShaderSource, ':')); Match(ShaderSource, ' '); while (*ShaderSource && *ShaderSource != '\n') { FString ParameterName = ParseIdentifier(ShaderSource); verify(ParameterName.Len() > 0); verify(Match(ShaderSource, '(')); ParseNumber(ShaderSource); verify(Match(ShaderSource, ',')); ParseNumber(ShaderSource); verify(Match(ShaderSource, ')')); if (Match(ShaderSource, '\n')) { break; } verify(Match(ShaderSource, ',')); } } // Packed Uniform Buffers copy lists & setup sizes for each UB/Precision entry if (FCStringAnsi::Strncmp(ShaderSource, PackedUBCopiesPrefix, PackedUBCopiesPrefixLen) == 0) { ShaderSource += PackedUBCopiesPrefixLen; while (*ShaderSource && *ShaderSource != '\n') { FMetalUniformBufferCopyInfo CopyInfo; CopyInfo.SourceUBIndex = ParseNumber(ShaderSource); verify(Match(ShaderSource, ':')); CopyInfo.SourceOffsetInFloats = ParseNumber(ShaderSource); verify(Match(ShaderSource, '-')); CopyInfo.DestUBIndex = ParseNumber(ShaderSource); verify(Match(ShaderSource, ':')); CopyInfo.DestUBTypeName = *ShaderSource++; CopyInfo.DestUBTypeIndex = CrossCompiler::PackedTypeNameToTypeIndex(CopyInfo.DestUBTypeName); verify(Match(ShaderSource, ':')); CopyInfo.DestOffsetInFloats = ParseNumber(ShaderSource); verify(Match(ShaderSource, ':')); CopyInfo.SizeInFloats = ParseNumber(ShaderSource); Header.UniformBuffersCopyInfo.Add(CopyInfo); auto& UniformBufferSize = PackedUniformBuffersSize.FindOrAdd(CopyInfo.DestUBIndex); uint16& Size = UniformBufferSize.FindOrAdd(CopyInfo.DestUBTypeName); Size = FMath::Max<uint16>(BytesPerComponent * (CopyInfo.DestOffsetInFloats + CopyInfo.SizeInFloats), Size); if (Match(ShaderSource, '\n')) { break; } verify(Match(ShaderSource, ',')); } } if (FCStringAnsi::Strncmp(ShaderSource, PackedUBGlobalCopiesPrefix, PackedUBGlobalCopiesPrefixLen) == 0) { ShaderSource += PackedUBGlobalCopiesPrefixLen; while (*ShaderSource && *ShaderSource != '\n') { FMetalUniformBufferCopyInfo CopyInfo; CopyInfo.SourceUBIndex = ParseNumber(ShaderSource); verify(Match(ShaderSource, ':')); CopyInfo.SourceOffsetInFloats = ParseNumber(ShaderSource); verify(Match(ShaderSource, '-')); CopyInfo.DestUBIndex = 0; CopyInfo.DestUBTypeName = *ShaderSource++; CopyInfo.DestUBTypeIndex = CrossCompiler::PackedTypeNameToTypeIndex(CopyInfo.DestUBTypeName); verify(Match(ShaderSource, ':')); CopyInfo.DestOffsetInFloats = ParseNumber(ShaderSource); verify(Match(ShaderSource, ':')); CopyInfo.SizeInFloats = ParseNumber(ShaderSource); Header.UniformBuffersCopyInfo.Add(CopyInfo); uint16& Size = PackedGlobalArraySize.FindOrAdd(CopyInfo.DestUBTypeName); Size = FMath::Max<uint16>(BytesPerComponent * (CopyInfo.DestOffsetInFloats + CopyInfo.SizeInFloats), Size); if (Match(ShaderSource, '\n')) { break; } verify(Match(ShaderSource, ',')); } } Header.Bindings.bHasRegularUniformBuffers = bHasRegularUniformBuffers; // Setup Packed Array info Header.Bindings.PackedGlobalArrays.Reserve(PackedGlobalArraySize.Num()); for (auto Iterator = PackedGlobalArraySize.CreateIterator(); Iterator; ++Iterator) { ANSICHAR TypeName = Iterator.Key(); uint16 Size = Iterator.Value(); Size = (Size + 0xf) & (~0xf); CrossCompiler::FPackedArrayInfo Info; Info.Size = Size; Info.TypeName = TypeName; Info.TypeIndex = CrossCompiler::PackedTypeNameToTypeIndex(TypeName); Header.Bindings.PackedGlobalArrays.Add(Info); } // Setup Packed Uniform Buffers info Header.Bindings.PackedUniformBuffers.Reserve(PackedUniformBuffersSize.Num()); for (auto Iterator = PackedUniformBuffersSize.CreateIterator(); Iterator; ++Iterator) { int BufferIndex = Iterator.Key(); auto& ArraySizes = Iterator.Value(); TArray<CrossCompiler::FPackedArrayInfo> InfoArray; InfoArray.Reserve(ArraySizes.Num()); for (auto IterSizes = ArraySizes.CreateIterator(); IterSizes; ++IterSizes) { ANSICHAR TypeName = IterSizes.Key(); uint16 Size = IterSizes.Value(); Size = (Size + 0xf) & (~0xf); CrossCompiler::FPackedArrayInfo Info; Info.Size = Size; Info.TypeName = TypeName; Info.TypeIndex = CrossCompiler::PackedTypeNameToTypeIndex(TypeName); InfoArray.Add(Info); } Header.Bindings.PackedUniformBuffers.Add(InfoArray); } // Then samplers. if (FCStringAnsi::Strncmp(ShaderSource, SamplersPrefix, SamplersPrefixLen) == 0) { ShaderSource += SamplersPrefixLen; while (*ShaderSource && *ShaderSource != '\n') { uint16 Offset = 0; uint16 NumSamplers = 0; FString ParameterName = ParseIdentifier(ShaderSource); verify(ParameterName.Len() > 0); verify(Match(ShaderSource, '(')); Offset = ParseNumber(ShaderSource); verify(Match(ShaderSource, ':')); NumSamplers = ParseNumber(ShaderSource); ParameterMap.AddParameterAllocation( *ParameterName, 0, Offset, NumSamplers ); Header.Bindings.NumSamplers = FMath::Max<uint8>( Header.Bindings.NumSamplers, Offset + NumSamplers ); if (Match(ShaderSource, '[')) { // Sampler States do { FString SamplerState = ParseIdentifier(ShaderSource); checkSlow(SamplerState.Len() != 0); ParameterMap.AddParameterAllocation( *SamplerState, 0, Offset, NumSamplers ); } while (Match(ShaderSource, ',')); verify(Match(ShaderSource, ']')); } verify(Match(ShaderSource, ')')); if (Match(ShaderSource, '\n')) { break; } // Skip the comma. verify(Match(ShaderSource, ',')); } } // Then UAVs (images in Metal) if (FCStringAnsi::Strncmp(ShaderSource, UAVsPrefix, UAVsPrefixLen) == 0) { ShaderSource += UAVsPrefixLen; while (*ShaderSource && *ShaderSource != '\n') { uint16 Offset = 0; uint16 NumUAVs = 0; FString ParameterName = ParseIdentifier(ShaderSource); verify(ParameterName.Len() > 0); verify(Match(ShaderSource, '(')); Offset = ParseNumber(ShaderSource); verify(Match(ShaderSource, ':')); NumUAVs = ParseNumber(ShaderSource); ParameterMap.AddParameterAllocation( *ParameterName, 0, Offset, NumUAVs ); Header.Bindings.NumUAVs = FMath::Max<uint8>( Header.Bindings.NumUAVs, Offset + NumUAVs ); verify(Match(ShaderSource, ')')); if (Match(ShaderSource, '\n')) { break; } // Skip the comma. verify(Match(ShaderSource, ',')); } } if (FCStringAnsi::Strncmp(ShaderSource, NumThreadsPrefix, NumThreadsPrefixLen) == 0) { ShaderSource += NumThreadsPrefixLen; Header.NumThreadsX = ParseNumber(ShaderSource); verify(Match(ShaderSource, ',')); Match(ShaderSource, ' '); Header.NumThreadsY = ParseNumber(ShaderSource); verify(Match(ShaderSource, ',')); Match(ShaderSource, ' '); Header.NumThreadsZ = ParseNumber(ShaderSource); verify(Match(ShaderSource, '\n')); } // Build the SRT for this shader. { // Build the generic SRT for this shader. FShaderResourceTable GenericSRT; BuildResourceTableMapping(ShaderInput.Environment.ResourceTableMap, ShaderInput.Environment.ResourceTableLayoutHashes, UsedUniformBufferSlots, ShaderOutput.ParameterMap, GenericSRT); // Copy over the bits indicating which resource tables are active. Header.Bindings.ShaderResourceTable.ResourceTableBits = GenericSRT.ResourceTableBits; Header.Bindings.ShaderResourceTable.ResourceTableLayoutHashes = GenericSRT.ResourceTableLayoutHashes; // Now build our token streams. BuildResourceTableTokenStream(GenericSRT.TextureMap, GenericSRT.MaxBoundResourceTable, Header.Bindings.ShaderResourceTable.TextureMap); BuildResourceTableTokenStream(GenericSRT.ShaderResourceViewMap, GenericSRT.MaxBoundResourceTable, Header.Bindings.ShaderResourceTable.ShaderResourceViewMap); BuildResourceTableTokenStream(GenericSRT.SamplerMap, GenericSRT.MaxBoundResourceTable, Header.Bindings.ShaderResourceTable.SamplerMap); BuildResourceTableTokenStream(GenericSRT.UnorderedAccessViewMap, GenericSRT.MaxBoundResourceTable, Header.Bindings.ShaderResourceTable.UnorderedAccessViewMap); Header.Bindings.NumUniformBuffers = FMath::Max((uint8)GetNumUniformBuffersUsed(GenericSRT), Header.Bindings.NumUniformBuffers); } const int32 MaxSamplers = GetFeatureLevelMaxTextureSamplers(ERHIFeatureLevel::ES3_1); if (Header.Bindings.NumSamplers > MaxSamplers) { ShaderOutput.bSucceeded = false; FShaderCompilerError* NewError = new(ShaderOutput.Errors) FShaderCompilerError(); NewError->StrippedErrorMessage = FString::Printf(TEXT("shader uses %d samplers exceeding the limit of %d"), Header.Bindings.NumSamplers, MaxSamplers); } else { #if METAL_OFFLINE_COMPILE // at this point, the shader source is ready to be compiled FString InputFilename = FPaths::CreateTempFilename(*FPaths::EngineIntermediateDir(), TEXT("ShaderIn"), TEXT("")); FString ObjFilename = InputFilename + TEXT(".o"); FString ArFilename = InputFilename + TEXT(".ar"); FString OutputFilename = InputFilename + TEXT(".lib"); InputFilename = InputFilename + TEXT(".metal"); // write out shader source FFileHelper::SaveStringToFile(FString(ShaderSource), *InputFilename); int32 ReturnCode = 0; FString Results; FString Errors; bool bHadError = true; // metal commandlines FString Params = FString::Printf(TEXT("-std=ios-metal1.0 %s -o %s"), *InputFilename, *ObjFilename); FPlatformProcess::ExecProcess( TEXT("/Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/usr/bin/metal"), *Params, &ReturnCode, &Results, &Errors ); // handle compile error if (ReturnCode != 0 || IFileManager::Get().FileSize(*ObjFilename) <= 0) { // FShaderCompilerError* Error = new(OutErrors) FShaderCompilerError(); // Error->ErrorFile = InputFilename; // Error->ErrorLineString = TEXT("0"); // Error->StrippedErrorMessage = Results + Errors; } else { Params = FString::Printf(TEXT("r %s %s"), *ArFilename, *ObjFilename); FPlatformProcess::ExecProcess( TEXT("/Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/usr/bin/metal-ar"), *Params, &ReturnCode, &Results, &Errors ); // handle compile error if (ReturnCode != 0 || IFileManager::Get().FileSize(*ArFilename) <= 0) { // FShaderCompilerError* Error = new(OutErrors) FShaderCompilerError(); // Error->ErrorFile = InputFilename; // Error->ErrorLineString = TEXT("0"); // Error->StrippedErrorMessage = Results + Errors; } else { Params = FString::Printf(TEXT("-o %s %s"), *OutputFilename, *ArFilename); FPlatformProcess::ExecProcess( TEXT("/Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/usr/bin/metallib"), *Params, &ReturnCode, &Results, &Errors ); // handle compile error if (ReturnCode != 0 || IFileManager::Get().FileSize(*OutputFilename) <= 0) { // FShaderCompilerError* Error = new(OutErrors) FShaderCompilerError(); // Error->ErrorFile = InputFilename; // Error->ErrorLineString = TEXT("0"); // Error->StrippedErrorMessage = Results + Errors; } else { bHadError = false; // Write out the header and compiled shader code FMemoryWriter Ar(ShaderOutput.Code, true); uint8 PrecompiledFlag = 1; Ar << PrecompiledFlag; Ar << Header; // load output TArray<uint8> CompiledShader; FFileHelper::LoadFileToArray(CompiledShader, *OutputFilename); // jam it into the output bytes Ar.Serialize(CompiledShader.GetData(), CompiledShader.Num()); ShaderOutput.NumInstructions = 0; ShaderOutput.NumTextureSamplers = Header.Bindings.NumSamplers; ShaderOutput.bSucceeded = true; } } } if (bHadError) { // Write out the header and shader source code. FMemoryWriter Ar(ShaderOutput.Code, true); uint8 PrecompiledFlag = 0; Ar << PrecompiledFlag; Ar << Header; Ar.Serialize((void*)ShaderSource, SourceLen + 1 - (ShaderSource - InShaderSource)); ShaderOutput.NumInstructions = 0; ShaderOutput.NumTextureSamplers = Header.Bindings.NumSamplers; ShaderOutput.bSucceeded = true; } IFileManager::Get().Delete(*InputFilename); IFileManager::Get().Delete(*ObjFilename); IFileManager::Get().Delete(*ArFilename); IFileManager::Get().Delete(*OutputFilename); #else // Write out the header and shader source code. FMemoryWriter Ar(ShaderOutput.Code, true); uint8 PrecompiledFlag = 0; Ar << PrecompiledFlag; Ar << Header; Ar.Serialize((void*)ShaderSource, SourceLen + 1 - (ShaderSource - InShaderSource)); ShaderOutput.NumInstructions = 0; ShaderOutput.NumTextureSamplers = Header.Bindings.NumSamplers; ShaderOutput.bSucceeded = true; #endif } }
BOOL CXMLElement::ParseString(LPCTSTR& strXML) { if ( ! ParseMatch( strXML, _T("<") ) ) return FALSE; if ( ! ParseIdentifier( strXML, m_sName ) ) return FALSE; while ( ! ParseMatch( strXML, _T(">") ) ) { if ( ParseMatch( strXML, _T("/") ) ) return ParseMatch( strXML, _T(">") ); if ( ! *strXML ) return FALSE; CXMLAttribute* pAttribute = new CXMLAttribute( this ); if ( ! pAttribute || ! pAttribute->ParseString( strXML ) ) { delete pAttribute; return FALSE; } CString strNameLower( pAttribute->m_sName ); strNameLower.MakeLower(); // Delete the old attribute if one exists CXMLAttribute* pExisting; if ( m_pAttributes.Lookup( strNameLower, pExisting ) ) delete pExisting; m_pAttributes.SetAt( strNameLower, pAttribute ); if ( ! m_pAttributesInsertion.Find( strNameLower ) ) m_pAttributesInsertion.AddTail( strNameLower ); // Track output order workaround } CString strClose = _T("</"); strClose += m_sName + '>'; for ( ;; ) { if ( ! *strXML ) return FALSE; LPCTSTR pszElement = _tcschr( strXML, '<' ); if ( ! pszElement || *pszElement != '<' ) return FALSE; if ( ParseMatch( strXML, _T("<![CDATA[") ) ) { pszElement = _tcsstr( strXML, _T("]]>") ); if ( ! pszElement || *pszElement != ']' ) return FALSE; if ( ! m_sValue.IsEmpty() && m_sValue.Right( 1 ) != ' ' ) m_sValue += ' '; m_sValue += Unescape( strXML, (int)( pszElement - strXML ) ); pszElement += 3; strXML = pszElement; } if ( pszElement > strXML ) { if ( ! m_sValue.IsEmpty() && m_sValue.Right( 1 ) != ' ' ) m_sValue += ' '; m_sValue += Unescape( strXML, (int)( pszElement - strXML ) ); strXML = pszElement; } if ( ParseMatch( strXML, strClose ) ) { break; } else if ( ParseMatch( strXML, _T("<!--") ) ) { pszElement = _tcsstr( strXML, _T("-->") ); if ( ! pszElement || *pszElement != '-' ) return FALSE; strXML = pszElement + 3; } else { CXMLElement* pElement = new CXMLElement( this ); if ( pElement->ParseString( strXML ) ) { m_pElements.AddTail( pElement ); } else { delete pElement; return FALSE; } } } return TRUE; }
bool FHlslccHeader::ReadInOut(const ANSICHAR*& ShaderSource, TArray<FInOut>& OutAttributes) { while (*ShaderSource && *ShaderSource != '\n') { FInOut Attribute; if (!ParseIdentifier(ShaderSource, Attribute.Type)) { return false; } if (Match(ShaderSource, '[')) { if (!ParseIntegerNumber(ShaderSource, Attribute.ArrayCount)) { return false; } if (!Match(ShaderSource, ']')) { return false; } } else { Attribute.ArrayCount = 0; } if (Match(ShaderSource, ';')) { if (!ParseSignedNumber(ShaderSource, Attribute.Index)) { return false; } } if (!Match(ShaderSource, ':')) { return false; } if (!ParseIdentifier(ShaderSource, Attribute.Name)) { return false; } // Optional array suffix if (Match(ShaderSource, '[')) { Attribute.Name += '['; while (*ShaderSource) { Attribute.Name += *ShaderSource; if (Match(ShaderSource, ']')) { break; } ++ShaderSource; } } OutAttributes.Add(Attribute); // Break if EOL if (Match(ShaderSource, '\n')) { break; } // Has to be a comma! if (Match(ShaderSource, ',')) { continue; } //#todo-rco: Need a log here //UE_LOG(ShaderCompilerCommon, Warning, TEXT("Invalid char '%c'"), *ShaderSource); return false; } return true; }
asCScriptNode *asCParser::ParseParameterList() { asCScriptNode *node = new asCScriptNode(snParameterList); sToken t1; GetToken(&t1); if( t1.type != ttOpenParanthesis ) { Error(ExpectedToken("(").AddressOf(), &t1); return node; } node->UpdateSourcePos(t1.pos, t1.length); GetToken(&t1); if( t1.type == ttCloseParanthesis ) { node->UpdateSourcePos(t1.pos, t1.length); // Statement block is finished return node; } else { RewindTo(&t1); for(;;) { // Parse data type node->AddChildLast(ParseType(true)); if( isSyntaxError ) return node; node->AddChildLast(ParseTypeMod(true)); if( isSyntaxError ) return node; // Parse identifier GetToken(&t1); if( t1.type == ttIdentifier ) { RewindTo(&t1); node->AddChildLast(ParseIdentifier()); if( isSyntaxError ) return node; GetToken(&t1); } // Check if list continues if( t1.type == ttCloseParanthesis ) { node->UpdateSourcePos(t1.pos, t1.length); return node; } else if( t1.type == ttListSeparator ) continue; else { Error(ExpectedTokens(")", ",").AddressOf(), &t1); return node; } } } return 0; }
internal bool ParseProperties(tokenizer *Tokenizer, window_properties *Properties) { if(RequireToken(Tokenizer, Token_Equals)) { if(RequireToken(Tokenizer, Token_OpenBrace)) { Properties->Scratchpad = -1; Properties->Display = -1; Properties->Space = -1; Properties->Float = -1; bool ValidState = true; while(ValidState) { token Token = GetToken(Tokenizer); switch(Token.Type) { case Token_SemiColon: { continue; } break; case Token_CloseBrace: { ValidState = false; } break; case Token_Identifier: { if(TokenEquals(Token, "float")) { std::string Value; if(ParseIdentifier(Tokenizer, &Value)) { if(Value == "true") Properties->Float = 1; else if(Value == "false") Properties->Float = 0; } } else if(TokenEquals(Token, "display")) { std::string Value; if(ParseIdentifier(Tokenizer, &Value)) Properties->Display = ConvertStringToInt(Value); } else if(TokenEquals(Token, "space")) { std::string Value; if(ParseIdentifier(Tokenizer, &Value)) Properties->Space = ConvertStringToInt(Value); } else if(TokenEquals(Token, "scratchpad")) { std::string Value; if(ParseIdentifier(Tokenizer, &Value)) { if(Value == "visible") Properties->Scratchpad = 1; else if(Value == "hidden") Properties->Scratchpad = 0; } } else if(TokenEquals(Token, "role")) { std::string Value; if(ParseIdentifier(Tokenizer, &Value)) Properties->Role = Value; } } break; default: { ReportInvalidRule("Expected token of type Token_Identifier: '" + std::string(Token.Text, Token.TextLength) + "'"); } break; } } return true; } else { ReportInvalidRule("Expected token '{'"); } } else { ReportInvalidRule("Expected token '='"); } return false; }
Lexems Parse( const std::string& file_data ) { Lexems result; unsigned int line= 1; std::string::const_iterator it= file_data.begin(); while( it != file_data.end() ) { const char c= *it; Lexem lexem; if( c == '/' ) { if( it+1 < file_data.end() ) { it++; if( *it == '/' ) { it++; while( it < file_data.end() && *it != '\n' ) it++; line++; continue; } else if( *it == '*' ) { it++; while( it < file_data.end() - 1 ) { if( *it == '\n' ) line++; else if( *it == '*' && *(it+1) == '/' ) { it+= 2; break; } it++; } continue; } else { lexem.type= Lexem::Type::Slash; lexem.text= "/"; } } } else if( c == '\n' ) { line++; it++; continue; } else if( isdigit(c) ) lexem= ParseNumericConstant( file_data, it ); else if( isalpha(c) ) lexem= ParseIdentifier( file_data, it ); else { bool found= false; for( int i= c_max_predifined_lexem_length; i >= 1; i-- ) { if( it + i <= file_data.end() ) { auto operator_lexem_it= c_operators_lexems.find( std::string( &*it, i ) ); if( operator_lexem_it != c_operators_lexems.end() ) { lexem.text= operator_lexem_it->first; lexem.type= operator_lexem_it->second; lexem.file_position= it - file_data.begin(); it+= i; found= true; break; } } } if( !found ) { // TODO - check for trash it++; continue; } } lexem.line= line; result.push_back(lexem); } // while not end return result; }
bool FHlslccHeader::Read(const ANSICHAR*& ShaderSource, int32 SourceLen) { #define DEF_PREFIX_STR(Str) \ static const ANSICHAR* Str##Prefix = "// @" #Str ": "; \ static const int32 Str##PrefixLen = FCStringAnsi::Strlen(Str##Prefix) DEF_PREFIX_STR(Inputs); DEF_PREFIX_STR(Outputs); DEF_PREFIX_STR(UniformBlocks); DEF_PREFIX_STR(Uniforms); DEF_PREFIX_STR(PackedGlobals); DEF_PREFIX_STR(PackedUB); DEF_PREFIX_STR(PackedUBCopies); DEF_PREFIX_STR(PackedUBGlobalCopies); DEF_PREFIX_STR(Samplers); DEF_PREFIX_STR(UAVs); DEF_PREFIX_STR(SamplerStates); DEF_PREFIX_STR(NumThreads); #undef DEF_PREFIX_STR // Skip any comments that come before the signature. while (FCStringAnsi::Strncmp(ShaderSource, "//", 2) == 0 && FCStringAnsi::Strncmp(ShaderSource + 2, " !", 2) != 0 && FCStringAnsi::Strncmp(ShaderSource + 2, " @", 2) != 0) { ShaderSource += 2; while (*ShaderSource && *ShaderSource++ != '\n') { // Do nothing } } // Read shader name if any if (FCStringAnsi::Strncmp(ShaderSource, "// !", 4) == 0) { ShaderSource += 4; while (*ShaderSource && *ShaderSource != '\n') { Name += (TCHAR)*ShaderSource; ++ShaderSource; } if (*ShaderSource == '\n') { ++ShaderSource; } } // Skip any comments that come before the signature. while (FCStringAnsi::Strncmp(ShaderSource, "//", 2) == 0 && FCStringAnsi::Strncmp(ShaderSource + 2, " @", 2) != 0) { ShaderSource += 2; while (*ShaderSource && *ShaderSource++ != '\n') { // Do nothing } } if (FCStringAnsi::Strncmp(ShaderSource, InputsPrefix, InputsPrefixLen) == 0) { ShaderSource += InputsPrefixLen; if (!ReadInOut(ShaderSource, Inputs)) { return false; } } if (FCStringAnsi::Strncmp(ShaderSource, OutputsPrefix, OutputsPrefixLen) == 0) { ShaderSource += OutputsPrefixLen; if (!ReadInOut(ShaderSource, Outputs)) { return false; } } if (FCStringAnsi::Strncmp(ShaderSource, UniformBlocksPrefix, UniformBlocksPrefixLen) == 0) { ShaderSource += UniformBlocksPrefixLen; while (*ShaderSource && *ShaderSource != '\n') { FAttribute UniformBlock; if (!ParseIdentifier(ShaderSource, UniformBlock.Name)) { return false; } if (!Match(ShaderSource, '(')) { return false; } if (!ParseIntegerNumber(ShaderSource, UniformBlock.Index)) { return false; } if (!Match(ShaderSource, ')')) { return false; } UniformBlocks.Add(UniformBlock); if (Match(ShaderSource, '\n')) { break; } if (Match(ShaderSource, ',')) { continue; } //#todo-rco: Need a log here //UE_LOG(ShaderCompilerCommon, Warning, TEXT("Invalid char '%c'"), *ShaderSource); return false; } } if (FCStringAnsi::Strncmp(ShaderSource, UniformsPrefix, UniformsPrefixLen) == 0) { // @todo-mobile: Will we ever need to support this code path? check(0); return false; /* ShaderSource += UniformsPrefixLen; while (*ShaderSource && *ShaderSource != '\n') { uint16 ArrayIndex = 0; uint16 Offset = 0; uint16 NumComponents = 0; FString ParameterName = ParseIdentifier(ShaderSource); verify(ParameterName.Len() > 0); verify(Match(ShaderSource, '(')); ArrayIndex = ParseNumber(ShaderSource); verify(Match(ShaderSource, ':')); Offset = ParseNumber(ShaderSource); verify(Match(ShaderSource, ':')); NumComponents = ParseNumber(ShaderSource); verify(Match(ShaderSource, ')')); ParameterMap.AddParameterAllocation( *ParameterName, ArrayIndex, Offset * BytesPerComponent, NumComponents * BytesPerComponent ); if (ArrayIndex < OGL_NUM_PACKED_UNIFORM_ARRAYS) { PackedUniformSize[ArrayIndex] = FMath::Max<uint16>( PackedUniformSize[ArrayIndex], BytesPerComponent * (Offset + NumComponents) ); } // Skip the comma. if (Match(ShaderSource, '\n')) { break; } verify(Match(ShaderSource, ',')); } Match(ShaderSource, '\n'); */ } // @PackedGlobals: Global0(h:0,1),Global1(h:4,1),Global2(h:8,1) if (FCStringAnsi::Strncmp(ShaderSource, PackedGlobalsPrefix, PackedGlobalsPrefixLen) == 0) { ShaderSource += PackedGlobalsPrefixLen; while (*ShaderSource && *ShaderSource != '\n') { FPackedGlobal PackedGlobal; if (!ParseIdentifier(ShaderSource, PackedGlobal.Name)) { return false; } if (!Match(ShaderSource, '(')) { return false; } PackedGlobal.PackedType = *ShaderSource++; if (!Match(ShaderSource, ':')) { return false; } if (!ParseIntegerNumber(ShaderSource, PackedGlobal.Offset)) { return false; } if (!Match(ShaderSource, ',')) { return false; } if (!ParseIntegerNumber(ShaderSource, PackedGlobal.Count)) { return false; } if (!Match(ShaderSource, ')')) { return false; } PackedGlobals.Add(PackedGlobal); // Break if EOL if (Match(ShaderSource, '\n')) { break; } // Has to be a comma! if (Match(ShaderSource, ',')) { continue; } //#todo-rco: Need a log here //UE_LOG(ShaderCompilerCommon, Warning, TEXT("Invalid char '%c'"), *ShaderSource); return false; } } // Packed Uniform Buffers (Multiple lines) // @PackedUB: CBuffer(0): CBMember0(0,1),CBMember1(1,1) while (FCStringAnsi::Strncmp(ShaderSource, PackedUBPrefix, PackedUBPrefixLen) == 0) { ShaderSource += PackedUBPrefixLen; FPackedUB PackedUB; if (!ParseIdentifier(ShaderSource, PackedUB.Attribute.Name)) { return false; } if (!Match(ShaderSource, '(')) { return false; } if (!ParseIntegerNumber(ShaderSource, PackedUB.Attribute.Index)) { return false; } if (!Match(ShaderSource, ')')) { return false; } if (!Match(ShaderSource, ':')) { return false; } if (!Match(ShaderSource, ' ')) { return false; } while (*ShaderSource && *ShaderSource != '\n') { FPackedUB::FMember Member; ParseIdentifier(ShaderSource, Member.Name); if (!Match(ShaderSource, '(')) { return false; } if (!ParseIntegerNumber(ShaderSource, Member.Offset)) { return false; } if (!Match(ShaderSource, ',')) { return false; } if (!ParseIntegerNumber(ShaderSource, Member.Count)) { return false; } if (!Match(ShaderSource, ')')) { return false; } PackedUB.Members.Add(Member); // Break if EOL if (Match(ShaderSource, '\n')) { break; } // Has to be a comma! if (Match(ShaderSource, ',')) { continue; } //#todo-rco: Need a log here //UE_LOG(ShaderCompilerCommon, Warning, TEXT("Invalid char '%c'"), *ShaderSource); return false; } PackedUBs.Add(PackedUB); } // @PackedUBCopies: 0:0-0:h:0:1,0:1-0:h:4:1,1:0-1:h:0:1 if (FCStringAnsi::Strncmp(ShaderSource, PackedUBCopiesPrefix, PackedUBCopiesPrefixLen) == 0) { ShaderSource += PackedUBCopiesPrefixLen; if (!ReadCopies(ShaderSource, false, PackedUBCopies)) { return false; } } // @PackedUBGlobalCopies: 0:0-h:12:1,0:1-h:16:1,1:0-h:20:1 if (FCStringAnsi::Strncmp(ShaderSource, PackedUBGlobalCopiesPrefix, PackedUBGlobalCopiesPrefixLen) == 0) { ShaderSource += PackedUBGlobalCopiesPrefixLen; if (!ReadCopies(ShaderSource, true, PackedUBGlobalCopies)) { return false; } } if (FCStringAnsi::Strncmp(ShaderSource, SamplersPrefix, SamplersPrefixLen) == 0) { ShaderSource += SamplersPrefixLen; while (*ShaderSource && *ShaderSource != '\n') { FSampler Sampler; if (!ParseIdentifier(ShaderSource, Sampler.Name)) { return false; } if (!Match(ShaderSource, '(')) { return false; } if (!ParseIntegerNumber(ShaderSource, Sampler.Offset)) { return false; } if (!Match(ShaderSource, ':')) { return false; } if (!ParseIntegerNumber(ShaderSource, Sampler.Count)) { return false; } if (Match(ShaderSource, '[')) { // Sampler States do { FString SamplerState; if (!ParseIdentifier(ShaderSource, SamplerState)) { return false; } Sampler.SamplerStates.Add(SamplerState); } while (Match(ShaderSource, ',')); if (!Match(ShaderSource, ']')) { return false; } } if (!Match(ShaderSource, ')')) { return false; } Samplers.Add(Sampler); // Break if EOL if (Match(ShaderSource, '\n')) { break; } // Has to be a comma! if (Match(ShaderSource, ',')) { continue; } //#todo-rco: Need a log here //UE_LOG(ShaderCompilerCommon, Warning, TEXT("Invalid char '%c'"), *ShaderSource); return false; } } if (FCStringAnsi::Strncmp(ShaderSource, UAVsPrefix, UAVsPrefixLen) == 0) { ShaderSource += UAVsPrefixLen; while (*ShaderSource && *ShaderSource != '\n') { FUAV UAV; if (!ParseIdentifier(ShaderSource, UAV.Name)) { return false; } if (!Match(ShaderSource, '(')) { return false; } if (!ParseIntegerNumber(ShaderSource, UAV.Offset)) { return false; } if (!Match(ShaderSource, ':')) { return false; } if (!ParseIntegerNumber(ShaderSource, UAV.Count)) { return false; } if (!Match(ShaderSource, ')')) { return false; } UAVs.Add(UAV); // Break if EOL if (Match(ShaderSource, '\n')) { break; } // Has to be a comma! if (Match(ShaderSource, ',')) { continue; } //#todo-rco: Need a log here //UE_LOG(ShaderCompilerCommon, Warning, TEXT("Invalid char '%c'"), *ShaderSource); return false; } } if (FCStringAnsi::Strncmp(ShaderSource, SamplerStatesPrefix, SamplerStatesPrefixLen) == 0) { ShaderSource += SamplerStatesPrefixLen; while (*ShaderSource && *ShaderSource != '\n') { FAttribute SamplerState; if (!ParseIntegerNumber(ShaderSource, SamplerState.Index)) { return false; } if (!Match(ShaderSource, ':')) { return false; } if (!ParseIdentifier(ShaderSource, SamplerState.Name)) { return false; } SamplerStates.Add(SamplerState); // Break if EOL if (Match(ShaderSource, '\n')) { break; } // Has to be a comma! if (Match(ShaderSource, ',')) { continue; } //#todo-rco: Need a log here //UE_LOG(ShaderCompilerCommon, Warning, TEXT("Invalid char '%c'"), *ShaderSource); return false; } } if (FCStringAnsi::Strncmp(ShaderSource, NumThreadsPrefix, NumThreadsPrefixLen) == 0) { ShaderSource += NumThreadsPrefixLen; if (!ParseIntegerNumber(ShaderSource, NumThreads[0])) { return false; } if (!Match(ShaderSource, ',')) { return false; } if (!Match(ShaderSource, ' ')) { return false; } if (!ParseIntegerNumber(ShaderSource, NumThreads[1])) { return false; } if (!Match(ShaderSource, ',')) { return false; } if (!Match(ShaderSource, ' ')) { return false; } if (!ParseIntegerNumber(ShaderSource, NumThreads[2])) { return false; } if (!Match(ShaderSource, '\n')) { return false; } } return true; }
char* Preprocessor::ParseLexem( char* start, char* end, Lexem& out ) { if( start == end ) return start; char current_char = *start; if( IsTrivial( current_char ) ) { out.Value += current_char; out.Type = TrivialTypes[Trivials.find_first_of( current_char )]; return ++start; } if( IsIdentifierStart( current_char ) ) return ParseIdentifier( start, end, out ); if( current_char == '#' ) { out.Value = "#"; ++start; if( *start == '#' ) { out.Value = "##"; out.Type = Lexem::IGNORE; return ( ++start ); } while( start != end && ( *start == ' ' || *start == '\t' ) ) ++start; if( start != end && IsIdentifierStart( *start ) ) start = ParseIdentifier( start, end, out ); out.Type = Lexem::PREPROCESSOR; return start; } if( IsNumber( current_char ) ) return ParseNumber( start, end, out ); if( current_char == '\"' ) return ParseStringLiteral( start, end, '\"', out ); if( current_char == '\'' ) return ParseStringLiteral( start, end, '\'', out ); // Todo: set optional ParseCharacterLiteral? if( current_char == '/' ) { // Need to see if it's a comment. ++start; if( start == end ) return start; if( *start == '*' ) return ParseBlockComment( start, end, out ); if( *start == '/' ) return ParseLineComment( start, end, out ); // Not a comment - let default code catch it as MISC --start; } if( current_char == '\\' ) { out.Type = Lexem::BACKSLASH; return ++start; } out.Value = std::string( 1, current_char ); out.Type = Lexem::IGNORE; return ++start; }