void CompileShader(const String& fileName)
{
String file = GetFileName(fileName);
XMLFile doc(context_);
File source(context_);
source.Open(fileName);
if (!doc.Load(source))
ErrorExit("Could not open input file " + fileName);
XMLElement shaders = doc.GetRoot("shaders");
if (!shaders)
ErrorExit("No shaders element in " + source.GetName());
if (compileVS_)
{
ShaderParser vsParser;
if (!vsParser.Parse(VS, shaders, defines_, defineValues_))
ErrorExit("VS: " + vsParser.GetErrorMessage());
const HashMap<String, unsigned>& combinations = vsParser.GetAllCombinations();
for (HashMap<String, unsigned>::ConstIterator i = combinations.Begin(); i != combinations.End(); ++i)
{
if (!compileVariation_ || i->first_ == variationName_)
{
ShaderCombination src = vsParser.GetCombination(i->first_);
CompiledVariation compile;
compile.type_ = VS;
compile.name_ = file;
compile.outFileName_ = outDir_ + file;
if (!src.name_.Empty())
{
compile.name_ += "_" + src.name_;
compile.outFileName_ += "_" + src.name_;
}
compile.outFileName_ += useSM3_ ? ".vs3" : ".vs2";
compile.defines_ = src.defines_;
compile.defineValues_ = src.defineValues_;
variations_.Push(compile);
workList_.Push(&variations_.Back());
}
}
}
if (compilePS_)
{
ShaderParser psParser;
if (!psParser.Parse(PS, shaders, defines_, defineValues_))
ErrorExit("PS: " + psParser.GetErrorMessage());
const HashMap<String, unsigned>& combinations = psParser.GetAllCombinations();
for (HashMap<String, unsigned>::ConstIterator i = combinations.Begin(); i != combinations.End(); ++i)
{
if (!compileVariation_ || i->first_ == variationName_)
{
ShaderCombination src = psParser.GetCombination(i->first_);
CompiledVariation compile;
compile.type_ = PS;
compile.name_ = file;
compile.outFileName_ = outDir_ + file;
if (!src.name_.Empty())
{
compile.name_ += "_" + src.name_;
compile.outFileName_ += "_" + src.name_;
}
compile.outFileName_ += useSM3_ ? ".ps3" : ".ps2";
compile.defines_ = src.defines_;
compile.defineValues_ = src.defineValues_;
variations_.Push(compile);
workList_.Push(&variations_.Back());
}
}
}
if (variations_.Empty())
{
PrintLine("No variations to compile");
return;
}
// Load the shader source code
{
String inputFileName = inDir_ + file + ".hlsl";
File hlslFile(context_, inputFileName);
if (!hlslFile.IsOpen())
ErrorExit("Could not open input file " + inputFileName);
hlslCode_.Clear();
while (!hlslFile.IsEof())
hlslCode_ += hlslFile.ReadLine() + "\n";
}
if (!compileVariation_)
{
// Create and start worker threads. Use all logical CPUs except one to not lock up the computer completely
unsigned numWorkerThreads = GetNumLogicalCPUs() - 1;
if (!numWorkerThreads)
numWorkerThreads = 1;
Vector<SharedPtr<WorkerThread> > workerThreads;
workerThreads.Resize(numWorkerThreads);
for (unsigned i = 0; i < workerThreads.Size(); ++i)
{
workerThreads[i] = new WorkerThread();
workerThreads[i]->Run();
}
// This will wait until the thread functions have stopped
for (unsigned i = 0; i < workerThreads.Size(); ++i)
workerThreads[i]->Stop();
}
else
{
WorkerThread dummyThread;
dummyThread.ThreadFunction();
}
// Check that all shaders compiled
for (List<CompiledVariation>::Iterator i = variations_.Begin(); i != variations_.End(); ++i)
{
if (!i->errorMsg_.Empty())
{
if (i->type_ == VS)
ErrorExit("Failed to compile vertex shader " + i->name_ + ": " + i->errorMsg_);
else
ErrorExit("Failed to compile pixel shader " + i->name_ + ": " + i->errorMsg_);
}
}
}
void ConfigParser::Parse( const IDataStream& Stream )
{
Array< SToken > Tokens;
SToken Token;
Token.m_TokenType = SToken::ET_Name;
char StrMark = 0; // Stores the character (either ' or ") that opened a string
SToken MacroToken;
List<int> ArrayCounters;
Array<char> CounterCharArray;
int LineCount = 1;
for(;;)
{
char c = Stream.ReadInt8();
// Skip the UTF-8 byte order mark if present.
if( UTF8_BOM_0 == static_cast<byte>( c ) )
{
CHECK( UTF8_BOM_1 == static_cast<byte>( Stream.ReadInt8() ) );
CHECK( UTF8_BOM_2 == static_cast<byte>( Stream.ReadInt8() ) );
c = Stream.ReadInt8();
}
if( Stream.EOS() )
{
if( Token.m_TokenString.Empty() )
{
Token.m_TokenType = SToken::ET_None;
Tokens.PushBack( Token );
DEBUGCATPRINTF( "Core", 2, "%s\n", SToken::m_TokenNames[ Token.m_TokenType ] );
}
else
{
Token.m_TokenString.PushBack( '\0' );
Tokens.PushBack( Token );
DEBUGCATPRINTF( "Core", 2, "%s: %s\n", SToken::m_TokenNames[ Token.m_TokenType ], Token.m_TokenString.GetData() );
Token.m_TokenString.Clear();
}
break;
}
if( c == '&' )
{
if( Token.m_TokenType == SToken::ET_Name )
{
// Increment the current counter and add it to the current name.
ASSERT( ArrayCounters.Size() > 0 );
List<int>::Iterator CounterIter = ArrayCounters.Back();
( *CounterIter )++;
SimpleString CounterString = SimpleString::PrintF( "%d", *CounterIter );
CounterCharArray.Clear();
CounterString.FillArray( CounterCharArray );
Token.m_TokenString.Append( CounterCharArray );
}
else if( Token.m_TokenType == SToken::ET_None )
{
// Add a new counter
// Push a counter token that will be replaced with the count int later.
ArrayCounters.PushBack( -1 );
Token.m_TokenType = SToken::ET_Counter;
}
else if( Token.m_TokenType == SToken::ET_String )
{
Token.m_TokenString.PushBack( c );
}
else
{
WARNDESC( "Unexpected character '&' in token." );
}
}
else if( c == '^' )
{
if( Token.m_TokenType == SToken::ET_Name )
{
// Add the current counter to the current name.
ASSERT( ArrayCounters.Size() > 0 );
List<int>::Iterator CounterIter = ArrayCounters.Back();
ASSERT( ( *CounterIter ) >= 0 );
SimpleString CounterString = SimpleString::PrintF( "%d", *CounterIter );
CounterCharArray.Clear();
CounterString.FillArray( CounterCharArray );
Token.m_TokenString.Append( CounterCharArray );
}
else if( Token.m_TokenType == SToken::ET_String )
{
Token.m_TokenString.PushBack( c );
}
else
{
WARNDESC( "Unexpected character '^' in token." );
}
}
else if( c == ' ' || c == '\t' )
{
switch( Token.m_TokenType )
{
case SToken::ET_None:
case SToken::ET_Equals:
// Ignore whitespace
break;
case SToken::ET_Name:
case SToken::ET_Context:
case SToken::ET_Macro:
if( Token.m_TokenString.Empty() )
{
// If the name is empty, ignore whitespace (before the name)
}
else
{
// Close current token, push it, and expect an equals
// If we're closing a macro, save it as such
if( Token.m_TokenType == SToken::ET_Macro )
{
MacroToken = Token;
}
Token.m_TokenString.PushBack( '\0' );
Tokens.PushBack( Token );
DEBUGCATPRINTF( "Core", 2, "%s: %s\n", SToken::m_TokenNames[ Token.m_TokenType ], Token.m_TokenString.GetData() );
Token.m_TokenString.Clear();
if( Token.m_TokenType == SToken::ET_Name )
{
Token.m_TokenType = SToken::ET_Equals;
}
else if( Token.m_TokenType == SToken::ET_Context || Token.m_TokenType == SToken::ET_Macro )
{
Token.m_TokenType = SToken::ET_Name;
}
}
break;
case SToken::ET_Bool:
case SToken::ET_Int:
case SToken::ET_Float:
// Close current token, push it, and expect nothing
Token.m_TokenString.PushBack( '\0' );
Tokens.PushBack( Token );
DEBUGCATPRINTF( "Core", 2, "%s: %s\n", SToken::m_TokenNames[ Token.m_TokenType ], Token.m_TokenString.GetData() );
Token.m_TokenString.Clear();
Token.m_TokenType = SToken::ET_None;
break;
case SToken::ET_String:
Token.m_TokenString.PushBack( c );
break;
default:
WARNDESC( "Unexpected token" );
break;
}
}
else if( c == '=' )
{
switch( Token.m_TokenType )
{
case SToken::ET_Name:
// Close current token, push it and an equals
Token.m_TokenString.PushBack( '\0' );
Tokens.PushBack( Token );
DEBUGCATPRINTF( "Core", 2, "%s: %s\n", SToken::m_TokenNames[ Token.m_TokenType ], Token.m_TokenString.GetData() );
Token.m_TokenString.Clear();
Token.m_TokenType = SToken::ET_Equals;
DEBUGCATPRINTF( "Core", 2, "%s\n", SToken::m_TokenNames[ Token.m_TokenType ] );
Tokens.PushBack( Token );
Token.m_TokenType = SToken::ET_None;
break;
case SToken::ET_Equals:
// Already expecting =, just push it
Tokens.PushBack( Token );
DEBUGCATPRINTF( "Core", 2, "%s\n", SToken::m_TokenNames[ Token.m_TokenType ] );
Token.m_TokenType = SToken::ET_None;
break;
case SToken::ET_String:
Token.m_TokenString.PushBack( c );
break;
default:
WARNDESC( "Unexpected token" );
break;
}
}
else if( c == '#' )
{
// # starts a comment
// Allow # inside a string
if( Token.m_TokenType == SToken::ET_String )
{
Token.m_TokenString.PushBack( c );
}
else
{
c = Stream.ReadInt8();
if( c == '!' ) // #! and !# indicate the start and end of block comments
{
while( !Stream.EOS() )
{
if( Stream.ReadInt8() == '!' && Stream.ReadInt8() == '#' )
{
break;
}
}
}
else
{
// Read to end of line
while( c != '\n' && c!= '\v' && !Stream.EOS() ) // Vertical tab stupidity again
{
c = Stream.ReadInt8();
}
// Change the context because we're on a new line
Token.m_TokenType = SToken::ET_Name;
++LineCount;
}
}
}
else if( c == '\\' && Token.m_TokenType == SToken::ET_String )
{
// Escape sequence, intended to insert linebreaks (and maybe other things in the future)
// Config string escape sequences are not the same as C++ escape sequences. They can be
// \\, \n, \?, \", or \xx (where x are hex digits, to specify any character by hex).
char next = Stream.ReadInt8();
if( next == 'n' )
{
Token.m_TokenString.PushBack( '\n' );
}
else if( next == '\"' )
{
Token.m_TokenString.PushBack( '\"' );
}
else if( next == '\\' )
{
Token.m_TokenString.PushBack( '\\' );
}
else if( next == 'x' )
{
char Hex = 0;
for( uint HexIndex = 0; HexIndex < 2; ++HexIndex )
{
next = Stream.ReadInt8();
ASSERT( IsHex( next ) );
Hex = ( Hex << 4 ) | GetHex( next );
}
Token.m_TokenString.PushBack( Hex );
}
else if( next == 'u' )
{
// First, extract a unicode code point (e.g. \u00d7 for U+00D7)
// NOTE: This only support the first Unicode plane, and is strict about
// using four characters, so \ud7 is not a valid substitute for \u00d7.
unicode_t CodePoint = 0;
for( uint UnicodeIndex = 0; UnicodeIndex < 4; ++UnicodeIndex )
{
next = Stream.ReadInt8();
ASSERT( IsHex( next ) );
CodePoint = ( CodePoint << 4 ) | GetHex( next );
}
// Then convert the two-byte code point to UTF-8.
Array<unicode_t> CodePointArray;
CodePointArray.PushBack( CodePoint );
const SimpleString UTF8String = SimpleString::SetUTF8( CodePointArray );
for( uint CharIndex = 0; CharIndex < UTF8String.Length(); ++CharIndex )
{
const char NextChar = UTF8String.GetChar( CharIndex );
Token.m_TokenString.PushBack( NextChar );
}
}
else
{
PRINTF( "Unrecognized escape sequence \\%c at line %d\n", next, LineCount );
WARNDESC( "Unrecognized escape sequence" );
}
}
else if( c == 0x0d )
{
// DOS linebreak is 0D 0A, so ignore and expect \n to follow
}
else if( c == '\0' )
{
// Don't know how these are getting in either, but ignore them
}
else if( c == '\n' || c == '\v' )
{
if( Token.m_TokenType == SToken::ET_Macro )
{
MacroToken = Token;
}
// Dunno how vertical tabs are getting in, but treat them as linebreaks
if( Token.m_TokenString.Empty() )
{
if( Token.m_TokenType != SToken::ET_Counter )
{
Token.m_TokenType = SToken::ET_None;
}
Tokens.PushBack( Token );
DEBUGCATPRINTF( "Core", 2, "%s\n", SToken::m_TokenNames[ Token.m_TokenType ] );
Token.m_TokenType = SToken::ET_Name;
}
else
{
Token.m_TokenString.PushBack( '\0' );
Tokens.PushBack( Token );
DEBUGCATPRINTF( "Core", 2, "%s: %s\n", SToken::m_TokenNames[ Token.m_TokenType ], Token.m_TokenString.GetData() );
Token.m_TokenString.Clear();
Token.m_TokenType = SToken::ET_Name;
}
++LineCount;
}
else if( c == '[' )
{
if( Token.m_TokenType == SToken::ET_String )
{
Token.m_TokenString.PushBack( c );
}
else
{
// We should only ever open a context when we're expecting a name
ASSERT( Token.m_TokenType == SToken::ET_Name );
Token.m_TokenType = SToken::ET_Context;
// Opening a new context, clear the macro token.
MacroToken = SToken();
}
}
else if( c == ']' )
{
// If we've already closed the context, ignore; else, push token
if( Token.m_TokenType == SToken::ET_String )
{
Token.m_TokenString.PushBack( c );
}
else
{
ASSERT( Token.m_TokenType == SToken::ET_Context );
Token.m_TokenString.PushBack( '\0' );
Tokens.PushBack( Token );
DEBUGCATPRINTF( "Core", 2, "%s: %s\n", SToken::m_TokenNames[ Token.m_TokenType ], Token.m_TokenString.GetData() );
Token.m_TokenString.Clear();
}
}
else if( c == '@' )
{
if( Token.m_TokenType == SToken::ET_String )
{
Token.m_TokenString.PushBack( c );
}
else
{
// We should only ever declare or insert a macro when we're expecting a name
ASSERT( Token.m_TokenType == SToken::ET_Name );
c = Stream.ReadInt8();
if( c == '@' )
{
// @@... means we're inserting the current macro into a name
// Make sure there is a current macro. If this fails, a macro probably
// wasn't opened in the current context.
ASSERT( MacroToken.m_TokenString.Size() > 0 );
const uint MacroLength = MacroToken.m_TokenString.Size();
for( uint MacroIndex = 0; MacroIndex < MacroLength; ++MacroIndex )
{
Token.m_TokenString.PushBack( MacroToken.m_TokenString[ MacroIndex ] );
}
}
else
{
// @... means we're declaring a new macro
Token.m_TokenType = SToken::ET_Macro;
if( c == ' ' || c == '\t' )
{
// Ignore whitespace at the front of macro
}
else
{
Token.m_TokenString.PushBack( c );
}
}
}
}
else
{
bool ClosedString = false;
InnerParse( c, StrMark, Token.m_TokenString, LineCount, Token.m_TokenType, &ClosedString );
if( ClosedString )
{
Tokens.PushBack( Token );
DEBUGCATPRINTF( "Core", 2, "%s: %s\n", SToken::m_TokenNames[ Token.m_TokenType ], Token.m_TokenString.GetData() );
Token.m_TokenString.Clear();
Token.m_TokenType = SToken::ET_None;
}
}
}
SimpleString Context = "";
// Tokens are made, now create config vars
for( uint i = 0; i < Tokens.Size(); ++i )
{
SToken& NameToken = Tokens[i];
SimpleString Name = "";
const char* ValueString = NULL;
if( NameToken.m_TokenType == SToken::ET_Name )
{
Name = NameToken.m_TokenString.GetData();
ASSERT( Tokens[ i + 1 ].m_TokenType == SToken::ET_Equals );
SToken& ValueToken = Tokens[ i + 2 ];
ValueString = ValueToken.m_TokenString.GetData();
if( Context != "" )
{
CATPRINTF( "Core", 2, "%s:", Context.CStr() );
}
CATPRINTF( "Core", 2, "%s: %s: %s\n", Name.CStr(), SToken::m_TokenNames[ ValueToken.m_TokenType ], ValueString );
switch( ValueToken.m_TokenType )
{
case SToken::ET_Bool:
{
// Just use the first character to determine truth
bool Value = false;
char first = ValueString[0];
if( first == 't' || first == 'T' )
{
Value = true;
}
ConfigManager::SetBool( Name, Value, Context );
}
break;
case SToken::ET_Int:
{
int Value = atoi( ValueString );
ConfigManager::SetInt( Name, Value, Context );
}
break;
case SToken::ET_Counter:
{
List<int>::Iterator NextCounterIter = ArrayCounters.Front();
( *NextCounterIter )++; // Add one to the last value we incremented, and that's the total for this array
ConfigManager::SetInt( Name, *NextCounterIter, Context );
ArrayCounters.PopFront();
}
break;
case SToken::ET_Float:
{
float Value = (float)atof( ValueString );
ConfigManager::SetFloat( Name, Value, Context );
}
break;
case SToken::ET_String:
{
// Make a permanent copy of the string
uint Length = (uint)strlen( ValueString );
char* pString = new char[ Length + 1];
memcpy_s( pString, Length + 1, ValueString, Length );
pString[ Length ] = '\0';
StringManager::AddString( StringManager::ESL_Permanent, pString );
ConfigManager::SetString( Name, pString, Context );
}
break;
default:
WARNDESC( "Unexpected token" );
break;
}
i += 2;
}
else if( NameToken.m_TokenType == SToken::ET_Context )
{
Context = NameToken.m_TokenString.GetData();
//CATPRINTF( "Core", 2, "Pushed context %s\n", Context.CStr() );
}
else
{
DEBUGCATPRINTF( "Core", 2, "Skipped unexpected token %s (expected ET_Name)\n", SToken::m_TokenNames[ NameToken.m_TokenType ] );
}
}
// Clean up
for( uint i = 0; i < Tokens.Size(); ++i )
{
Tokens[i].m_TokenString.Clear();
}
Tokens.Clear();
}