/// Resolve a set of shader preprocessor options from the associated index.
///
/// @param[in] shaderType Type of shader.
/// @param[in] index Option set index.
/// @param[out] rToggleNames List of enabled shader toggles.
/// @param[out] rSelectPairs List shader selection pair values.
void Shader::Options::GetOptionSetFromIndex(
RShader::EType shaderType,
size_t index,
DynArray< Name >& rToggleNames,
DynArray< SelectPair >& rSelectPairs ) const
{
HELIUM_ASSERT( static_cast< size_t >( shaderType ) < static_cast< size_t >( RShader::TYPE_MAX ) );
rToggleNames.Resize( 0 );
rSelectPairs.Resize( 0 );
uint32_t shaderTypeMask = ( 1 << shaderType );
size_t shaderToggleCount = m_toggles.GetSize();
for( size_t shaderToggleIndex = 0; shaderToggleIndex < shaderToggleCount; ++shaderToggleIndex )
{
const Toggle& rShaderToggle = m_toggles[ shaderToggleIndex ];
if( !( rShaderToggle.shaderTypeFlags & shaderTypeMask ) )
{
continue;
}
if( index & 0x1 )
{
HELIUM_VERIFY( rToggleNames.New( rShaderToggle.name ) );
}
index >>= 1;
}
size_t shaderSelectCount = m_selects.GetSize();
for( size_t shaderSelectIndex = 0; shaderSelectIndex < shaderSelectCount; ++shaderSelectIndex )
{
const Select& rShaderSelect = m_selects[ shaderSelectIndex ];
if( !( rShaderSelect.shaderTypeFlags & shaderTypeMask ) )
{
continue;
}
const DynArray< Name >& rShaderSelectChoices = rShaderSelect.choices;
size_t shaderSelectChoiceCount = rShaderSelectChoices.GetSize();
size_t selectIndexMultiplier = shaderSelectChoiceCount + rShaderSelect.bOptional;
size_t selectIndex = index % selectIndexMultiplier;
index /= selectIndexMultiplier;
if( !rShaderSelect.bOptional || selectIndex != shaderSelectChoiceCount )
{
SelectPair* pSelectPair = rSelectPairs.New();
HELIUM_ASSERT( pSelectPair );
pSelectPair->name = rShaderSelect.name;
pSelectPair->choice = rShaderSelectChoices[ selectIndex ];
}
}
}
void Helium::ResourceHandler::SaveObjectToPersistentDataBuffer( Reflect::Object *_object, DynArray< uint8_t > &_buffer )
{
_buffer.Resize(0);
if (!_object)
{
return;
}
Cache::WriteCacheObjectToBuffer(*_object, _buffer);
}
void Helium::Cache::WriteCacheObjectToBuffer( Reflect::Object &_object, DynArray< uint8_t > &_buffer )
{
#if USE_XML_FOR_CACHE_DATA
{
tstringstream xml_out_ss;
Reflect::ArchiveXML xml_out(new Reflect::TCharStream(&xml_out_ss, false), true);
xml_out.WriteFileHeader();
xml_out.WriteSingleObject(_object);
xml_out.WriteFileFooter();
xml_out.Close();
tstring xml_str;
xml_str = xml_out_ss.str();
if (xml_str.size() > 0)
{
_buffer.Resize(xml_str.size() * sizeof(tchar_t));
memcpy(&_buffer[0], xml_str.data(), xml_str.size() * sizeof(tchar_t));
}
}
#else
{
std::stringstream ss_out;
Reflect::ArchiveBinary binary_out(new Reflect::CharStream(&ss_out, false, Helium::ByteOrders::LittleEndian, Helium::Reflect::CharacterEncodings::UTF_16), true);
binary_out.SerializeInstance( &_object );
// This is not an efficient way to do this
std::string str_out;
str_out = ss_out.str();
if (!str_out.empty())
{
_buffer.Resize(str_out.size());
memcpy(&_buffer[0], str_out.data(), str_out.size());
}
}
#endif
}
void StructureDynArrayData::Serialize( ArchiveT& archive )
{
const Structure *structure = GetInternalStructure();
DynArray< ObjectPtr > components;
components.Resize( structure->m_DynArrayAdapter->GetSize(GetInternalPtr(structure)) );
for (size_t i = 0; i < components.GetSize(); ++i)
{
components[i] = structure->m_DynArrayAdapter->GetItem(GetInternalPtr(structure), i, m_Instance, m_Field);
}
archive.SerializeArray( components );
}
/// Gather all existing resource handlers.
///
/// This will iterate through all sub-types of ResourceHandler and add their template objects to a dynamic array.
///
/// @param[out] rResourceHandlers List of existing resource handlers.
void ResourceHandler::GetAllResourceHandlers( DynArray< ResourceHandler* >& rResourceHandlers )
{
rResourceHandlers.Resize( 0 );
const GameObjectType* pResourceHandlerType = GetStaticType();
HELIUM_ASSERT( pResourceHandlerType );
GameObjectType::ConstIterator typeEnd = GameObjectType::GetTypeEnd();
for( GameObjectType::ConstIterator typeIterator = GameObjectType::GetTypeBegin(); typeIterator != typeEnd; ++typeIterator )
{
const GameObjectType& rType = *typeIterator;
if( &rType != pResourceHandlerType && rType.GetClass()->IsType( pResourceHandlerType->GetClass() ) )
{
ResourceHandler* pHandler = Reflect::AssertCast< ResourceHandler >( rType.GetTemplate() );
HELIUM_ASSERT( pHandler );
rResourceHandlers.Push( pHandler );
}
}
}
/// Helper function for compiling a shader for a specific profile.
///
/// @param[in] pVariant Shader variant for which we are compiling.
/// @param[in] pPreprocessor Platform preprocessor to use for compiling.
/// @param[in] platformIndex Platform index.
/// @param[in] shaderProfileIndex Index of the target shader profile.
/// @param[in] shaderType Type of shader to compile.
/// @param[in] pShaderSourceData Buffer in which the shader source code is stored.
/// @param[in] shaderSourceSize Size of the shader source buffer, in bytes.
/// @param[in] rTokens Array specifying preprocessor tokens to pass to the shader compiler.
/// @param[out] rCompiledCodeBuffer Buffer in which the compiled code will be stored.
///
/// @return True if compiling was successful, false if not.
bool ShaderVariantResourceHandler::CompileShader(
ShaderVariant* pVariant,
PlatformPreprocessor* pPreprocessor,
size_t platformIndex,
size_t shaderProfileIndex,
RShader::EType shaderType,
const void* pShaderSourceData,
size_t shaderSourceSize,
const DynArray< PlatformPreprocessor::ShaderToken >& rTokens,
DynArray< uint8_t >& rCompiledCodeBuffer )
{
HELIUM_ASSERT( pVariant );
HELIUM_ASSERT( pPreprocessor );
HELIUM_ASSERT( static_cast< size_t >( shaderType ) < static_cast< size_t >( RShader::TYPE_MAX ) );
HELIUM_ASSERT( pShaderSourceData || shaderSourceSize == 0 );
HELIUM_UNREF( platformIndex ); // Variable is used for logging only.
rCompiledCodeBuffer.Resize( 0 );
#if HELIUM_ENABLE_TRACE
DynArray< String > errorMessages;
#endif
Path shaderFilePath;
if ( !File::GetDataDirectory( shaderFilePath ) )
{
HELIUM_TRACE(
TRACE_ERROR,
TXT( "ShaderVariantResourceHandler: Failed to obtain data directory." ) );
return false;
}
shaderFilePath += pVariant->GetPath().GetParent().ToFilePathString().GetData();
bool bCompileResult = pPreprocessor->CompileShader(
shaderFilePath,
shaderProfileIndex,
shaderType,
pShaderSourceData,
shaderSourceSize,
rTokens.GetData(),
rTokens.GetSize(),
rCompiledCodeBuffer
#if HELIUM_ENABLE_TRACE
, &errorMessages
#else
, NULL
#endif
);
if( !bCompileResult )
{
rCompiledCodeBuffer.Resize( 0 );
#if HELIUM_ENABLE_TRACE
String tokenList;
#if HELIUM_UNICODE
String convertedToken;
#endif
size_t tokenCount = rTokens.GetSize();
for( size_t tokenIndex = 0; tokenIndex < tokenCount; ++tokenIndex )
{
tokenList += TXT( ' ' );
#if HELIUM_UNICODE
StringConverter< char, tchar_t >::Convert( convertedToken, rTokens[ tokenIndex ].name );
tokenList += convertedToken;
#else
tokenList += rTokens[ tokenIndex ].name;
#endif
}
size_t errorCount = errorMessages.GetSize();
HELIUM_TRACE(
TRACE_ERROR,
( TXT( "ShaderVariantResourceHandler: Failed to compile \"%s\" for platform %" ) TPRIuSZ
TXT( ", profile %" ) TPRIuSZ TXT( "; %" ) TPRIuSZ TXT( " errors (tokens:%s):\n" ) ),
*pVariant->GetPath().ToString(),
platformIndex,
shaderProfileIndex,
errorCount,
*tokenList );
for( size_t errorIndex = 0; errorIndex < errorCount; ++errorIndex )
{
HELIUM_TRACE( TRACE_ERROR, TXT( "- %s\n" ), *errorMessages[ errorIndex ] );
}
#endif // HELIUM_ENABLE_TRACE
}
return bCompileResult;
}
/// @copydoc ResourceHandler::CacheResource()
bool ShaderVariantResourceHandler::CacheResource(
ObjectPreprocessor* pObjectPreprocessor,
Resource* pResource,
const String& rSourceFilePath )
{
HELIUM_ASSERT( pObjectPreprocessor );
HELIUM_ASSERT( pResource );
ShaderVariant* pVariant = Reflect::AssertCast< ShaderVariant >( pResource );
// Parse the shader type and user option index from the variant name.
Name variantName = pVariant->GetName();
const tchar_t* pVariantNameString = *variantName;
HELIUM_ASSERT( pVariantNameString );
tchar_t shaderTypeCharacter = pVariantNameString[ 0 ];
HELIUM_ASSERT( shaderTypeCharacter != TXT( '\0' ) );
RShader::EType shaderType;
switch( shaderTypeCharacter )
{
case TXT( 'v' ):
{
shaderType = RShader::TYPE_VERTEX;
break;
}
case TXT( 'p' ):
{
shaderType = RShader::TYPE_PIXEL;
break;
}
default:
{
HELIUM_TRACE(
TRACE_ERROR,
( TXT( "ShaderVariantResourceHandler: Failed to determine shader type from the name of object " )
TXT( "\"%s\".\n" ) ),
*pVariant->GetPath().ToString() );
return false;
}
}
uint32_t userOptionIndex = 0;
++pVariantNameString;
int parseResult;
#if HELIUM_UNICODE
#if HELIUM_CC_CL
parseResult = swscanf_s( pVariantNameString, TXT( "%" ) TSCNu32, &userOptionIndex );
#else
parseResult = swscanf( pVariantNameString, TXT( "%" ) TSCNu32, &userOptionIndex );
#endif
#else
#if HELIUM_CC_CL
parseResult = sscanf_s( pVariantNameString, TXT( "%" ) TSCNu32, &userOptionIndex );
#else
parseResult = sscanf( pVariantNameString, TXT( "%" ) TSCNu32, &userOptionIndex );
#endif
#endif
if( parseResult != 1 )
{
HELIUM_TRACE(
TRACE_ERROR,
( TXT( "ShaderVariantResourceHandler: Failed to parse user shader option set index from the name of " )
TXT( "option \"%s\".\n" ) ),
*pVariant->GetPath().ToString() );
return false;
}
// Get the parent shader.
Shader* pShader = Reflect::AssertCast< Shader >( pVariant->GetOwner() );
HELIUM_ASSERT( pShader );
HELIUM_ASSERT( pShader->GetAnyFlagSet( GameObject::FLAG_PRECACHED ) );
// Acquire the user preprocessor option set associated with the target shader type and user option set index.
const Shader::Options& rUserOptions = pShader->GetUserOptions();
DynArray< Name > toggleNames;
DynArray< Shader::SelectPair > selectPairs;
rUserOptions.GetOptionSetFromIndex( shaderType, userOptionIndex, toggleNames, selectPairs );
DynArray< PlatformPreprocessor::ShaderToken > shaderTokens;
size_t userToggleNameCount = toggleNames.GetSize();
for( size_t toggleNameIndex = 0; toggleNameIndex < userToggleNameCount; ++toggleNameIndex )
{
PlatformPreprocessor::ShaderToken* pToken = shaderTokens.New();
HELIUM_ASSERT( pToken );
StringConverter< tchar_t, char >::Convert( pToken->name, *toggleNames[ toggleNameIndex ] );
pToken->definition = "1";
}
size_t userSelectPairCount = selectPairs.GetSize();
for( size_t selectPairIndex = 0; selectPairIndex < userSelectPairCount; ++selectPairIndex )
{
const Shader::SelectPair& rPair = selectPairs[ selectPairIndex ];
PlatformPreprocessor::ShaderToken* pToken = shaderTokens.New();
HELIUM_ASSERT( pToken );
StringConverter< tchar_t, char >::Convert( pToken->name, *rPair.name );
pToken->definition = "1";
pToken = shaderTokens.New();
HELIUM_ASSERT( pToken );
StringConverter< tchar_t, char >::Convert( pToken->name, *rPair.choice );
pToken->definition = "1";
}
size_t userShaderTokenCount = shaderTokens.GetSize();
// Load the entire shader resource into memory.
FileStream* pSourceFileStream = File::Open( rSourceFilePath, FileStream::MODE_READ );
if( !pSourceFileStream )
{
HELIUM_TRACE(
TRACE_ERROR,
( TXT( "ShaderVariantResourceHandler: Source file for shader variant resource \"%s\" failed to open " )
TXT( "properly.\n" ) ),
*pVariant->GetPath().ToString() );
return false;
}
int64_t size64 = pSourceFileStream->GetSize();
HELIUM_ASSERT( size64 != -1 );
HELIUM_ASSERT( static_cast< uint64_t >( size64 ) <= static_cast< size_t >( -1 ) );
if( size64 > static_cast< uint64_t >( static_cast< size_t >( -1 ) ) )
{
HELIUM_TRACE(
TRACE_ERROR,
( TXT( "ShaderVariantResourceHandler: Source file for shader resource \"%s\" is too large to fit " )
TXT( "into memory for preprocessing.\n" ) ),
*pShader->GetPath().ToString() );
delete pSourceFileStream;
return false;
}
size_t size = static_cast< size_t >( size64 );
DefaultAllocator allocator;
void* pShaderSource = allocator.Allocate( size );
HELIUM_ASSERT( pShaderSource );
if( !pShaderSource )
{
HELIUM_TRACE(
TRACE_ERROR,
( TXT( "ShaderVariantResourceHandler: Failed to allocate %" ) TPRIuSZ TXT( " bytes for loading the " )
TXT( "source data of \"%s\" for preprocessing.\n" ) ),
size,
*pShader->GetPath().ToString() );
delete pSourceFileStream;
return false;
}
BufferedStream( pSourceFileStream ).Read( pShaderSource, 1, size );
delete pSourceFileStream;
// Compile each variant of system options for each shader profile in each supported target platform.
const Shader::Options& rSystemOptions = pShader->GetSystemOptions();
size_t systemOptionSetCount = rSystemOptions.ComputeOptionSetCount( shaderType );
if( systemOptionSetCount > UINT32_MAX )
{
HELIUM_TRACE(
TRACE_ERROR,
( TXT( "ShaderVariantResourceHandler: System option set count (%" ) TPRIuSZ TXT( ") in shader \"%s\" " )
TXT( "exceeds the maximum supported (%" ) TPRIuSZ TXT( ").\n" ) ),
systemOptionSetCount,
*pShader->GetPath().ToString(),
static_cast< size_t >( UINT32_MAX ) );
allocator.Free( pShaderSource );
return false;
}
uint32_t systemOptionSetCount32 = static_cast< uint32_t >( systemOptionSetCount );
for( size_t platformIndex = 0; platformIndex < static_cast< size_t >( Cache::PLATFORM_MAX ); ++platformIndex )
{
PlatformPreprocessor* pPreprocessor = pObjectPreprocessor->GetPlatformPreprocessor(
static_cast< Cache::EPlatform >( platformIndex ) );
if( !pPreprocessor )
{
continue;
}
Resource::PreprocessedData& rPreprocessedData = pVariant->GetPreprocessedData(
static_cast< Cache::EPlatform >( platformIndex ) );
ShaderVariant::PersistentResourceData persistentResourceData;
persistentResourceData.m_resourceCount = systemOptionSetCount32;
SaveObjectToPersistentDataBuffer(&persistentResourceData, rPreprocessedData.persistentDataBuffer);
size_t shaderProfileCount = pPreprocessor->GetShaderProfileCount();
size_t shaderCount = shaderProfileCount * systemOptionSetCount;
DynArray< DynArray< uint8_t > >& rSubDataBuffers = rPreprocessedData.subDataBuffers;
rSubDataBuffers.Reserve( shaderCount );
rSubDataBuffers.Resize( 0 );
rSubDataBuffers.Resize( shaderCount );
rSubDataBuffers.Trim();
rPreprocessedData.bLoaded = true;
}
// DynArray< uint8_t > compiledCodeBuffer;
// DynArray< ShaderConstantBufferInfo > constantBuffers, pcSm4ConstantBuffers;
// DynArray< ShaderSamplerInfo > samplerInputs;
// DynArray< ShaderTextureInfo > textureInputs;
CompiledShaderData csd_pc_sm4;
for( size_t systemOptionSetIndex = 0; systemOptionSetIndex < systemOptionSetCount; ++systemOptionSetIndex )
{
rSystemOptions.GetOptionSetFromIndex( shaderType, systemOptionSetIndex, toggleNames, selectPairs );
size_t systemToggleNameCount = toggleNames.GetSize();
for( size_t toggleNameIndex = 0; toggleNameIndex < systemToggleNameCount; ++toggleNameIndex )
{
PlatformPreprocessor::ShaderToken* pToken = shaderTokens.New();
HELIUM_ASSERT( pToken );
StringConverter< tchar_t, char >::Convert( pToken->name, *toggleNames[ toggleNameIndex ] );
pToken->definition = "1";
}
size_t systemSelectPairCount = selectPairs.GetSize();
for( size_t selectPairIndex = 0; selectPairIndex < systemSelectPairCount; ++selectPairIndex )
{
const Shader::SelectPair& rPair = selectPairs[ selectPairIndex ];
PlatformPreprocessor::ShaderToken* pToken = shaderTokens.New();
HELIUM_ASSERT( pToken );
StringConverter< tchar_t, char >::Convert( pToken->name, *rPair.name );
pToken->definition = "1";
pToken = shaderTokens.New();
HELIUM_ASSERT( pToken );
StringConverter< tchar_t, char >::Convert( pToken->name, *rPair.choice );
pToken->definition = "1";
}
// Compile for PC shader model 4 first so that we can get the constant buffer information.
PlatformPreprocessor* pPreprocessor = pObjectPreprocessor->GetPlatformPreprocessor( Cache::PLATFORM_PC );
HELIUM_ASSERT( pPreprocessor );
csd_pc_sm4.compiledCodeBuffer.Resize( 0 );
bool bCompiled = CompileShader(
pVariant,
pPreprocessor,
Cache::PLATFORM_PC,
ShaderProfile::PC_SM4,
shaderType,
pShaderSource,
size,
shaderTokens,
csd_pc_sm4.compiledCodeBuffer );
if( !bCompiled )
{
HELIUM_TRACE(
TRACE_ERROR,
( TXT( "ShaderVariantResourceHandler: Failed to compile shader for PC shader model 4, which is " )
TXT( "needed for reflection purposes. Additional shader targets will not be built.\n" ) ) );
}
else
{
csd_pc_sm4.constantBuffers.Resize( 0 );
csd_pc_sm4.samplerInputs.Resize( 0 );
csd_pc_sm4.textureInputs.Resize( 0 );
bool bReadConstantBuffers = pPreprocessor->FillShaderReflectionData(
ShaderProfile::PC_SM4,
csd_pc_sm4.compiledCodeBuffer.GetData(),
csd_pc_sm4.compiledCodeBuffer.GetSize(),
csd_pc_sm4.constantBuffers,
csd_pc_sm4.samplerInputs,
csd_pc_sm4.textureInputs );
if( !bReadConstantBuffers )
{
HELIUM_TRACE(
TRACE_ERROR,
( TXT( "ShaderVariantResourceHandler: Failed to read reflection information for PC shader " )
TXT( "model 4. Additional shader targets will not be built.\n" ) ) );
}
else
{
Resource::PreprocessedData& rPcPreprocessedData = pVariant->GetPreprocessedData(
Cache::PLATFORM_PC );
DynArray< DynArray< uint8_t > >& rPcSubDataBuffers = rPcPreprocessedData.subDataBuffers;
DynArray< uint8_t >& rPcSm4SubDataBuffer =
rPcSubDataBuffers[ ShaderProfile::PC_SM4 * systemOptionSetCount + systemOptionSetIndex ];
Cache::WriteCacheObjectToBuffer(csd_pc_sm4, rPcSm4SubDataBuffer);
// FOR EACH PLATFORM
for( size_t platformIndex = 0;
platformIndex < static_cast< size_t >( Cache::PLATFORM_MAX );
++platformIndex )
{
PlatformPreprocessor* pPreprocessor = pObjectPreprocessor->GetPlatformPreprocessor(
static_cast< Cache::EPlatform >( platformIndex ) );
if( !pPreprocessor )
{
continue;
}
// GET PLATFORM'S SUBDATA BUFFER
Resource::PreprocessedData& rPreprocessedData = pVariant->GetPreprocessedData(
static_cast< Cache::EPlatform >( platformIndex ) );
DynArray< DynArray< uint8_t > >& rSubDataBuffers = rPreprocessedData.subDataBuffers;
size_t shaderProfileCount = pPreprocessor->GetShaderProfileCount();
for( size_t shaderProfileIndex = 0;
shaderProfileIndex < shaderProfileCount;
++shaderProfileIndex )
{
CompiledShaderData csd;
// Already cached PC shader model 4...
if( shaderProfileIndex == ShaderProfile::PC_SM4 && platformIndex == Cache::PLATFORM_PC )
{
continue;
}
bCompiled = CompileShader(
pVariant,
pPreprocessor,
platformIndex,
shaderProfileIndex,
shaderType,
pShaderSource,
size,
shaderTokens,
csd.compiledCodeBuffer );
if( !bCompiled )
{
continue;
}
csd.constantBuffers = csd_pc_sm4.constantBuffers;
csd.samplerInputs.Resize( 0 );
csd.textureInputs.Resize( 0 );
bReadConstantBuffers = pPreprocessor->FillShaderReflectionData(
shaderProfileIndex,
csd.compiledCodeBuffer.GetData(),
csd.compiledCodeBuffer.GetSize(),
csd.constantBuffers,
csd.samplerInputs,
csd.textureInputs );
if( !bReadConstantBuffers )
{
continue;
}
DynArray< uint8_t >& rTargetSubDataBuffer =
rSubDataBuffers[ shaderProfileIndex * systemOptionSetCount + systemOptionSetIndex ];
Cache::WriteCacheObjectToBuffer(csd, rTargetSubDataBuffer);
}
}
}
}
// Trim the system tokens off the shader token list for the next pass.
shaderTokens.Resize( userShaderTokenCount );
}
allocator.Free( pShaderSource );
return true;
}