std::string DetailsColumn::Size( const Helium::FilePath& path )
{
Status status;
status.Read( path.Get().c_str() );
int64_t size = status.m_Size;
std::stringstream printSize;
if ( size == 0 )
{
printSize << "0 KB";
}
else if ( size <= 1024 )
{
printSize << "1 KB";
}
else
{
size = size / 1024;
printSize << size << " KB";
}
return printSize.str().c_str();
}
/// Initialize this cache.
///
/// This will verify the existence of the given files and prepare for cache loading. Note that no file loading is
/// performed at this time.
///
/// Once initialization is performed, the table of contents must be loaded using BeginLoadToc().
///
/// @param[in] name Name identifying this cache.
/// @param[in] platform Cache platform identifier.
/// @param[in] pTocFileName FilePath name of the table of contents file.
/// @param[in] pCacheFileName FilePath name of the cache file.
///
/// @return True if initialization was successful, false if not.
///
/// @see Shutdown(), BeginLoadToc()
bool Cache::Initialize( Name name, EPlatform platform, const tchar_t* pTocFileName, const tchar_t* pCacheFileName )
{
HELIUM_ASSERT( !name.IsEmpty() );
HELIUM_ASSERT( static_cast< size_t >( platform ) < static_cast< size_t >( PLATFORM_MAX ) );
HELIUM_ASSERT( pTocFileName );
HELIUM_ASSERT( pCacheFileName );
Shutdown();
m_name = name;
m_platform = platform;
Status status;
status.Read( pTocFileName );
int64_t tocSize64 = status.m_Size;
if( tocSize64 != -1 && static_cast< uint64_t >( tocSize64 ) >= UINT32_MAX )
{
HELIUM_TRACE(
TraceLevels::Error,
TXT( "Cache::Initialize(): TOC file \"%s\" exceeds the maximum allowed size for TOC files (2 GB).\n" ),
pTocFileName );
return false;
}
m_tocFileName = pTocFileName;
m_cacheFileName = pCacheFileName;
m_tocSize = static_cast< uint32_t >( tocSize64 );
HELIUM_ASSERT( !m_pEntryPool );
m_pEntryPool = new ObjectPool< Entry >( ENTRY_POOL_BLOCK_SIZE );
HELIUM_ASSERT( m_pEntryPool );
return true;
}
/// Update processing of object property preloading for a given load request.
///
/// @param[in] pRequest Load request to process.
///
/// @return True if object property preloading for the given load request has completed, false if not.
bool LoosePackageLoader::TickDeserialize( LoadRequest* pRequest )
{
HELIUM_ASSERT( pRequest );
HELIUM_ASSERT( !( pRequest->flags & LOAD_FLAG_PROPERTY_PRELOADED ) );
Asset* pObject = pRequest->spObject;
HELIUM_ASSERT( pRequest->index < m_objects.GetSize() );
SerializedObjectData& rObjectData = m_objects[ pRequest->index ];
// Wait for the template and owner objects to load.
AssetLoader* pAssetLoader = AssetLoader::GetStaticInstance();
HELIUM_ASSERT( pAssetLoader );
if( !rObjectData.templatePath.IsEmpty() )
{
if( IsValid( pRequest->templateLoadId ) )
{
if( !pAssetLoader->TryFinishLoad( pRequest->templateLoadId, pRequest->spTemplate ) )
{
return false;
}
SetInvalid( pRequest->templateLoadId );
}
if( !pRequest->spTemplate )
{
HELIUM_TRACE(
TraceLevels::Error,
TXT( "LoosePackageLoader: Failed to load template object for \"%s\".\n" ),
*rObjectData.objectPath.ToString() );
if( pObject )
{
pObject->SetFlags( Asset::FLAG_PRELOADED | Asset::FLAG_LINKED );
pObject->ConditionalFinalizeLoad();
}
pRequest->flags |= LOAD_FLAG_PRELOADED | LOAD_FLAG_ERROR;
return true;
}
}
HELIUM_ASSERT( IsInvalid( pRequest->templateLoadId ) );
Asset* pTemplate = pRequest->spTemplate;
AssetPath ownerPath = rObjectData.objectPath.GetParent();
if( !ownerPath.IsEmpty() )
{
if( IsValid( pRequest->ownerLoadId ) )
{
if( !pAssetLoader->TryFinishLoad( pRequest->ownerLoadId, pRequest->spOwner ) )
{
return false;
}
SetInvalid( pRequest->ownerLoadId );
}
if( !pRequest->spOwner )
{
HELIUM_TRACE(
TraceLevels::Error,
TXT( "LoosePackageLoader: Failed to load owner object for \"%s\".\n" ),
*rObjectData.objectPath.ToString() );
if( pObject )
{
pObject->SetFlags( Asset::FLAG_PRELOADED | Asset::FLAG_LINKED );
pObject->ConditionalFinalizeLoad();
}
pRequest->flags |= LOAD_FLAG_PRELOADED | LOAD_FLAG_ERROR;
return true;
}
}
HELIUM_ASSERT( IsInvalid( pRequest->ownerLoadId ) );
Asset* pOwner = pRequest->spOwner;
AssetType* pType = pRequest->spType;
HELIUM_ASSERT( pType );
HELIUM_ASSERT( !pOwner || pOwner->IsFullyLoaded() );
HELIUM_ASSERT( !pTemplate || pTemplate->IsFullyLoaded() );
AsyncLoader& rAsyncLoader = AsyncLoader::GetStaticInstance();
FilePath object_file_path = m_packageDirPath + *rObjectData.objectPath.GetName() + TXT( "." ) + Persist::ArchiveExtensions[ Persist::ArchiveTypes::Json ];
bool load_properties_from_file = true;
size_t object_file_size = 0;
if ( !IsValid( pRequest->asyncFileLoadId ) )
{
if (!object_file_path.IsFile())
{
if (pType->GetMetaClass()->IsType( Reflect::GetMetaClass< Resource >() ))
{
HELIUM_TRACE(
TraceLevels::Info,
TXT( "LoosePackageLoader::TickDeserialize(): No object file found for resource \"%s\". Expected file location: \"%s\". This is normal for newly added resources.\n" ),
*rObjectData.objectPath.ToString(),
*object_file_path);
// We will allow continuing to load using all default properties. This behavior is to support dropping resources into the
// data property and autogenerating objects from them.
load_properties_from_file = false;
}
else
{
HELIUM_TRACE(
TraceLevels::Warning,
TXT( "LoosePackageLoader::TickDeserialize(): No object file found for object \"%s\". Expected file location: \"%s\"\n" ),
*rObjectData.objectPath.ToString(),
*object_file_path);
}
}
else
{
Status status;
status.Read( object_file_path.Get().c_str() );
int64_t i64_object_file_size = status.m_Size;
if( i64_object_file_size == -1 )
{
HELIUM_TRACE(
TraceLevels::Warning,
TXT( "LoosePackageLoader::TickDeserialize(): Could not get file size for object file of object \"%s\". Expected file location: \"%s\"\n" ),
*rObjectData.objectPath.ToString(),
*object_file_path );
}
else if( i64_object_file_size == 0 )
{
HELIUM_TRACE(
TraceLevels::Warning,
TXT( "LoosePackageLoader::TickDeserialize(): Object file \"%s\" for objct \"%s\" is empty.\n" ),
*object_file_path,
*rObjectData.objectPath.ToString() );
}
else if( static_cast< uint64_t >( i64_object_file_size ) > static_cast< uint64_t >( ~static_cast< size_t >( 0 ) ) )
{
HELIUM_TRACE(
TraceLevels::Error,
( TXT( "LoosePackageLoader::TickDeserialize(): Object file \"%s\" exceeds the maximum size supported by " )
TXT( "the current platform (file size: %" ) PRIu64 TXT( " bytes; max supported: %" ) PRIuSZ
TXT( " bytes).\n" ) ),
object_file_path.c_str(),
static_cast< uint64_t >( i64_object_file_size ),
~static_cast< size_t >( 0 ) );
}
else
{
object_file_size = static_cast< size_t >(i64_object_file_size);
}
}
if (!load_properties_from_file)
{
HELIUM_ASSERT(!object_file_size);
}
else if (!object_file_size)
{
pRequest->flags |= LOAD_FLAG_PRELOADED | LOAD_FLAG_ERROR;
return true;
}
else
{
HELIUM_ASSERT( !pRequest->pAsyncFileLoadBuffer );
pRequest->pAsyncFileLoadBuffer = DefaultAllocator().Allocate( object_file_size );
HELIUM_ASSERT( pRequest->pAsyncFileLoadBuffer );
pRequest->asyncFileLoadBufferSize = object_file_size;
pRequest->asyncFileLoadId = rAsyncLoader.QueueRequest(
pRequest->pAsyncFileLoadBuffer,
String(object_file_path.c_str()),
0,
pRequest->asyncFileLoadBufferSize);
}
}
size_t bytesRead = 0;
if (load_properties_from_file)
{
HELIUM_ASSERT( IsValid( pRequest->asyncFileLoadId ) );
if ( !rAsyncLoader.TrySyncRequest( pRequest->asyncFileLoadId, bytesRead ) )
{
return false;
}
}
/////// POINT OF NO RETURN: We *will* return true after this point, and the object *will* be finished preloading,
/////// for good or for bad.
SetInvalid(pRequest->asyncFileLoadId);
bool object_creation_failure = false;
// If we already had an existing object, make sure the type and template match.
if( pObject )
{
const AssetType* pExistingType = pObject->GetAssetType();
HELIUM_ASSERT( pExistingType );
if( pExistingType != pType )
{
HELIUM_TRACE(
TraceLevels::Error,
( TXT( "LoosePackageLoader: Cannot load \"%s\" using the existing object as the types do not match " )
TXT( "(existing type: \"%s\"; serialized type: \"%s\".\n" ) ),
*rObjectData.objectPath.ToString(),
*pExistingType->GetName(),
*pType->GetName() );
pObject->SetFlags( Asset::FLAG_PRELOADED | Asset::FLAG_LINKED );
pObject->ConditionalFinalizeLoad();
object_creation_failure = true;
}
}
else
{
bool bCreateResult = false;
if (pRequest->forceReload)
{
// Create the object.
bCreateResult = Asset::CreateObject(
pRequest->spObject,
pType,
Name( NULL_NAME ),
NULL,
pTemplate );
}
else
{
// Create the object.
bCreateResult = Asset::CreateObject(
pRequest->spObject,
pType,
rObjectData.objectPath.GetName(),
pOwner,
pTemplate );
}
if( !bCreateResult )
{
HELIUM_TRACE(
TraceLevels::Error,
TXT( "LoosePackageLoader: Failed to create \"%s\" during loading.\n" ),
*rObjectData.objectPath.ToString() );
object_creation_failure = true;
}
pObject = pRequest->spObject;
HELIUM_ASSERT( pObject );
}
if (load_properties_from_file && !object_creation_failure)
{
// Sanity checks for file load, then success path
HELIUM_ASSERT( bytesRead == pRequest->asyncFileLoadBufferSize );
if( IsInvalid( bytesRead ) )
{
HELIUM_TRACE(
TraceLevels::Error,
TXT( "LoosePackageLoader: Failed to read the contents of object file \"%s\" in async load request \"%d\".\n" ),
object_file_path.c_str(),
pRequest->asyncFileLoadId );
}
else if( bytesRead != pRequest->asyncFileLoadBufferSize )
{
HELIUM_TRACE(
TraceLevels::Warning,
( TXT( "LoosePackageLoader: Attempted to read %" ) PRIuSZ TXT( " bytes from object file \"%s\", " )
TXT( "but only %" ) PRIuSZ TXT( " bytes were read.\n" ) ),
pRequest->asyncFileLoadBufferSize,
object_file_path.c_str(),
bytesRead );
}
else
{
StaticMemoryStream archiveStream ( pRequest->pAsyncFileLoadBuffer, pRequest->asyncFileLoadBufferSize );
HELIUM_TRACE(
TraceLevels::Info,
TXT( "LoosePackageLoader: Reading %s. pResolver = %x\n"),
object_file_path.c_str(),
pRequest->pResolver);
DynamicArray< Reflect::ObjectPtr > objects;
objects.Push( pRequest->spObject.Get() ); // use existing objects
Persist::ArchiveReaderJson::ReadFromStream( archiveStream, objects, pRequest->pResolver );
HELIUM_ASSERT( objects[0].Get() == pRequest->spObject.Get() );
}
}
if (load_properties_from_file)
{
DefaultAllocator().Free(pRequest->pAsyncFileLoadBuffer);
pRequest->pAsyncFileLoadBuffer = NULL;
pRequest->asyncFileLoadBufferSize = 0;
}
pRequest->flags |= LOAD_FLAG_PROPERTY_PRELOADED;
if( object_creation_failure )
{
HELIUM_TRACE(
TraceLevels::Error,
TXT( "LoosePackageLoader: Deserialization of object \"%s\" failed.\n" ),
*rObjectData.objectPath.ToString() );
pObject->SetFlags( Asset::FLAG_PRELOADED | Asset::FLAG_LINKED );
pObject->ConditionalFinalizeLoad();
pRequest->flags |= LOAD_FLAG_ERROR;
}
else if( !pObject->IsDefaultTemplate() )
{
// If the object is a resource (not including the default template object for resource types), attempt to begin
// loading any existing persistent resource data stored in the object cache.
Resource* pResource = Reflect::SafeCast< Resource >( pObject );
if( pResource )
{
Name objectCacheName = Name( HELIUM_ASSET_CACHE_NAME );
CacheManager& rCacheManager = CacheManager::GetStaticInstance();
Cache* pCache = rCacheManager.GetCache( objectCacheName );
HELIUM_ASSERT( pCache );
pCache->EnforceTocLoad();
const Cache::Entry* pEntry = pCache->FindEntry( rObjectData.objectPath, 0 );
if( pEntry && pEntry->size != 0 )
{
HELIUM_ASSERT( IsInvalid( pRequest->persistentResourceDataLoadId ) );
HELIUM_ASSERT( !pRequest->pCachedObjectDataBuffer );
pRequest->pCachedObjectDataBuffer =
static_cast< uint8_t* >( DefaultAllocator().Allocate( pEntry->size ) );
HELIUM_ASSERT( pRequest->pCachedObjectDataBuffer );
pRequest->cachedObjectDataBufferSize = pEntry->size;
AsyncLoader& rAsyncLoader = AsyncLoader::GetStaticInstance();
pRequest->persistentResourceDataLoadId = rAsyncLoader.QueueRequest(
pRequest->pCachedObjectDataBuffer,
pCache->GetCacheFileName(),
pEntry->offset,
pEntry->size );
HELIUM_ASSERT( IsValid( pRequest->persistentResourceDataLoadId ) );
}
}
}
if( IsInvalid( pRequest->persistentResourceDataLoadId ) )
{
// No persistent resource data needs to be loaded.
pObject->SetFlags( Asset::FLAG_PRELOADED );
pRequest->flags |= LOAD_FLAG_PERSISTENT_RESOURCE_PRELOADED;
}
// Asset is now preloaded.
return true;
}
/// Add or update an entry in the cache.
///
/// @param[in] path GameObject path.
/// @param[in] subDataIndex Sub-data index associated with the cached data.
/// @param[in] pData Data to cache.
/// @param[in] timestamp Timestamp value to associate with the entry in the cache.
/// @param[in] size Number of bytes to cache.
///
/// @return True if the cache was updated successfully, false if not.
bool Cache::CacheEntry(
GameObjectPath path,
uint32_t subDataIndex,
const void* pData,
int64_t timestamp,
uint32_t size )
{
HELIUM_ASSERT( pData || size == 0 );
Status status;
status.Read( m_cacheFileName.GetData() );
int64_t cacheFileSize = status.m_Size;
uint64_t entryOffset = ( cacheFileSize == -1 ? 0 : static_cast< uint64_t >( cacheFileSize ) );
HELIUM_ASSERT( m_pEntryPool );
Entry* pEntryUpdate = m_pEntryPool->Allocate();
HELIUM_ASSERT( pEntryUpdate );
pEntryUpdate->offset = entryOffset;
pEntryUpdate->timestamp = timestamp;
pEntryUpdate->path = path;
pEntryUpdate->subDataIndex = subDataIndex;
pEntryUpdate->size = size;
uint64_t originalOffset = 0;
int64_t originalTimestamp = 0;
uint32_t originalSize = 0;
EntryKey key;
key.path = path;
key.subDataIndex = subDataIndex;
EntryMapType::Accessor entryAccessor;
bool bNewEntry = m_entryMap.Insert( entryAccessor, KeyValue< EntryKey, Entry* >( key, pEntryUpdate ) );
if( bNewEntry )
{
HELIUM_TRACE( TraceLevels::Info, TXT( "Cache: Adding \"%s\" to cache \"%s\".\n" ), *path.ToString(), *m_cacheFileName );
m_entries.Push( pEntryUpdate );
}
else
{
HELIUM_TRACE( TraceLevels::Info, TXT( "Cache: Updating \"%s\" in cache \"%s\".\n" ), *path.ToString(), *m_cacheFileName );
m_pEntryPool->Release( pEntryUpdate );
pEntryUpdate = entryAccessor->Second();
HELIUM_ASSERT( pEntryUpdate );
originalOffset = pEntryUpdate->offset;
originalTimestamp = pEntryUpdate->timestamp;
originalSize = pEntryUpdate->size;
if( originalSize < size )
{
pEntryUpdate->offset = entryOffset;
}
else
{
entryOffset = originalOffset;
}
pEntryUpdate->timestamp = timestamp;
pEntryUpdate->size = size;
}
AsyncLoader& rLoader = AsyncLoader::GetStaticInstance();
rLoader.Lock();
bool bCacheSuccess = true;
FileStream* pCacheStream = FileStream::OpenFileStream( m_cacheFileName, FileStream::MODE_WRITE, false );
if( !pCacheStream )
{
HELIUM_TRACE( TraceLevels::Error, TXT( "Cache: Failed to open cache \"%s\" for writing.\n" ), *m_cacheFileName );
bCacheSuccess = false;
}
else
{
HELIUM_TRACE(
TraceLevels::Info,
TXT( "Cache: Caching \"%s\" to \"%s\" (%" ) TPRIu32 TXT( " bytes @ offset %" ) TPRIu64 TXT( ").\n" ),
*path.ToString(),
*m_cacheFileName,
size,
entryOffset );
uint64_t seekOffset = static_cast< uint64_t >( pCacheStream->Seek(
static_cast< int64_t >( entryOffset ),
SeekOrigins::SEEK_ORIGIN_BEGIN ) );
if( seekOffset != entryOffset )
{
HELIUM_TRACE( TraceLevels::Error, TXT( "Cache: Cache file offset seek failed.\n" ) );
if( bNewEntry )
{
m_entries.Pop();
m_entryMap.Remove( entryAccessor );
m_pEntryPool->Release( pEntryUpdate );
}
else
{
pEntryUpdate->offset = originalOffset;
pEntryUpdate->timestamp = originalTimestamp;
pEntryUpdate->size = originalSize;
}
bCacheSuccess = false;
}
else
{
size_t writeSize = pCacheStream->Write( pData, 1, size );
if( writeSize != size )
{
HELIUM_TRACE(
TraceLevels::Error,
( TXT( "Cache: Failed to write %" ) TPRIu32 TXT( " bytes to cache \"%s\" (%" ) TPRIuSZ
TXT( " bytes written).\n" ) ),
size,
*m_cacheFileName,
writeSize );
if( bNewEntry )
{
m_entries.Pop();
m_entryMap.Remove( entryAccessor );
m_pEntryPool->Release( pEntryUpdate );
}
else
{
pEntryUpdate->offset = originalOffset;
pEntryUpdate->timestamp = originalTimestamp;
pEntryUpdate->size = originalSize;
}
bCacheSuccess = false;
}
else
{
HELIUM_TRACE( TraceLevels::Info, TXT( "Cache: Rewriting TOC file \"%s\".\n" ), *m_tocFileName );
FileStream* pTocStream = FileStream::OpenFileStream( m_tocFileName, FileStream::MODE_WRITE, true );
if( !pTocStream )
{
HELIUM_TRACE( TraceLevels::Error, TXT( "Cache: Failed to open TOC \"%s\" for writing.\n" ), *m_tocFileName );
}
else
{
BufferedStream* pBufferedStream = new BufferedStream( pTocStream );
HELIUM_ASSERT( pBufferedStream );
pBufferedStream->Write( &TOC_MAGIC, sizeof( TOC_MAGIC ), 1 );
pBufferedStream->Write( &sm_Version, sizeof( sm_Version ), 1 );
uint32_t entryCount = static_cast< uint32_t >( m_entries.GetSize() );
pBufferedStream->Write( &entryCount, sizeof( entryCount ), 1 );
String entryPath;
uint_fast32_t entryCountFast = entryCount;
for( uint_fast32_t entryIndex = 0; entryIndex < entryCountFast; ++entryIndex )
{
Entry* pEntry = m_entries[ entryIndex ];
HELIUM_ASSERT( pEntry );
pEntry->path.ToString( entryPath );
HELIUM_ASSERT( entryPath.GetSize() < UINT16_MAX );
uint16_t pathSize = static_cast< uint16_t >( entryPath.GetSize() );
pBufferedStream->Write( &pathSize, sizeof( pathSize ), 1 );
pBufferedStream->Write( *entryPath, sizeof( tchar_t ), pathSize );
pBufferedStream->Write( &pEntry->subDataIndex, sizeof( pEntry->subDataIndex ), 1 );
pBufferedStream->Write( &pEntry->offset, sizeof( pEntry->offset ), 1 );
pBufferedStream->Write( &pEntry->timestamp, sizeof( pEntry->timestamp ), 1 );
pBufferedStream->Write( &pEntry->size, sizeof( pEntry->size ), 1 );
}
delete pBufferedStream;
delete pTocStream;
}
}
}
delete pCacheStream;
}
rLoader.Unlock();
return bCacheSuccess;
}
/// @copydoc AssetLoader::CacheObject()
bool LooseAssetLoader::CacheObject( Asset* pAsset, bool bEvictPlatformPreprocessedResourceData )
{
HELIUM_ASSERT( pAsset );
HELIUM_TRACE(
TraceLevels::Info,
TXT( "LooseAssetLoader::CacheObject(): Caching asset %s.\n" ), *pAsset->GetPath().ToString() );
// Don't cache broken objects or packages.
if( pAsset->GetAnyFlagSet( Asset::FLAG_BROKEN ) || pAsset->IsPackage() )
{
return false;
}
// Make sure we have an object preprocessor instance with which to cache the object.
AssetPreprocessor* pAssetPreprocessor = AssetPreprocessor::GetStaticInstance();
if( !pAssetPreprocessor )
{
HELIUM_TRACE(
TraceLevels::Warning,
TXT( "LooseAssetLoader::CacheObject(): Missing AssetPreprocessor to use for caching.\n" ) );
return false;
}
// User configuration objects should not be cached.
AssetPath objectPath = pAsset->GetPath();
Config& rConfig = Config::GetStaticInstance();
// Only cache the files we care about
if ( rConfig.IsAssetPathInUserConfigPackage(objectPath) )
{
return false;
}
int64_t objectTimestamp = pAsset->GetAssetFileTimeStamp();
if( !pAsset->IsDefaultTemplate() )
{
Resource* pResource = Reflect::SafeCast< Resource >( pAsset );
if( pResource )
{
AssetPath baseResourcePath = pResource->GetPath();
HELIUM_ASSERT( !baseResourcePath.IsPackage() );
for( ; ; )
{
AssetPath parentPath = baseResourcePath.GetParent();
if( parentPath.IsEmpty() || parentPath.IsPackage() )
{
break;
}
baseResourcePath = parentPath;
}
FilePath sourceFilePath;
if ( !FileLocations::GetDataDirectory( sourceFilePath ) )
{
HELIUM_TRACE(
TraceLevels::Warning,
TXT( "LooseAssetLoader::CacheObject(): Could not obtain data directory.\n" ) );
return false;
}
sourceFilePath += baseResourcePath.ToFilePathString().GetData();
Status stat;
stat.Read( sourceFilePath.Get().c_str() );
int64_t sourceFileTimestamp = stat.m_ModifiedTime;
if( sourceFileTimestamp > objectTimestamp )
{
objectTimestamp = sourceFileTimestamp;
}
}
}
// Cache the object.
bool bSuccess = pAssetPreprocessor->CacheObject(
pAsset,
objectTimestamp,
bEvictPlatformPreprocessedResourceData );
if( !bSuccess )
{
HELIUM_TRACE(
TraceLevels::Error,
TXT( "LooseAssetLoader: Failed to cache object \"%s\".\n" ),
*objectPath.ToString() );
}
return bSuccess;
}
