/** Retrieves the derived chunk from the derived data cache. */
void FAsyncStreamDerivedChunkWorker::DoWork()
{
DECLARE_SCOPE_CYCLE_COUNTER(TEXT("FAsyncStreamDerivedChunkWorker::DoWork"), STAT_AsyncStreamDerivedChunkWorker_DoWork, STATGROUP_StreamingDetails);
UE_LOG(LogAudio, Verbose, TEXT("Start of ASync DDC Chunk read for key: %s"), *DerivedDataKey);
TArray<uint8> DerivedChunkData;
if (GetDerivedDataCacheRef().GetSynchronous(*DerivedDataKey, DerivedChunkData))
{
FMemoryReader Ar(DerivedChunkData, true);
int32 ChunkSize = 0;
Ar << ChunkSize;
checkf(ChunkSize == ExpectedChunkSize, TEXT("ChunkSize(%d) != ExpectedSize(%d)"), ChunkSize, ExpectedChunkSize);
Ar.Serialize(DestChunkData, ChunkSize);
}
else
{
bRequestFailed = true;
}
FPlatformMisc::MemoryBarrier();
ThreadSafeCounter->Decrement();
UE_LOG(LogAudio, Verbose, TEXT("End of ASync DDC Chunk read for key: %s"), *DerivedDataKey);
}
FByteBulkData* UNavCollision::GetCookedData(FName Format)
{
if (IsTemplate())
{
return NULL;
}
bool bContainedData = CookedFormatData.Contains(Format);
FByteBulkData* Result = &CookedFormatData.GetFormat(Format);
if (!bContainedData)
{
if (FPlatformProperties::RequiresCookedData())
{
UE_LOG(LogNavigation, Fatal, TEXT("Attempt to build nav collision data for %s when we are unable to. This platform requires cooked packages."), *GetPathName());
}
TArray<uint8> OutData;
FDerivedDataNavCollisionCooker* DerivedNavCollisionData = new FDerivedDataNavCollisionCooker(Format, this);
if (DerivedNavCollisionData->CanBuild()
&& GetDerivedDataCacheRef().GetSynchronous(DerivedNavCollisionData, OutData))
{
if (OutData.Num())
{
Result->Lock(LOCK_READ_WRITE);
FMemory::Memcpy(Result->Realloc(OutData.Num()), OutData.GetData(), OutData.Num());
Result->Unlock();
}
}
}
check(Result);
return Result->GetBulkDataSize() > 0 ? Result : NULL; // we don't return empty bulk data...but we save it to avoid thrashing the DDC
}
FByteBulkData* UBodySetup::GetCookedData(FName Format)
{
if (IsTemplate())
{
return NULL;
}
IInterface_CollisionDataProvider* CDP = Cast<IInterface_CollisionDataProvider>(GetOuter());
// If there is nothing to cook or if we are reading data from a cooked package for an asset with no collision,
// we want to return here
if ((AggGeom.ConvexElems.Num() == 0 && CDP == NULL) || !bHasCookedCollisionData)
{
return NULL;
}
FFormatContainer* UseCookedData = CookedFormatDataOverride ? CookedFormatDataOverride : &CookedFormatData;
bool bContainedData = UseCookedData->Contains(Format);
FByteBulkData* Result = &UseCookedData->GetFormat(Format);
#if WITH_PHYSX
if (!bContainedData)
{
if (FPlatformProperties::RequiresCookedData())
{
UE_LOG(LogPhysics, Error, TEXT("Attempt to build physics data for %s when we are unable to. This platform requires cooked packages."), *GetPathName());
}
if (AggGeom.ConvexElems.Num() == 0 && (CDP == NULL || CDP->ContainsPhysicsTriMeshData(bMeshCollideAll) == false))
{
return NULL;
}
#if WITH_RUNTIME_PHYSICS_COOKING || WITH_EDITOR
TArray<uint8> OutData;
FDerivedDataPhysXCooker* DerivedPhysXData = new FDerivedDataPhysXCooker(Format, this);
if (DerivedPhysXData->CanBuild())
{
#if WITH_EDITOR
GetDerivedDataCacheRef().GetSynchronous(DerivedPhysXData, OutData);
#elif WITH_RUNTIME_PHYSICS_COOKING
DerivedPhysXData->Build(OutData);
#endif
if (OutData.Num())
{
Result->Lock(LOCK_READ_WRITE);
FMemory::Memcpy(Result->Realloc(OutData.Num()), OutData.GetData(), OutData.Num());
Result->Unlock();
}
}
else
#endif
{
UE_LOG(LogPhysics, Warning, TEXT("Attempt to build physics data for %s when we are unable to."), *GetPathName());
}
}
#endif // WITH_PHYSX
check(Result);
return Result->GetBulkDataSize() > 0 ? Result : NULL; // we don't return empty bulk data...but we save it to avoid thrashing the DDC
}
FByteBulkData* USoundWave::GetCompressedData(FName Format)
{
if (IsTemplate() || IsRunningDedicatedServer())
{
return NULL;
}
bool bContainedData = CompressedFormatData.Contains(Format);
FByteBulkData* Result = &CompressedFormatData.GetFormat(Format);
if (!bContainedData)
{
if (!FPlatformProperties::RequiresCookedData() && GetDerivedDataCache())
{
TArray<uint8> OutData;
FDerivedAudioDataCompressor* DeriveAudioData = new FDerivedAudioDataCompressor(this, Format);
GetDerivedDataCacheRef().GetSynchronous(DeriveAudioData, OutData);
if (OutData.Num())
{
Result->Lock(LOCK_READ_WRITE);
FMemory::Memcpy(Result->Realloc(OutData.Num()), OutData.GetData(), OutData.Num());
Result->Unlock();
}
}
else
{
UE_LOG(LogAudio, Error, TEXT("Attempt to access the DDC when there is none available on sound '%s', format = %s. Should have been cooked."), *GetFullName(), *Format.ToString());
}
}
check(Result);
return Result->GetBulkDataSize() > 0 ? Result : NULL; // we don't return empty bulk data...but we save it to avoid thrashing the DDC
}
bool FNetworkPlatformFile::IsInLocalDirectory(const FString& Filename)
{
if (!bHasLoadedDDCDirectories)
{
// need to be careful here to avoid initializing the DDC from the wrong thread or using LocalDirectories while it is being initialized
FScopeLock ScopeLock(&LocalDirectoriesCriticalSection);
if (IsInGameThread() && GConfig && GConfig->IsReadyForUse())
{
// one time DDC directory initialization
// add any DDC directories to our list of local directories (local = inner platform file, it may
// actually live on a server, but it will use the platform's file system)
if (GetDerivedDataCache())
{
TArray<FString> DdcDirectories;
GetDerivedDataCacheRef().GetDirectories(DdcDirectories);
LocalDirectories.Append(DdcDirectories);
}
FPlatformMisc::MemoryBarrier();
bHasLoadedDDCDirectories = true;
}
return IsInLocalDirectoryUnGuarded(Filename);
}
// once the DDC is initialized, we don't need to lock a critical section anymore
return IsInLocalDirectoryUnGuarded(Filename);
}
bool FStreamedAudioPlatformData::TryInlineChunkData()
{
TArray<uint32> AsyncHandles;
TArray<uint8> TempData;
FDerivedDataCacheInterface& DDC = GetDerivedDataCacheRef();
BeginLoadDerivedChunks(Chunks, 0, AsyncHandles);
for (int32 ChunkIndex = 0; ChunkIndex < Chunks.Num(); ++ChunkIndex)
{
FStreamedAudioChunk& Chunk = Chunks[ChunkIndex];
if (Chunk.DerivedDataKey.IsEmpty() == false)
{
uint32 AsyncHandle = AsyncHandles[ChunkIndex];
DDC.WaitAsynchronousCompletion(AsyncHandle);
if (DDC.GetAsynchronousResults(AsyncHandle, TempData))
{
int32 ChunkSize = 0;
FMemoryReader Ar(TempData, /*bIsPersistent=*/ true);
Ar << ChunkSize;
Chunk.BulkData.Lock(LOCK_READ_WRITE);
void* ChunkData = Chunk.BulkData.Realloc(ChunkSize);
Ar.Serialize(ChunkData, ChunkSize);
Chunk.BulkData.Unlock();
Chunk.DerivedDataKey.Empty();
}
else
{
return false;
}
TempData.Reset();
}
}
return true;
}
/** Saves the platform's shader map to the DDC. */
void SaveGlobalShaderMapToDerivedDataCache(EShaderPlatform Platform)
{
TArray<uint8> SaveData;
FMemoryWriter Ar(SaveData, true);
SerializeGlobalShaders(Ar, GGlobalShaderMap[Platform]);
FGlobalShaderMapId ShaderMapId(Platform);
GetDerivedDataCacheRef().Put(*GetGlobalShaderMapKeyString(ShaderMapId, Platform), SaveData);
}
TRefCountPtr<FReflectionCaptureEncodedHDRDerivedData> FReflectionCaptureEncodedHDRDerivedData::GenerateEncodedHDRData(const FReflectionCaptureFullHDRDerivedData& FullHDRData, const FGuid& StateId, float Brightness)
{
TRefCountPtr<FReflectionCaptureEncodedHDRDerivedData> EncodedHDRData = new FReflectionCaptureEncodedHDRDerivedData();
const FString KeyString = GetDDCKeyString(StateId);
if (!GetDerivedDataCacheRef().GetSynchronous(*KeyString, EncodedHDRData->CapturedData))
{
EncodedHDRData->GenerateFromDerivedDataSource(FullHDRData, Brightness);
if (EncodedHDRData->CapturedData.Num() > 0)
{
GetDerivedDataCacheRef().Put(*KeyString, EncodedHDRData->CapturedData);
}
}
check(EncodedHDRData->CapturedData.Num() > 0);
INC_MEMORY_STAT_BY(STAT_ReflectionCaptureMemory,EncodedHDRData->CapturedData.GetAllocatedSize());
return EncodedHDRData;
}
/**
* Executes async DDC gets for chunks stored in the derived data cache.
* @param Chunks - Chunks to retrieve.
* @param FirstChunkToLoad - Index of the first chunk to retrieve.
* @param OutHandles - Handles to the asynchronous DDC gets.
*/
static void BeginLoadDerivedChunks(TIndirectArray<FStreamedAudioChunk>& Chunks, int32 FirstChunkToLoad, TArray<uint32>& OutHandles)
{
FDerivedDataCacheInterface& DDC = GetDerivedDataCacheRef();
OutHandles.AddZeroed(Chunks.Num());
for (int32 ChunkIndex = FirstChunkToLoad; ChunkIndex < Chunks.Num(); ++ChunkIndex)
{
const FStreamedAudioChunk& Chunk = Chunks[ChunkIndex];
if (!Chunk.DerivedDataKey.IsEmpty())
{
OutHandles[ChunkIndex] = DDC.GetAsynchronous(*Chunk.DerivedDataKey);
}
}
}
bool FStreamedAudioPlatformData::AreDerivedChunksAvailable() const
{
bool bChunksAvailable = true;
FDerivedDataCacheInterface& DDC = GetDerivedDataCacheRef();
for (int32 ChunkIndex = 0; bChunksAvailable && ChunkIndex < Chunks.Num(); ++ChunkIndex)
{
const FStreamedAudioChunk& Chunk = Chunks[ChunkIndex];
if (Chunk.DerivedDataKey.IsEmpty() == false)
{
bChunksAvailable = DDC.CachedDataProbablyExists(*Chunk.DerivedDataKey);
}
}
return bChunksAvailable;
}
/**
* Stores derived data in the DDC.
* @param DerivedData - The data to store in the DDC.
* @param DerivedDataKeySuffix - The key suffix at which to store derived data.
*/
static void PutDerivedDataInCache(
FStreamedAudioPlatformData* DerivedData,
const FString& DerivedDataKeySuffix
)
{
TArray<uint8> RawDerivedData;
FString DerivedDataKey;
// Build the key with which to cache derived data.
GetStreamedAudioDerivedDataKeyFromSuffix(DerivedDataKeySuffix, DerivedDataKey);
FString LogString;
if (UE_LOG_ACTIVE(LogAudio,Verbose))
{
LogString = FString::Printf(
TEXT("Storing Streamed Audio in DDC:\n Key: %s\n Format: %s\n"),
*DerivedDataKey,
*DerivedData->AudioFormat.ToString()
);
}
// Write out individual chunks to the derived data cache.
const int32 ChunkCount = DerivedData->Chunks.Num();
for (int32 ChunkIndex = 0; ChunkIndex < ChunkCount; ++ChunkIndex)
{
FString ChunkDerivedDataKey;
FStreamedAudioChunk& Chunk = DerivedData->Chunks[ChunkIndex];
GetStreamedAudioDerivedChunkKey(ChunkIndex, Chunk, DerivedDataKeySuffix, ChunkDerivedDataKey);
if (UE_LOG_ACTIVE(LogAudio,Verbose))
{
LogString += FString::Printf(TEXT(" Chunk%d %d bytes %s\n"),
ChunkIndex,
Chunk.BulkData.GetBulkDataSize(),
*ChunkDerivedDataKey
);
}
Chunk.StoreInDerivedDataCache(ChunkDerivedDataKey);
}
// Store derived data.
FMemoryWriter Ar(RawDerivedData, /*bIsPersistent=*/ true);
DerivedData->Serialize(Ar, NULL);
GetDerivedDataCacheRef().Put(*DerivedDataKey, RawDerivedData);
UE_LOG(LogAudio,Verbose,TEXT("%s Derived Data: %d bytes"),*LogString,RawDerivedData.Num());
}
void FStreamedAudioChunk::StoreInDerivedDataCache(const FString& InDerivedDataKey)
{
int32 BulkDataSizeInBytes = BulkData.GetBulkDataSize();
check(BulkDataSizeInBytes > 0);
TArray<uint8> DerivedData;
FMemoryWriter Ar(DerivedData, /*bIsPersistent=*/ true);
Ar << BulkDataSizeInBytes;
{
void* BulkChunkData = BulkData.Lock(LOCK_READ_ONLY);
Ar.Serialize(BulkChunkData, BulkDataSizeInBytes);
BulkData.Unlock();
}
GetDerivedDataCacheRef().Put(*InDerivedDataKey, DerivedData);
DerivedDataKey = InDerivedDataKey;
BulkData.RemoveBulkData();
}
/** Does the work to cache derived data. Safe to call from any thread. */
void DoWork()
{
TArray<uint8> RawDerivedData;
bool bForceRebuild = (CacheFlags & EStreamedAudioCacheFlags::ForceRebuild) != 0;
bool bInlineChunks = (CacheFlags & EStreamedAudioCacheFlags::InlineChunks) != 0;
bool bForDDC = (CacheFlags & EStreamedAudioCacheFlags::ForDDCBuild) != 0;
bool bAllowAsyncBuild = (CacheFlags & EStreamedAudioCacheFlags::AllowAsyncBuild) != 0;
if (!bForceRebuild && GetDerivedDataCacheRef().GetSynchronous(*DerivedData->DerivedDataKey, RawDerivedData))
{
FMemoryReader Ar(RawDerivedData, /*bIsPersistent=*/ true);
DerivedData->Serialize(Ar, NULL);
bSucceeded = true;
if (bForDDC)
{
for (int32 Index = 0; Index < DerivedData->NumChunks; ++Index)
{
if (!DerivedData->TryLoadChunk(Index, NULL))
{
bSucceeded = false;
break;
}
}
}
else if (bInlineChunks)
{
bSucceeded = DerivedData->TryInlineChunkData();
}
else
{
bSucceeded = DerivedData->AreDerivedChunksAvailable();
}
}
else if (bAllowAsyncBuild)
{
BuildStreamedAudio();
}
}
TShaderMap<FGlobalShaderType>* GetGlobalShaderMap(EShaderPlatform Platform, bool bRefreshShaderMap)
{
// No global shaders needed on dedicated server
if (FPlatformProperties::IsServerOnly())
{
if (!GGlobalShaderMap[Platform])
{
GGlobalShaderMap[Platform] = new TShaderMap<FGlobalShaderType>();
return GGlobalShaderMap[Platform];
}
return NULL;
}
if (bRefreshShaderMap)
{
// delete the current global shader map
delete GGlobalShaderMap[Platform];
GGlobalShaderMap[Platform] = NULL;
// make sure we look for updated shader source files
FlushShaderFileCache();
}
// If the global shader map hasn't been created yet, create it.
if(!GGlobalShaderMap[Platform])
{
// GetGlobalShaderMap is called the first time during startup in the main thread.
check(IsInGameThread());
FScopedSlowTask SlowTask(70);
// verify that all shader source files are intact
SlowTask.EnterProgressFrame(20);
VerifyShaderSourceFiles();
GGlobalShaderMap[Platform] = new TShaderMap<FGlobalShaderType>();
bool bLoadedFromCacheFile = false;
// Try to load the global shaders from a local cache file if it exists
// This method is used exclusively with cooked content, since the DDC is not present
if (FPlatformProperties::RequiresCookedData())
{
SlowTask.EnterProgressFrame(50);
TArray<uint8> GlobalShaderData;
FString GlobalShaderCacheFilename = FPaths::GetRelativePathToRoot() / GetGlobalShaderCacheFilename(Platform);
FPaths::MakeStandardFilename(GlobalShaderCacheFilename);
bLoadedFromCacheFile = FFileHelper::LoadFileToArray(GlobalShaderData, *GlobalShaderCacheFilename, FILEREAD_Silent);
if (!bLoadedFromCacheFile)
{
// Handle this gracefully and exit.
FString SandboxPath = IFileManager::Get().ConvertToAbsolutePathForExternalAppForWrite(*GlobalShaderCacheFilename);
// This can be too early to localize in some situations.
const FText Message = FText::Format( NSLOCTEXT("Engine", "GlobalShaderCacheFileMissing", "The global shader cache file '{0}' is missing.\n\nYou're running a version of the application built to load COOKED content only, however no COOKED content was found. Consider cooking content for this build, or build and run the UNCOOKED version of the application instead."), FText::FromString( SandboxPath ) );
if (FPlatformProperties::SupportsWindowedMode())
{
UE_LOG(LogMaterial, Error, TEXT("%s"), *Message.ToString());
FMessageDialog::Open(EAppMsgType::Ok, Message);
FPlatformMisc::RequestExit(false);
return NULL;
}
else
{
UE_LOG(LogMaterial, Fatal, TEXT("%s"), *Message.ToString());
}
}
FMemoryReader MemoryReader(GlobalShaderData);
SerializeGlobalShaders(MemoryReader, GGlobalShaderMap[Platform]);
}
// Uncooked platform
else
{
FGlobalShaderMapId ShaderMapId(Platform);
TArray<uint8> CachedData;
SlowTask.EnterProgressFrame(40);
const FString DataKey = GetGlobalShaderMapKeyString(ShaderMapId, Platform);
// Find the shader map in the derived data cache
SlowTask.EnterProgressFrame(10);
if (GetDerivedDataCacheRef().GetSynchronous(*DataKey, CachedData))
{
FMemoryReader Ar(CachedData, true);
// Deserialize from the cached data
SerializeGlobalShaders(Ar, GGlobalShaderMap[Platform]);
}
}
// If any shaders weren't loaded, compile them now.
VerifyGlobalShaders(Platform, bLoadedFromCacheFile);
extern int32 GCreateShadersOnLoad;
if (GCreateShadersOnLoad && Platform == GMaxRHIShaderPlatform)
{
for (TMap<FShaderType*, TRefCountPtr<FShader> >::TConstIterator ShaderIt(GGlobalShaderMap[Platform]->GetShaders()); ShaderIt; ++ShaderIt)
{
FShader* Shader = ShaderIt.Value();
if (Shader)
{
Shader->InitializeResource();
}
}
}
}
return GGlobalShaderMap[Platform];
}
void UReflectionCaptureComponent::PostLoad()
{
Super::PostLoad();
// If we're loading on a platform that doesn't require cooked data, attempt to load missing data from the DDC
if (!FPlatformProperties::RequiresCookedData())
{
// Only check the DDC if we don't already have it loaded
// If we are loading cooked then FullHDRDerivedData might be setup already (FullHDRDerivedData is set in Serialize)
if (!FullHDRDerivedData)
{
FullHDRDerivedData = new FReflectionCaptureFullHDRDerivedData();
// Attempt to load the full HDR data from the DDC
if (!GetDerivedDataCacheRef().GetSynchronous(*FReflectionCaptureFullHDRDerivedData::GetDDCKeyString(StateId), FullHDRDerivedData->CompressedCapturedData))
{
bDerivedDataDirty = true;
delete FullHDRDerivedData;
FullHDRDerivedData = NULL;
if (!FApp::CanEverRender())
{
// Warn, especially when running the DDC commandlet to build a DDC for the binary version of UE4.
UE_LOG(LogMaterial, Warning, TEXT("Reflection capture was loaded without any valid capture data and will be black. This can happen if the DDC was not up to date during cooking. Load the map in the editor once before cooking to fix. %s."), *GetFullName());
}
}
}
// If we have full HDR data but not encoded HDR data, generate the encoded data now
if (FullHDRDerivedData
&& !EncodedHDRDerivedData
&& GRHIFeatureLevel == ERHIFeatureLevel::ES2)
{
EncodedHDRDerivedData = FReflectionCaptureEncodedHDRDerivedData::GenerateEncodedHDRData(*FullHDRDerivedData, StateId, Brightness);
}
}
// Initialize rendering resources for the current feature level, and toss data only needed by other feature levels
if (FullHDRDerivedData && GRHIFeatureLevel >= ERHIFeatureLevel::SM4)
{
// Don't need encoded HDR data for rendering on this feature level
INC_MEMORY_STAT_BY(STAT_ReflectionCaptureMemory,FullHDRDerivedData->CompressedCapturedData.GetAllocatedSize());
EncodedHDRDerivedData = NULL;
if (GRHIFeatureLevel == ERHIFeatureLevel::SM4)
{
SM4FullHDRCubemapTexture = new FReflectionTextureCubeResource();
SM4FullHDRCubemapTexture->SetupParameters(GReflectionCaptureSize, FMath::CeilLogTwo(GReflectionCaptureSize) + 1, PF_FloatRGBA, &FullHDRDerivedData->GetCapturedDataForSM4Load());
BeginInitResource(SM4FullHDRCubemapTexture);
}
}
else if (EncodedHDRDerivedData && GRHIFeatureLevel == ERHIFeatureLevel::ES2)
{
if (FPlatformProperties::RequiresCookedData())
{
INC_MEMORY_STAT_BY(STAT_ReflectionCaptureMemory,EncodedHDRDerivedData->CapturedData.GetAllocatedSize());
}
// Create a cubemap texture out of the encoded HDR data
EncodedHDRCubemapTexture = new FReflectionTextureCubeResource();
EncodedHDRCubemapTexture->SetupParameters(GReflectionCaptureSize, FMath::CeilLogTwo(GReflectionCaptureSize) + 1, PF_B8G8R8A8, &EncodedHDRDerivedData->CapturedData);
BeginInitResource(EncodedHDRCubemapTexture);
// Don't need the full HDR data for rendering on this feature level
delete FullHDRDerivedData;
FullHDRDerivedData = NULL;
}
// PostLoad was called, so we can make a proper decision based on visibility of whether or not to update
ReflectionCapturesToUpdateNewlyCreated.Remove(this);
// Add ourselves to the global list of reflection captures that need to be uploaded to the scene or recaptured
if (bVisible)
{
ReflectionCapturesToUpdateForLoad.AddUnique(this);
bCaptureDirty = true;
}
}
int32 UDerivedDataCacheCommandlet::Main( const FString& Params )
{
TArray<FString> Tokens, Switches;
ParseCommandLine(*Params, Tokens, Switches);
bool bFillCache = Switches.Contains("FILL"); // do the equivalent of a "loadpackage -all" to fill the DDC
bool bStartupOnly = Switches.Contains("STARTUPONLY"); // regardless of any other flags, do not iterate packages
// Subsets for parallel processing
uint32 SubsetMod = 0;
uint32 SubsetTarget = MAX_uint32;
FParse::Value(*Params, TEXT("SubsetMod="), SubsetMod);
FParse::Value(*Params, TEXT("SubsetTarget="), SubsetTarget);
bool bDoSubset = SubsetMod > 0 && SubsetTarget < SubsetMod;
double FindProcessedPackagesTime = 0.0;
double GCTime = 0.0;
if (!bStartupOnly && bFillCache)
{
FCoreDelegates::PackageCreatedForLoad.AddUObject(this, &UDerivedDataCacheCommandlet::MaybeMarkPackageAsAlreadyLoaded);
TArray<FString> FilesInPath;
Tokens.Empty(2);
Tokens.Add(FString("*") + FPackageName::GetAssetPackageExtension());
Tokens.Add(FString("*") + FPackageName::GetMapPackageExtension());
uint8 PackageFilter = NORMALIZE_DefaultFlags;
if ( Switches.Contains(TEXT("MAPSONLY")) )
{
PackageFilter |= NORMALIZE_ExcludeContentPackages;
}
if ( !Switches.Contains(TEXT("DEV")) )
{
PackageFilter |= NORMALIZE_ExcludeDeveloperPackages;
}
// assume the first token is the map wildcard/pathname
TArray<FString> Unused;
for ( int32 TokenIndex = 0; TokenIndex < Tokens.Num(); TokenIndex++ )
{
TArray<FString> TokenFiles;
if ( !NormalizePackageNames( Unused, TokenFiles, Tokens[TokenIndex], PackageFilter) )
{
UE_LOG(LogDerivedDataCacheCommandlet, Display, TEXT("No packages found for parameter %i: '%s'"), TokenIndex, *Tokens[TokenIndex]);
continue;
}
FilesInPath += TokenFiles;
}
if ( FilesInPath.Num() == 0 )
{
UE_LOG(LogDerivedDataCacheCommandlet, Warning, TEXT("No files found."));
}
ITargetPlatformManagerModule* TPM = GetTargetPlatformManager();
const TArray<ITargetPlatform*>& Platforms = TPM->GetActiveTargetPlatforms();
for (int32 Index = 0; Index < Platforms.Num(); Index++)
{
TArray<FName> DesiredShaderFormats;
Platforms[Index]->GetShaderFormats(DesiredShaderFormats);
for (int32 FormatIndex = 0; FormatIndex < DesiredShaderFormats.Num(); FormatIndex++)
{
const EShaderPlatform TargetPlatform = ShaderFormatToLegacyShaderPlatform(DesiredShaderFormats[FormatIndex]);
// Kick off global shader compiles for each target platform
GetGlobalShaderMap(TargetPlatform);
}
}
const int32 GCInterval = 100;
int32 NumProcessedSinceLastGC = GCInterval;
bool bLastPackageWasMap = true; // 'true' is to prime the ProcessedPackages list
TSet<FString> ProcessedPackages;
UE_LOG(LogDerivedDataCacheCommandlet, Display, TEXT("%d packages to load..."), FilesInPath.Num());
for( int32 FileIndex = FilesInPath.Num() - 1; ; FileIndex-- )
{
// Keep track of which packages have already been processed along with the map.
if (NumProcessedSinceLastGC >= GCInterval || bLastPackageWasMap || FileIndex == FilesInPath.Num() - 1)
{
const double FindProcessedPackagesStartTime = FPlatformTime::Seconds();
TArray<UObject *> ObjectsInOuter;
GetObjectsWithOuter(NULL, ObjectsInOuter, false);
for( int32 Index = 0; Index < ObjectsInOuter.Num(); Index++ )
{
UPackage* Pkg = Cast<UPackage>(ObjectsInOuter[Index]);
if (!Pkg)
{
continue;
}
FString Filename;
if (FPackageName::DoesPackageExist(Pkg->GetName(), NULL, &Filename))
{
if (!ProcessedPackages.Contains(Filename))
{
ProcessedPackages.Add(Filename);
PackagesToNotReload.Add(Pkg->GetName());
Pkg->PackageFlags |= PKG_ReloadingForCooker;
{
TArray<UObject *> ObjectsInPackage;
GetObjectsWithOuter(Pkg, ObjectsInPackage, true);
for( int32 IndexPackage = 0; IndexPackage < ObjectsInPackage.Num(); IndexPackage++ )
{
ObjectsInPackage[IndexPackage]->CookerWillNeverCookAgain();
}
}
}
}
}
FindProcessedPackagesTime += FPlatformTime::Seconds() - FindProcessedPackagesStartTime;
}
if (NumProcessedSinceLastGC >= GCInterval || FileIndex < 0 || bLastPackageWasMap)
{
const double StartGCTime = FPlatformTime::Seconds();
if (NumProcessedSinceLastGC >= GCInterval || FileIndex < 0)
{
UE_LOG(LogDerivedDataCacheCommandlet, Display, TEXT("GC (Full)..."));
CollectGarbage( RF_Native );
NumProcessedSinceLastGC = 0;
}
else
{
UE_LOG(LogDerivedDataCacheCommandlet, Display, TEXT("GC..."));
CollectGarbage( RF_Native | RF_Standalone );
}
GCTime += FPlatformTime::Seconds() - StartGCTime;
bLastPackageWasMap = false;
}
if (FileIndex < 0)
{
break;
}
const FString& Filename = FilesInPath[FileIndex];
if (ProcessedPackages.Contains(Filename))
{
continue;
}
if (bDoSubset)
{
const FString& PackageName = FPackageName::PackageFromPath(*Filename);
if (FCrc::StrCrc_DEPRECATED(*PackageName.ToUpper()) % SubsetMod != SubsetTarget)
{
continue;
}
}
UE_LOG(LogDerivedDataCacheCommandlet, Display, TEXT("Loading (%d) %s"), FilesInPath.Num() - FileIndex, *Filename );
UPackage* Package = LoadPackage( NULL, *Filename, LOAD_None );
if( Package == NULL )
{
UE_LOG(LogDerivedDataCacheCommandlet, Error, TEXT("Error loading %s!"), *Filename );
}
else
{
bLastPackageWasMap = Package->ContainsMap();
NumProcessedSinceLastGC++;
}
}
}
IConsoleManager::Get().ProcessUserConsoleInput(TEXT("Tex.DerivedDataTimings"), *GWarn, NULL);
UE_LOG(LogDerivedDataCacheCommandlet, Display, TEXT("Waiting for shaders to finish."));
GShaderCompilingManager->FinishAllCompilation();
UE_LOG(LogDerivedDataCacheCommandlet, Display, TEXT("Done waiting for shaders to finish."));
GetDerivedDataCacheRef().WaitForQuiescence(true);
UE_LOG(LogDerivedDataCacheCommandlet, Display, TEXT("%.2lfs spent looking for processed packages, %.2lfs spent on GC."), FindProcessedPackagesTime, GCTime);
return 0;
}
bool FStreamedAudioPlatformData::TryLoadChunk(int32 ChunkIndex, uint8** OutChunkData)
{
bool bCachedChunk = false;
FStreamedAudioChunk& Chunk = Chunks[ChunkIndex];
#if WITH_EDITORONLY_DATA
// Begin async DDC retrieval
FDerivedDataCacheInterface& DDC = GetDerivedDataCacheRef();
uint32 AsyncHandle = 0;
if (!Chunk.DerivedDataKey.IsEmpty())
{
AsyncHandle = DDC.GetAsynchronous(*Chunk.DerivedDataKey);
}
#endif // #if WITH_EDITORONLY_DATA
// Load chunk from bulk data if available.
if (Chunk.BulkData.GetBulkDataSize() > 0)
{
if (OutChunkData)
{
if (*OutChunkData == NULL)
{
*OutChunkData = static_cast<uint8*>(FMemory::Malloc(Chunk.BulkData.GetBulkDataSize()));
}
Chunk.BulkData.GetCopy((void**)OutChunkData);
}
bCachedChunk = true;
}
#if WITH_EDITORONLY_DATA
// Wait for async DDC to complete
if (Chunk.DerivedDataKey.IsEmpty() == false)
{
TArray<uint8> TempData;
DDC.WaitAsynchronousCompletion(AsyncHandle);
if (DDC.GetAsynchronousResults(AsyncHandle, TempData))
{
int32 ChunkSize = 0;
FMemoryReader Ar(TempData, /*bIsPersistent=*/ true);
Ar << ChunkSize;
if (ChunkSize != Chunk.DataSize)
{
UE_LOG(LogAudio, Warning,
TEXT("Chunk %d of %s SoundWave has invalid data in the DDC. Got %d bytes, expected %d. Key=%s"),
ChunkIndex,
*AudioFormat.ToString(),
ChunkSize,
Chunk.DataSize,
*Chunk.DerivedDataKey
);
}
bCachedChunk = true;
if (OutChunkData)
{
if (*OutChunkData == NULL)
{
*OutChunkData = static_cast<uint8*>(FMemory::Malloc(ChunkSize));
}
Ar.Serialize(*OutChunkData, ChunkSize);
}
}
}
#endif // #if WITH_EDITORONLY_DATA
return bCachedChunk;
}
