/** Async worker that checks the cache backend and if that fails, calls the deriver to build the data and then puts the results to the cache **/
void DoWork()
{
bool bGetResult;
{
INC_DWORD_STAT(STAT_DDC_NumGets);
STAT(double ThisTime = 0);
{
SCOPE_SECONDS_COUNTER(ThisTime);
bGetResult = FDerivedDataBackend::Get().GetRoot().GetCachedData(*CacheKey, Data);
}
INC_FLOAT_STAT_BY(STAT_DDC_SyncGetTime, bSynchronousForStats ? (float)ThisTime : 0.0f);
}
if (bGetResult)
{
check(Data.Num());
bSuccess = true;
delete DataDeriver;
DataDeriver = NULL;
}
else if (DataDeriver)
{
{
INC_DWORD_STAT(STAT_DDC_NumBuilds);
STAT(double ThisTime = 0);
{
SCOPE_SECONDS_COUNTER(ThisTime);
bSuccess = DataDeriver->Build(Data);
}
INC_FLOAT_STAT_BY(STAT_DDC_SyncBuildTime, bSynchronousForStats ? (float)ThisTime : 0.0f);
}
delete DataDeriver;
DataDeriver = NULL;
if (bSuccess)
{
check(Data.Num());
INC_DWORD_STAT(STAT_DDC_NumPuts);
STAT(double ThisTime = 0);
{
SCOPE_SECONDS_COUNTER(ThisTime);
FDerivedDataBackend::Get().GetRoot().PutCachedData(*CacheKey, Data, true);
}
INC_FLOAT_STAT_BY(STAT_DDC_PutTime, bSynchronousForStats ? (float)ThisTime : 0.0f);
}
}
if (!bSuccess)
{
Data.Empty();
}
FDerivedDataBackend::Get().AddToAsyncCompletionCounter(-1);
}
virtual void Put(const TCHAR* CacheKey, TArray<uint8>& Data, bool bPutEvenIfExists = false) override
{
STAT(double ThisTime = 0);
{
SCOPE_SECONDS_COUNTER(ThisTime);
FDerivedDataBackend::Get().GetRoot().PutCachedData(CacheKey, Data, bPutEvenIfExists);
}
INC_FLOAT_STAT_BY(STAT_DDC_PutTime,(float)ThisTime);
INC_DWORD_STAT(STAT_DDC_NumPuts);
}
/**
* Constructor, initializing all member variables. It also reads .ini settings and caches them and creates the
* database connection object and attempts to connect if bUseTaskPerfTracking is set.
*/
FTaskPerfTracker::FTaskPerfTracker()
: FTaskDatabase()
, bUseTaskPerfTracking(FALSE)
, TimeSpentTalkingWithDB(0)
{
#if !UDK
// Read the ini settings and store them in the struct to aid debugging.
GConfig->GetBool(TEXT("TaskPerfTracking"), TEXT("bUseTaskPerfTracking" ), bUseTaskPerfTracking, GEngineIni);
// Only do work if task tracking is enabled.
if( bUseTaskPerfTracking )
{
// only attempt to get the data when we want to use the TaskPerfTracking
verify(GConfig->GetString( TEXT("TaskPerfTracking"), TEXT("ConnectionString"), ConnectionString, GEngineIni ));
verify(GConfig->GetString( TEXT("TaskPerfTracking"), TEXT("RemoteConnectionIP"), RemoteConnectionIP, GEngineIni ));
verify(GConfig->GetString( TEXT("TaskPerfTracking"), TEXT("RemoteConnectionStringOverride"), RemoteConnectionStringOverride, GEngineIni ));
// Track time spent talking with DB to ensure we don't introduce nasty stalls.
SCOPE_SECONDS_COUNTER(TimeSpentTalkingWithDB);
// Create the connection object; needs to be deleted via "delete".
Connection = FDataBaseConnection::CreateObject();
// Try to open connection to DB - this is a synchronous operation.
if( Connection->Open( *ConnectionString, *RemoteConnectionIP, *RemoteConnectionStringOverride ) == TRUE )
{
warnf(NAME_DevDataBase,TEXT("Connection to \"%s\" or \"%s\" succeeded"), *ConnectionString, *RemoteConnectionIP);
// Create format string for calling procedure.
FormatString = FString(TEXT("EXEC BEGINRUN "));
#if _DEBUG
FormatString += TEXT(" @ConfigName='DEBUG', ");
#elif FINAL_RELEASE_DEBUGCONSOLE
FormatString += TEXT(" @ConfigName='FINAL_RELEASE_DEBUGCONSOLE', ");
#elif FINAL_RELEASE
FormatString += TEXT(" @ConfigName='FINAL_RELEASE', ");
#else
FormatString += TEXT(" @ConfigName='RELEASE', ");
#endif
FormatString += FString(TEXT("@GameName='")) + GGameName + TEXT("', @MachineName='") + appComputerName() + TEXT("', ");
FormatString += FString(TEXT("@CmdLine='")) + appCmdLine() + TEXT("', @UserName='******', ");
FormatString += FString(TEXT("@TaskDescription='%s', @TaskParameter='%s', @Changelist=")) + appItoa(GBuiltFromChangeList);
}
// Connection failed :(
else
{
warnf(NAME_DevDataBase,TEXT("Connection to \"%s\" or \"%s\" failed"), *ConnectionString, *RemoteConnectionIP);
// Only delete object - no need to close as connection failed.
delete Connection;
Connection = NULL;
}
}
#endif //#if !UDK
}
virtual bool CachedDataProbablyExists(const TCHAR* CacheKey) override
{
bool bResult;
INC_DWORD_STAT(STAT_DDC_NumExist);
STAT(double ThisTime = 0);
{
SCOPE_SECONDS_COUNTER(ThisTime);
bResult = FDerivedDataBackend::Get().GetRoot().CachedDataProbablyExists(CacheKey);
}
INC_FLOAT_STAT_BY(STAT_DDC_ExistTime, (float)ThisTime);
return bResult;
}
virtual void WaitAsynchronousCompletion(uint32 Handle) override
{
STAT(double ThisTime = 0);
{
SCOPE_SECONDS_COUNTER(ThisTime);
FAsyncTask<FBuildAsyncWorker>* AsyncTask = NULL;
{
FScopeLock ScopeLock(&SynchronizationObject);
AsyncTask = PendingTasks.FindRef(Handle);
}
check(AsyncTask);
AsyncTask->EnsureCompletion();
}
INC_FLOAT_STAT_BY(STAT_DDC_ASyncWaitTime,(float)ThisTime);
}
void FShaderResource::InitializePixelShaderRHI()
{
if (!IsInitialized())
{
STAT(double ShaderInitializationTime = 0);
{
SCOPE_CYCLE_COUNTER(STAT_Shaders_FrameRTShaderInitForRenderingTime);
SCOPE_SECONDS_COUNTER(ShaderInitializationTime);
InitResourceFromPossiblyParallelRendering();
}
INC_FLOAT_STAT_BY(STAT_Shaders_TotalRTShaderInitForRenderingTime,(float)ShaderInitializationTime);
}
checkSlow(IsInitialized());
}
/**
* Frees allocation associated with passed in pointer.
*
* @param Pointer Pointer to free.
*/
void FBestFitAllocator::Free( void* Pointer )
{
SCOPE_SECONDS_COUNTER(TimeSpentInAllocator);
// Look up pointer in TMap.
FMemoryChunk* MatchingChunk = PointerToChunkMap.FindRef( (PTRINT) Pointer );
check( MatchingChunk );
// Remove the entry
PointerToChunkMap.Remove((PTRINT) Pointer);
// Update usage stats in a thread safe way.
appInterlockedAdd( &AllocatedMemorySize, -MatchingChunk->Size );
appInterlockedAdd( &AvailableMemorySize, +MatchingChunk->Size );
// Free the chunk.
FreeChunk(MatchingChunk);
}
virtual void WaitAsynchronousCompletion(uint32 Handle) override
{
STAT(double ThisTime = 0);
{
SCOPE_SECONDS_COUNTER(ThisTime);
FScopeLock ScopeLock(&SynchronizationObject);
for (TSet<FDerivedDataRollup*>::TIterator Iter(PendingRollups); Iter; ++Iter)
{
if ((*Iter)->Contains(Handle))
{
(*Iter)->WaitAsynchronousCompletion(Handle);
return;
}
}
}
INC_FLOAT_STAT_BY(STAT_DDC_ASyncWaitTime,(float)ThisTime);
Super::WaitAsynchronousCompletion(Handle);
}
void FAsyncIOSystemBase::InternalRead( IFileHandle* FileHandle, int64 Offset, int64 Size, void* Dest )
{
DECLARE_SCOPE_CYCLE_COUNTER(TEXT("FAsyncIOSystemBase::InternalRead"), STAT_AsyncIOSystemBase_InternalRead, STATGROUP_AsyncIO_Verbose);
FScopeLock ScopeLock( ExclusiveReadCriticalSection );
STAT(double ReadTime = 0);
{
SCOPE_SECONDS_COUNTER(ReadTime);
PlatformReadDoNotCallDirectly( FileHandle, Offset, Size, Dest );
}
INC_FLOAT_STAT_BY(STAT_AsyncIO_PlatformReadTime,(float)ReadTime);
// The platform might actually read more than Size due to aligning and internal min read sizes
// though we only really care about throttling requested bandwidth as it's not very accurate
// to begin with.
STAT(ConstrainBandwidth(Size, ReadTime));
}
void FPackageDependencyInfo::ResolveCircularDependencies(FPackageDependencyTrackingInfo* InPkgInfo)
{
double Seconds = 0.0f;
{
SCOPE_SECONDS_COUNTER(Seconds);
NumResolvePasses = 0;
NumResolveIterations = 0;
NumCirculars = 0;
ResolveCircularDependenciesInnerFast();
}
UE_LOG(LogPackageDependencyInfo, Verbose, TEXT("Resolve circular dependencies for package %s took %.1f ms (Iter=%i, Passes=%i, Circulars=%i)"),
*InPkgInfo->PackageName,
Seconds*1000.0f,
NumResolveIterations,
NumResolvePasses,
NumCirculars );
}
const FHullShaderRHIRef& FShaderResource::GetHullShader()
{
checkSlow(Target.Frequency == SF_Hull);
if (!IsInitialized())
{
STAT(double ShaderInitializationTime = 0);
{
SCOPE_CYCLE_COUNTER(STAT_Shaders_FrameRTShaderInitForRenderingTime);
SCOPE_SECONDS_COUNTER(ShaderInitializationTime);
InitResourceFromPossiblyParallelRendering();
}
INC_FLOAT_STAT_BY(STAT_Shaders_TotalRTShaderInitForRenderingTime,(float)ShaderInitializationTime);
}
checkSlow(IsInitialized());
return HullShader;
}
/**
* Adds a task to track to the DB.
*
* @param Task Name of task
* @param TaskParameter Additional information, e.g. name of map if task is lighting rebuild
* @param DurationInSeconds Duration of task in seconds
*/
void FTaskPerfTracker::AddTask( const TCHAR* Task, const TCHAR* TaskParameter, FLOAT DurationInSeconds )
{
// A valid connection means that tracking is enabled.
if( Connection )
{
// Track time spent talking with DB to ensure we don't introduce nasty stalls.
SCOPE_SECONDS_COUNTER(TimeSpentTalkingWithDB);
// Execute BeginRun with baked format string.
FDataBaseRecordSet* RecordSet = NULL;
if( Connection->Execute( *FString::Printf( *FormatString, Task, TaskParameter ), RecordSet ) && RecordSet )
{
// Retrieve RunID from recordset. It's the return value of the EXEC.
INT RunID = RecordSet->GetInt(TEXT("Return Value"));
// Add stat entry.
Connection->Execute( *FString::Printf(TEXT("EXEC ADDRUNDATA @RunID=%i, @StatGroupName='Ungrouped', @StatName='Duration', @StatValue='%f'"), RunID, DurationInSeconds) );
// End the run, marking it as 'passed'.
Connection->Execute( *FString::Printf(TEXT("EXEC ENDRUN @RunID=%i, @ResultDescription='Passed'"), RunID ) );
delete RecordSet;
RecordSet = NULL;
}
}
}
/**
* Tries to reallocate texture memory in-place (without relocating),
* by adjusting the base address of the allocation but keeping the end address the same.
*
* @param OldBaseAddress Pointer to the original allocation
* @param NewBaseAddress New desired baseaddress for the allocation (adjusting the size so the end stays the same)
* @returns TRUE if it succeeded
**/
UBOOL FBestFitAllocator::Reallocate( void* OldBaseAddress, void* NewBaseAddress )
{
SCOPE_SECONDS_COUNTER(TimeSpentInAllocator);
// Look up pointer in TMap.
FMemoryChunk* MatchingChunk = PointerToChunkMap.FindRef( PTRINT(OldBaseAddress) );
check( MatchingChunk && PTRINT(OldBaseAddress) == PTRINT(MatchingChunk->Base) );
INT MemoryAdjustment = Abs<INT>(PTRINT(NewBaseAddress) - PTRINT(OldBaseAddress));
// Are we growing the allocation?
if ( PTRINT(NewBaseAddress) < PTRINT(OldBaseAddress) )
{
// Is there enough free memory immediately before this chunk?
FMemoryChunk* PrevChunk = MatchingChunk->PreviousChunk;
if ( PrevChunk && PrevChunk->bIsAvailable && PrevChunk->Size >= MemoryAdjustment )
{
PointerToChunkMap.Remove( PTRINT(OldBaseAddress) );
// Shrink the previous and grow the current chunk.
PrevChunk->Size -= MemoryAdjustment;
MatchingChunk->Base -= MemoryAdjustment;
MatchingChunk->Size += MemoryAdjustment;
check(PTRINT(NewBaseAddress) == PTRINT(MatchingChunk->Base));
PointerToChunkMap.Set( PTRINT(NewBaseAddress), MatchingChunk );
if ( PrevChunk->Size == 0 )
{
delete PrevChunk;
}
// Update usage stats in a thread safe way.
appInterlockedAdd( &AllocatedMemorySize, +MemoryAdjustment );
appInterlockedAdd( &AvailableMemorySize, -MemoryAdjustment );
return TRUE;
}
}
else
{
// We're shrinking the allocation.
check( MemoryAdjustment <= MatchingChunk->Size );
FMemoryChunk* PrevChunk = MatchingChunk->PreviousChunk;
if ( PrevChunk )
{
// Shrink the current chunk.
MatchingChunk->Base += MemoryAdjustment;
MatchingChunk->Size -= MemoryAdjustment;
// Grow the previous chunk.
INT OriginalPrevSize = PrevChunk->Size;
PrevChunk->Size += MemoryAdjustment;
// If the previous chunk was "in use", split it and insert a 2nd free chunk.
if ( !PrevChunk->bIsAvailable )
{
Split( PrevChunk, OriginalPrevSize );
}
}
else
{
// This was the first chunk, split it.
Split( MatchingChunk, MemoryAdjustment );
// We're going to use the new chunk. Mark it as "used memory".
MatchingChunk = MatchingChunk->NextChunk;
MatchingChunk->UnlinkFree();
// Make the original chunk "free memory".
FreeChunk( MatchingChunk->PreviousChunk );
}
check(PTRINT(NewBaseAddress) == PTRINT(MatchingChunk->Base));
PointerToChunkMap.Remove( PTRINT(OldBaseAddress) );
PointerToChunkMap.Set( PTRINT(NewBaseAddress), MatchingChunk );
// Update usage stats in a thread safe way.
appInterlockedAdd( &AllocatedMemorySize, -MemoryAdjustment );
appInterlockedAdd( &AvailableMemorySize, +MemoryAdjustment );
return TRUE;
}
return FALSE;
}
/**
* Allocate physical memory.
*
* @param AllocationSize Size of allocation
* @param bAllowFailure Whether to allow allocation failure or not
* @return Pointer to allocated memory
*/
void* FBestFitAllocator::Allocate( INT AllocationSize, UBOOL bAllowFailure )
{
SCOPE_SECONDS_COUNTER(TimeSpentInAllocator);
check( FirstChunk );
// Make sure everything is appropriately aligned.
AllocationSize = Align( AllocationSize, AllocationAlignment );
// Perform a "best fit" search, returning first perfect fit if there is one.
FMemoryChunk* CurrentChunk = FirstFreeChunk;
FMemoryChunk* BestChunk = NULL;
while( CurrentChunk )
{
// Check whether chunk is available and large enough to hold allocation.
check( CurrentChunk->bIsAvailable );
if( CurrentChunk->Size >= AllocationSize )
{
// Compare with current best chunk if one exists.
if( BestChunk )
{
// Tighter fits are preferred.
if( CurrentChunk->Size < BestChunk->Size )
{
BestChunk = CurrentChunk;
}
}
// No existing best chunk so use this one.
else
{
BestChunk = CurrentChunk;
}
// We have a perfect fit, no need to iterate further.
if( BestChunk->Size == AllocationSize )
{
break;
}
}
CurrentChunk = CurrentChunk->NextFreeChunk;
}
// Dump allocation info and return NULL if we weren't able to satisfy allocation request.
if( !BestChunk )
{
if ( !bAllowFailure )
{
#if !FINAL_RELEASE
DumpAllocs();
debugf(TEXT("Ran out of memory for allocation in best-fit allocator of size %i KByte"), AllocationSize / 1024);
GLog->FlushThreadedLogs();
#endif
}
return NULL;
}
// Mark as being in use.
BestChunk->UnlinkFree();
// Split chunk to avoid waste.
if( BestChunk->Size > AllocationSize )
{
Split( BestChunk, AllocationSize );
}
// Ensure that everything's in range.
check( (BestChunk->Base + BestChunk->Size) <= (MemoryBase + MemorySize) );
check( BestChunk->Base >= MemoryBase );
// Update usage stats in a thread safe way.
appInterlockedAdd( &AllocatedMemorySize, +BestChunk->Size );
appInterlockedAdd( &AvailableMemorySize, -BestChunk->Size );
// Keep track of mapping and return pointer.
PointerToChunkMap.Set( (PTRINT) BestChunk->Base, BestChunk );
return BestChunk->Base;
}
void FAsyncIOSystemBase::FulfillCompressedRead( const FAsyncIORequest& IORequest, IFileHandle* FileHandle )
{
DECLARE_SCOPE_CYCLE_COUNTER(TEXT("FAsyncIOSystemBase::FulfillCompressedRead"), STAT_AsyncIOSystemBase_FulfillCompressedRead, STATGROUP_AsyncIO_Verbose);
if (GbLogAsyncLoading == true)
{
LogIORequest(TEXT("FulfillCompressedRead"), IORequest);
}
// Initialize variables.
FAsyncUncompress* Uncompressor = NULL;
uint8* UncompressedBuffer = (uint8*) IORequest.Dest;
// First compression chunk contains information about total size so we skip that one.
int32 CurrentChunkIndex = 1;
int32 CurrentBufferIndex = 0;
bool bHasProcessedAllData = false;
// read the first two ints, which will contain the magic bytes (to detect byteswapping)
// and the original size the chunks were compressed from
int64 HeaderData[2];
int32 HeaderSize = sizeof(HeaderData);
InternalRead(FileHandle, IORequest.Offset, HeaderSize, HeaderData);
RETURN_IF_EXIT_REQUESTED;
// if the magic bytes don't match, then we are byteswapped (or corrupted)
bool bIsByteswapped = HeaderData[0] != PACKAGE_FILE_TAG;
// if its potentially byteswapped, make sure it's not just corrupted
if (bIsByteswapped)
{
// if it doesn't equal the swapped version, then data is corrupted
if (HeaderData[0] != PACKAGE_FILE_TAG_SWAPPED)
{
UE_LOG(LogStreaming, Warning, TEXT("Detected data corruption [header] trying to read %lld bytes at offset %lld from '%s'. Please delete file and recook."),
IORequest.UncompressedSize,
IORequest.Offset ,
*IORequest.FileName );
check(0);
FPlatformMisc::HandleIOFailure(*IORequest.FileName);
}
// otherwise, we have a valid byteswapped file, so swap the chunk size
else
{
HeaderData[1] = BYTESWAP_ORDER64(HeaderData[1]);
}
}
int32 CompressionChunkSize = HeaderData[1];
// handle old packages that don't have the chunk size in the header, in which case
// we can use the old hardcoded size
if (CompressionChunkSize == PACKAGE_FILE_TAG)
{
CompressionChunkSize = LOADING_COMPRESSION_CHUNK_SIZE;
}
// calculate the number of chunks based on the size they were compressed from
int32 TotalChunkCount = (IORequest.UncompressedSize + CompressionChunkSize - 1) / CompressionChunkSize + 1;
// allocate chunk info data based on number of chunks
FCompressedChunkInfo* CompressionChunks = (FCompressedChunkInfo*)FMemory::Malloc(sizeof(FCompressedChunkInfo) * TotalChunkCount);
int32 ChunkInfoSize = (TotalChunkCount) * sizeof(FCompressedChunkInfo);
void* CompressedBuffer[2] = { 0, 0 };
// Read table of compression chunks after seeking to offset (after the initial header data)
InternalRead( FileHandle, IORequest.Offset + HeaderSize, ChunkInfoSize, CompressionChunks );
RETURN_IF_EXIT_REQUESTED;
// Handle byte swapping. This is required for opening a cooked file on the PC.
int64 CalculatedUncompressedSize = 0;
if (bIsByteswapped)
{
for( int32 ChunkIndex=0; ChunkIndex<TotalChunkCount; ChunkIndex++ )
{
CompressionChunks[ChunkIndex].CompressedSize = BYTESWAP_ORDER64(CompressionChunks[ChunkIndex].CompressedSize);
CompressionChunks[ChunkIndex].UncompressedSize = BYTESWAP_ORDER64(CompressionChunks[ChunkIndex].UncompressedSize);
if (ChunkIndex > 0)
{
CalculatedUncompressedSize += CompressionChunks[ChunkIndex].UncompressedSize;
}
}
}
else
{
for( int32 ChunkIndex=1; ChunkIndex<TotalChunkCount; ChunkIndex++ )
{
CalculatedUncompressedSize += CompressionChunks[ChunkIndex].UncompressedSize;
}
}
if (CompressionChunks[0].UncompressedSize != CalculatedUncompressedSize)
{
UE_LOG(LogStreaming, Warning, TEXT("Detected data corruption [incorrect uncompressed size] calculated %i bytes, requested %i bytes at offset %i from '%s'. Please delete file and recook."),
CalculatedUncompressedSize,
IORequest.UncompressedSize,
IORequest.Offset ,
*IORequest.FileName );
check(0);
FPlatformMisc::HandleIOFailure(*IORequest.FileName);
}
if (ChunkInfoSize + HeaderSize + CompressionChunks[0].CompressedSize > IORequest.Size )
{
UE_LOG(LogStreaming, Warning, TEXT("Detected data corruption [undershoot] trying to read %lld bytes at offset %lld from '%s'. Please delete file and recook."),
IORequest.UncompressedSize,
IORequest.Offset ,
*IORequest.FileName );
check(0);
FPlatformMisc::HandleIOFailure(*IORequest.FileName);
}
if (IORequest.UncompressedSize != CalculatedUncompressedSize)
{
UE_LOG(LogStreaming, Warning, TEXT("Detected data corruption [incorrect uncompressed size] calculated %lld bytes, requested %lld bytes at offset %lld from '%s'. Please delete file and recook."),
CalculatedUncompressedSize,
IORequest.UncompressedSize,
IORequest.Offset ,
*IORequest.FileName );
check(0);
FPlatformMisc::HandleIOFailure(*IORequest.FileName);
}
// Figure out maximum size of compressed data chunk.
int64 MaxCompressedSize = 0;
for (int32 ChunkIndex = 1; ChunkIndex < TotalChunkCount; ChunkIndex++)
{
MaxCompressedSize = FMath::Max(MaxCompressedSize, CompressionChunks[ChunkIndex].CompressedSize);
// Verify the all chunks are 'full size' until the last one...
if (CompressionChunks[ChunkIndex].UncompressedSize < CompressionChunkSize)
{
if (ChunkIndex != (TotalChunkCount - 1))
{
checkf(0, TEXT("Calculated too many chunks: %d should be last, there are %d from '%s'"), ChunkIndex, TotalChunkCount, *IORequest.FileName);
}
}
check( CompressionChunks[ChunkIndex].UncompressedSize <= CompressionChunkSize );
}
int32 Padding = 0;
// Allocate memory for compressed data.
CompressedBuffer[0] = FMemory::Malloc( MaxCompressedSize + Padding );
CompressedBuffer[1] = FMemory::Malloc( MaxCompressedSize + Padding );
// Initial read request.
InternalRead( FileHandle, FileHandle->Tell(), CompressionChunks[CurrentChunkIndex].CompressedSize, CompressedBuffer[CurrentBufferIndex] );
RETURN_IF_EXIT_REQUESTED;
// Loop till we're done decompressing all data.
while( !bHasProcessedAllData )
{
FAsyncTask<FAsyncUncompress> UncompressTask(
IORequest.CompressionFlags,
UncompressedBuffer,
CompressionChunks[CurrentChunkIndex].UncompressedSize,
CompressedBuffer[CurrentBufferIndex],
CompressionChunks[CurrentChunkIndex].CompressedSize,
(Padding > 0)
);
#if BLOCK_ON_DECOMPRESSION
UncompressTask.StartSynchronousTask();
#else
UncompressTask.StartBackgroundTask();
#endif
// Advance destination pointer.
UncompressedBuffer += CompressionChunks[CurrentChunkIndex].UncompressedSize;
// Check whether we are already done reading.
if( CurrentChunkIndex < TotalChunkCount-1 )
{
// Can't postincrement in if statement as we need it to remain at valid value for one more loop iteration to finish
// the decompression.
CurrentChunkIndex++;
// Swap compression buffers to read into.
CurrentBufferIndex = 1 - CurrentBufferIndex;
// Read more data.
InternalRead( FileHandle, FileHandle->Tell(), CompressionChunks[CurrentChunkIndex].CompressedSize, CompressedBuffer[CurrentBufferIndex] );
RETURN_IF_EXIT_REQUESTED;
}
// We were already done reading the last time around so we are done processing now.
else
{
bHasProcessedAllData = true;
}
//@todo async loading: should use event for this
STAT(double UncompressorWaitTime = 0);
{
SCOPE_SECONDS_COUNTER(UncompressorWaitTime);
UncompressTask.EnsureCompletion(); // just decompress on this thread if it isn't started yet
}
INC_FLOAT_STAT_BY(STAT_AsyncIO_UncompressorWaitTime,(float)UncompressorWaitTime);
}
FMemory::Free(CompressionChunks);
FMemory::Free(CompressedBuffer[0]);
FMemory::Free(CompressedBuffer[1] );
}
void FNetworkPlatformFile::InitializeAfterSetActive()
{
double NetworkFileStartupTime = 0.0;
{
SCOPE_SECONDS_COUNTER(NetworkFileStartupTime);
// send the filenames and timestamps to the server
FNetworkFileArchive Payload(NFS_Messages::GetFileList);
FillGetFileList(Payload, false);
// send the directories over, and wait for a response
FArrayReader Response;
if (!SendPayloadAndReceiveResponse(Payload, Response))
{
delete Transport;
return;
}
else
{
// receive the cooked version information
int32 ServerPackageVersion = 0;
int32 ServerPackageLicenseeVersion = 0;
ProcessServerInitialResponse(Response, ServerPackageVersion, ServerPackageLicenseeVersion);
// receive a list of the cache files and their timestamps
TMap<FString, FDateTime> ServerCachedFiles;
Response << ServerCachedFiles;
bool bDeleteAllFiles = true;
// Check the stored cooked version
FString CookedVersionFile = FPaths::GeneratedConfigDir() / TEXT("CookedVersion.txt");
if (InnerPlatformFile->FileExists(*CookedVersionFile) == true)
{
IFileHandle* FileHandle = InnerPlatformFile->OpenRead(*CookedVersionFile);
if (FileHandle != NULL)
{
int32 StoredPackageCookedVersion;
int32 StoredPackageCookedLicenseeVersion;
if (FileHandle->Read((uint8*)&StoredPackageCookedVersion, sizeof(int32)) == true)
{
if (FileHandle->Read((uint8*)&StoredPackageCookedLicenseeVersion, sizeof(int32)) == true)
{
if ((ServerPackageVersion == StoredPackageCookedVersion) &&
(ServerPackageLicenseeVersion == StoredPackageCookedLicenseeVersion))
{
bDeleteAllFiles = false;
}
else
{
UE_LOG(LogNetworkPlatformFile, Display,
TEXT("Engine version mismatch: Server %d.%d, Stored %d.%d\n"),
ServerPackageVersion, ServerPackageLicenseeVersion,
StoredPackageCookedVersion, StoredPackageCookedLicenseeVersion);
}
}
}
delete FileHandle;
}
}
else
{
UE_LOG(LogNetworkPlatformFile, Display, TEXT("Cooked version file missing: %s\n"), *CookedVersionFile);
}
if (bDeleteAllFiles == true)
{
// Make sure the config file exists...
InnerPlatformFile->CreateDirectoryTree(*(FPaths::GeneratedConfigDir()));
// Update the cooked version file
IFileHandle* FileHandle = InnerPlatformFile->OpenWrite(*CookedVersionFile);
if (FileHandle != NULL)
{
FileHandle->Write((const uint8*)&ServerPackageVersion, sizeof(int32));
FileHandle->Write((const uint8*)&ServerPackageLicenseeVersion, sizeof(int32));
delete FileHandle;
}
}
// list of directories to skip
TArray<FString> DirectoriesToSkip;
TArray<FString> DirectoriesToNotRecurse;
// use the timestamp grabbing visitor to get all the content times
FLocalTimestampDirectoryVisitor Visitor(*InnerPlatformFile, DirectoriesToSkip, DirectoriesToNotRecurse, false);
TArray<FString> RootContentPaths;
FPackageName::QueryRootContentPaths( RootContentPaths );
for( TArray<FString>::TConstIterator RootPathIt( RootContentPaths ); RootPathIt; ++RootPathIt )
{
const FString& RootPath = *RootPathIt;
const FString& ContentFolder = FPackageName::LongPackageNameToFilename(RootPath);
InnerPlatformFile->IterateDirectory( *ContentFolder, Visitor);
}
// delete out of date files using the server cached files
for (TMap<FString, FDateTime>::TIterator It(ServerCachedFiles); It; ++It)
{
bool bDeleteFile = bDeleteAllFiles;
FString ServerFile = It.Key();
// Convert the filename to the client version
ConvertServerFilenameToClientFilename(ServerFile);
// Set it in the visitor file times list
Visitor.FileTimes.Add(ServerFile, FDateTime::MinValue());
if (bDeleteFile == false)
{
// Check the time stamps...
// get local time
FDateTime LocalTime = InnerPlatformFile->GetTimeStamp(*ServerFile);
// If local time == MinValue than the file does not exist in the cache.
if (LocalTime != FDateTime::MinValue())
{
FDateTime ServerTime = It.Value();
// delete if out of date
// We will use 1.0 second as the tolerance to cover any platform differences in resolution
FTimespan TimeDiff = LocalTime - ServerTime;
double TimeDiffInSeconds = TimeDiff.GetTotalSeconds();
bDeleteFile = (TimeDiffInSeconds > 1.0) || (TimeDiffInSeconds < -1.0);
if (bDeleteFile == true)
{
if (InnerPlatformFile->FileExists(*ServerFile) == true)
{
UE_LOG(LogNetworkPlatformFile, Display, TEXT("Deleting cached file: TimeDiff %5.3f, %s"), TimeDiffInSeconds, *It.Key());
}
else
{
// It's a directory
bDeleteFile = false;
}
}
}
}
if (bDeleteFile == true)
{
InnerPlatformFile->DeleteFile(*ServerFile);
}
}
// Any content files we have locally that were not cached, delete them
for (TMap<FString, FDateTime>::TIterator It(Visitor.FileTimes); It; ++It)
{
if (It.Value() != FDateTime::MinValue())
{
// This was *not* found in the server file list... delete it
UE_LOG(LogNetworkPlatformFile, Display, TEXT("Deleting cached file: %s"), *It.Key());
InnerPlatformFile->DeleteFile(*It.Key());
}
}
// make sure we can sync a file
FString TestSyncFile = FPaths::Combine(*(FPaths::EngineDir()), TEXT("Config/BaseEngine.ini"));
InnerPlatformFile->SetReadOnly(*TestSyncFile, false);
InnerPlatformFile->DeleteFile(*TestSyncFile);
if (InnerPlatformFile->FileExists(*TestSyncFile))
{
UE_LOG(LogNetworkPlatformFile, Fatal, TEXT("Could not delete file sync test file %s."), *TestSyncFile);
}
EnsureFileIsLocal(TestSyncFile);
if (!InnerPlatformFile->FileExists(*TestSyncFile) || InnerPlatformFile->FileSize(*TestSyncFile) < 1)
{
UE_LOG(LogNetworkPlatformFile, Fatal, TEXT("Could not sync test file %s."), *TestSyncFile);
}
}
}
FPlatformMisc::LowLevelOutputDebugStringf(TEXT("Network file startup time: %5.3f seconds\n"), NetworkFileStartupTime);
}
// Compiles a blueprint.
void FKismet2CompilerModule::CompileBlueprint(class UBlueprint* Blueprint, const FKismetCompilerOptions& CompileOptions, FCompilerResultsLog& Results, FBlueprintCompileReinstancer* ParentReinstancer, TArray<UObject*>* ObjLoaded)
{
SCOPE_SECONDS_COUNTER(GBlueprintCompileTime);
BP_SCOPED_COMPILER_EVENT_STAT(EKismetCompilerStats_CompileTime);
Results.SetSourceName(Blueprint->GetName());
const bool bIsBrandNewBP = (Blueprint->SkeletonGeneratedClass == NULL) && (Blueprint->GeneratedClass == NULL) && (Blueprint->ParentClass != NULL) && !CompileOptions.bIsDuplicationInstigated;
for ( IBlueprintCompiler* Compiler : Compilers )
{
Compiler->PreCompile(Blueprint);
}
if (CompileOptions.CompileType != EKismetCompileType::Cpp)
{
BP_SCOPED_COMPILER_EVENT_STAT(EKismetCompilerStats_CompileSkeletonClass);
FBlueprintCompileReinstancer SkeletonReinstancer(Blueprint->SkeletonGeneratedClass);
FCompilerResultsLog SkeletonResults;
SkeletonResults.bSilentMode = true;
FKismetCompilerOptions SkeletonCompileOptions;
SkeletonCompileOptions.CompileType = EKismetCompileType::SkeletonOnly;
CompileBlueprintInner(Blueprint, SkeletonCompileOptions, SkeletonResults, ObjLoaded);
}
// If this was a full compile, take appropriate actions depending on the success of failure of the compile
if( CompileOptions.IsGeneratedClassCompileType() )
{
BP_SCOPED_COMPILER_EVENT_STAT(EKismetCompilerStats_CompileGeneratedClass);
// Perform the full compile
CompileBlueprintInner(Blueprint, CompileOptions, Results, ObjLoaded);
if (Results.NumErrors == 0)
{
// Blueprint is error free. Go ahead and fix up debug info
Blueprint->Status = (0 == Results.NumWarnings) ? BS_UpToDate : BS_UpToDateWithWarnings;
Blueprint->BlueprintSystemVersion = UBlueprint::GetCurrentBlueprintSystemVersion();
// Reapply breakpoints to the bytecode of the new class
for (int32 Index = 0; Index < Blueprint->Breakpoints.Num(); ++Index)
{
UBreakpoint* Breakpoint = Blueprint->Breakpoints[Index];
FKismetDebugUtilities::ReapplyBreakpoint(Breakpoint);
}
}
else
{
// Should never get errors from a brand new blueprint!
ensure(!bIsBrandNewBP || (Results.NumErrors == 0));
// There were errors. Compile the generated class to have function stubs
Blueprint->Status = BS_Error;
static const FBoolConfigValueHelper ReinstanceOnlyWhenNecessary(TEXT("Kismet"), TEXT("bReinstanceOnlyWhenNecessary"), GEngineIni);
// Reinstance objects here, so we can preserve their memory layouts to reinstance them again
if( ParentReinstancer != NULL )
{
ParentReinstancer->UpdateBytecodeReferences();
if(!Blueprint->bIsRegeneratingOnLoad)
{
ParentReinstancer->ReinstanceObjects(!ReinstanceOnlyWhenNecessary);
}
}
FBlueprintCompileReinstancer StubReinstancer(Blueprint->GeneratedClass);
// Toss the half-baked class and generate a stubbed out skeleton class that can be used
FCompilerResultsLog StubResults;
StubResults.bSilentMode = true;
FKismetCompilerOptions StubCompileOptions(CompileOptions);
StubCompileOptions.CompileType = EKismetCompileType::StubAfterFailure;
CompileBlueprintInner(Blueprint, StubCompileOptions, StubResults, ObjLoaded);
StubReinstancer.UpdateBytecodeReferences();
if( !Blueprint->bIsRegeneratingOnLoad )
{
StubReinstancer.ReinstanceObjects(!ReinstanceOnlyWhenNecessary);
}
}
}
for ( IBlueprintCompiler* Compiler : Compilers )
{
Compiler->PostCompile(Blueprint);
}
UPackage* Package = Blueprint->GetOutermost();
if( Package )
{
UMetaData* MetaData = Package->GetMetaData();
MetaData->RemoveMetaDataOutsidePackage();
}
}
/**
* Serializes and compresses/ uncompresses data. This is a shared helper function for compression
* support. The data is saved in a way compatible with FIOSystem::LoadCompressedData.
*
* @note: the way this code works needs to be in line with FIOSystem::LoadCompressedData implementations
* @note: the way this code works needs to be in line with FAsyncIOSystemBase::FulfillCompressedRead
*
* @param V Data pointer to serialize data from/to, or a FileReader if bTreatBufferAsFileReader is true
* @param Length Length of source data if we're saving, unused otherwise
* @param Flags Flags to control what method to use for [de]compression and optionally control memory vs speed when compressing
* @param bTreatBufferAsFileReader true if V is actually an FArchive, which is used when saving to read data - helps to avoid single huge allocations of source data
*/
void FArchive::SerializeCompressed( void* V, int64 Length, ECompressionFlags Flags, bool bTreatBufferAsFileReader )
{
if( IsLoading() )
{
// Serialize package file tag used to determine endianess.
FCompressedChunkInfo PackageFileTag;
PackageFileTag.CompressedSize = 0;
PackageFileTag.UncompressedSize = 0;
*this << PackageFileTag;
bool bWasByteSwapped = PackageFileTag.CompressedSize != PACKAGE_FILE_TAG;
// Read in base summary.
FCompressedChunkInfo Summary;
*this << Summary;
if (bWasByteSwapped)
{
check( PackageFileTag.CompressedSize == PACKAGE_FILE_TAG_SWAPPED );
Summary.CompressedSize = BYTESWAP_ORDER64(Summary.CompressedSize);
Summary.UncompressedSize = BYTESWAP_ORDER64(Summary.UncompressedSize);
PackageFileTag.UncompressedSize = BYTESWAP_ORDER64(PackageFileTag.UncompressedSize);
}
else
{
check( PackageFileTag.CompressedSize == PACKAGE_FILE_TAG );
}
// Handle change in compression chunk size in backward compatible way.
int64 LoadingCompressionChunkSize = PackageFileTag.UncompressedSize;
if (LoadingCompressionChunkSize == PACKAGE_FILE_TAG)
{
LoadingCompressionChunkSize = LOADING_COMPRESSION_CHUNK_SIZE;
}
// Figure out how many chunks there are going to be based on uncompressed size and compression chunk size.
int64 TotalChunkCount = (Summary.UncompressedSize + LoadingCompressionChunkSize - 1) / LoadingCompressionChunkSize;
// Allocate compression chunk infos and serialize them, keeping track of max size of compression chunks used.
FCompressedChunkInfo* CompressionChunks = new FCompressedChunkInfo[TotalChunkCount];
int64 MaxCompressedSize = 0;
for( int32 ChunkIndex=0; ChunkIndex<TotalChunkCount; ChunkIndex++ )
{
*this << CompressionChunks[ChunkIndex];
if (bWasByteSwapped)
{
CompressionChunks[ChunkIndex].CompressedSize = BYTESWAP_ORDER64( CompressionChunks[ChunkIndex].CompressedSize );
CompressionChunks[ChunkIndex].UncompressedSize = BYTESWAP_ORDER64( CompressionChunks[ChunkIndex].UncompressedSize );
}
MaxCompressedSize = FMath::Max( CompressionChunks[ChunkIndex].CompressedSize, MaxCompressedSize );
}
int64 Padding = 0;
// Set up destination pointer and allocate memory for compressed chunk[s] (one at a time).
uint8* Dest = (uint8*) V;
void* CompressedBuffer = FMemory::Malloc( MaxCompressedSize + Padding );
// Iterate over all chunks, serialize them into memory and decompress them directly into the destination pointer
for( int64 ChunkIndex=0; ChunkIndex<TotalChunkCount; ChunkIndex++ )
{
const FCompressedChunkInfo& Chunk = CompressionChunks[ChunkIndex];
// Read compressed data.
Serialize( CompressedBuffer, Chunk.CompressedSize );
// Decompress into dest pointer directly.
verify( FCompression::UncompressMemory( Flags, Dest, Chunk.UncompressedSize, CompressedBuffer, Chunk.CompressedSize, (Padding > 0) ? true : false ) );
// And advance it by read amount.
Dest += Chunk.UncompressedSize;
}
// Free up allocated memory.
FMemory::Free( CompressedBuffer );
delete [] CompressionChunks;
}
else if( IsSaving() )
{
SCOPE_SECONDS_COUNTER(GArchiveSerializedCompressedSavingTime);
check( Length > 0 );
// Serialize package file tag used to determine endianess in LoadCompressedData.
FCompressedChunkInfo PackageFileTag;
PackageFileTag.CompressedSize = PACKAGE_FILE_TAG;
PackageFileTag.UncompressedSize = GSavingCompressionChunkSize;
*this << PackageFileTag;
// Figure out how many chunks there are going to be based on uncompressed size and compression chunk size.
int64 TotalChunkCount = (Length + GSavingCompressionChunkSize - 1) / GSavingCompressionChunkSize + 1;
// Keep track of current position so we can later seek back and overwrite stub compression chunk infos.
int64 StartPosition = Tell();
// Allocate compression chunk infos and serialize them so we can later overwrite the data.
FCompressedChunkInfo* CompressionChunks = new FCompressedChunkInfo[TotalChunkCount];
for( int64 ChunkIndex=0; ChunkIndex<TotalChunkCount; ChunkIndex++ )
{
*this << CompressionChunks[ChunkIndex];
}
// The uncompressd size is equal to the passed in length.
CompressionChunks[0].UncompressedSize = Length;
// Zero initialize compressed size so we can update it during chunk compression.
CompressionChunks[0].CompressedSize = 0;
#if WITH_MULTI_THREADED_COMPRESSION
#define MAX_COMPRESSION_JOBS (16)
// Don't scale more than 16x to avoid going overboard wrt temporary memory.
FAsyncTask<FAsyncCompressionChunk> AsyncChunks[MAX_COMPRESSION_JOBS];
// used to keep track of which job is the next one we need to retire
int32 AsyncChunkIndex[MAX_COMPRESSION_JOBS]={0};
static uint32 GNumUnusedThreads_SerializeCompressed = -1;
if (GNumUnusedThreads_SerializeCompressed == (uint32)-1)
{
// one-time initialization
GNumUnusedThreads_SerializeCompressed = 1;
// if we should use all available cores then we want to compress with all
if( FParse::Param(FCommandLine::Get(), TEXT("USEALLAVAILABLECORES")) == true )
{
GNumUnusedThreads_SerializeCompressed = 0;
}
}
// Maximum number of concurrent async tasks we're going to kick off. This is based on the number of processors
// available in the system.
int32 MaxConcurrentAsyncChunks = FMath::Clamp<int32>( FPlatformMisc::NumberOfCores() - GNumUnusedThreads_SerializeCompressed, 1, MAX_COMPRESSION_JOBS );
if (FParse::Param(FCommandLine::Get(), TEXT("MTCHILD")))
{
// throttle this back when doing MT cooks
MaxConcurrentAsyncChunks = FMath::Min<int32>( MaxConcurrentAsyncChunks,4 );
}
// Number of chunks left to finalize.
int64 NumChunksLeftToFinalize = (Length + GSavingCompressionChunkSize - 1) / GSavingCompressionChunkSize;
// Number of chunks left to kick off
int64 NumChunksLeftToKickOff = NumChunksLeftToFinalize;
// Start at index 1 as first chunk info is summary.
int64 CurrentChunkIndex = 1;
// Start at index 1 as first chunk info is summary.
int64 RetireChunkIndex = 1;
// Number of bytes remaining to kick off compression for.
int64 BytesRemainingToKickOff = Length;
// Pointer to src data if buffer is memory pointer, NULL if it's a FArchive.
uint8* SrcBuffer = bTreatBufferAsFileReader ? NULL : (uint8*)V;
check(!bTreatBufferAsFileReader || ((FArchive*)V)->IsLoading());
check(NumChunksLeftToFinalize);
// Loop while there is work left to do based on whether we have finalized all chunks yet.
while( NumChunksLeftToFinalize )
{
// If true we are waiting for async tasks to complete and should wait to complete some
// if there are no async tasks finishing this iteration.
bool bNeedToWaitForAsyncTask = false;
// Try to kick off async tasks if there are chunks left to kick off.
if( NumChunksLeftToKickOff )
{
// Find free index based on looking at uncompressed size. We can't use the thread counter
// for this as that might be a chunk ready for finalization.
int32 FreeIndex = INDEX_NONE;
for( int32 i=0; i<MaxConcurrentAsyncChunks; i++ )
{
if( !AsyncChunkIndex[i] )
{
FreeIndex = i;
check(AsyncChunks[FreeIndex].IsIdle()); // this is not supposed to be in use
break;
}
}
// Kick off async compression task if we found a chunk for it.
if( FreeIndex != INDEX_NONE )
{
FAsyncCompressionChunk& NewChunk = AsyncChunks[FreeIndex].GetTask();
// 2 times the uncompressed size should be more than enough; the compressed data shouldn't be that much larger
NewChunk.CompressedSize = 2 * GSavingCompressionChunkSize;
// Allocate compressed buffer placeholder on first use.
if( NewChunk.CompressedBuffer == NULL )
{
NewChunk.CompressedBuffer = FMemory::Malloc( NewChunk.CompressedSize );
}
// By default everything is chunked up into GSavingCompressionChunkSize chunks.
NewChunk.UncompressedSize = FMath::Min( BytesRemainingToKickOff, (int64)GSavingCompressionChunkSize );
check(NewChunk.UncompressedSize>0);
// Need to serialize source data if passed in pointer is an FArchive.
if( bTreatBufferAsFileReader )
{
// Allocate memory on first use. We allocate the maximum amount to allow reuse.
if( !NewChunk.UncompressedBuffer )
{
NewChunk.UncompressedBuffer = FMemory::Malloc(GSavingCompressionChunkSize);
}
((FArchive*)V)->Serialize(NewChunk.UncompressedBuffer, NewChunk.UncompressedSize);
}
// Advance src pointer by amount to be compressed.
else
{
NewChunk.UncompressedBuffer = SrcBuffer;
SrcBuffer += NewChunk.UncompressedSize;
}
// Update status variables for tracking how much work is left, what to do next.
BytesRemainingToKickOff -= NewChunk.UncompressedSize;
AsyncChunkIndex[FreeIndex] = CurrentChunkIndex++;
NewChunk.Flags = Flags;
NumChunksLeftToKickOff--;
AsyncChunks[FreeIndex].StartBackgroundTask();
}
// No chunks were available to use, complete some
else
{
bNeedToWaitForAsyncTask = true;
}
}
// Index of oldest chunk, needed as we need to serialize in order.
int32 OldestAsyncChunkIndex = INDEX_NONE;
for( int32 i=0; i<MaxConcurrentAsyncChunks; i++ )
{
check(AsyncChunkIndex[i] == 0 || AsyncChunkIndex[i] >= RetireChunkIndex);
check(AsyncChunkIndex[i] < RetireChunkIndex + MaxConcurrentAsyncChunks);
if (AsyncChunkIndex[i] == RetireChunkIndex)
{
OldestAsyncChunkIndex = i;
}
}
check(OldestAsyncChunkIndex != INDEX_NONE); // the retire chunk better be outstanding
bool ChunkReady;
if (bNeedToWaitForAsyncTask)
{
// This guarantees that the async work has finished, doing it on this thread if it hasn't been started
AsyncChunks[OldestAsyncChunkIndex].EnsureCompletion();
ChunkReady = true;
}
else
{
ChunkReady = AsyncChunks[OldestAsyncChunkIndex].IsDone();
}
if (ChunkReady)
{
FAsyncCompressionChunk& DoneChunk = AsyncChunks[OldestAsyncChunkIndex].GetTask();
// Serialize the data via archive.
Serialize( DoneChunk.CompressedBuffer, DoneChunk.CompressedSize );
// Update associated chunk.
int64 CompressionChunkIndex = RetireChunkIndex++;
check(CompressionChunkIndex<TotalChunkCount);
CompressionChunks[CompressionChunkIndex].CompressedSize = DoneChunk.CompressedSize;
CompressionChunks[CompressionChunkIndex].UncompressedSize = DoneChunk.UncompressedSize;
// Keep track of total compressed size, stored in first chunk.
CompressionChunks[0].CompressedSize += DoneChunk.CompressedSize;
// Clean up chunk. Src and dst buffer are not touched as the contain allocations we keep till the end.
AsyncChunkIndex[OldestAsyncChunkIndex] = 0;
DoneChunk.CompressedSize = 0;
DoneChunk.UncompressedSize = 0;
// Finalized one :)
NumChunksLeftToFinalize--;
bNeedToWaitForAsyncTask = false;
}
}
// Free intermediate buffer storage.
for( int32 i=0; i<MaxConcurrentAsyncChunks; i++ )
{
// Free temporary compressed buffer storage.
FMemory::Free( AsyncChunks[i].GetTask().CompressedBuffer );
AsyncChunks[i].GetTask().CompressedBuffer = NULL;
// Free temporary uncompressed buffer storage if data was serialized in.
if( bTreatBufferAsFileReader )
{
FMemory::Free( AsyncChunks[i].GetTask().UncompressedBuffer );
AsyncChunks[i].GetTask().UncompressedBuffer = NULL;
}
}
#else
// Set up source pointer amount of data to copy (in bytes)
uint8* Src;
// allocate memory to read into
if (bTreatBufferAsFileReader)
{
Src = (uint8*)FMemory::Malloc(GSavingCompressionChunkSize);
check(((FArchive*)V)->IsLoading());
}
else
{
Src = (uint8*) V;
}
int64 BytesRemaining = Length;
// Start at index 1 as first chunk info is summary.
int32 CurrentChunkIndex = 1;
// 2 times the uncompressed size should be more than enough; the compressed data shouldn't be that much larger
int64 CompressedBufferSize = 2 * GSavingCompressionChunkSize;
void* CompressedBuffer = FMemory::Malloc( CompressedBufferSize );
while( BytesRemaining > 0 )
{
int64 BytesToCompress = FMath::Min( BytesRemaining, (int64)GSavingCompressionChunkSize );
int64 CompressedSize = CompressedBufferSize;
// read in the next chunk from the reader
if (bTreatBufferAsFileReader)
{
((FArchive*)V)->Serialize(Src, BytesToCompress);
}
check(CompressedSize < INT_MAX);
int32 CompressedSizeInt = (int32)CompressedSize;
verify( FCompression::CompressMemory( Flags, CompressedBuffer, CompressedSizeInt, Src, BytesToCompress ) );
CompressedSize = CompressedSizeInt;
// move to next chunk if not reading from file
if (!bTreatBufferAsFileReader)
{
Src += BytesToCompress;
}
Serialize( CompressedBuffer, CompressedSize );
// Keep track of total compressed size, stored in first chunk.
CompressionChunks[0].CompressedSize += CompressedSize;
// Update current chunk.
check(CurrentChunkIndex<TotalChunkCount);
CompressionChunks[CurrentChunkIndex].CompressedSize = CompressedSize;
CompressionChunks[CurrentChunkIndex].UncompressedSize = BytesToCompress;
CurrentChunkIndex++;
BytesRemaining -= GSavingCompressionChunkSize;
}
// free the buffer we read into
if (bTreatBufferAsFileReader)
{
FMemory::Free(Src);
}
// Free allocated memory.
FMemory::Free( CompressedBuffer );
#endif
// Overrwrite chunk infos by seeking to the beginning, serializing the data and then
// seeking back to the end.
auto EndPosition = Tell();
// Seek to the beginning.
Seek( StartPosition );
// Serialize chunk infos.
for( int32 ChunkIndex=0; ChunkIndex<TotalChunkCount; ChunkIndex++ )
{
*this << CompressionChunks[ChunkIndex];
}
// Seek back to end.
Seek( EndPosition );
// Free intermediate data.
delete [] CompressionChunks;
}
}
