/** Async worker that checks the cache backend and if that fails, calls the deriver to build the data and then puts the results to the cache **/ void DoWork() { bool bGetResult; { INC_DWORD_STAT(STAT_DDC_NumGets); STAT(double ThisTime = 0); { SCOPE_SECONDS_COUNTER(ThisTime); bGetResult = FDerivedDataBackend::Get().GetRoot().GetCachedData(*CacheKey, Data); } INC_FLOAT_STAT_BY(STAT_DDC_SyncGetTime, bSynchronousForStats ? (float)ThisTime : 0.0f); } if (bGetResult) { check(Data.Num()); bSuccess = true; delete DataDeriver; DataDeriver = NULL; } else if (DataDeriver) { { INC_DWORD_STAT(STAT_DDC_NumBuilds); STAT(double ThisTime = 0); { SCOPE_SECONDS_COUNTER(ThisTime); bSuccess = DataDeriver->Build(Data); } INC_FLOAT_STAT_BY(STAT_DDC_SyncBuildTime, bSynchronousForStats ? (float)ThisTime : 0.0f); } delete DataDeriver; DataDeriver = NULL; if (bSuccess) { check(Data.Num()); INC_DWORD_STAT(STAT_DDC_NumPuts); STAT(double ThisTime = 0); { SCOPE_SECONDS_COUNTER(ThisTime); FDerivedDataBackend::Get().GetRoot().PutCachedData(*CacheKey, Data, true); } INC_FLOAT_STAT_BY(STAT_DDC_PutTime, bSynchronousForStats ? (float)ThisTime : 0.0f); } } if (!bSuccess) { Data.Empty(); } FDerivedDataBackend::Get().AddToAsyncCompletionCounter(-1); }
virtual void Put(const TCHAR* CacheKey, TArray<uint8>& Data, bool bPutEvenIfExists = false) override { STAT(double ThisTime = 0); { SCOPE_SECONDS_COUNTER(ThisTime); FDerivedDataBackend::Get().GetRoot().PutCachedData(CacheKey, Data, bPutEvenIfExists); } INC_FLOAT_STAT_BY(STAT_DDC_PutTime,(float)ThisTime); INC_DWORD_STAT(STAT_DDC_NumPuts); }
/** * Constructor, initializing all member variables. It also reads .ini settings and caches them and creates the * database connection object and attempts to connect if bUseTaskPerfTracking is set. */ FTaskPerfTracker::FTaskPerfTracker() : FTaskDatabase() , bUseTaskPerfTracking(FALSE) , TimeSpentTalkingWithDB(0) { #if !UDK // Read the ini settings and store them in the struct to aid debugging. GConfig->GetBool(TEXT("TaskPerfTracking"), TEXT("bUseTaskPerfTracking" ), bUseTaskPerfTracking, GEngineIni); // Only do work if task tracking is enabled. if( bUseTaskPerfTracking ) { // only attempt to get the data when we want to use the TaskPerfTracking verify(GConfig->GetString( TEXT("TaskPerfTracking"), TEXT("ConnectionString"), ConnectionString, GEngineIni )); verify(GConfig->GetString( TEXT("TaskPerfTracking"), TEXT("RemoteConnectionIP"), RemoteConnectionIP, GEngineIni )); verify(GConfig->GetString( TEXT("TaskPerfTracking"), TEXT("RemoteConnectionStringOverride"), RemoteConnectionStringOverride, GEngineIni )); // Track time spent talking with DB to ensure we don't introduce nasty stalls. SCOPE_SECONDS_COUNTER(TimeSpentTalkingWithDB); // Create the connection object; needs to be deleted via "delete". Connection = FDataBaseConnection::CreateObject(); // Try to open connection to DB - this is a synchronous operation. if( Connection->Open( *ConnectionString, *RemoteConnectionIP, *RemoteConnectionStringOverride ) == TRUE ) { warnf(NAME_DevDataBase,TEXT("Connection to \"%s\" or \"%s\" succeeded"), *ConnectionString, *RemoteConnectionIP); // Create format string for calling procedure. FormatString = FString(TEXT("EXEC BEGINRUN ")); #if _DEBUG FormatString += TEXT(" @ConfigName='DEBUG', "); #elif FINAL_RELEASE_DEBUGCONSOLE FormatString += TEXT(" @ConfigName='FINAL_RELEASE_DEBUGCONSOLE', "); #elif FINAL_RELEASE FormatString += TEXT(" @ConfigName='FINAL_RELEASE', "); #else FormatString += TEXT(" @ConfigName='RELEASE', "); #endif FormatString += FString(TEXT("@GameName='")) + GGameName + TEXT("', @MachineName='") + appComputerName() + TEXT("', "); FormatString += FString(TEXT("@CmdLine='")) + appCmdLine() + TEXT("', @UserName='******', "); FormatString += FString(TEXT("@TaskDescription='%s', @TaskParameter='%s', @Changelist=")) + appItoa(GBuiltFromChangeList); } // Connection failed :( else { warnf(NAME_DevDataBase,TEXT("Connection to \"%s\" or \"%s\" failed"), *ConnectionString, *RemoteConnectionIP); // Only delete object - no need to close as connection failed. delete Connection; Connection = NULL; } } #endif //#if !UDK }
virtual bool CachedDataProbablyExists(const TCHAR* CacheKey) override { bool bResult; INC_DWORD_STAT(STAT_DDC_NumExist); STAT(double ThisTime = 0); { SCOPE_SECONDS_COUNTER(ThisTime); bResult = FDerivedDataBackend::Get().GetRoot().CachedDataProbablyExists(CacheKey); } INC_FLOAT_STAT_BY(STAT_DDC_ExistTime, (float)ThisTime); return bResult; }
virtual void WaitAsynchronousCompletion(uint32 Handle) override { STAT(double ThisTime = 0); { SCOPE_SECONDS_COUNTER(ThisTime); FAsyncTask<FBuildAsyncWorker>* AsyncTask = NULL; { FScopeLock ScopeLock(&SynchronizationObject); AsyncTask = PendingTasks.FindRef(Handle); } check(AsyncTask); AsyncTask->EnsureCompletion(); } INC_FLOAT_STAT_BY(STAT_DDC_ASyncWaitTime,(float)ThisTime); }
void FShaderResource::InitializePixelShaderRHI() { if (!IsInitialized()) { STAT(double ShaderInitializationTime = 0); { SCOPE_CYCLE_COUNTER(STAT_Shaders_FrameRTShaderInitForRenderingTime); SCOPE_SECONDS_COUNTER(ShaderInitializationTime); InitResourceFromPossiblyParallelRendering(); } INC_FLOAT_STAT_BY(STAT_Shaders_TotalRTShaderInitForRenderingTime,(float)ShaderInitializationTime); } checkSlow(IsInitialized()); }
/** * Frees allocation associated with passed in pointer. * * @param Pointer Pointer to free. */ void FBestFitAllocator::Free( void* Pointer ) { SCOPE_SECONDS_COUNTER(TimeSpentInAllocator); // Look up pointer in TMap. FMemoryChunk* MatchingChunk = PointerToChunkMap.FindRef( (PTRINT) Pointer ); check( MatchingChunk ); // Remove the entry PointerToChunkMap.Remove((PTRINT) Pointer); // Update usage stats in a thread safe way. appInterlockedAdd( &AllocatedMemorySize, -MatchingChunk->Size ); appInterlockedAdd( &AvailableMemorySize, +MatchingChunk->Size ); // Free the chunk. FreeChunk(MatchingChunk); }
virtual void WaitAsynchronousCompletion(uint32 Handle) override { STAT(double ThisTime = 0); { SCOPE_SECONDS_COUNTER(ThisTime); FScopeLock ScopeLock(&SynchronizationObject); for (TSet<FDerivedDataRollup*>::TIterator Iter(PendingRollups); Iter; ++Iter) { if ((*Iter)->Contains(Handle)) { (*Iter)->WaitAsynchronousCompletion(Handle); return; } } } INC_FLOAT_STAT_BY(STAT_DDC_ASyncWaitTime,(float)ThisTime); Super::WaitAsynchronousCompletion(Handle); }
void FAsyncIOSystemBase::InternalRead( IFileHandle* FileHandle, int64 Offset, int64 Size, void* Dest ) { DECLARE_SCOPE_CYCLE_COUNTER(TEXT("FAsyncIOSystemBase::InternalRead"), STAT_AsyncIOSystemBase_InternalRead, STATGROUP_AsyncIO_Verbose); FScopeLock ScopeLock( ExclusiveReadCriticalSection ); STAT(double ReadTime = 0); { SCOPE_SECONDS_COUNTER(ReadTime); PlatformReadDoNotCallDirectly( FileHandle, Offset, Size, Dest ); } INC_FLOAT_STAT_BY(STAT_AsyncIO_PlatformReadTime,(float)ReadTime); // The platform might actually read more than Size due to aligning and internal min read sizes // though we only really care about throttling requested bandwidth as it's not very accurate // to begin with. STAT(ConstrainBandwidth(Size, ReadTime)); }
void FPackageDependencyInfo::ResolveCircularDependencies(FPackageDependencyTrackingInfo* InPkgInfo) { double Seconds = 0.0f; { SCOPE_SECONDS_COUNTER(Seconds); NumResolvePasses = 0; NumResolveIterations = 0; NumCirculars = 0; ResolveCircularDependenciesInnerFast(); } UE_LOG(LogPackageDependencyInfo, Verbose, TEXT("Resolve circular dependencies for package %s took %.1f ms (Iter=%i, Passes=%i, Circulars=%i)"), *InPkgInfo->PackageName, Seconds*1000.0f, NumResolveIterations, NumResolvePasses, NumCirculars ); }
const FHullShaderRHIRef& FShaderResource::GetHullShader() { checkSlow(Target.Frequency == SF_Hull); if (!IsInitialized()) { STAT(double ShaderInitializationTime = 0); { SCOPE_CYCLE_COUNTER(STAT_Shaders_FrameRTShaderInitForRenderingTime); SCOPE_SECONDS_COUNTER(ShaderInitializationTime); InitResourceFromPossiblyParallelRendering(); } INC_FLOAT_STAT_BY(STAT_Shaders_TotalRTShaderInitForRenderingTime,(float)ShaderInitializationTime); } checkSlow(IsInitialized()); return HullShader; }
/** * Adds a task to track to the DB. * * @param Task Name of task * @param TaskParameter Additional information, e.g. name of map if task is lighting rebuild * @param DurationInSeconds Duration of task in seconds */ void FTaskPerfTracker::AddTask( const TCHAR* Task, const TCHAR* TaskParameter, FLOAT DurationInSeconds ) { // A valid connection means that tracking is enabled. if( Connection ) { // Track time spent talking with DB to ensure we don't introduce nasty stalls. SCOPE_SECONDS_COUNTER(TimeSpentTalkingWithDB); // Execute BeginRun with baked format string. FDataBaseRecordSet* RecordSet = NULL; if( Connection->Execute( *FString::Printf( *FormatString, Task, TaskParameter ), RecordSet ) && RecordSet ) { // Retrieve RunID from recordset. It's the return value of the EXEC. INT RunID = RecordSet->GetInt(TEXT("Return Value")); // Add stat entry. Connection->Execute( *FString::Printf(TEXT("EXEC ADDRUNDATA @RunID=%i, @StatGroupName='Ungrouped', @StatName='Duration', @StatValue='%f'"), RunID, DurationInSeconds) ); // End the run, marking it as 'passed'. Connection->Execute( *FString::Printf(TEXT("EXEC ENDRUN @RunID=%i, @ResultDescription='Passed'"), RunID ) ); delete RecordSet; RecordSet = NULL; } } }
/** * Tries to reallocate texture memory in-place (without relocating), * by adjusting the base address of the allocation but keeping the end address the same. * * @param OldBaseAddress Pointer to the original allocation * @param NewBaseAddress New desired baseaddress for the allocation (adjusting the size so the end stays the same) * @returns TRUE if it succeeded **/ UBOOL FBestFitAllocator::Reallocate( void* OldBaseAddress, void* NewBaseAddress ) { SCOPE_SECONDS_COUNTER(TimeSpentInAllocator); // Look up pointer in TMap. FMemoryChunk* MatchingChunk = PointerToChunkMap.FindRef( PTRINT(OldBaseAddress) ); check( MatchingChunk && PTRINT(OldBaseAddress) == PTRINT(MatchingChunk->Base) ); INT MemoryAdjustment = Abs<INT>(PTRINT(NewBaseAddress) - PTRINT(OldBaseAddress)); // Are we growing the allocation? if ( PTRINT(NewBaseAddress) < PTRINT(OldBaseAddress) ) { // Is there enough free memory immediately before this chunk? FMemoryChunk* PrevChunk = MatchingChunk->PreviousChunk; if ( PrevChunk && PrevChunk->bIsAvailable && PrevChunk->Size >= MemoryAdjustment ) { PointerToChunkMap.Remove( PTRINT(OldBaseAddress) ); // Shrink the previous and grow the current chunk. PrevChunk->Size -= MemoryAdjustment; MatchingChunk->Base -= MemoryAdjustment; MatchingChunk->Size += MemoryAdjustment; check(PTRINT(NewBaseAddress) == PTRINT(MatchingChunk->Base)); PointerToChunkMap.Set( PTRINT(NewBaseAddress), MatchingChunk ); if ( PrevChunk->Size == 0 ) { delete PrevChunk; } // Update usage stats in a thread safe way. appInterlockedAdd( &AllocatedMemorySize, +MemoryAdjustment ); appInterlockedAdd( &AvailableMemorySize, -MemoryAdjustment ); return TRUE; } } else { // We're shrinking the allocation. check( MemoryAdjustment <= MatchingChunk->Size ); FMemoryChunk* PrevChunk = MatchingChunk->PreviousChunk; if ( PrevChunk ) { // Shrink the current chunk. MatchingChunk->Base += MemoryAdjustment; MatchingChunk->Size -= MemoryAdjustment; // Grow the previous chunk. INT OriginalPrevSize = PrevChunk->Size; PrevChunk->Size += MemoryAdjustment; // If the previous chunk was "in use", split it and insert a 2nd free chunk. if ( !PrevChunk->bIsAvailable ) { Split( PrevChunk, OriginalPrevSize ); } } else { // This was the first chunk, split it. Split( MatchingChunk, MemoryAdjustment ); // We're going to use the new chunk. Mark it as "used memory". MatchingChunk = MatchingChunk->NextChunk; MatchingChunk->UnlinkFree(); // Make the original chunk "free memory". FreeChunk( MatchingChunk->PreviousChunk ); } check(PTRINT(NewBaseAddress) == PTRINT(MatchingChunk->Base)); PointerToChunkMap.Remove( PTRINT(OldBaseAddress) ); PointerToChunkMap.Set( PTRINT(NewBaseAddress), MatchingChunk ); // Update usage stats in a thread safe way. appInterlockedAdd( &AllocatedMemorySize, -MemoryAdjustment ); appInterlockedAdd( &AvailableMemorySize, +MemoryAdjustment ); return TRUE; } return FALSE; }
/** * Allocate physical memory. * * @param AllocationSize Size of allocation * @param bAllowFailure Whether to allow allocation failure or not * @return Pointer to allocated memory */ void* FBestFitAllocator::Allocate( INT AllocationSize, UBOOL bAllowFailure ) { SCOPE_SECONDS_COUNTER(TimeSpentInAllocator); check( FirstChunk ); // Make sure everything is appropriately aligned. AllocationSize = Align( AllocationSize, AllocationAlignment ); // Perform a "best fit" search, returning first perfect fit if there is one. FMemoryChunk* CurrentChunk = FirstFreeChunk; FMemoryChunk* BestChunk = NULL; while( CurrentChunk ) { // Check whether chunk is available and large enough to hold allocation. check( CurrentChunk->bIsAvailable ); if( CurrentChunk->Size >= AllocationSize ) { // Compare with current best chunk if one exists. if( BestChunk ) { // Tighter fits are preferred. if( CurrentChunk->Size < BestChunk->Size ) { BestChunk = CurrentChunk; } } // No existing best chunk so use this one. else { BestChunk = CurrentChunk; } // We have a perfect fit, no need to iterate further. if( BestChunk->Size == AllocationSize ) { break; } } CurrentChunk = CurrentChunk->NextFreeChunk; } // Dump allocation info and return NULL if we weren't able to satisfy allocation request. if( !BestChunk ) { if ( !bAllowFailure ) { #if !FINAL_RELEASE DumpAllocs(); debugf(TEXT("Ran out of memory for allocation in best-fit allocator of size %i KByte"), AllocationSize / 1024); GLog->FlushThreadedLogs(); #endif } return NULL; } // Mark as being in use. BestChunk->UnlinkFree(); // Split chunk to avoid waste. if( BestChunk->Size > AllocationSize ) { Split( BestChunk, AllocationSize ); } // Ensure that everything's in range. check( (BestChunk->Base + BestChunk->Size) <= (MemoryBase + MemorySize) ); check( BestChunk->Base >= MemoryBase ); // Update usage stats in a thread safe way. appInterlockedAdd( &AllocatedMemorySize, +BestChunk->Size ); appInterlockedAdd( &AvailableMemorySize, -BestChunk->Size ); // Keep track of mapping and return pointer. PointerToChunkMap.Set( (PTRINT) BestChunk->Base, BestChunk ); return BestChunk->Base; }
void FAsyncIOSystemBase::FulfillCompressedRead( const FAsyncIORequest& IORequest, IFileHandle* FileHandle ) { DECLARE_SCOPE_CYCLE_COUNTER(TEXT("FAsyncIOSystemBase::FulfillCompressedRead"), STAT_AsyncIOSystemBase_FulfillCompressedRead, STATGROUP_AsyncIO_Verbose); if (GbLogAsyncLoading == true) { LogIORequest(TEXT("FulfillCompressedRead"), IORequest); } // Initialize variables. FAsyncUncompress* Uncompressor = NULL; uint8* UncompressedBuffer = (uint8*) IORequest.Dest; // First compression chunk contains information about total size so we skip that one. int32 CurrentChunkIndex = 1; int32 CurrentBufferIndex = 0; bool bHasProcessedAllData = false; // read the first two ints, which will contain the magic bytes (to detect byteswapping) // and the original size the chunks were compressed from int64 HeaderData[2]; int32 HeaderSize = sizeof(HeaderData); InternalRead(FileHandle, IORequest.Offset, HeaderSize, HeaderData); RETURN_IF_EXIT_REQUESTED; // if the magic bytes don't match, then we are byteswapped (or corrupted) bool bIsByteswapped = HeaderData[0] != PACKAGE_FILE_TAG; // if its potentially byteswapped, make sure it's not just corrupted if (bIsByteswapped) { // if it doesn't equal the swapped version, then data is corrupted if (HeaderData[0] != PACKAGE_FILE_TAG_SWAPPED) { UE_LOG(LogStreaming, Warning, TEXT("Detected data corruption [header] trying to read %lld bytes at offset %lld from '%s'. Please delete file and recook."), IORequest.UncompressedSize, IORequest.Offset , *IORequest.FileName ); check(0); FPlatformMisc::HandleIOFailure(*IORequest.FileName); } // otherwise, we have a valid byteswapped file, so swap the chunk size else { HeaderData[1] = BYTESWAP_ORDER64(HeaderData[1]); } } int32 CompressionChunkSize = HeaderData[1]; // handle old packages that don't have the chunk size in the header, in which case // we can use the old hardcoded size if (CompressionChunkSize == PACKAGE_FILE_TAG) { CompressionChunkSize = LOADING_COMPRESSION_CHUNK_SIZE; } // calculate the number of chunks based on the size they were compressed from int32 TotalChunkCount = (IORequest.UncompressedSize + CompressionChunkSize - 1) / CompressionChunkSize + 1; // allocate chunk info data based on number of chunks FCompressedChunkInfo* CompressionChunks = (FCompressedChunkInfo*)FMemory::Malloc(sizeof(FCompressedChunkInfo) * TotalChunkCount); int32 ChunkInfoSize = (TotalChunkCount) * sizeof(FCompressedChunkInfo); void* CompressedBuffer[2] = { 0, 0 }; // Read table of compression chunks after seeking to offset (after the initial header data) InternalRead( FileHandle, IORequest.Offset + HeaderSize, ChunkInfoSize, CompressionChunks ); RETURN_IF_EXIT_REQUESTED; // Handle byte swapping. This is required for opening a cooked file on the PC. int64 CalculatedUncompressedSize = 0; if (bIsByteswapped) { for( int32 ChunkIndex=0; ChunkIndex<TotalChunkCount; ChunkIndex++ ) { CompressionChunks[ChunkIndex].CompressedSize = BYTESWAP_ORDER64(CompressionChunks[ChunkIndex].CompressedSize); CompressionChunks[ChunkIndex].UncompressedSize = BYTESWAP_ORDER64(CompressionChunks[ChunkIndex].UncompressedSize); if (ChunkIndex > 0) { CalculatedUncompressedSize += CompressionChunks[ChunkIndex].UncompressedSize; } } } else { for( int32 ChunkIndex=1; ChunkIndex<TotalChunkCount; ChunkIndex++ ) { CalculatedUncompressedSize += CompressionChunks[ChunkIndex].UncompressedSize; } } if (CompressionChunks[0].UncompressedSize != CalculatedUncompressedSize) { UE_LOG(LogStreaming, Warning, TEXT("Detected data corruption [incorrect uncompressed size] calculated %i bytes, requested %i bytes at offset %i from '%s'. Please delete file and recook."), CalculatedUncompressedSize, IORequest.UncompressedSize, IORequest.Offset , *IORequest.FileName ); check(0); FPlatformMisc::HandleIOFailure(*IORequest.FileName); } if (ChunkInfoSize + HeaderSize + CompressionChunks[0].CompressedSize > IORequest.Size ) { UE_LOG(LogStreaming, Warning, TEXT("Detected data corruption [undershoot] trying to read %lld bytes at offset %lld from '%s'. Please delete file and recook."), IORequest.UncompressedSize, IORequest.Offset , *IORequest.FileName ); check(0); FPlatformMisc::HandleIOFailure(*IORequest.FileName); } if (IORequest.UncompressedSize != CalculatedUncompressedSize) { UE_LOG(LogStreaming, Warning, TEXT("Detected data corruption [incorrect uncompressed size] calculated %lld bytes, requested %lld bytes at offset %lld from '%s'. Please delete file and recook."), CalculatedUncompressedSize, IORequest.UncompressedSize, IORequest.Offset , *IORequest.FileName ); check(0); FPlatformMisc::HandleIOFailure(*IORequest.FileName); } // Figure out maximum size of compressed data chunk. int64 MaxCompressedSize = 0; for (int32 ChunkIndex = 1; ChunkIndex < TotalChunkCount; ChunkIndex++) { MaxCompressedSize = FMath::Max(MaxCompressedSize, CompressionChunks[ChunkIndex].CompressedSize); // Verify the all chunks are 'full size' until the last one... if (CompressionChunks[ChunkIndex].UncompressedSize < CompressionChunkSize) { if (ChunkIndex != (TotalChunkCount - 1)) { checkf(0, TEXT("Calculated too many chunks: %d should be last, there are %d from '%s'"), ChunkIndex, TotalChunkCount, *IORequest.FileName); } } check( CompressionChunks[ChunkIndex].UncompressedSize <= CompressionChunkSize ); } int32 Padding = 0; // Allocate memory for compressed data. CompressedBuffer[0] = FMemory::Malloc( MaxCompressedSize + Padding ); CompressedBuffer[1] = FMemory::Malloc( MaxCompressedSize + Padding ); // Initial read request. InternalRead( FileHandle, FileHandle->Tell(), CompressionChunks[CurrentChunkIndex].CompressedSize, CompressedBuffer[CurrentBufferIndex] ); RETURN_IF_EXIT_REQUESTED; // Loop till we're done decompressing all data. while( !bHasProcessedAllData ) { FAsyncTask<FAsyncUncompress> UncompressTask( IORequest.CompressionFlags, UncompressedBuffer, CompressionChunks[CurrentChunkIndex].UncompressedSize, CompressedBuffer[CurrentBufferIndex], CompressionChunks[CurrentChunkIndex].CompressedSize, (Padding > 0) ); #if BLOCK_ON_DECOMPRESSION UncompressTask.StartSynchronousTask(); #else UncompressTask.StartBackgroundTask(); #endif // Advance destination pointer. UncompressedBuffer += CompressionChunks[CurrentChunkIndex].UncompressedSize; // Check whether we are already done reading. if( CurrentChunkIndex < TotalChunkCount-1 ) { // Can't postincrement in if statement as we need it to remain at valid value for one more loop iteration to finish // the decompression. CurrentChunkIndex++; // Swap compression buffers to read into. CurrentBufferIndex = 1 - CurrentBufferIndex; // Read more data. InternalRead( FileHandle, FileHandle->Tell(), CompressionChunks[CurrentChunkIndex].CompressedSize, CompressedBuffer[CurrentBufferIndex] ); RETURN_IF_EXIT_REQUESTED; } // We were already done reading the last time around so we are done processing now. else { bHasProcessedAllData = true; } //@todo async loading: should use event for this STAT(double UncompressorWaitTime = 0); { SCOPE_SECONDS_COUNTER(UncompressorWaitTime); UncompressTask.EnsureCompletion(); // just decompress on this thread if it isn't started yet } INC_FLOAT_STAT_BY(STAT_AsyncIO_UncompressorWaitTime,(float)UncompressorWaitTime); } FMemory::Free(CompressionChunks); FMemory::Free(CompressedBuffer[0]); FMemory::Free(CompressedBuffer[1] ); }
void FNetworkPlatformFile::InitializeAfterSetActive() { double NetworkFileStartupTime = 0.0; { SCOPE_SECONDS_COUNTER(NetworkFileStartupTime); // send the filenames and timestamps to the server FNetworkFileArchive Payload(NFS_Messages::GetFileList); FillGetFileList(Payload, false); // send the directories over, and wait for a response FArrayReader Response; if (!SendPayloadAndReceiveResponse(Payload, Response)) { delete Transport; return; } else { // receive the cooked version information int32 ServerPackageVersion = 0; int32 ServerPackageLicenseeVersion = 0; ProcessServerInitialResponse(Response, ServerPackageVersion, ServerPackageLicenseeVersion); // receive a list of the cache files and their timestamps TMap<FString, FDateTime> ServerCachedFiles; Response << ServerCachedFiles; bool bDeleteAllFiles = true; // Check the stored cooked version FString CookedVersionFile = FPaths::GeneratedConfigDir() / TEXT("CookedVersion.txt"); if (InnerPlatformFile->FileExists(*CookedVersionFile) == true) { IFileHandle* FileHandle = InnerPlatformFile->OpenRead(*CookedVersionFile); if (FileHandle != NULL) { int32 StoredPackageCookedVersion; int32 StoredPackageCookedLicenseeVersion; if (FileHandle->Read((uint8*)&StoredPackageCookedVersion, sizeof(int32)) == true) { if (FileHandle->Read((uint8*)&StoredPackageCookedLicenseeVersion, sizeof(int32)) == true) { if ((ServerPackageVersion == StoredPackageCookedVersion) && (ServerPackageLicenseeVersion == StoredPackageCookedLicenseeVersion)) { bDeleteAllFiles = false; } else { UE_LOG(LogNetworkPlatformFile, Display, TEXT("Engine version mismatch: Server %d.%d, Stored %d.%d\n"), ServerPackageVersion, ServerPackageLicenseeVersion, StoredPackageCookedVersion, StoredPackageCookedLicenseeVersion); } } } delete FileHandle; } } else { UE_LOG(LogNetworkPlatformFile, Display, TEXT("Cooked version file missing: %s\n"), *CookedVersionFile); } if (bDeleteAllFiles == true) { // Make sure the config file exists... InnerPlatformFile->CreateDirectoryTree(*(FPaths::GeneratedConfigDir())); // Update the cooked version file IFileHandle* FileHandle = InnerPlatformFile->OpenWrite(*CookedVersionFile); if (FileHandle != NULL) { FileHandle->Write((const uint8*)&ServerPackageVersion, sizeof(int32)); FileHandle->Write((const uint8*)&ServerPackageLicenseeVersion, sizeof(int32)); delete FileHandle; } } // list of directories to skip TArray<FString> DirectoriesToSkip; TArray<FString> DirectoriesToNotRecurse; // use the timestamp grabbing visitor to get all the content times FLocalTimestampDirectoryVisitor Visitor(*InnerPlatformFile, DirectoriesToSkip, DirectoriesToNotRecurse, false); TArray<FString> RootContentPaths; FPackageName::QueryRootContentPaths( RootContentPaths ); for( TArray<FString>::TConstIterator RootPathIt( RootContentPaths ); RootPathIt; ++RootPathIt ) { const FString& RootPath = *RootPathIt; const FString& ContentFolder = FPackageName::LongPackageNameToFilename(RootPath); InnerPlatformFile->IterateDirectory( *ContentFolder, Visitor); } // delete out of date files using the server cached files for (TMap<FString, FDateTime>::TIterator It(ServerCachedFiles); It; ++It) { bool bDeleteFile = bDeleteAllFiles; FString ServerFile = It.Key(); // Convert the filename to the client version ConvertServerFilenameToClientFilename(ServerFile); // Set it in the visitor file times list Visitor.FileTimes.Add(ServerFile, FDateTime::MinValue()); if (bDeleteFile == false) { // Check the time stamps... // get local time FDateTime LocalTime = InnerPlatformFile->GetTimeStamp(*ServerFile); // If local time == MinValue than the file does not exist in the cache. if (LocalTime != FDateTime::MinValue()) { FDateTime ServerTime = It.Value(); // delete if out of date // We will use 1.0 second as the tolerance to cover any platform differences in resolution FTimespan TimeDiff = LocalTime - ServerTime; double TimeDiffInSeconds = TimeDiff.GetTotalSeconds(); bDeleteFile = (TimeDiffInSeconds > 1.0) || (TimeDiffInSeconds < -1.0); if (bDeleteFile == true) { if (InnerPlatformFile->FileExists(*ServerFile) == true) { UE_LOG(LogNetworkPlatformFile, Display, TEXT("Deleting cached file: TimeDiff %5.3f, %s"), TimeDiffInSeconds, *It.Key()); } else { // It's a directory bDeleteFile = false; } } } } if (bDeleteFile == true) { InnerPlatformFile->DeleteFile(*ServerFile); } } // Any content files we have locally that were not cached, delete them for (TMap<FString, FDateTime>::TIterator It(Visitor.FileTimes); It; ++It) { if (It.Value() != FDateTime::MinValue()) { // This was *not* found in the server file list... delete it UE_LOG(LogNetworkPlatformFile, Display, TEXT("Deleting cached file: %s"), *It.Key()); InnerPlatformFile->DeleteFile(*It.Key()); } } // make sure we can sync a file FString TestSyncFile = FPaths::Combine(*(FPaths::EngineDir()), TEXT("Config/BaseEngine.ini")); InnerPlatformFile->SetReadOnly(*TestSyncFile, false); InnerPlatformFile->DeleteFile(*TestSyncFile); if (InnerPlatformFile->FileExists(*TestSyncFile)) { UE_LOG(LogNetworkPlatformFile, Fatal, TEXT("Could not delete file sync test file %s."), *TestSyncFile); } EnsureFileIsLocal(TestSyncFile); if (!InnerPlatformFile->FileExists(*TestSyncFile) || InnerPlatformFile->FileSize(*TestSyncFile) < 1) { UE_LOG(LogNetworkPlatformFile, Fatal, TEXT("Could not sync test file %s."), *TestSyncFile); } } } FPlatformMisc::LowLevelOutputDebugStringf(TEXT("Network file startup time: %5.3f seconds\n"), NetworkFileStartupTime); }
// Compiles a blueprint. void FKismet2CompilerModule::CompileBlueprint(class UBlueprint* Blueprint, const FKismetCompilerOptions& CompileOptions, FCompilerResultsLog& Results, FBlueprintCompileReinstancer* ParentReinstancer, TArray<UObject*>* ObjLoaded) { SCOPE_SECONDS_COUNTER(GBlueprintCompileTime); BP_SCOPED_COMPILER_EVENT_STAT(EKismetCompilerStats_CompileTime); Results.SetSourceName(Blueprint->GetName()); const bool bIsBrandNewBP = (Blueprint->SkeletonGeneratedClass == NULL) && (Blueprint->GeneratedClass == NULL) && (Blueprint->ParentClass != NULL) && !CompileOptions.bIsDuplicationInstigated; for ( IBlueprintCompiler* Compiler : Compilers ) { Compiler->PreCompile(Blueprint); } if (CompileOptions.CompileType != EKismetCompileType::Cpp) { BP_SCOPED_COMPILER_EVENT_STAT(EKismetCompilerStats_CompileSkeletonClass); FBlueprintCompileReinstancer SkeletonReinstancer(Blueprint->SkeletonGeneratedClass); FCompilerResultsLog SkeletonResults; SkeletonResults.bSilentMode = true; FKismetCompilerOptions SkeletonCompileOptions; SkeletonCompileOptions.CompileType = EKismetCompileType::SkeletonOnly; CompileBlueprintInner(Blueprint, SkeletonCompileOptions, SkeletonResults, ObjLoaded); } // If this was a full compile, take appropriate actions depending on the success of failure of the compile if( CompileOptions.IsGeneratedClassCompileType() ) { BP_SCOPED_COMPILER_EVENT_STAT(EKismetCompilerStats_CompileGeneratedClass); // Perform the full compile CompileBlueprintInner(Blueprint, CompileOptions, Results, ObjLoaded); if (Results.NumErrors == 0) { // Blueprint is error free. Go ahead and fix up debug info Blueprint->Status = (0 == Results.NumWarnings) ? BS_UpToDate : BS_UpToDateWithWarnings; Blueprint->BlueprintSystemVersion = UBlueprint::GetCurrentBlueprintSystemVersion(); // Reapply breakpoints to the bytecode of the new class for (int32 Index = 0; Index < Blueprint->Breakpoints.Num(); ++Index) { UBreakpoint* Breakpoint = Blueprint->Breakpoints[Index]; FKismetDebugUtilities::ReapplyBreakpoint(Breakpoint); } } else { // Should never get errors from a brand new blueprint! ensure(!bIsBrandNewBP || (Results.NumErrors == 0)); // There were errors. Compile the generated class to have function stubs Blueprint->Status = BS_Error; static const FBoolConfigValueHelper ReinstanceOnlyWhenNecessary(TEXT("Kismet"), TEXT("bReinstanceOnlyWhenNecessary"), GEngineIni); // Reinstance objects here, so we can preserve their memory layouts to reinstance them again if( ParentReinstancer != NULL ) { ParentReinstancer->UpdateBytecodeReferences(); if(!Blueprint->bIsRegeneratingOnLoad) { ParentReinstancer->ReinstanceObjects(!ReinstanceOnlyWhenNecessary); } } FBlueprintCompileReinstancer StubReinstancer(Blueprint->GeneratedClass); // Toss the half-baked class and generate a stubbed out skeleton class that can be used FCompilerResultsLog StubResults; StubResults.bSilentMode = true; FKismetCompilerOptions StubCompileOptions(CompileOptions); StubCompileOptions.CompileType = EKismetCompileType::StubAfterFailure; CompileBlueprintInner(Blueprint, StubCompileOptions, StubResults, ObjLoaded); StubReinstancer.UpdateBytecodeReferences(); if( !Blueprint->bIsRegeneratingOnLoad ) { StubReinstancer.ReinstanceObjects(!ReinstanceOnlyWhenNecessary); } } } for ( IBlueprintCompiler* Compiler : Compilers ) { Compiler->PostCompile(Blueprint); } UPackage* Package = Blueprint->GetOutermost(); if( Package ) { UMetaData* MetaData = Package->GetMetaData(); MetaData->RemoveMetaDataOutsidePackage(); } }
/** * Serializes and compresses/ uncompresses data. This is a shared helper function for compression * support. The data is saved in a way compatible with FIOSystem::LoadCompressedData. * * @note: the way this code works needs to be in line with FIOSystem::LoadCompressedData implementations * @note: the way this code works needs to be in line with FAsyncIOSystemBase::FulfillCompressedRead * * @param V Data pointer to serialize data from/to, or a FileReader if bTreatBufferAsFileReader is true * @param Length Length of source data if we're saving, unused otherwise * @param Flags Flags to control what method to use for [de]compression and optionally control memory vs speed when compressing * @param bTreatBufferAsFileReader true if V is actually an FArchive, which is used when saving to read data - helps to avoid single huge allocations of source data */ void FArchive::SerializeCompressed( void* V, int64 Length, ECompressionFlags Flags, bool bTreatBufferAsFileReader ) { if( IsLoading() ) { // Serialize package file tag used to determine endianess. FCompressedChunkInfo PackageFileTag; PackageFileTag.CompressedSize = 0; PackageFileTag.UncompressedSize = 0; *this << PackageFileTag; bool bWasByteSwapped = PackageFileTag.CompressedSize != PACKAGE_FILE_TAG; // Read in base summary. FCompressedChunkInfo Summary; *this << Summary; if (bWasByteSwapped) { check( PackageFileTag.CompressedSize == PACKAGE_FILE_TAG_SWAPPED ); Summary.CompressedSize = BYTESWAP_ORDER64(Summary.CompressedSize); Summary.UncompressedSize = BYTESWAP_ORDER64(Summary.UncompressedSize); PackageFileTag.UncompressedSize = BYTESWAP_ORDER64(PackageFileTag.UncompressedSize); } else { check( PackageFileTag.CompressedSize == PACKAGE_FILE_TAG ); } // Handle change in compression chunk size in backward compatible way. int64 LoadingCompressionChunkSize = PackageFileTag.UncompressedSize; if (LoadingCompressionChunkSize == PACKAGE_FILE_TAG) { LoadingCompressionChunkSize = LOADING_COMPRESSION_CHUNK_SIZE; } // Figure out how many chunks there are going to be based on uncompressed size and compression chunk size. int64 TotalChunkCount = (Summary.UncompressedSize + LoadingCompressionChunkSize - 1) / LoadingCompressionChunkSize; // Allocate compression chunk infos and serialize them, keeping track of max size of compression chunks used. FCompressedChunkInfo* CompressionChunks = new FCompressedChunkInfo[TotalChunkCount]; int64 MaxCompressedSize = 0; for( int32 ChunkIndex=0; ChunkIndex<TotalChunkCount; ChunkIndex++ ) { *this << CompressionChunks[ChunkIndex]; if (bWasByteSwapped) { CompressionChunks[ChunkIndex].CompressedSize = BYTESWAP_ORDER64( CompressionChunks[ChunkIndex].CompressedSize ); CompressionChunks[ChunkIndex].UncompressedSize = BYTESWAP_ORDER64( CompressionChunks[ChunkIndex].UncompressedSize ); } MaxCompressedSize = FMath::Max( CompressionChunks[ChunkIndex].CompressedSize, MaxCompressedSize ); } int64 Padding = 0; // Set up destination pointer and allocate memory for compressed chunk[s] (one at a time). uint8* Dest = (uint8*) V; void* CompressedBuffer = FMemory::Malloc( MaxCompressedSize + Padding ); // Iterate over all chunks, serialize them into memory and decompress them directly into the destination pointer for( int64 ChunkIndex=0; ChunkIndex<TotalChunkCount; ChunkIndex++ ) { const FCompressedChunkInfo& Chunk = CompressionChunks[ChunkIndex]; // Read compressed data. Serialize( CompressedBuffer, Chunk.CompressedSize ); // Decompress into dest pointer directly. verify( FCompression::UncompressMemory( Flags, Dest, Chunk.UncompressedSize, CompressedBuffer, Chunk.CompressedSize, (Padding > 0) ? true : false ) ); // And advance it by read amount. Dest += Chunk.UncompressedSize; } // Free up allocated memory. FMemory::Free( CompressedBuffer ); delete [] CompressionChunks; } else if( IsSaving() ) { SCOPE_SECONDS_COUNTER(GArchiveSerializedCompressedSavingTime); check( Length > 0 ); // Serialize package file tag used to determine endianess in LoadCompressedData. FCompressedChunkInfo PackageFileTag; PackageFileTag.CompressedSize = PACKAGE_FILE_TAG; PackageFileTag.UncompressedSize = GSavingCompressionChunkSize; *this << PackageFileTag; // Figure out how many chunks there are going to be based on uncompressed size and compression chunk size. int64 TotalChunkCount = (Length + GSavingCompressionChunkSize - 1) / GSavingCompressionChunkSize + 1; // Keep track of current position so we can later seek back and overwrite stub compression chunk infos. int64 StartPosition = Tell(); // Allocate compression chunk infos and serialize them so we can later overwrite the data. FCompressedChunkInfo* CompressionChunks = new FCompressedChunkInfo[TotalChunkCount]; for( int64 ChunkIndex=0; ChunkIndex<TotalChunkCount; ChunkIndex++ ) { *this << CompressionChunks[ChunkIndex]; } // The uncompressd size is equal to the passed in length. CompressionChunks[0].UncompressedSize = Length; // Zero initialize compressed size so we can update it during chunk compression. CompressionChunks[0].CompressedSize = 0; #if WITH_MULTI_THREADED_COMPRESSION #define MAX_COMPRESSION_JOBS (16) // Don't scale more than 16x to avoid going overboard wrt temporary memory. FAsyncTask<FAsyncCompressionChunk> AsyncChunks[MAX_COMPRESSION_JOBS]; // used to keep track of which job is the next one we need to retire int32 AsyncChunkIndex[MAX_COMPRESSION_JOBS]={0}; static uint32 GNumUnusedThreads_SerializeCompressed = -1; if (GNumUnusedThreads_SerializeCompressed == (uint32)-1) { // one-time initialization GNumUnusedThreads_SerializeCompressed = 1; // if we should use all available cores then we want to compress with all if( FParse::Param(FCommandLine::Get(), TEXT("USEALLAVAILABLECORES")) == true ) { GNumUnusedThreads_SerializeCompressed = 0; } } // Maximum number of concurrent async tasks we're going to kick off. This is based on the number of processors // available in the system. int32 MaxConcurrentAsyncChunks = FMath::Clamp<int32>( FPlatformMisc::NumberOfCores() - GNumUnusedThreads_SerializeCompressed, 1, MAX_COMPRESSION_JOBS ); if (FParse::Param(FCommandLine::Get(), TEXT("MTCHILD"))) { // throttle this back when doing MT cooks MaxConcurrentAsyncChunks = FMath::Min<int32>( MaxConcurrentAsyncChunks,4 ); } // Number of chunks left to finalize. int64 NumChunksLeftToFinalize = (Length + GSavingCompressionChunkSize - 1) / GSavingCompressionChunkSize; // Number of chunks left to kick off int64 NumChunksLeftToKickOff = NumChunksLeftToFinalize; // Start at index 1 as first chunk info is summary. int64 CurrentChunkIndex = 1; // Start at index 1 as first chunk info is summary. int64 RetireChunkIndex = 1; // Number of bytes remaining to kick off compression for. int64 BytesRemainingToKickOff = Length; // Pointer to src data if buffer is memory pointer, NULL if it's a FArchive. uint8* SrcBuffer = bTreatBufferAsFileReader ? NULL : (uint8*)V; check(!bTreatBufferAsFileReader || ((FArchive*)V)->IsLoading()); check(NumChunksLeftToFinalize); // Loop while there is work left to do based on whether we have finalized all chunks yet. while( NumChunksLeftToFinalize ) { // If true we are waiting for async tasks to complete and should wait to complete some // if there are no async tasks finishing this iteration. bool bNeedToWaitForAsyncTask = false; // Try to kick off async tasks if there are chunks left to kick off. if( NumChunksLeftToKickOff ) { // Find free index based on looking at uncompressed size. We can't use the thread counter // for this as that might be a chunk ready for finalization. int32 FreeIndex = INDEX_NONE; for( int32 i=0; i<MaxConcurrentAsyncChunks; i++ ) { if( !AsyncChunkIndex[i] ) { FreeIndex = i; check(AsyncChunks[FreeIndex].IsIdle()); // this is not supposed to be in use break; } } // Kick off async compression task if we found a chunk for it. if( FreeIndex != INDEX_NONE ) { FAsyncCompressionChunk& NewChunk = AsyncChunks[FreeIndex].GetTask(); // 2 times the uncompressed size should be more than enough; the compressed data shouldn't be that much larger NewChunk.CompressedSize = 2 * GSavingCompressionChunkSize; // Allocate compressed buffer placeholder on first use. if( NewChunk.CompressedBuffer == NULL ) { NewChunk.CompressedBuffer = FMemory::Malloc( NewChunk.CompressedSize ); } // By default everything is chunked up into GSavingCompressionChunkSize chunks. NewChunk.UncompressedSize = FMath::Min( BytesRemainingToKickOff, (int64)GSavingCompressionChunkSize ); check(NewChunk.UncompressedSize>0); // Need to serialize source data if passed in pointer is an FArchive. if( bTreatBufferAsFileReader ) { // Allocate memory on first use. We allocate the maximum amount to allow reuse. if( !NewChunk.UncompressedBuffer ) { NewChunk.UncompressedBuffer = FMemory::Malloc(GSavingCompressionChunkSize); } ((FArchive*)V)->Serialize(NewChunk.UncompressedBuffer, NewChunk.UncompressedSize); } // Advance src pointer by amount to be compressed. else { NewChunk.UncompressedBuffer = SrcBuffer; SrcBuffer += NewChunk.UncompressedSize; } // Update status variables for tracking how much work is left, what to do next. BytesRemainingToKickOff -= NewChunk.UncompressedSize; AsyncChunkIndex[FreeIndex] = CurrentChunkIndex++; NewChunk.Flags = Flags; NumChunksLeftToKickOff--; AsyncChunks[FreeIndex].StartBackgroundTask(); } // No chunks were available to use, complete some else { bNeedToWaitForAsyncTask = true; } } // Index of oldest chunk, needed as we need to serialize in order. int32 OldestAsyncChunkIndex = INDEX_NONE; for( int32 i=0; i<MaxConcurrentAsyncChunks; i++ ) { check(AsyncChunkIndex[i] == 0 || AsyncChunkIndex[i] >= RetireChunkIndex); check(AsyncChunkIndex[i] < RetireChunkIndex + MaxConcurrentAsyncChunks); if (AsyncChunkIndex[i] == RetireChunkIndex) { OldestAsyncChunkIndex = i; } } check(OldestAsyncChunkIndex != INDEX_NONE); // the retire chunk better be outstanding bool ChunkReady; if (bNeedToWaitForAsyncTask) { // This guarantees that the async work has finished, doing it on this thread if it hasn't been started AsyncChunks[OldestAsyncChunkIndex].EnsureCompletion(); ChunkReady = true; } else { ChunkReady = AsyncChunks[OldestAsyncChunkIndex].IsDone(); } if (ChunkReady) { FAsyncCompressionChunk& DoneChunk = AsyncChunks[OldestAsyncChunkIndex].GetTask(); // Serialize the data via archive. Serialize( DoneChunk.CompressedBuffer, DoneChunk.CompressedSize ); // Update associated chunk. int64 CompressionChunkIndex = RetireChunkIndex++; check(CompressionChunkIndex<TotalChunkCount); CompressionChunks[CompressionChunkIndex].CompressedSize = DoneChunk.CompressedSize; CompressionChunks[CompressionChunkIndex].UncompressedSize = DoneChunk.UncompressedSize; // Keep track of total compressed size, stored in first chunk. CompressionChunks[0].CompressedSize += DoneChunk.CompressedSize; // Clean up chunk. Src and dst buffer are not touched as the contain allocations we keep till the end. AsyncChunkIndex[OldestAsyncChunkIndex] = 0; DoneChunk.CompressedSize = 0; DoneChunk.UncompressedSize = 0; // Finalized one :) NumChunksLeftToFinalize--; bNeedToWaitForAsyncTask = false; } } // Free intermediate buffer storage. for( int32 i=0; i<MaxConcurrentAsyncChunks; i++ ) { // Free temporary compressed buffer storage. FMemory::Free( AsyncChunks[i].GetTask().CompressedBuffer ); AsyncChunks[i].GetTask().CompressedBuffer = NULL; // Free temporary uncompressed buffer storage if data was serialized in. if( bTreatBufferAsFileReader ) { FMemory::Free( AsyncChunks[i].GetTask().UncompressedBuffer ); AsyncChunks[i].GetTask().UncompressedBuffer = NULL; } } #else // Set up source pointer amount of data to copy (in bytes) uint8* Src; // allocate memory to read into if (bTreatBufferAsFileReader) { Src = (uint8*)FMemory::Malloc(GSavingCompressionChunkSize); check(((FArchive*)V)->IsLoading()); } else { Src = (uint8*) V; } int64 BytesRemaining = Length; // Start at index 1 as first chunk info is summary. int32 CurrentChunkIndex = 1; // 2 times the uncompressed size should be more than enough; the compressed data shouldn't be that much larger int64 CompressedBufferSize = 2 * GSavingCompressionChunkSize; void* CompressedBuffer = FMemory::Malloc( CompressedBufferSize ); while( BytesRemaining > 0 ) { int64 BytesToCompress = FMath::Min( BytesRemaining, (int64)GSavingCompressionChunkSize ); int64 CompressedSize = CompressedBufferSize; // read in the next chunk from the reader if (bTreatBufferAsFileReader) { ((FArchive*)V)->Serialize(Src, BytesToCompress); } check(CompressedSize < INT_MAX); int32 CompressedSizeInt = (int32)CompressedSize; verify( FCompression::CompressMemory( Flags, CompressedBuffer, CompressedSizeInt, Src, BytesToCompress ) ); CompressedSize = CompressedSizeInt; // move to next chunk if not reading from file if (!bTreatBufferAsFileReader) { Src += BytesToCompress; } Serialize( CompressedBuffer, CompressedSize ); // Keep track of total compressed size, stored in first chunk. CompressionChunks[0].CompressedSize += CompressedSize; // Update current chunk. check(CurrentChunkIndex<TotalChunkCount); CompressionChunks[CurrentChunkIndex].CompressedSize = CompressedSize; CompressionChunks[CurrentChunkIndex].UncompressedSize = BytesToCompress; CurrentChunkIndex++; BytesRemaining -= GSavingCompressionChunkSize; } // free the buffer we read into if (bTreatBufferAsFileReader) { FMemory::Free(Src); } // Free allocated memory. FMemory::Free( CompressedBuffer ); #endif // Overrwrite chunk infos by seeking to the beginning, serializing the data and then // seeking back to the end. auto EndPosition = Tell(); // Seek to the beginning. Seek( StartPosition ); // Serialize chunk infos. for( int32 ChunkIndex=0; ChunkIndex<TotalChunkCount; ChunkIndex++ ) { *this << CompressionChunks[ChunkIndex]; } // Seek back to end. Seek( EndPosition ); // Free intermediate data. delete [] CompressionChunks; } }