/**
* Starts the async process of checking the cache and if the item is present, retrieving the cached results (version for internal use by rollups)
* @param CacheKey Key to identify the data
* @param Rollup Rollup pointer, if this is part of a rollup
* @param Handle The handle that can be used for PollAsynchronousCompletion, WaitAsynchronousCompletion and GetAsynchronousResults
**/
void GetAsynchronousForRollup(const TCHAR* CacheKey, uint32 Handle)
{
FScopeLock ScopeLock(&SynchronizationObject);
UE_LOG(LogDerivedDataCache, Verbose, TEXT("GetAsynchronous(handle) %s"), CacheKey);
FAsyncTask<FBuildAsyncWorker>* AsyncTask = new FAsyncTask<FBuildAsyncWorker>((FDerivedDataPluginInterface*)NULL, CacheKey, false);
check(!PendingTasks.Contains(Handle));
PendingTasks.Add(Handle,AsyncTask);
AddToAsyncCompletionCounter(1);
AsyncTask->StartBackgroundTask();
}
virtual bool PollAsynchronousCompletion(uint32 Handle) override
{
FAsyncTask<FBuildAsyncWorker>* AsyncTask = NULL;
{
FScopeLock ScopeLock(&SynchronizationObject);
AsyncTask = PendingTasks.FindRef(Handle);
}
check(AsyncTask);
return AsyncTask->IsDone();
}
virtual uint32 GetAsynchronous(const TCHAR* CacheKey, IDerivedDataRollup* Rollup = NULL) override
{
check(!Rollup); // this needs to be handled by someone else, if rollups are disabled, then it should be NULL
FScopeLock ScopeLock(&SynchronizationObject);
UE_LOG(LogDerivedDataCache, Verbose, TEXT("GetAsynchronous %s"), CacheKey);
uint32 Handle = NextHandle();
FAsyncTask<FBuildAsyncWorker>* AsyncTask = new FAsyncTask<FBuildAsyncWorker>((FDerivedDataPluginInterface*)NULL, CacheKey, false);
check(!PendingTasks.Contains(Handle));
PendingTasks.Add(Handle, AsyncTask);
AddToAsyncCompletionCounter(1);
AsyncTask->StartBackgroundTask();
return Handle;
}
virtual void WaitAsynchronousCompletion(uint32 Handle) override
{
STAT(double ThisTime = 0);
{
SCOPE_SECONDS_COUNTER(ThisTime);
FAsyncTask<FBuildAsyncWorker>* AsyncTask = NULL;
{
FScopeLock ScopeLock(&SynchronizationObject);
AsyncTask = PendingTasks.FindRef(Handle);
}
check(AsyncTask);
AsyncTask->EnsureCompletion();
}
INC_FLOAT_STAT_BY(STAT_DDC_ASyncWaitTime,(float)ThisTime);
}
virtual bool GetAsynchronousResults(uint32 Handle, TArray<uint8>& OutData) override
{
FAsyncTask<FBuildAsyncWorker>* AsyncTask = NULL;
{
FScopeLock ScopeLock(&SynchronizationObject);
PendingTasks.RemoveAndCopyValue(Handle,AsyncTask);
}
check(AsyncTask);
if (!AsyncTask->GetTask().bSuccess)
{
delete AsyncTask;
return false;
}
OutData = AsyncTask->GetTask().Data;
delete AsyncTask;
check(OutData.Num());
return true;
}
virtual uint32 GetAsynchronous(FDerivedDataPluginInterface* DataDeriver)
{
FScopeLock ScopeLock(&SynchronizationObject);
uint32 Handle = NextHandle();
FString CacheKey = FDerivedDataCache::BuildCacheKey(DataDeriver);
UE_LOG(LogDerivedDataCache, Verbose, TEXT("GetAsynchronous %s"), *CacheKey);
bool bSync = !DataDeriver->IsBuildThreadsafe();
FAsyncTask<FBuildAsyncWorker>* AsyncTask = new FAsyncTask<FBuildAsyncWorker>(DataDeriver, *CacheKey, bSync);
check(!PendingTasks.Contains(Handle));
PendingTasks.Add(Handle,AsyncTask);
AddToAsyncCompletionCounter(1);
if (!bSync)
{
AsyncTask->StartBackgroundTask();
}
else
{
AsyncTask->StartSynchronousTask();
}
// Must return a valid handle
check(Handle != 0);
return Handle;
}
void Serialize(int64 DesiredPosition, void* V, int64 Length)
{
const int32 CompressionBlockSize = PakEntry.CompressionBlockSize;
uint32 CompressionBlockIndex = DesiredPosition / CompressionBlockSize;
uint8* WorkingBuffers[2];
int64 DirectCopyStart = DesiredPosition % PakEntry.CompressionBlockSize;
FAsyncTask<FPakUncompressTask> UncompressTask;
FCompressionScratchBuffers& ScratchSpace = FCompressionScratchBuffers::Get();
bool bStartedUncompress = false;
int64 WorkingBufferRequiredSize = FCompression::CompressMemoryBound((ECompressionFlags)PakEntry.CompressionMethod,CompressionBlockSize);
WorkingBufferRequiredSize = EncryptionPolicy::AlignReadRequest(WorkingBufferRequiredSize);
ScratchSpace.EnsureBufferSpace(CompressionBlockSize, WorkingBufferRequiredSize*2);
WorkingBuffers[0] = ScratchSpace.ScratchBuffer;
WorkingBuffers[1] = ScratchSpace.ScratchBuffer + WorkingBufferRequiredSize;
while (Length > 0)
{
const FPakCompressedBlock& Block = PakEntry.CompressionBlocks[CompressionBlockIndex];
int64 Pos = CompressionBlockIndex * CompressionBlockSize;
int64 CompressedBlockSize = Block.CompressedEnd-Block.CompressedStart;
int64 UncompressedBlockSize = FMath::Min<int64>(PakEntry.UncompressedSize-Pos, PakEntry.CompressionBlockSize);
int64 ReadSize = EncryptionPolicy::AlignReadRequest(CompressedBlockSize);
int64 WriteSize = FMath::Min<int64>(UncompressedBlockSize - DirectCopyStart, Length);
PakReader->Seek(Block.CompressedStart);
PakReader->Serialize(WorkingBuffers[CompressionBlockIndex & 1],ReadSize);
if (bStartedUncompress)
{
UncompressTask.EnsureCompletion();
bStartedUncompress = false;
}
FPakUncompressTask& TaskDetails = UncompressTask.GetTask();
if (DirectCopyStart == 0 && Length >= CompressionBlockSize)
{
// Block can be decompressed directly into output buffer
TaskDetails.Flags = (ECompressionFlags)PakEntry.CompressionMethod;
TaskDetails.UncompressedBuffer = (uint8*)V;
TaskDetails.UncompressedSize = UncompressedBlockSize;
TaskDetails.CompressedBuffer = WorkingBuffers[CompressionBlockIndex & 1];
TaskDetails.CompressedSize = CompressedBlockSize;
TaskDetails.CopyOut = nullptr;
}
else
{
// Block needs to be copied from a working buffer
TaskDetails.Flags = (ECompressionFlags)PakEntry.CompressionMethod;
TaskDetails.UncompressedBuffer = (uint8*)ScratchSpace.TempBuffer;
TaskDetails.UncompressedSize = UncompressedBlockSize;
TaskDetails.CompressedBuffer = WorkingBuffers[CompressionBlockIndex & 1];
TaskDetails.CompressedSize = CompressedBlockSize;
TaskDetails.CopyOut = V;
TaskDetails.CopyOffset = DirectCopyStart;
TaskDetails.CopyLength = WriteSize;
}
if (Length == WriteSize)
{
UncompressTask.StartSynchronousTask();
}
else
{
UncompressTask.StartBackgroundTask();
}
bStartedUncompress = true;
V = (void*)((uint8*)V + WriteSize);
Length -= WriteSize;
DirectCopyStart = 0;
++CompressionBlockIndex;
}
if(bStartedUncompress)
{
UncompressTask.EnsureCompletion();
}
}
