virtual void SubmitAndFreeContextContainer(int32 Index, int32 Num) override
{
if (Index == 0)
{
check((IsInRenderingThread() || IsInRHIThread()));
OwningDevice->GetDefaultCommandContext().FlushCommands();
}
// Add the current lists for execution (now or possibly later)
for (int32 i = 0; i < CommandLists.Num(); ++i)
{
OwningDevice->PendingCommandLists.Add(CommandLists[i]);
OwningDevice->PendingCommandListsTotalWorkCommands +=
CommandLists[i].GetCurrentOwningContext()->numClears +
CommandLists[i].GetCurrentOwningContext()->numCopies +
CommandLists[i].GetCurrentOwningContext()->numDraws;
}
CommandLists.Reset();
// Submission occurs when a batch is finished
const bool FinalCommandListInBatch = Index == (Num - 1);
if (FinalCommandListInBatch && OwningDevice->PendingCommandLists.Num() > 0)
{
#if SUPPORTS_MEMORY_RESIDENCY
OwningDevice->GetOwningRHI()->GetResourceResidencyManager().MakeResident();
#endif
OwningDevice->GetCommandListManager().ExecuteCommandLists(OwningDevice->PendingCommandLists);
OwningDevice->PendingCommandLists.Reset();
OwningDevice->PendingCommandListsTotalWorkCommands = 0;
}
delete this;
}
void FD3D12CommandContext::RHIBeginFrame()
{
check(IsDefaultContext());
RHIPrivateBeginFrame();
FD3D12Device* Device = GetParentDevice();
FD3D12GlobalOnlineHeap& SamplerHeap = Device->GetGlobalSamplerHeap();
const uint32 NumContexts = Device->GetNumContexts();
if (SamplerHeap.DescriptorTablesDirty())
{
//Rearrange the set for better look-up performance
SamplerHeap.GetUniqueDescriptorTables().Compact();
}
for (uint32 i = 0; i < NumContexts; ++i)
{
Device->GetCommandContext(i).StateCache.GetDescriptorCache()->BeginFrame();
}
Device->GetGlobalSamplerHeap().ToggleDescriptorTablesDirtyFlag(false);
UniformBufferBeginFrame();
OwningRHI.GPUProfilingData.BeginFrame(&OwningRHI);
}
virtual void FinishContext() override
{
// We never "Finish" the default context. It gets submitted when FlushCommands() is called.
check(!CmdContext->IsDefaultContext());
CmdContext->Finish(CommandLists);
OwningDevice->ReleaseCommandContext(CmdContext);
CmdContext = nullptr;
}
FD3D12CommandListHandle FD3D12CommandContext::FlushCommands(bool WaitForCompletion)
{
// We should only be flushing the default context
check(IsDefaultContext());
FD3D12Device* Device = GetParentDevice();
const bool bExecutePendingWork = GCommandListBatchingMode == CLB_AggressiveBatching || GetParentDevice()->IsGPUIdle();
const bool bHasPendingWork = bExecutePendingWork && (Device->PendingCommandListsTotalWorkCommands > 0);
const bool bHasDoneWork = HasDoneWork() || bHasPendingWork;
// Only submit a command list if it does meaningful work or the flush is expected to wait for completion.
if (WaitForCompletion || bHasDoneWork)
{
#if SUPPORTS_MEMORY_RESIDENCY
Device->GetOwningRHI()->GetResourceResidencyManager().MakeResident();
#endif
// Close the current command list
CloseCommandList();
if (bHasPendingWork)
{
// Submit all pending command lists and the current command list
Device->PendingCommandLists.Add(CommandListHandle);
GetCommandListManager().ExecuteCommandLists(Device->PendingCommandLists, WaitForCompletion);
Device->PendingCommandLists.Reset();
Device->PendingCommandListsTotalWorkCommands = 0;
}
else
{
// Just submit the current command list
CommandListHandle.Execute(WaitForCompletion);
}
// Get a new command list to replace the one we submitted for execution.
// Restore the state from the previous command list.
OpenCommandList(true);
}
return CommandListHandle;
}
void FD3D12BuddyAllocator::Initialize()
{
FD3D12Device* Device = GetParentDevice();
FD3D12Adapter* Adapter = Device->GetParentAdapter();
if (AllocationStrategy == eBuddyAllocationStrategy::kPlacedResourceStrategy)
{
D3D12_HEAP_PROPERTIES HeapProps = CD3DX12_HEAP_PROPERTIES(HeapType);
HeapProps.CreationNodeMask = GetNodeMask();
HeapProps.VisibleNodeMask = GetVisibilityMask();
D3D12_HEAP_DESC Desc = {};
Desc.SizeInBytes = MaxBlockSize;
Desc.Properties = HeapProps;
Desc.Alignment = 0;
Desc.Flags = HeapFlags;
ID3D12Heap* Heap = nullptr;
VERIFYD3D12RESULT(Adapter->GetD3DDevice()->CreateHeap(&Desc, IID_PPV_ARGS(&Heap)));
SetName(Heap, L"Placed Resource Allocator Backing Heap");
BackingHeap = new FD3D12Heap(GetParentDevice(), GetVisibilityMask());
BackingHeap->SetHeap(Heap);
if (IsCPUWritable(HeapType) == false)
{
BackingHeap->BeginTrackingResidency(Desc.SizeInBytes);
}
}
else
{
VERIFYD3D12RESULT(Adapter->CreateBuffer(HeapType, GetNodeMask(), GetVisibilityMask(), MaxBlockSize, BackingResource.GetInitReference(), ResourceFlags));
SetName(BackingResource, L"Resource Allocator Underlying Buffer");
if (IsCPUWritable(HeapType))
{
BackingResource->Map();
}
}
}
inline FD3D12UnorderedAccessView* CreateUAV(D3D12_UNORDERED_ACCESS_VIEW_DESC& Desc, ResourceType* Resource, bool bNeedsCounterResource)
{
if (Resource == nullptr)
{
return nullptr;
}
FD3D12Adapter* Adapter = Resource->GetParentDevice()->GetParentAdapter();
return Adapter->CreateLinkedViews<ResourceType, FD3D12UnorderedAccessView>(Resource, [bNeedsCounterResource, &Desc](ResourceType* Resource)
{
FD3D12Device* Device = Resource->GetParentDevice();
FD3D12Resource* CounterResource = nullptr;
if (bNeedsCounterResource)
{
const GPUNodeMask Node = Device->GetNodeMask();
Device->GetParentAdapter()->CreateBuffer(D3D12_HEAP_TYPE_DEFAULT, Node, Node, 4, &CounterResource, D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS);
}
return new FD3D12UnorderedAccessView(Device, &Desc, &Resource->ResourceLocation, CounterResource);
});
}
virtual IRHICommandContext* GetContext() override
{
check(CmdContext == nullptr);
CmdContext = OwningDevice->ObtainCommandContext();
check(CmdContext != nullptr);
if (CmdContext->CommandListHandle == nullptr)
{
CmdContext->OpenCommandList();
}
CmdContext->ClearState();
return CmdContext;
}
void FD3D12CommandContext::RHIEndFrame()
{
check(IsDefaultContext());
OwningRHI.GPUProfilingData.EndFrame();
FD3D12Device* Device = GetParentDevice();
Device->FirstFrameSeen = true;
Device->GetDeferredDeletionQueue().ReleaseResources();
Device->GetDefaultBufferAllocator().CleanupFreeBlocks();
const uint32 NumContexts = Device->GetNumContexts();
for (uint32 i = 0; i < NumContexts; ++i)
{
Device->GetCommandContext(i).EndFrame();
}
Device->GetDefaultUploadHeapAllocator().CleanUpAllocations();
Device->GetTextureAllocator().CleanUpAllocations();
Device->GetDefaultBufferAllocator().CleanupFreeBlocks();
Device->GetDefaultFastAllocatorPool().CleanUpPages(10);
{
FScopeLock Lock(Device->GetBufferInitFastAllocator().GetCriticalSection());
Device->GetBufferInitFastAllocatorPool().CleanUpPages(10);
}
// The Texture streaming threads share a pool
{
FD3D12ThreadSafeFastAllocator* pCurrentHelperThreadDynamicHeapAllocator = nullptr;
for (uint32 i = 0; i < FD3D12DynamicRHI::GetD3DRHI()->NumThreadDynamicHeapAllocators; ++i)
{
pCurrentHelperThreadDynamicHeapAllocator = FD3D12DynamicRHI::GetD3DRHI()->ThreadDynamicHeapAllocatorArray[i];
if (pCurrentHelperThreadDynamicHeapAllocator)
{
FScopeLock Lock(pCurrentHelperThreadDynamicHeapAllocator->GetCriticalSection());
pCurrentHelperThreadDynamicHeapAllocator->GetPool()->CleanUpPages(10);
break;
}
}
}
#if SUPPORTS_MEMORY_RESIDENCY
GetResourceResidencyManager().Process();
#endif
DXGI_QUERY_VIDEO_MEMORY_INFO LocalVideoMemoryInfo;
FD3D12DynamicRHI::GetD3DRHI()->GetLocalVideoMemoryInfo(&LocalVideoMemoryInfo);
int64 budget = LocalVideoMemoryInfo.Budget;
int64 AvailableSpace = budget - int64(LocalVideoMemoryInfo.CurrentUsage);
SET_MEMORY_STAT(STAT_D3D12UsedVideoMemory, LocalVideoMemoryInfo.CurrentUsage);
SET_MEMORY_STAT(STAT_D3D12AvailableVideoMemory, AvailableSpace);
SET_MEMORY_STAT(STAT_D3D12TotalVideoMemory, budget);
}
void* FD3D12DynamicRHI::LockBuffer(FRHICommandListImmediate* RHICmdList, BufferType* Buffer, uint32 Offset, uint32 Size, EResourceLockMode LockMode)
{
#if STATS
LockBufferCalls++;
SCOPE_CYCLE_COUNTER(STAT_D3D12LockBufferTime);
INC_DWORD_STAT_BY(STAT_D3D12LockBufferCalls, LockBufferCalls);
#endif
FD3D12LockedResource& LockedData = Buffer->LockedData;
check(LockedData.bLocked == false);
FD3D12Device* Device = GetRHIDevice();
FD3D12Adapter& Adapter = GetAdapter();
// Determine whether the buffer is dynamic or not.
const bool bIsDynamic = (Buffer->GetUsage() & BUF_AnyDynamic) ? true : false;
void* Data = nullptr;
if (bIsDynamic)
{
check(LockMode == RLM_WriteOnly);
BufferType* CurrentBuffer = Buffer;
// Update all of the resources in the LDA chain
while (CurrentBuffer)
{
// Allocate a new resource
// If on the RenderThread, queue up a command on the RHIThread to rename this buffer at the correct time
if (ShouldDeferBufferLockOperation(RHICmdList))
{
FRHICommandRenameUploadBuffer<BufferType>* Command = new (RHICmdList->AllocCommand<FRHICommandRenameUploadBuffer<BufferType>>()) FRHICommandRenameUploadBuffer<BufferType>(CurrentBuffer, Device);
Data = Adapter.GetUploadHeapAllocator().AllocUploadResource(Buffer->GetSize(), Buffer->BufferAlignment, Command->NewResource);
}
else
{
FD3D12ResourceLocation Location(CurrentBuffer->GetParentDevice());
Data = Adapter.GetUploadHeapAllocator().AllocUploadResource(Buffer->GetSize(), Buffer->BufferAlignment, Location);
CurrentBuffer->Rename(Location);
}
CurrentBuffer = CurrentBuffer->GetNextObject();
}
}
else
{
FD3D12Resource* pResource = Buffer->ResourceLocation.GetResource();
// Locking for read must occur immediately so we can't queue up the operations later.
if (LockMode == RLM_ReadOnly)
{
LockedData.bLockedForReadOnly = true;
// If the static buffer is being locked for reading, create a staging buffer.
FD3D12Resource* StagingBuffer = nullptr;
const GPUNodeMask Node = Device->GetNodeMask();
VERIFYD3D12RESULT(Adapter.CreateBuffer(D3D12_HEAP_TYPE_READBACK, Node, Node, Offset + Size, &StagingBuffer));
// Copy the contents of the buffer to the staging buffer.
{
const auto& pfnCopyContents = [&]()
{
FD3D12CommandContext& DefaultContext = Device->GetDefaultCommandContext();
FD3D12CommandListHandle& hCommandList = DefaultContext.CommandListHandle;
FScopeResourceBarrier ScopeResourceBarrierSource(hCommandList, pResource, pResource->GetDefaultResourceState(), D3D12_RESOURCE_STATE_COPY_SOURCE, 0);
// Don't need to transition upload heaps
DefaultContext.numCopies++;
hCommandList->CopyBufferRegion(
StagingBuffer->GetResource(),
0,
pResource->GetResource(),
Offset, Size);
hCommandList.UpdateResidency(StagingBuffer);
hCommandList.UpdateResidency(pResource);
DefaultContext.FlushCommands(true);
};
if (ShouldDeferBufferLockOperation(RHICmdList))
{
// Sync when in the render thread implementation
check(IsInRHIThread() == false);
RHICmdList->ImmediateFlush(EImmediateFlushType::FlushRHIThread);
pfnCopyContents();
}
else
{
check(IsInRHIThread());
pfnCopyContents();
}
}
LockedData.ResourceLocation.AsStandAlone(StagingBuffer, Size);
Data = LockedData.ResourceLocation.GetMappedBaseAddress();
}
else
{
// If the static buffer is being locked for writing, allocate memory for the contents to be written to.
Data = Device->GetDefaultFastAllocator().Allocate<FD3D12ScopeLock>(Size, D3D12_TEXTURE_DATA_PLACEMENT_ALIGNMENT, &LockedData.ResourceLocation);
}
}
LockedData.LockedOffset = Offset;
LockedData.LockedPitch = Size;
LockedData.bLocked = true;
// Return the offset pointer
check(Data != nullptr);
return Data;
}
BufferType* FD3D12Adapter::CreateRHIBuffer(FRHICommandListImmediate* RHICmdList,
const D3D12_RESOURCE_DESC& InDesc,
uint32 Alignment,
uint32 Stride,
uint32 Size,
uint32 InUsage,
FRHIResourceCreateInfo& CreateInfo,
bool SkipCreate)
{
const bool bIsDynamic = (InUsage & BUF_AnyDynamic) ? true : false;
BufferType* BufferOut = CreateLinkedObject<BufferType>([&](FD3D12Device* Device)
{
BufferType* NewBuffer = new BufferType(Device, Stride, Size, InUsage);
NewBuffer->BufferAlignment = Alignment;
if (SkipCreate == false)
{
AllocateBuffer(Device, InDesc, Size, InUsage, CreateInfo, Alignment, NewBuffer->ResourceLocation);
}
return NewBuffer;
});
if (CreateInfo.ResourceArray)
{
if (bIsDynamic == false && BufferOut->ResourceLocation.IsValid())
{
check(Size == CreateInfo.ResourceArray->GetResourceDataSize());
// Get an upload heap and initialize data
FD3D12ResourceLocation SrcResourceLoc(BufferOut->GetParentDevice());
void* pData = SrcResourceLoc.GetParentDevice()->GetDefaultFastAllocator().Allocate<FD3D12ScopeLock>(Size, 4UL, &SrcResourceLoc);
check(pData);
FMemory::Memcpy(pData, CreateInfo.ResourceArray->GetResourceData(), Size);
const auto& pfnUpdateBuffer = [&]()
{
BufferType* CurrentBuffer = BufferOut;
while (CurrentBuffer != nullptr)
{
FD3D12Resource* Destination = CurrentBuffer->ResourceLocation.GetResource();
FD3D12Device* Device = Destination->GetParentDevice();
FD3D12CommandListHandle& hCommandList = Device->GetDefaultCommandContext().CommandListHandle;
// Copy from the temporary upload heap to the default resource
{
// Writable structured bufferes are sometimes initialized with inital data which means they sometimes need tracking.
FConditionalScopeResourceBarrier ConditionalScopeResourceBarrier(hCommandList, Destination, D3D12_RESOURCE_STATE_COPY_DEST, 0);
Device->GetDefaultCommandContext().numCopies++;
hCommandList->CopyBufferRegion(
Destination->GetResource(),
CurrentBuffer->ResourceLocation.GetOffsetFromBaseOfResource(),
SrcResourceLoc.GetResource()->GetResource(),
SrcResourceLoc.GetOffsetFromBaseOfResource(), Size);
hCommandList.UpdateResidency(Destination);
}
CurrentBuffer = CurrentBuffer->GetNextObject();
}
};
//TODO: This should be a deferred op like the buffer lock/unlocks
// We only need to synchronize when creating default resource buffers (because we need a command list to initialize them)
if (RHICmdList)
{
FScopedRHIThreadStaller StallRHIThread(*RHICmdList);
pfnUpdateBuffer();
}
else
{
pfnUpdateBuffer();
}
}
// Discard the resource array's contents.
CreateInfo.ResourceArray->Discard();
}
return BufferOut;
}