bool WrappedVulkan::Serialise_vkGetDeviceQueue(Serialiser *localSerialiser, VkDevice device,
uint32_t queueFamilyIndex, uint32_t queueIndex,
VkQueue *pQueue)
{
SERIALISE_ELEMENT(ResourceId, devId, GetResID(device));
SERIALISE_ELEMENT(uint32_t, familyIdx, m_SupportedQueueFamily);
SERIALISE_ELEMENT(uint32_t, idx, queueIndex);
SERIALISE_ELEMENT(ResourceId, queueId, GetResID(*pQueue));
if(m_State == READING)
{
device = GetResourceManager()->GetLiveHandle<VkDevice>(devId);
VkQueue queue;
ObjDisp(device)->GetDeviceQueue(Unwrap(device), familyIdx, idx, &queue);
GetResourceManager()->WrapResource(Unwrap(device), queue);
GetResourceManager()->AddLiveResource(queueId, queue);
if(familyIdx == m_QueueFamilyIdx)
{
m_Queue = queue;
// we can now submit any cmds that were queued (e.g. from creating debug
// manager on vkCreateDevice)
SubmitCmds();
}
}
return true;
}
void WrappedVulkan::vkDestroyDevice(VkDevice device, const VkAllocationCallbacks* pAllocator)
{
// flush out any pending commands/semaphores
SubmitCmds();
SubmitSemaphores();
FlushQ();
// MULTIDEVICE this function will need to check if the device is the one we
// used for debugmanager/cmd pool etc, and only remove child queues and
// resources (instead of doing full resource manager shutdown).
// Or will we have a debug manager per-device?
RDCASSERT(m_Device == device);
// delete all debug manager objects
SAFE_DELETE(m_DebugManager);
// since we didn't create proper registered resources for our command buffers,
// they won't be taken down properly with the pool. So we release them (just our
// data) here.
for(size_t i=0; i < m_InternalCmds.freecmds.size(); i++)
GetResourceManager()->ReleaseWrappedResource(m_InternalCmds.freecmds[i]);
// destroy our command pool
if(m_InternalCmds.cmdpool != VK_NULL_HANDLE)
{
ObjDisp(m_Device)->DestroyCommandPool(Unwrap(m_Device), Unwrap(m_InternalCmds.cmdpool), NULL);
GetResourceManager()->ReleaseWrappedResource(m_InternalCmds.cmdpool);
}
for(size_t i=0; i < m_InternalCmds.freesems.size(); i++)
{
ObjDisp(m_Device)->DestroySemaphore(Unwrap(m_Device), Unwrap(m_InternalCmds.freesems[i]), NULL);
GetResourceManager()->ReleaseWrappedResource(m_InternalCmds.freesems[i]);
}
m_InternalCmds.Reset();
m_QueueFamilyIdx = ~0U;
m_Queue = VK_NULL_HANDLE;
// destroy the API device immediately. There should be no more
// resources left in the resource manager device/physical device/instance.
// Anything we created should be gone and anything the application created
// should be deleted by now.
// If there were any leaks, we will leak them ourselves in vkDestroyInstance
// rather than try to delete API objects after the device has gone
ObjDisp(m_Device)->DestroyDevice(Unwrap(m_Device), pAllocator);
GetResourceManager()->ReleaseWrappedResource(m_Device);
m_Device = VK_NULL_HANDLE;
m_PhysicalDevice = VK_NULL_HANDLE;
}
void WrappedVulkan::Shutdown()
{
// flush out any pending commands
SubmitCmds();
FlushQ();
// since we didn't create proper registered resources for our command buffers,
// they won't be taken down properly with the pool. So we release them (just our
// data) here.
for(size_t i=0; i < m_InternalCmds.freecmds.size(); i++)
GetResourceManager()->ReleaseWrappedResource(m_InternalCmds.freecmds[i]);
// destroy the pool
ObjDisp(m_Device)->DestroyCommandPool(Unwrap(m_Device), Unwrap(m_InternalCmds.cmdpool), NULL);
GetResourceManager()->ReleaseWrappedResource(m_InternalCmds.cmdpool);
// we do more in Shutdown than the equivalent vkDestroyInstance since on replay there's
// no explicit vkDestroyDevice, we destroy the device here then the instance
// destroy any replay objects that aren't specifically to do with the frame capture
for(size_t i=0; i < m_CleanupMems.size(); i++)
{
ObjDisp(m_Device)->FreeMemory(Unwrap(m_Device), Unwrap(m_CleanupMems[i]), NULL);
GetResourceManager()->ReleaseWrappedResource(m_CleanupMems[i]);
}
m_CleanupMems.clear();
// destroy debug manager and any objects it created
SAFE_DELETE(m_DebugManager);
if(ObjDisp(m_Instance)->DestroyDebugReportCallbackEXT && m_DbgMsgCallback != VK_NULL_HANDLE)
ObjDisp(m_Instance)->DestroyDebugReportCallbackEXT(Unwrap(m_Instance), m_DbgMsgCallback, NULL);
// need to store the unwrapped device and instance to destroy the
// API object after resource manager shutdown
VkInstance inst = Unwrap(m_Instance);
VkDevice dev = Unwrap(m_Device);
const VkLayerDispatchTable *vt = ObjDisp(m_Device);
const VkLayerInstanceDispatchTable *vit = ObjDisp(m_Instance);
// this destroys the wrapped objects for the devices and instances
m_ResourceManager->Shutdown();
delete GetWrapped(m_Device);
delete GetWrapped(m_Instance);
m_PhysicalDevice = VK_NULL_HANDLE;
m_Device = VK_NULL_HANDLE;
m_Instance = VK_NULL_HANDLE;
m_PhysicalDevices.clear();
for(size_t i=0; i < m_QueueFamilies.size(); i++)
delete[] m_QueueFamilies[i];
m_QueueFamilies.clear();
// finally destroy device then instance
vt->DestroyDevice(dev, NULL);
vit->DestroyInstance(inst, NULL);
}
void WrappedVulkan::vkGetDeviceQueue(VkDevice device, uint32_t queueFamilyIndex,
uint32_t queueIndex, VkQueue *pQueue)
{
ObjDisp(device)->GetDeviceQueue(Unwrap(device), queueFamilyIndex, queueIndex, pQueue);
if(m_SetDeviceLoaderData)
m_SetDeviceLoaderData(m_Device, *pQueue);
else
SetDispatchTableOverMagicNumber(device, *pQueue);
RDCASSERT(m_State >= WRITING);
{
// it's perfectly valid for enumerate type functions to return the same handle
// each time. If that happens, we will already have a wrapper created so just
// return the wrapped object to the user and do nothing else
if(m_QueueFamilies[queueFamilyIndex][queueIndex] != VK_NULL_HANDLE)
{
*pQueue = m_QueueFamilies[queueFamilyIndex][queueIndex];
}
else
{
ResourceId id = GetResourceManager()->WrapResource(Unwrap(device), *pQueue);
{
Chunk *chunk = NULL;
{
CACHE_THREAD_SERIALISER();
SCOPED_SERIALISE_CONTEXT(GET_DEVICE_QUEUE);
Serialise_vkGetDeviceQueue(localSerialiser, device, queueFamilyIndex, queueIndex, pQueue);
chunk = scope.Get();
}
VkResourceRecord *record = GetResourceManager()->AddResourceRecord(*pQueue);
RDCASSERT(record);
VkResourceRecord *instrecord = GetRecord(m_Instance);
// treat queues as pool members of the instance (ie. freed when the instance dies)
{
instrecord->LockChunks();
instrecord->pooledChildren.push_back(record);
instrecord->UnlockChunks();
}
record->AddChunk(chunk);
}
m_QueueFamilies[queueFamilyIndex][queueIndex] = *pQueue;
if(queueFamilyIndex == m_QueueFamilyIdx)
{
m_Queue = *pQueue;
// we can now submit any cmds that were queued (e.g. from creating debug
// manager on vkCreateDevice)
SubmitCmds();
}
}
}
}
VkResult WrappedVulkan::vkQueuePresentKHR(
VkQueue queue,
const VkPresentInfoKHR* pPresentInfo)
{
if(m_State == WRITING_IDLE)
{
RenderDoc::Inst().Tick();
GetResourceManager()->FlushPendingDirty();
}
m_FrameCounter++; // first present becomes frame #1, this function is at the end of the frame
if(pPresentInfo->swapchainCount > 1 && (m_FrameCounter % 100) == 0)
{
RDCWARN("Presenting multiple swapchains at once - only first will be processed");
}
vector<VkSwapchainKHR> unwrappedSwaps;
vector<VkSemaphore> unwrappedSems;
VkPresentInfoKHR unwrappedInfo = *pPresentInfo;
for(uint32_t i=0; i < unwrappedInfo.swapchainCount; i++)
unwrappedSwaps.push_back(Unwrap(unwrappedInfo.pSwapchains[i]));
for(uint32_t i=0; i < unwrappedInfo.waitSemaphoreCount; i++)
unwrappedSems.push_back(Unwrap(unwrappedInfo.pWaitSemaphores[i]));
unwrappedInfo.pSwapchains = unwrappedInfo.swapchainCount ? &unwrappedSwaps[0] : NULL;
unwrappedInfo.pWaitSemaphores = unwrappedInfo.waitSemaphoreCount ? &unwrappedSems[0] : NULL;
// Don't support any extensions for present info
RDCASSERT(pPresentInfo->pNext == NULL);
VkResourceRecord *swaprecord = GetRecord(pPresentInfo->pSwapchains[0]);
RDCASSERT(swaprecord->swapInfo);
SwapchainInfo &swapInfo = *swaprecord->swapInfo;
bool activeWindow = RenderDoc::Inst().IsActiveWindow(LayerDisp(m_Instance), swapInfo.wndHandle);
// need to record which image was last flipped so we can get the correct backbuffer
// for a thumbnail in EndFrameCapture
swapInfo.lastPresent = pPresentInfo->pImageIndices[0];
m_LastSwap = swaprecord->GetResourceID();
VkImage backbuffer = swapInfo.images[pPresentInfo->pImageIndices[0]].im;
if(m_State == WRITING_IDLE)
{
m_FrameTimes.push_back(m_FrameTimer.GetMilliseconds());
m_TotalTime += m_FrameTimes.back();
m_FrameTimer.Restart();
// update every second
if(m_TotalTime > 1000.0)
{
m_MinFrametime = 10000.0;
m_MaxFrametime = 0.0;
m_AvgFrametime = 0.0;
m_TotalTime = 0.0;
for(size_t i=0; i < m_FrameTimes.size(); i++)
{
m_AvgFrametime += m_FrameTimes[i];
if(m_FrameTimes[i] < m_MinFrametime)
m_MinFrametime = m_FrameTimes[i];
if(m_FrameTimes[i] > m_MaxFrametime)
m_MaxFrametime = m_FrameTimes[i];
}
m_AvgFrametime /= double(m_FrameTimes.size());
m_FrameTimes.clear();
}
uint32_t overlay = RenderDoc::Inst().GetOverlayBits();
if(overlay & eRENDERDOC_Overlay_Enabled)
{
VkRenderPass rp = swapInfo.rp;
VkImage im = swapInfo.images[pPresentInfo->pImageIndices[0]].im;
VkFramebuffer fb = swapInfo.images[pPresentInfo->pImageIndices[0]].fb;
VkLayerDispatchTable *vt = ObjDisp(GetDev());
TextPrintState textstate = { GetNextCmd(), rp, fb, swapInfo.extent.width, swapInfo.extent.height };
VkCommandBufferBeginInfo beginInfo = { VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, NULL, VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT };
VkResult vkr = vt->BeginCommandBuffer(Unwrap(textstate.cmd), &beginInfo);
RDCASSERTEQUAL(vkr, VK_SUCCESS);
VkImageMemoryBarrier bbBarrier = {
VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, NULL,
0, 0, VK_IMAGE_LAYOUT_PRESENT_SRC_KHR, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL,
VK_QUEUE_FAMILY_IGNORED, VK_QUEUE_FAMILY_IGNORED,
Unwrap(im),
{ VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1 }
};
bbBarrier.srcAccessMask = VK_ACCESS_ALL_READ_BITS;
bbBarrier.dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT;
DoPipelineBarrier(textstate.cmd, 1, &bbBarrier);
GetDebugManager()->BeginText(textstate);
if(activeWindow)
{
vector<RENDERDOC_InputButton> keys = RenderDoc::Inst().GetCaptureKeys();
string overlayText = "Vulkan. ";
for(size_t i=0; i < keys.size(); i++)
{
if(i > 0)
overlayText += ", ";
overlayText += ToStr::Get(keys[i]);
}
if(!keys.empty())
overlayText += " to capture.";
if(overlay & eRENDERDOC_Overlay_FrameNumber)
{
overlayText += StringFormat::Fmt(" Frame: %d.", m_FrameCounter);
}
if(overlay & eRENDERDOC_Overlay_FrameRate)
{
overlayText += StringFormat::Fmt(" %.2lf ms (%.2lf .. %.2lf) (%.0lf FPS)",
m_AvgFrametime, m_MinFrametime, m_MaxFrametime, 1000.0f/m_AvgFrametime);
}
float y=0.0f;
if(!overlayText.empty())
{
GetDebugManager()->RenderText(textstate, 0.0f, y, overlayText.c_str());
y += 1.0f;
}
if(overlay & eRENDERDOC_Overlay_CaptureList)
{
GetDebugManager()->RenderText(textstate, 0.0f, y, "%d Captures saved.\n", (uint32_t)m_FrameRecord.size());
y += 1.0f;
uint64_t now = Timing::GetUnixTimestamp();
for(size_t i=0; i < m_FrameRecord.size(); i++)
{
if(now - m_FrameRecord[i].frameInfo.captureTime < 20)
{
GetDebugManager()->RenderText(textstate, 0.0f, y, "Captured frame %d.\n", m_FrameRecord[i].frameInfo.frameNumber);
y += 1.0f;
}
}
}
#if !defined(RELEASE)
GetDebugManager()->RenderText(textstate, 0.0f, y, "%llu chunks - %.2f MB", Chunk::NumLiveChunks(), float(Chunk::TotalMem())/1024.0f/1024.0f);
y += 1.0f;
#endif
}
else
{
vector<RENDERDOC_InputButton> keys = RenderDoc::Inst().GetFocusKeys();
string str = "Vulkan. Inactive swapchain.";
for(size_t i=0; i < keys.size(); i++)
{
if(i == 0)
str += " ";
else
str += ", ";
str += ToStr::Get(keys[i]);
}
if(!keys.empty())
str += " to cycle between swapchains";
GetDebugManager()->RenderText(textstate, 0.0f, 0.0f, str.c_str());
}
GetDebugManager()->EndText(textstate);
std::swap(bbBarrier.oldLayout, bbBarrier.newLayout);
bbBarrier.srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT;
bbBarrier.dstAccessMask = VK_ACCESS_ALL_READ_BITS;
DoPipelineBarrier(textstate.cmd, 1, &bbBarrier);
ObjDisp(textstate.cmd)->EndCommandBuffer(Unwrap(textstate.cmd));
SubmitCmds();
FlushQ();
}
}
VkResult vkr = ObjDisp(queue)->QueuePresentKHR(Unwrap(queue), &unwrappedInfo);
if(!activeWindow)
return vkr;
RenderDoc::Inst().SetCurrentDriver(RDC_Vulkan);
// kill any current capture that isn't application defined
if(m_State == WRITING_CAPFRAME && !m_AppControlledCapture)
RenderDoc::Inst().EndFrameCapture(LayerDisp(m_Instance), swapInfo.wndHandle);
if(RenderDoc::Inst().ShouldTriggerCapture(m_FrameCounter) && m_State == WRITING_IDLE)
{
RenderDoc::Inst().StartFrameCapture(LayerDisp(m_Instance), swapInfo.wndHandle);
m_AppControlledCapture = false;
}
return vkr;
}
VkResult WrappedVulkan::vkQueuePresentKHR(VkQueue queue, const VkPresentInfoKHR *pPresentInfo)
{
if(m_State == WRITING_IDLE)
{
RenderDoc::Inst().Tick();
GetResourceManager()->FlushPendingDirty();
}
m_FrameCounter++; // first present becomes frame #1, this function is at the end of the frame
if(pPresentInfo->swapchainCount > 1 && (m_FrameCounter % 100) == 0)
{
RDCWARN("Presenting multiple swapchains at once - only first will be processed");
}
vector<VkSwapchainKHR> unwrappedSwaps;
vector<VkSemaphore> unwrappedSems;
VkPresentInfoKHR unwrappedInfo = *pPresentInfo;
for(uint32_t i = 0; i < unwrappedInfo.swapchainCount; i++)
unwrappedSwaps.push_back(Unwrap(unwrappedInfo.pSwapchains[i]));
for(uint32_t i = 0; i < unwrappedInfo.waitSemaphoreCount; i++)
unwrappedSems.push_back(Unwrap(unwrappedInfo.pWaitSemaphores[i]));
unwrappedInfo.pSwapchains = unwrappedInfo.swapchainCount ? &unwrappedSwaps[0] : NULL;
unwrappedInfo.pWaitSemaphores = unwrappedInfo.waitSemaphoreCount ? &unwrappedSems[0] : NULL;
// Don't support any extensions for present info
RDCASSERT(pPresentInfo->pNext == NULL);
VkResourceRecord *swaprecord = GetRecord(pPresentInfo->pSwapchains[0]);
RDCASSERT(swaprecord->swapInfo);
SwapchainInfo &swapInfo = *swaprecord->swapInfo;
bool activeWindow = RenderDoc::Inst().IsActiveWindow(LayerDisp(m_Instance), swapInfo.wndHandle);
// need to record which image was last flipped so we can get the correct backbuffer
// for a thumbnail in EndFrameCapture
swapInfo.lastPresent = pPresentInfo->pImageIndices[0];
m_LastSwap = swaprecord->GetResourceID();
if(m_State == WRITING_IDLE)
{
uint32_t overlay = RenderDoc::Inst().GetOverlayBits();
if(overlay & eRENDERDOC_Overlay_Enabled)
{
VkRenderPass rp = swapInfo.rp;
VkImage im = swapInfo.images[pPresentInfo->pImageIndices[0]].im;
VkFramebuffer fb = swapInfo.images[pPresentInfo->pImageIndices[0]].fb;
VkLayerDispatchTable *vt = ObjDisp(GetDev());
TextPrintState textstate = {
GetNextCmd(), rp, fb, swapInfo.extent.width, swapInfo.extent.height, swapInfo.format};
VkCommandBufferBeginInfo beginInfo = {VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, NULL,
VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT};
VkResult vkr = vt->BeginCommandBuffer(Unwrap(textstate.cmd), &beginInfo);
RDCASSERTEQUAL(vkr, VK_SUCCESS);
VkImageMemoryBarrier bbBarrier = {VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
NULL,
0,
0,
VK_IMAGE_LAYOUT_PRESENT_SRC_KHR,
VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL,
VK_QUEUE_FAMILY_IGNORED,
VK_QUEUE_FAMILY_IGNORED,
Unwrap(im),
{VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}};
bbBarrier.srcAccessMask = VK_ACCESS_ALL_READ_BITS;
bbBarrier.dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT;
DoPipelineBarrier(textstate.cmd, 1, &bbBarrier);
GetDebugManager()->BeginText(textstate);
int flags = activeWindow ? RenderDoc::eOverlay_ActiveWindow : 0;
string overlayText = RenderDoc::Inst().GetOverlayText(RDC_Vulkan, m_FrameCounter, flags);
if(!overlayText.empty())
GetDebugManager()->RenderText(textstate, 0.0f, 0.0f, overlayText.c_str());
GetDebugManager()->EndText(textstate);
std::swap(bbBarrier.oldLayout, bbBarrier.newLayout);
bbBarrier.srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT;
bbBarrier.dstAccessMask = VK_ACCESS_ALL_READ_BITS;
DoPipelineBarrier(textstate.cmd, 1, &bbBarrier);
ObjDisp(textstate.cmd)->EndCommandBuffer(Unwrap(textstate.cmd));
SubmitCmds();
FlushQ();
}
}
VkResult vkr = ObjDisp(queue)->QueuePresentKHR(Unwrap(queue), &unwrappedInfo);
if(!activeWindow)
return vkr;
RenderDoc::Inst().SetCurrentDriver(RDC_Vulkan);
// kill any current capture that isn't application defined
if(m_State == WRITING_CAPFRAME && !m_AppControlledCapture)
RenderDoc::Inst().EndFrameCapture(LayerDisp(m_Instance), swapInfo.wndHandle);
if(RenderDoc::Inst().ShouldTriggerCapture(m_FrameCounter) && m_State == WRITING_IDLE)
{
RenderDoc::Inst().StartFrameCapture(LayerDisp(m_Instance), swapInfo.wndHandle);
m_AppControlledCapture = false;
}
return vkr;
}
