Fx mesh viewer being opened repeatedly causing crashes on D3D12

baldurk · Sep 29, 2023 · 2886f4a · 2886f4a
1 parent c46a9d0
commit 2886f4a
Show file tree

Hide file tree

Showing 5 changed files with 33 additions and 14 deletions.
diff --git a/renderdoc/driver/d3d12/d3d12_debug.cpp b/renderdoc/driver/d3d12/d3d12_debug.cpp
@@ -137,13 +137,15 @@ D3D12DebugManager::D3D12DebugManager(WrappedID3D12Device *wrapper)
 
   HRESULT hr = S_OK;
 
+  const uint32_t rtvCount = 1024;
+
   D3D12_DESCRIPTOR_HEAP_DESC desc;
   desc.Flags = D3D12_DESCRIPTOR_HEAP_FLAG_NONE;
   desc.NodeMask = 1;
-  desc.NumDescriptors = 1024;
+  desc.NumDescriptors = rtvCount;
   desc.Type = D3D12_DESCRIPTOR_HEAP_TYPE_RTV;
 
-  RDCCOMPILE_ASSERT(FIRST_WIN_RTV + 256 < 1024, "Increase size of RTV heap");
+  RDCCOMPILE_ASSERT(LAST_WIN_RTV < rtvCount, "Increase size of RTV heap");
 
   hr = m_pDevice->CreateDescriptorHeap(&desc, __uuidof(ID3D12DescriptorHeap), (void **)&rtvHeap);
   m_pDevice->InternalRef();
@@ -155,10 +157,12 @@ D3D12DebugManager::D3D12DebugManager(WrappedID3D12Device *wrapper)
 
   rm->SetInternalResource(rtvHeap);
 
-  desc.NumDescriptors = 64;
+  const uint32_t dsvCount = 80;
+
+  desc.NumDescriptors = dsvCount;
   desc.Type = D3D12_DESCRIPTOR_HEAP_TYPE_DSV;
 
-  RDCCOMPILE_ASSERT(FIRST_WIN_DSV + 32 < 64, "Increase size of DSV heap");
+  RDCCOMPILE_ASSERT(LAST_WIN_DSV < dsvCount, "Increase size of DSV heap");
 
   hr = m_pDevice->CreateDescriptorHeap(&desc, __uuidof(ID3D12DescriptorHeap), (void **)&dsvHeap);
   m_pDevice->InternalRef();

diff --git a/renderdoc/driver/d3d12/d3d12_debug.h b/renderdoc/driver/d3d12/d3d12_debug.h
@@ -94,6 +94,7 @@ enum RTVSlot
   FIRST_TMP_RTV,
   LAST_TMP_RTV = FIRST_TMP_RTV + 16,
   FIRST_WIN_RTV,
+  LAST_WIN_RTV = FIRST_WIN_RTV + 768,
 };
 
 enum SamplerSlot
@@ -111,6 +112,7 @@ enum DSVSlot
   MSAA_DSV,
   TMP_DSV,
   FIRST_WIN_DSV,
+  LAST_WIN_DSV = FIRST_WIN_DSV + 64,
 };
 
 struct MeshDisplayPipelines

diff --git a/renderdoc/driver/d3d12/d3d12_outputwindow.cpp b/renderdoc/driver/d3d12/d3d12_outputwindow.cpp
@@ -248,26 +248,28 @@ uint64_t D3D12Replay::MakeOutputWindow(WindowingData window, bool depth)
 
   outw.bbIdx = 0;
 
-  outw.rtv = GetDebugManager()->GetCPUHandle(FIRST_WIN_RTV);
-  outw.rtv.ptr += SIZE_T(m_OutputWindowID) * sizeof(D3D12Descriptor);
+  outw.rtvId = m_OutputWindowIDs.back();
+  m_OutputWindowIDs.pop_back();
 
-  outw.dsv = GetDebugManager()->GetCPUHandle(FIRST_WIN_DSV);
-  outw.dsv.ptr += SIZE_T(m_DSVID) * sizeof(D3D12Descriptor);
+  outw.rtv = GetDebugManager()->GetCPUHandle(RTVSlot(outw.rtvId));
 
   outw.col = NULL;
   outw.colResolve = NULL;
   outw.MakeRTV(depth);
 
+  outw.dsvId = 0;
   outw.depth = NULL;
   if(depth)
   {
+    outw.dsvId = m_DSVIDs.back();
+    m_DSVIDs.pop_back();
+    outw.dsv = GetDebugManager()->GetCPUHandle(DSVSlot(outw.dsvId));
+
     outw.MakeDSV();
-    m_DSVID++;
   }
 
-  uint64_t id = m_OutputWindowID++;
-  m_OutputWindows[id] = outw;
-  return id;
+  m_OutputWindows[outw.rtvId] = outw;
+  return outw.rtvId;
 }
 
 void D3D12Replay::DestroyOutputWindow(uint64_t id)
@@ -278,6 +280,10 @@ void D3D12Replay::DestroyOutputWindow(uint64_t id)
 
   OutputWindow &outw = it->second;
 
+  m_OutputWindowIDs.push_back(outw.rtvId);
+  if(outw.dsvId != 0)
+    m_DSVIDs.push_back(outw.dsvId);
+
   m_pDevice->FlushLists(true);
 
   SAFE_RELEASE(outw.swap);

diff --git a/renderdoc/driver/d3d12/d3d12_replay.cpp b/renderdoc/driver/d3d12/d3d12_replay.cpp
@@ -143,6 +143,11 @@ void D3D12Replay::CreateResources()
 {
   m_DebugManager = new D3D12DebugManager(m_pDevice);
 
+  for(uint64_t i = FIRST_WIN_RTV; i <= LAST_WIN_RTV; i++)
+    m_OutputWindowIDs.insert(0, i);
+  for(uint64_t i = FIRST_WIN_DSV; i <= LAST_WIN_DSV; i++)
+    m_DSVIDs.insert(0, i);
+
   if(RenderDoc::Inst().IsReplayApp())
   {
     CreateSOBuffers();

diff --git a/renderdoc/driver/d3d12/d3d12_replay.h b/renderdoc/driver/d3d12/d3d12_replay.h
@@ -374,6 +374,8 @@ class D3D12Replay : public IReplayDriver
     D3D12_CPU_DESCRIPTOR_HANDLE dsv;
     D3D12_RESOURCE_DESC bbDesc;
 
+    uint64_t rtvId, dsvId;
+
     WrappedID3D12Device *dev;
 
     void MakeRTV(bool multisampled);
@@ -386,8 +388,8 @@ class D3D12Replay : public IReplayDriver
   float m_OutputHeight = 1.0f;
   D3D12_VIEWPORT m_OutputViewport;
 
-  uint64_t m_OutputWindowID = 1;
-  uint64_t m_DSVID = 0;
+  rdcarray<uint64_t> m_OutputWindowIDs;
+  rdcarray<uint64_t> m_DSVIDs;
   uint64_t m_CurrentOutputWindow = 0;
   std::map<uint64_t, OutputWindow> m_OutputWindows;