finalize display reset bug

src/platform/common.h

@@ -187,6 +187,19 @@ struct encode_device_t {
 
   virtual int convert(platf::img_t &img) = 0;
 
+  /**
+   * Release any shared_ptr<display_t> the device holds onto.
+   *
+   * The display is only required while the encode device is being initialized (to read the
+   * source dimensions/rotation). Holding it afterwards needlessly keeps the capture display
+   * alive: when a display mode change invalidates the capture device, destroying this encode
+   * device can block for the full duration of the mode switch, and as long as that teardown
+   * holds a display reference the capture thread's reinit wait (which spins until the display
+   * use_count drops to 1) cannot make progress. Dropping the reference here lets capture
+   * reinitialize immediately while the heavy GPU teardown finishes in the background.
+   */
+  virtual void release_display() {}
+
   video::sunshine_colorspace_t colorspace;
 };
 

src/platform/windows/display_vram.cpp

@@ -434,6 +434,13 @@ public:
     return 0;
   }
 
+  // Drop the shared_ptr<display_t> kept only for initialization. After init the display is no
+  // longer needed for steady-state conversion, so releasing it lets the capture thread reinit
+  // the display even while this (potentially slow to destroy) device is still being torn down.
+  void release_display() {
+    display.reset();
+  }
+
   void apply_colorspace(const ::video::sunshine_colorspace_t &colorspace) {
     auto color_vectors = ::video::color_vectors_from_colorspace(colorspace);
 
@@ -813,6 +820,10 @@ public:
     return base.convert(img_base);
   }
 
+  void release_display() override {
+    base.release_display();
+  }
+
   void apply_colorspace() override {
     base.apply_colorspace(colorspace);
   }
@@ -933,6 +944,10 @@ public:
     return base.convert(img_base);
   }
 
+  void release_display() override {
+    base.release_display();
+  }
+
 private:
   d3d_base_encode_device base;
   std::unique_ptr<nvenc::nvenc_d3d11> nvenc_d3d;

src/video.cpp

@@ -343,6 +343,12 @@ public:
     return device->convert(img);
   }
 
+  void release_display() override {
+    if (device) {
+      device->release_display();
+    }
+  }
+
   void request_idr_frame() override {
     if (device && device->frame) {
       auto &frame = device->frame;
@@ -414,6 +420,12 @@ public:
     return device->convert(img);
   }
 
+  void release_display() override {
+    if (device) {
+      device->release_display();
+    }
+  }
+
   void request_idr_frame() override {
     force_idr = true;
   }
@@ -1734,13 +1746,17 @@ void encode_run(int &frame_nr, // Store progress of the frame number
   std::uint64_t video_rtp_remainder = 0;
 
   bool requested_idr_frame = true;
+  bool decouple_teardown = false;
   while (true) {
     if (shutdown_event->peek()) {
       break;
     }
     if (reinit_event.peek() || video_reset_events->peek()) {
       BOOST_LOG(info) << "Video encode loop pausing for display reinit or reset"sv;
-      video_reset_events->pop();
+      if (video_reset_events->peek()) {
+        video_reset_events->pop();
+      }
+      decouple_teardown = true;
       break;
     }
     if (!images->running()) {
@@ -1831,6 +1847,33 @@ void encode_run(int &frame_nr, // Store progress of the frame number
 
     session->request_normal_frame();
   }
+
+  // When pausing for a display reinit, the capture thread is blocked waiting for every other
+  // shared_ptr<display_t> to be released (it spins until use_count() == 1). This encode thread
+  // holds three of those references: the by-value 'disp' parameter, the display kept inside the
+  // encode device, and the caller's display in capture(). Destroying the session here can block
+  // for the full duration of a guest display mode change (the encoder D3D device cross-references
+  // capture textures that are invalidated mid-switch), which would keep all of those references
+  // alive and stall reinitialization for several seconds until an external video reset arrives.
+  //
+  // Instead, drop the display references this thread can reach immediately, then move the heavy
+  // session teardown onto a detached thread so the capture thread can reinitialize right away
+  // while the GPU resources are released in the background.
+  if (decouple_teardown && session) {
+    session->release_display();
+    disp.reset();
+
+    std::thread([session = std::move(session)]() mutable {
+      auto teardown_start = std::chrono::steady_clock::now();
+      session.reset();
+      auto elapsed_ms = std::chrono::duration_cast<std::chrono::milliseconds>(
+                            std::chrono::steady_clock::now() - teardown_start)
+                            .count();
+      if (elapsed_ms >= 500) {
+        BOOST_LOG(info) << "Background encode session teardown took " << elapsed_ms << "ms"sv;
+      }
+    }).detach();
+  }
 }
 
 std::unique_ptr<platf::encode_device_t>

src/video.h

@@ -194,6 +194,12 @@ struct encode_session_t {
 
   virtual int convert(platf::img_t &img) = 0;
 
+  /**
+   * Release the shared_ptr<display_t> held by the underlying encode device so the capture
+   * thread can reinitialize the display without waiting for this session to be destroyed.
+   */
+  virtual void release_display() {}
+
   virtual void request_idr_frame() = 0;
 
   virtual void request_normal_frame() = 0;