Extend packet header with frame processing latency

src/platform/common.h

@@ -186,6 +186,8 @@ namespace platf {
     std::int32_t pixel_pitch {};
     std::int32_t row_pitch {};
 
+    std::optional<std::chrono::steady_clock::time_point> frame_timestamp;
+
     virtual ~img_t() = default;
   };
 

src/platform/linux/cuda.cu

@@ -1,5 +1,6 @@
 // #include <algorithm>
 #include <helper_math.h>
+#include <chrono>
 #include <limits>
 #include <memory>
 #include <optional>
@@ -31,8 +32,10 @@ using namespace std::literals;
 
 //////////////////// Special desclarations
 /**
- * NVCC segfaults when including <chrono>
- * Therefore, some declarations need to be added explicitely
+ * NVCC tends to have problems with standard headers.
+ * Don't include common.h, instead use bare minimum
+ * of standard headers and duplicate declarations of necessary classes.
+ * Not pretty and extremely error-prone, fix at earliest convenience.
  */
 namespace platf {
 struct img_t: std::enable_shared_from_this<img_t> {
@@ -43,6 +46,8 @@ public:
   std::int32_t pixel_pitch {};
   std::int32_t row_pitch {};
 
+  std::optional<std::chrono::steady_clock::time_point> frame_timestamp;
+
   virtual ~img_t() = default;
 };
 } // namespace platf

src/platform/windows/display_ram.cpp

@@ -1,4 +1,6 @@
 #include "display.h"
+
+#include "misc.h"
 #include "src/main.h"
 
 namespace platf {
@@ -192,6 +194,12 @@ namespace platf::dxgi {
       return capture_e::timeout;
     }
 
+    std::optional<std::chrono::steady_clock::time_point> frame_timestamp;
+    if (auto qpc_displayed = std::max(frame_info.LastPresentTime.QuadPart, frame_info.LastMouseUpdateTime.QuadPart)) {
+      // Translate QueryPerformanceCounter() value to steady_clock time point
+      frame_timestamp = std::chrono::steady_clock::now() - qpc_time_difference(qpc_counter(), qpc_displayed);
+    }
+
     if (frame_info.PointerShapeBufferSize > 0) {
       auto &img_data = cursor.img_data;
 
@@ -307,6 +315,10 @@ namespace platf::dxgi {
       blend_cursor(cursor, *img);
     }
 
+    if (img) {
+      img->frame_timestamp = frame_timestamp;
+    }
+
     return capture_e::ok;
   }
 

src/platform/windows/display_vram.cpp

@@ -11,6 +11,7 @@ extern "C" {
 }
 
 #include "display.h"
+#include "misc.h"
 #include "src/main.h"
 #include "src/video.h"
 
@@ -894,6 +895,12 @@ namespace platf::dxgi {
       return capture_e::timeout;
     }
 
+    std::optional<std::chrono::steady_clock::time_point> frame_timestamp;
+    if (auto qpc_displayed = std::max(frame_info.LastPresentTime.QuadPart, frame_info.LastMouseUpdateTime.QuadPart)) {
+      // Translate QueryPerformanceCounter() value to steady_clock time point
+      frame_timestamp = std::chrono::steady_clock::now() - qpc_time_difference(qpc_counter(), qpc_displayed);
+    }
+
     if (frame_info.PointerShapeBufferSize > 0) {
       DXGI_OUTDUPL_POINTER_SHAPE_INFO shape_info {};
 
@@ -1239,6 +1246,10 @@ namespace platf::dxgi {
       old_surface_delayed_destruction.reset();
     }
 
+    if (img_out) {
+      img_out->frame_timestamp = frame_timestamp;
+    }
+
     return capture_e::ok;
   }
 

src/platform/windows/misc.cpp

@@ -1020,4 +1020,25 @@ namespace platf {
 
     return std::make_unique<qos_t>(flow_id);
   }
+  int64_t
+  qpc_counter() {
+    LARGE_INTEGER performace_counter;
+    if (QueryPerformanceCounter(&performace_counter)) return performace_counter.QuadPart;
+    return 0;
+  }
+
+  std::chrono::nanoseconds
+  qpc_time_difference(int64_t performance_counter1, int64_t performance_counter2) {
+    auto get_frequency = []() {
+      LARGE_INTEGER frequency;
+      frequency.QuadPart = 0;
+      QueryPerformanceFrequency(&frequency);
+      return frequency.QuadPart;
+    };
+    static const double frequency = get_frequency();
+    if (frequency) {
+      return std::chrono::nanoseconds((int64_t) ((performance_counter1 - performance_counter2) * frequency / std::nano::den));
+    }
+    return {};
+  }
 }  // namespace platf

src/platform/windows/misc.h

@@ -1,5 +1,7 @@
 #ifndef SUNSHINE_WINDOWS_MISC_H
 #define SUNSHINE_WINDOWS_MISC_H
+
+#include <chrono>
 #include <string_view>
 #include <windows.h>
 #include <winnt.h>
@@ -9,6 +11,12 @@ namespace platf {
   print_status(const std::string_view &prefix, HRESULT status);
   HDESK
   syncThreadDesktop();
+
+  int64_t
+  qpc_counter();
+
+  std::chrono::nanoseconds
+  qpc_time_difference(int64_t performance_counter1, int64_t performance_counter2);
 }  // namespace platf
 
 #endif

src/stream.cpp

@@ -8,6 +8,8 @@
 #include <fstream>
 #include <openssl/err.h>
 
+#include <boost/endian/arithmetic.hpp>
+
 extern "C" {
 #include <moonlight-common-c/src/RtpAudioQueue.h>
 #include <moonlight-common-c/src/Video.h>
@@ -74,7 +76,11 @@ namespace stream {
     }
 
     std::uint8_t headerType;  // Always 0x01 for short headers
-    std::uint8_t unknown[2];
+
+    // Sunshine extension
+    // Frame processing latency, in 1/10 ms units
+    //     zero when the frame is repeated or there is no backend implementation
+    boost::endian::little_uint16_at frame_processing_latency;
 
     // Currently known values:
     // 1 = Normal P-frame
@@ -1013,6 +1019,19 @@ namespace stream {
       frame_header.headerType = 0x01;  // Short header type
       frame_header.frameType = (av_packet->flags & AV_PKT_FLAG_KEY) ? 2 : 1;
 
+      if (packet->frame_timestamp) {
+        auto duration_to_latency = [](const std::chrono::steady_clock::duration &duration) {
+          const auto duration_us = std::chrono::duration_cast<std::chrono::microseconds>(duration).count();
+          return (uint16_t) std::clamp<decltype(duration_us)>((duration_us + 50) / 100, 0, std::numeric_limits<uint16_t>::max());
+        };
+
+        uint16_t latency = duration_to_latency(std::chrono::steady_clock::now() - *packet->frame_timestamp);
+        frame_header.frame_processing_latency = latency;
+      }
+      else {
+        frame_header.frame_processing_latency = 0;
+      }
+
       std::copy_n((uint8_t *) &frame_header, sizeof(frame_header), std::back_inserter(payload_new));
       std::copy(std::begin(payload), std::end(payload), std::back_inserter(payload_new));
 

src/video.cpp

@@ -870,6 +870,7 @@ namespace video {
         if (img_out) {
           // trim allocated but unused portion of the pool based on timeouts
           trim_imgs();
+          img_out->frame_timestamp.reset();
           return true;
         }
         else {
@@ -989,7 +990,7 @@ namespace video {
   }
 
   int
-  encode(int64_t frame_nr, session_t &session, frame_t::pointer frame, safe::mail_raw_t::queue_t<packet_t> &packets, void *channel_data) {
+  encode(int64_t frame_nr, session_t &session, frame_t::pointer frame, safe::mail_raw_t::queue_t<packet_t> &packets, void *channel_data, const std::optional<std::chrono::steady_clock::time_point> &frame_timestamp) {
     frame->pts = frame_nr;
 
     auto &ctx = session.ctx;
@@ -1042,6 +1043,10 @@ namespace video {
           std::string_view((char *) std::begin(sps._new), sps._new.size()));
       }
 
+      if (av_packet && av_packet->pts == frame_nr) {
+        packet->frame_timestamp = frame_timestamp;
+      }
+
       packet->replacements = &session.replacements;
       packet->channel_data = channel_data;
       packets->raise(std::move(packet));
@@ -1402,9 +1407,12 @@ namespace video {
         idr_events->pop();
       }
 
+      std::optional<std::chrono::steady_clock::time_point> frame_timestamp;
+
       // Encode at a minimum of 10 FPS to avoid image quality issues with static content
       if (!frame->key_frame || images->peek()) {
         if (auto img = images->pop(100ms)) {
+          frame_timestamp = img->frame_timestamp;
           if (session->device->convert(*img)) {
             BOOST_LOG(error) << "Could not convert image"sv;
             return;
@@ -1415,7 +1423,7 @@ namespace video {
         }
       }
 
-      if (encode(frame_nr++, *session, frame, packets, channel_data)) {
+      if (encode(frame_nr++, *session, frame, packets, channel_data, frame_timestamp)) {
         BOOST_LOG(error) << "Could not encode video packet"sv;
         return;
       }
@@ -1600,7 +1608,12 @@ namespace video {
             continue;
           }
 
-          if (encode(ctx->frame_nr++, pos->session, frame, ctx->packets, ctx->channel_data)) {
+          std::optional<std::chrono::steady_clock::time_point> frame_timestamp;
+          if (img) {
+            frame_timestamp = img->frame_timestamp;
+          }
+
+          if (encode(ctx->frame_nr++, pos->session, frame, ctx->packets, ctx->channel_data, frame_timestamp)) {
             BOOST_LOG(error) << "Could not encode video packet"sv;
             ctx->shutdown_event->raise(true);
 
@@ -1625,6 +1638,7 @@ namespace video {
 
       auto pull_free_image_callback = [&img](std::shared_ptr<platf::img_t> &img_out) -> bool {
         img_out = img;
+        img_out->frame_timestamp.reset();
         return true;
       };
 
@@ -1813,7 +1827,7 @@ namespace video {
 
     auto packets = mail::man->queue<packet_t>(mail::video_packets);
     while (!packets->peek()) {
-      if (encode(1, *session, frame, packets, nullptr)) {
+      if (encode(1, *session, frame, packets, nullptr, {})) {
         return -1;
       }
     }

src/video.h

@@ -48,6 +48,8 @@ namespace video {
     AVPacket *av_packet;
     std::vector<replace_t> *replacements;
     void *channel_data;
+
+    std::optional<std::chrono::steady_clock::time_point> frame_timestamp;
   };
 
   using packet_t = std::unique_ptr<packet_raw_t>;