From bd533dc31bbee0a2bf4a4ca7e9d3ff5085a546bb Mon Sep 17 00:00:00 2001 From: ns6089 <61738816+ns6089@users.noreply.github.com> Date: Tue, 18 Apr 2023 19:03:52 +0300 Subject: [PATCH] Extend packet header with frame processing latency --- src/platform/common.h | 2 ++ src/platform/linux/cuda.cu | 9 +++++++-- src/platform/windows/display_ram.cpp | 12 ++++++++++++ src/platform/windows/display_vram.cpp | 11 +++++++++++ src/platform/windows/misc.cpp | 21 +++++++++++++++++++++ src/platform/windows/misc.h | 8 ++++++++ src/stream.cpp | 21 ++++++++++++++++++++- src/video.cpp | 22 ++++++++++++++++++---- src/video.h | 2 ++ 9 files changed, 101 insertions(+), 7 deletions(-) diff --git a/src/platform/common.h b/src/platform/common.h index 3d611179..62c1d60f 100644 --- a/src/platform/common.h +++ b/src/platform/common.h @@ -186,6 +186,8 @@ namespace platf { std::int32_t pixel_pitch {}; std::int32_t row_pitch {}; + std::optional frame_timestamp; + virtual ~img_t() = default; }; diff --git a/src/platform/linux/cuda.cu b/src/platform/linux/cuda.cu index ba630c03..85f9ee4a 100644 --- a/src/platform/linux/cuda.cu +++ b/src/platform/linux/cuda.cu @@ -1,5 +1,6 @@ // #include #include +#include #include #include #include @@ -31,8 +32,10 @@ using namespace std::literals; //////////////////// Special desclarations /** - * NVCC segfaults when including - * Therefore, some declarations need to be added explicitely + * NVCC tends to have problems with standard headers. + * Don't include common.h, instead use bare minimum + * of standard headers and duplicate declarations of necessary classes. + * Not pretty and extremely error-prone, fix at earliest convenience. */ namespace platf { struct img_t: std::enable_shared_from_this { @@ -43,6 +46,8 @@ public: std::int32_t pixel_pitch {}; std::int32_t row_pitch {}; + std::optional frame_timestamp; + virtual ~img_t() = default; }; } // namespace platf diff --git a/src/platform/windows/display_ram.cpp b/src/platform/windows/display_ram.cpp index 04ea0753..bfd90c5d 100644 --- a/src/platform/windows/display_ram.cpp +++ b/src/platform/windows/display_ram.cpp @@ -1,4 +1,6 @@ #include "display.h" + +#include "misc.h" #include "src/main.h" namespace platf { @@ -192,6 +194,12 @@ namespace platf::dxgi { return capture_e::timeout; } + std::optional frame_timestamp; + if (auto qpc_displayed = std::max(frame_info.LastPresentTime.QuadPart, frame_info.LastMouseUpdateTime.QuadPart)) { + // Translate QueryPerformanceCounter() value to steady_clock time point + frame_timestamp = std::chrono::steady_clock::now() - qpc_time_difference(qpc_counter(), qpc_displayed); + } + if (frame_info.PointerShapeBufferSize > 0) { auto &img_data = cursor.img_data; @@ -307,6 +315,10 @@ namespace platf::dxgi { blend_cursor(cursor, *img); } + if (img) { + img->frame_timestamp = frame_timestamp; + } + return capture_e::ok; } diff --git a/src/platform/windows/display_vram.cpp b/src/platform/windows/display_vram.cpp index 93cef64e..9ea507f4 100644 --- a/src/platform/windows/display_vram.cpp +++ b/src/platform/windows/display_vram.cpp @@ -11,6 +11,7 @@ extern "C" { } #include "display.h" +#include "misc.h" #include "src/main.h" #include "src/video.h" @@ -894,6 +895,12 @@ namespace platf::dxgi { return capture_e::timeout; } + std::optional frame_timestamp; + if (auto qpc_displayed = std::max(frame_info.LastPresentTime.QuadPart, frame_info.LastMouseUpdateTime.QuadPart)) { + // Translate QueryPerformanceCounter() value to steady_clock time point + frame_timestamp = std::chrono::steady_clock::now() - qpc_time_difference(qpc_counter(), qpc_displayed); + } + if (frame_info.PointerShapeBufferSize > 0) { DXGI_OUTDUPL_POINTER_SHAPE_INFO shape_info {}; @@ -1239,6 +1246,10 @@ namespace platf::dxgi { old_surface_delayed_destruction.reset(); } + if (img_out) { + img_out->frame_timestamp = frame_timestamp; + } + return capture_e::ok; } diff --git a/src/platform/windows/misc.cpp b/src/platform/windows/misc.cpp index 67dae2e1..6b529cd9 100644 --- a/src/platform/windows/misc.cpp +++ b/src/platform/windows/misc.cpp @@ -1020,4 +1020,25 @@ namespace platf { return std::make_unique(flow_id); } + int64_t + qpc_counter() { + LARGE_INTEGER performace_counter; + if (QueryPerformanceCounter(&performace_counter)) return performace_counter.QuadPart; + return 0; + } + + std::chrono::nanoseconds + qpc_time_difference(int64_t performance_counter1, int64_t performance_counter2) { + auto get_frequency = []() { + LARGE_INTEGER frequency; + frequency.QuadPart = 0; + QueryPerformanceFrequency(&frequency); + return frequency.QuadPart; + }; + static const double frequency = get_frequency(); + if (frequency) { + return std::chrono::nanoseconds((int64_t) ((performance_counter1 - performance_counter2) * frequency / std::nano::den)); + } + return {}; + } } // namespace platf \ No newline at end of file diff --git a/src/platform/windows/misc.h b/src/platform/windows/misc.h index 6fe68ac2..016580fc 100644 --- a/src/platform/windows/misc.h +++ b/src/platform/windows/misc.h @@ -1,5 +1,7 @@ #ifndef SUNSHINE_WINDOWS_MISC_H #define SUNSHINE_WINDOWS_MISC_H + +#include #include #include #include @@ -9,6 +11,12 @@ namespace platf { print_status(const std::string_view &prefix, HRESULT status); HDESK syncThreadDesktop(); + + int64_t + qpc_counter(); + + std::chrono::nanoseconds + qpc_time_difference(int64_t performance_counter1, int64_t performance_counter2); } // namespace platf #endif \ No newline at end of file diff --git a/src/stream.cpp b/src/stream.cpp index 65490593..016f9592 100644 --- a/src/stream.cpp +++ b/src/stream.cpp @@ -8,6 +8,8 @@ #include #include +#include + extern "C" { #include #include @@ -74,7 +76,11 @@ namespace stream { } std::uint8_t headerType; // Always 0x01 for short headers - std::uint8_t unknown[2]; + + // Sunshine extension + // Frame processing latency, in 1/10 ms units + // zero when the frame is repeated or there is no backend implementation + boost::endian::little_uint16_at frame_processing_latency; // Currently known values: // 1 = Normal P-frame @@ -1013,6 +1019,19 @@ namespace stream { frame_header.headerType = 0x01; // Short header type frame_header.frameType = (av_packet->flags & AV_PKT_FLAG_KEY) ? 2 : 1; + if (packet->frame_timestamp) { + auto duration_to_latency = [](const std::chrono::steady_clock::duration &duration) { + const auto duration_us = std::chrono::duration_cast(duration).count(); + return (uint16_t) std::clamp((duration_us + 50) / 100, 0, std::numeric_limits::max()); + }; + + uint16_t latency = duration_to_latency(std::chrono::steady_clock::now() - *packet->frame_timestamp); + frame_header.frame_processing_latency = latency; + } + else { + frame_header.frame_processing_latency = 0; + } + std::copy_n((uint8_t *) &frame_header, sizeof(frame_header), std::back_inserter(payload_new)); std::copy(std::begin(payload), std::end(payload), std::back_inserter(payload_new)); diff --git a/src/video.cpp b/src/video.cpp index 93c5633d..cac50837 100644 --- a/src/video.cpp +++ b/src/video.cpp @@ -870,6 +870,7 @@ namespace video { if (img_out) { // trim allocated but unused portion of the pool based on timeouts trim_imgs(); + img_out->frame_timestamp.reset(); return true; } else { @@ -989,7 +990,7 @@ namespace video { } int - encode(int64_t frame_nr, session_t &session, frame_t::pointer frame, safe::mail_raw_t::queue_t &packets, void *channel_data) { + encode(int64_t frame_nr, session_t &session, frame_t::pointer frame, safe::mail_raw_t::queue_t &packets, void *channel_data, const std::optional &frame_timestamp) { frame->pts = frame_nr; auto &ctx = session.ctx; @@ -1042,6 +1043,10 @@ namespace video { std::string_view((char *) std::begin(sps._new), sps._new.size())); } + if (av_packet && av_packet->pts == frame_nr) { + packet->frame_timestamp = frame_timestamp; + } + packet->replacements = &session.replacements; packet->channel_data = channel_data; packets->raise(std::move(packet)); @@ -1402,9 +1407,12 @@ namespace video { idr_events->pop(); } + std::optional frame_timestamp; + // Encode at a minimum of 10 FPS to avoid image quality issues with static content if (!frame->key_frame || images->peek()) { if (auto img = images->pop(100ms)) { + frame_timestamp = img->frame_timestamp; if (session->device->convert(*img)) { BOOST_LOG(error) << "Could not convert image"sv; return; @@ -1415,7 +1423,7 @@ namespace video { } } - if (encode(frame_nr++, *session, frame, packets, channel_data)) { + if (encode(frame_nr++, *session, frame, packets, channel_data, frame_timestamp)) { BOOST_LOG(error) << "Could not encode video packet"sv; return; } @@ -1600,7 +1608,12 @@ namespace video { continue; } - if (encode(ctx->frame_nr++, pos->session, frame, ctx->packets, ctx->channel_data)) { + std::optional frame_timestamp; + if (img) { + frame_timestamp = img->frame_timestamp; + } + + if (encode(ctx->frame_nr++, pos->session, frame, ctx->packets, ctx->channel_data, frame_timestamp)) { BOOST_LOG(error) << "Could not encode video packet"sv; ctx->shutdown_event->raise(true); @@ -1625,6 +1638,7 @@ namespace video { auto pull_free_image_callback = [&img](std::shared_ptr &img_out) -> bool { img_out = img; + img_out->frame_timestamp.reset(); return true; }; @@ -1813,7 +1827,7 @@ namespace video { auto packets = mail::man->queue(mail::video_packets); while (!packets->peek()) { - if (encode(1, *session, frame, packets, nullptr)) { + if (encode(1, *session, frame, packets, nullptr, {})) { return -1; } } diff --git a/src/video.h b/src/video.h index d3eb1a95..37f4a728 100644 --- a/src/video.h +++ b/src/video.h @@ -48,6 +48,8 @@ namespace video { AVPacket *av_packet; std::vector *replacements; void *channel_data; + + std::optional frame_timestamp; }; using packet_t = std::unique_ptr;