[gnome-remote-desktop] rdp: Defer cairo region creation if possible

[Jonas Ådahl <jadahl src gnome org>]

commit 39f0e6ff014ca3715de3e65df41ccf8ef3532a04
Author: Pascal Nowack <Pascal Nowack gmx de>
Date:   Thu Jan 13 15:32:12 2022 +0100

    rdp: Defer cairo region creation if possible
    
    NVENC does not make use of damage regions, but xfreerdp uses damage
    regions to cut out unnecessary operations to display the decoded frame.
    
    When NVENC encodes a frame, CUDA kernels can still run and data can
    still be transferred between the Host and the GPU, without affecting
    the NVENC operation.
    With the split up of the NVENC encode operation done in the last
    commit, run the cairo damage region creation, while NVENC encodes the
    frame.
    As the damage checking is already done, the damage region just needs to
    be simplified and transferred to the host.
    When this task is done, the graphics thread will wait for NVENC to
    finish the encoding operation.

 src/grd-rdp-graphics-pipeline.c | 36 ++++++++++++++++++++++++++++++------
 src/grd-rdp-graphics-pipeline.h |  2 --
 src/grd-session-rdp.c           | 40 ++++++++++++++++++----------------------
 3 files changed, 48 insertions(+), 30 deletions(-)
---
diff --git a/src/grd-rdp-graphics-pipeline.c b/src/grd-rdp-graphics-pipeline.c
index 088b92e4..fa7d6ad5 100644
--- a/src/grd-rdp-graphics-pipeline.c
+++ b/src/grd-rdp-graphics-pipeline.c
@@ -21,10 +21,12 @@
 
 #include "grd-rdp-graphics-pipeline.h"
 
+#include <cairo/cairo.h>
 #include <winpr/sysinfo.h>
 
 #include "grd-hwaccel-nvidia.h"
 #include "grd-rdp-buffer.h"
+#include "grd-rdp-damage-detector.h"
 #include "grd-rdp-frame-info.h"
 #include "grd-rdp-gfx-surface.h"
 #include "grd-rdp-network-autodetection.h"
@@ -389,7 +391,6 @@ static gboolean
 refresh_gfx_surface_avc420 (GrdRdpGraphicsPipeline *graphics_pipeline,
                             HWAccelContext         *hwaccel_context,
                             GrdRdpSurface          *rdp_surface,
-                            cairo_region_t         *region,
                             GrdRdpBuffer           *buffer,
                             int64_t                *enc_time_us)
 {
@@ -406,6 +407,7 @@ refresh_gfx_surface_avc420 (GrdRdpGraphicsPipeline *graphics_pipeline,
   uint16_t surface_height = rdp_surface->height;
   uint16_t aligned_width;
   uint16_t aligned_height;
+  cairo_region_t *region;
   uint32_t surface_serial;
   int64_t enc_ack_time_us;
   int i;
@@ -428,11 +430,23 @@ refresh_gfx_surface_avc420 (GrdRdpGraphicsPipeline *graphics_pipeline,
       return FALSE;
     }
 
+  region = grd_rdp_damage_detector_get_damage_region (rdp_surface->detector);
+  if (!region)
+    {
+      grd_session_rdp_notify_error (graphics_pipeline->session_rdp,
+                                    GRD_SESSION_RDP_ERROR_GRAPHICS_SUBSYSTEM_FAILED);
+      grd_hwaccel_nvidia_avc420_retrieve_bitstream (graphics_pipeline->hwaccel_nvidia,
+                                                    hwaccel_context->encode_session_id,
+                                                    NULL, NULL);
+      return FALSE;
+    }
+
   if (!grd_hwaccel_nvidia_avc420_retrieve_bitstream (graphics_pipeline->hwaccel_nvidia,
                                                      hwaccel_context->encode_session_id,
                                                      &avc420.data, &avc420.length))
     {
       g_warning ("[RDP.RDPGFX] Failed to retrieve AVC420 bitstream");
+      cairo_region_destroy (region);
       return FALSE;
     }
 
@@ -504,6 +518,7 @@ refresh_gfx_surface_avc420 (GrdRdpGraphicsPipeline *graphics_pipeline,
   g_free (avc420.data);
   g_free (avc420.meta.quantQualityVals);
   g_free (avc420.meta.regionRects);
+  cairo_region_destroy (region);
 
   return TRUE;
 }
@@ -642,7 +657,6 @@ rfx_progressive_write_message (RFX_MESSAGE *rfx_message,
 static gboolean
 refresh_gfx_surface_rfx_progressive (GrdRdpGraphicsPipeline *graphics_pipeline,
                                      GrdRdpSurface          *rdp_surface,
-                                     cairo_region_t         *region,
                                      GrdRdpBuffer           *buffer,
                                      int64_t                *enc_time_us)
 {
@@ -655,6 +669,7 @@ refresh_gfx_surface_rfx_progressive (GrdRdpGraphicsPipeline *graphics_pipeline,
   RDPGFX_START_FRAME_PDU cmd_start = {0};
   RDPGFX_END_FRAME_PDU cmd_end = {0};
   gboolean needs_progressive_header = FALSE;
+  cairo_region_t *region;
   cairo_rectangle_int_t cairo_rect;
   RFX_RECT *rfx_rects, *rfx_rect;
   int n_rects;
@@ -665,6 +680,14 @@ refresh_gfx_surface_rfx_progressive (GrdRdpGraphicsPipeline *graphics_pipeline,
   int64_t enc_ack_time_us;
   int i;
 
+  region = grd_rdp_damage_detector_get_damage_region (rdp_surface->detector);
+  if (!region)
+    {
+      grd_session_rdp_notify_error (
+        session_rdp, GRD_SESSION_RDP_ERROR_GRAPHICS_SUBSYSTEM_FAILED);
+      return FALSE;
+    }
+
   graphics_pipeline->rfx_context->mode = RLGR1;
   if (!rdp_surface->valid)
     {
@@ -722,6 +745,7 @@ refresh_gfx_surface_rfx_progressive (GrdRdpGraphicsPipeline *graphics_pipeline,
     {
       g_warning ("[RDP.RDPGFX] rfx_progressive_write_message() failed");
       rfx_message_free (graphics_pipeline->rfx_context, rfx_message);
+      cairo_region_destroy (region);
       return FALSE;
     }
   rfx_message_free (graphics_pipeline->rfx_context, rfx_message);
@@ -761,6 +785,8 @@ refresh_gfx_surface_rfx_progressive (GrdRdpGraphicsPipeline *graphics_pipeline,
 
   *enc_time_us = enc_ack_time_us;
 
+  cairo_region_destroy (region);
+
   return TRUE;
 }
 
@@ -881,7 +907,6 @@ ensure_rtt_receivement (GrdRdpGraphicsPipeline *graphics_pipeline)
 void
 grd_rdp_graphics_pipeline_refresh_gfx (GrdRdpGraphicsPipeline *graphics_pipeline,
                                        GrdRdpSurface          *rdp_surface,
-                                       cairo_region_t         *region,
                                        GrdRdpBuffer           *buffer)
 {
   RdpgfxServerContext *rdpgfx_context = graphics_pipeline->rdpgfx_context;
@@ -918,13 +943,12 @@ grd_rdp_graphics_pipeline_refresh_gfx (GrdRdpGraphicsPipeline *graphics_pipeline
     {
       g_assert (hwaccel_context->api == HW_ACCEL_API_NVENC);
       success = refresh_gfx_surface_avc420 (graphics_pipeline, hwaccel_context,
-                                            rdp_surface, region, buffer,
-                                            &enc_time_us);
+                                            rdp_surface, buffer, &enc_time_us);
     }
   else
     {
       success = refresh_gfx_surface_rfx_progressive (graphics_pipeline, rdp_surface,
-                                                     region, buffer, &enc_time_us);
+                                                     buffer, &enc_time_us);
     }
 
   if (success)
diff --git a/src/grd-rdp-graphics-pipeline.h b/src/grd-rdp-graphics-pipeline.h
index c67aa108..e3346cf3 100644
--- a/src/grd-rdp-graphics-pipeline.h
+++ b/src/grd-rdp-graphics-pipeline.h
@@ -20,7 +20,6 @@
 #ifndef GRD_RDP_GRAPHICS_PIPELINE_H
 #define GRD_RDP_GRAPHICS_PIPELINE_H
 
-#include <cairo/cairo.h>
 #include <freerdp/server/rdpgfx.h>
 #include <glib-object.h>
 
@@ -61,7 +60,6 @@ void grd_rdp_graphics_pipeline_notify_new_round_trip_time (GrdRdpGraphicsPipelin
 
 void grd_rdp_graphics_pipeline_refresh_gfx (GrdRdpGraphicsPipeline *graphics_pipeline,
                                             GrdRdpSurface          *rdp_surface,
-                                            cairo_region_t         *region,
                                             GrdRdpBuffer           *buffer);
 
 #endif /* GRD_RDP_GRAPHICS_PIPELINE_H */
diff --git a/src/grd-session-rdp.c b/src/grd-session-rdp.c
index cde858f2..17a5cd53 100644
--- a/src/grd-session-rdp.c
+++ b/src/grd-session-rdp.c
@@ -166,7 +166,6 @@ close_session_idle (gpointer user_data);
 static void
 rdp_peer_refresh_region (GrdSessionRdp  *session_rdp,
                          GrdRdpSurface  *rdp_surface,
-                         cairo_region_t *region,
                          GrdRdpBuffer   *buffer);
 
 static gboolean
@@ -249,7 +248,6 @@ take_or_encode_frame (GrdSessionRdp *session_rdp,
 {
   uint16_t width = buffer->width;
   uint16_t height = buffer->height;
-  cairo_region_t *region;
 
   g_clear_pointer (&rdp_surface->pending_framebuffer, grd_rdp_buffer_release);
   maybe_resize_graphics_output_buffer (session_rdp, width, height);
@@ -284,8 +282,6 @@ take_or_encode_frame (GrdSessionRdp *session_rdp,
       !is_rdp_peer_flag_set (session_rdp, RDP_PEER_PENDING_GFX_INIT) &&
       !rdp_surface->encoding_suspended)
     {
-      gboolean is_region_damaged;
-
       if (!grd_rdp_damage_detector_submit_new_framebuffer (rdp_surface->detector,
                                                            buffer))
         {
@@ -295,20 +291,8 @@ take_or_encode_frame (GrdSessionRdp *session_rdp,
           return;
         }
 
-      is_region_damaged =
-        grd_rdp_damage_detector_is_region_damaged (rdp_surface->detector);
-
-      region = grd_rdp_damage_detector_get_damage_region (rdp_surface->detector);
-      if (!region)
-        {
-          grd_session_rdp_notify_error (
-            session_rdp, GRD_SESSION_RDP_ERROR_GRAPHICS_SUBSYSTEM_FAILED);
-          return;
-        }
-      if (is_region_damaged)
-        rdp_peer_refresh_region (session_rdp, rdp_surface, region, buffer);
-
-      cairo_region_destroy (region);
+      if (grd_rdp_damage_detector_is_region_damaged (rdp_surface->detector))
+        rdp_peer_refresh_region (session_rdp, rdp_surface, buffer);
     }
   else
     {
@@ -680,7 +664,6 @@ get_current_monitor_config (GrdSessionRdp  *session_rdp,
 static void
 rdp_peer_refresh_gfx (GrdSessionRdp  *session_rdp,
                       GrdRdpSurface  *rdp_surface,
-                      cairo_region_t *region,
                       GrdRdpBuffer   *buffer)
 {
   freerdp_peer *peer = session_rdp->peer;
@@ -704,7 +687,7 @@ rdp_peer_refresh_gfx (GrdSessionRdp  *session_rdp,
     }
 
   grd_rdp_graphics_pipeline_refresh_gfx (graphics_pipeline,
-                                         rdp_surface, region, buffer);
+                                         rdp_surface, buffer);
 }
 
 static void
@@ -1229,15 +1212,26 @@ rdp_peer_refresh_raw (GrdSessionRdp  *session_rdp,
 static void
 rdp_peer_refresh_region (GrdSessionRdp  *session_rdp,
                          GrdRdpSurface  *rdp_surface,
-                         cairo_region_t *region,
                          GrdRdpBuffer   *buffer)
 {
   freerdp_peer *peer = session_rdp->peer;
   RdpPeerContext *rdp_peer_context = (RdpPeerContext *) peer->context;
   rdpSettings *rdp_settings = peer->settings;
+  cairo_region_t *region = NULL;
+
+  if (!rdp_settings->SupportGraphicsPipeline)
+    {
+      region = grd_rdp_damage_detector_get_damage_region (rdp_surface->detector);
+      if (!region)
+        {
+          grd_session_rdp_notify_error (
+            session_rdp, GRD_SESSION_RDP_ERROR_GRAPHICS_SUBSYSTEM_FAILED);
+          return;
+        }
+    }
 
   if (rdp_settings->SupportGraphicsPipeline)
-    rdp_peer_refresh_gfx (session_rdp, rdp_surface, region, buffer);
+    rdp_peer_refresh_gfx (session_rdp, rdp_surface, buffer);
   else if (rdp_settings->RemoteFxCodec)
     rdp_peer_refresh_rfx (session_rdp, rdp_surface, region, buffer);
   else if (rdp_settings->NSCodec)
@@ -1245,6 +1239,8 @@ rdp_peer_refresh_region (GrdSessionRdp  *session_rdp,
   else
     rdp_peer_refresh_raw (session_rdp, rdp_surface, region, buffer);
 
+  g_clear_pointer (&region, cairo_region_destroy);
+
   ++rdp_peer_context->frame_id;
 }