[mutter] shaped-texture: Fix damage propagation for rotated transforms with viewport

[Marge Bot <marge-bot src gnome org>]

commit 7d3e2c5f9c09fae50450df691eeba0f067de8cff
Author: Robert Mader <robert mader posteo de>
Date:   Tue Apr 20 14:56:56 2021 +0200

    shaped-texture: Fix damage propagation for rotated transforms with viewport
    
    When a viewport source rect or destination size is set, `stex->dst_width`
    gives us the the cropped and/or scaled size. At this step, we need the
    uncropped/unscaled size however.
    
    Note: this is only ever relevant if buffer transform and viewport are
    used together - otherwise the values are the same.
    
    Part-of: <https://gitlab.gnome.org/GNOME/mutter/-/merge_requests/1836>

 src/compositor/meta-shaped-texture.c | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)
---
diff --git a/src/compositor/meta-shaped-texture.c b/src/compositor/meta-shaped-texture.c
index 6f6d85345f..6a8af828f0 100644
--- a/src/compositor/meta-shaped-texture.c
+++ b/src/compositor/meta-shaped-texture.c
@@ -989,6 +989,8 @@ meta_shaped_texture_update_area (MetaShapedTexture     *stex,
                                  cairo_rectangle_int_t *clip)
 {
   MetaMonitorTransform inverted_transform;
+  int scaled_and_transformed_width;
+  int scaled_and_transformed_height;
 
   if (stex->texture == NULL)
     return FALSE;
@@ -1005,12 +1007,21 @@ meta_shaped_texture_update_area (MetaShapedTexture     *stex,
                                META_ROUNDING_STRATEGY_GROW,
                                clip);
 
+  if (meta_monitor_transform_is_rotated (stex->transform))
+    {
+      scaled_and_transformed_width = stex->tex_height / stex->buffer_scale;
+      scaled_and_transformed_height = stex->tex_width / stex->buffer_scale;
+    }
+  else
+    {
+      scaled_and_transformed_width = stex->tex_width / stex->buffer_scale;
+      scaled_and_transformed_height = stex->tex_height / stex->buffer_scale;
+    }
   inverted_transform = meta_monitor_transform_invert (stex->transform);
-  meta_shaped_texture_ensure_size_valid (stex);
   meta_rectangle_transform (clip,
                             inverted_transform,
-                            stex->dst_width,
-                            stex->dst_height,
+                            scaled_and_transformed_width,
+                            scaled_and_transformed_height,
                             clip);
 
   if (stex->has_viewport_src_rect || stex->has_viewport_dst_size)
@@ -1031,8 +1042,8 @@ meta_shaped_texture_update_area (MetaShapedTexture     *stex,
           viewport = (graphene_rect_t) {
             .origin.x = 0,
             .origin.y = 0,
-            .size.width = stex->tex_width / stex->buffer_scale,
-            .size.height = stex->tex_height / stex->buffer_scale,
+            .size.width = scaled_and_transformed_width,
+            .size.height = scaled_and_transformed_height,
           };
         }