[glib: 1/5] gvariant-serialiser: Prevent unbounded recursion in is_normal()

[Philip Withnall <pwithnall src gnome org>]

commit 77233f6f0779fe0c1cb48861d7deded4ae413567
Author: Sebastian Wilhelmi <wilhelmi google com>
Date:   Thu Jan 6 20:50:34 2022 +0000

    gvariant-serialiser: Prevent unbounded recursion in is_normal()
    
    This fixes a bug in 7c4e6e9fbe473de0401c778c6b0c4aad27d5145a.
    
    The original approach in that commit accidentally only checked the depth
    at the leaf nodes in the variant tree, whereas actually the depth should
    be checked before recursing to avoid stack overflow.
    
    It neglected to consider that `g_variant_serialised_is_normal()` would
    be recursed into by some of the `DISPATCH(_is_normal)` cases. When that
    happened, the depth check was after the recursion so couldn’t prevent a
    stack overflow.
    
    Fixes: #2572

 glib/gvariant-serialiser.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)
---
diff --git a/glib/gvariant-serialiser.c b/glib/gvariant-serialiser.c
index 832a8fdc2..7b13381b6 100644
--- a/glib/gvariant-serialiser.c
+++ b/glib/gvariant-serialiser.c
@@ -1587,6 +1587,9 @@ g_variant_serialised_byteswap (GVariantSerialised serialised)
 gboolean
 g_variant_serialised_is_normal (GVariantSerialised serialised)
 {
+  if (serialised.depth >= G_VARIANT_MAX_RECURSION_DEPTH)
+    return FALSE;
+
   DISPATCH_CASES (serialised.type_info,
 
                   return gvs_/**/,/**/_is_normal (serialised);
@@ -1595,8 +1598,6 @@ g_variant_serialised_is_normal (GVariantSerialised serialised)
 
   if (serialised.data == NULL)
     return FALSE;
-  if (serialised.depth >= G_VARIANT_MAX_RECURSION_DEPTH)
-    return FALSE;
 
   /* some hard-coded terminal cases */
   switch (g_variant_type_info_get_type_char (serialised.type_info))