[glib/wip/3v1n0/support-can-fail-tests: 1/3] meson: Support tests that can fail under certain conditions

[Marco Trevisan <marcotrevi src gnome org>]

commit 8c2eebe9f59647ed8299a6840a333eeae5c4b80e
Author: Marco Trevisan (Treviño) <mail 3v1n0 net>
Date:   Wed Oct 19 20:08:15 2022 +0200

    meson: Support tests that can fail under certain conditions
    
    We have tests that are failing under certain environments, but it's
    difficult to handle them because:
     - for some environments we just allow all the tests to fail: DANGEROUS
     - when we don't allow failures we have flacky tests: A CI pain
    
    So, to avoid this add a way to wrap tests with a script that just marks
    the test as skipped in case of failures.
    
    Note that timeout for wrapped tests won't take advantage of the native
    meson tests multiplier, but we can accept that given that we'd likely to
    fail anyways.

 gio/tests/meson.build     | 70 ++++++++++++++++++++++++++++++++++++-----
 glib/tests/meson.build    | 70 +++++++++++++++++++++++++++++++++++++----
 gmodule/tests/meson.build | 35 ++++++++++++++++++++-
 gobject/tests/meson.build | 79 +++++++++++++++++++++++++++++++++++++++++------
 meson.build               |  4 ++-
 5 files changed, 234 insertions(+), 24 deletions(-)
---
diff --git a/gio/tests/meson.build b/gio/tests/meson.build
index 57fc7d3cc6..ed52db48bb 100644
--- a/gio/tests/meson.build
+++ b/gio/tests/meson.build
@@ -154,9 +154,9 @@ test_extra_programs = {
   'gsubprocess-testprog' : {},
 }
 
-python_tests = [
-  'codegen.py',
-]
+python_tests = {
+  'codegen.py' : {},
+}
 
 test_env = environment(common_test_env)
 test_env.set('G_TEST_SRCDIR', meson.current_source_dir())
@@ -905,21 +905,48 @@ foreach test_name, extra_args : gio_tests
     install_tag: 'tests',
     install: install,
   )
+  args = extra_args.get('args', [])
+  depends = extra_args.get('depends', [])
 
   suite = ['gio'] + extra_args.get('suite', [])
   timeout = suite.contains('slow') ? test_timeout_slow : test_timeout
   local_test_env = test_env
 
+  if extra_args.get('can_fail', false)
+    if have_sh
+      depends += [exe]
+      args = [
+        '-c',
+        'timeout -k @0@ "@1@" "$@" || exit 77'.format(timeout, exe.full_path()),
+        '--',
+        args
+      ]
+      exe = sh_program
+      timeout = -1
+    elif cmd_exe_program.found()
+      depends += [exe]
+      args = [
+        '/c',
+        '"@0@" "@1@" || exit 77'.format(exe.full_path(), ' '.join(args)),
+      ]
+      exe = cmd_exe_program
+    else
+      warning('Test @0@ cannot be ran reliably, skipping it'.format(test_name))
+      continue
+    endif
+  endif
+
   foreach var, value : extra_args.get('env', {})
     local_test_env.append(var, value)
   endforeach
 
   test(test_name, exe,
+    args: args,
     env : local_test_env,
     timeout : timeout,
     suite : suite,
     is_parallel : extra_args.get('is_parallel', true),
-    depends : extra_args.get('depends', []),
+    depends : depends,
     should_fail : extra_args.get('should_fail', false),
   )
 endforeach
@@ -937,12 +964,41 @@ foreach program_name, extra_args : test_extra_programs
   )
 endforeach
 
-foreach test_name : python_tests
+foreach test_name, extra_args : python_tests
+  exe = python
+  args = ['-B', files(test_name)]
+  timeout = test_timeout
+
+  if extra_args.get('can_fail', false)
+    if have_sh
+      depends += [exe]
+      args = [
+        '-c',
+        'timeout -k @0@ "@1@" "$@" || exit 77'.format(timeout, exe.full_path()),
+        '--',
+        args
+      ]
+      exe = sh_program
+      timeout = -1
+    elif cmd_exe_program.found()
+      depends += [exe]
+      args = [
+        '/c',
+        '"@0@" "@1@" || exit 77'.format(exe.full_path(), ' '.join(args)),
+      ]
+      exe = cmd_exe_program
+    else
+      warning('Test @0@ cannot be ran reliably, skipping it'.format(test_name))
+      continue
+    endif
+  endif
+
   test(
     test_name,
-    python,
-    args: ['-B', files(test_name)],
+    exe,
+    args: args,
     env: test_env,
+    timeout: timeout,
     suite: ['gio', 'no-valgrind'],
   )
 
diff --git a/glib/tests/meson.build b/glib/tests/meson.build
index 8b5c58b8c5..bf15dae931 100644
--- a/glib/tests/meson.build
+++ b/glib/tests/meson.build
@@ -293,13 +293,42 @@ foreach test_name, extra_args : glib_tests
     install_tag: 'tests',
     install: install,
   )
+  args = extra_args.get('args', [])
+  depends = extra_args.get('depends', [])
 
   suite = ['glib'] + extra_args.get('suite', [])
   timeout = suite.contains('slow') ? test_timeout_slow : test_timeout
+
+  if extra_args.get('can_fail', false)
+    if have_sh
+      depends += [exe]
+      args = [
+        '-c',
+        'timeout -k @0@ "@1@" "$@" || exit 77'.format(timeout, exe.full_path()),
+        '--',
+        args
+      ]
+      exe = sh_program
+      timeout = -1
+    elif cmd_exe_program.found()
+      depends += [exe]
+      args = [
+        '/c',
+        '"@0@" "@1@" || exit 77'.format(exe.full_path(), ' '.join(args)),
+      ]
+      exe = cmd_exe_program
+    else
+      warning('Test @0@ cannot be ran reliably, skipping it'.format(test_name))
+      continue
+    endif
+  endif
+
   test(test_name, exe,
+    args: args,
     env : test_env,
     timeout : timeout,
     suite : suite,
+    depends : depends,
     should_fail : extra_args.get('should_fail', false),
   )
 endforeach
@@ -312,9 +341,9 @@ if installed_tests_enabled
   )
 endif
 
-python_tests = [
-  'assert-msg-test.py',
-]
+python_tests = {
+  'assert-msg-test.py' : {},
+}
 
 executable('assert-msg-test', ['assert-msg-test.c'],
   c_args : test_cargs,
@@ -325,12 +354,41 @@ executable('assert-msg-test', ['assert-msg-test.c'],
   win_subsystem : extra_args.get('win_subsystem', 'console'),
 )
 
-foreach test_name : python_tests
+foreach test_name, extra_args : python_tests
+  exe = python
+  args = ['-B', files(test_name)]
+  timeout = test_timeout
+
+  if extra_args.get('can_fail', false)
+    if have_sh
+      depends += [exe]
+      args = [
+        '-c',
+        'timeout -k @0@ "@1@" "$@" || exit 77'.format(timeout, exe.full_path()),
+        '--',
+        args
+      ]
+      exe = sh_program
+      timeout = -1
+    elif cmd_exe_program.found()
+      depends += [exe]
+      args = [
+        '/c',
+        '"@0@" "@1@" || exit 77'.format(exe.full_path(), ' '.join(args)),
+      ]
+      exe = cmd_exe_program
+    else
+      warning('Test @0@ cannot be ran reliably, skipping it'.format(test_name))
+      continue
+    endif
+  endif
+
   test(
     test_name,
-    python,
-    args: ['-B', files(test_name)],
+    exe,
+    args: args,
     env: test_env,
+    timeout: timeout,
     suite: ['glib', 'no-valgrind'],
   )
 
diff --git a/gmodule/tests/meson.build b/gmodule/tests/meson.build
index a751f3185b..b2e34b296e 100644
--- a/gmodule/tests/meson.build
+++ b/gmodule/tests/meson.build
@@ -92,8 +92,41 @@ foreach test_name, extra_args : gmodule_tests
     install_tag: 'tests',
     install: install,
   )
+  args = extra_args.get('args', [])
+  depends = extra_args.get('depends', [])
 
   suite = ['gmodule'] + extra_args.get('suite', [])
   timeout = suite.contains('slow') ? test_timeout_slow : test_timeout
-  test(test_name, exe, env : test_env, timeout : timeout, suite : suite)
+
+  if extra_args.get('can_fail', false)
+    if have_sh
+      depends += [exe]
+      args = [
+        '-c',
+        'timeout -k @0@ "@1@" "$@" || exit 77'.format(timeout, exe.full_path()),
+        '--',
+        args
+      ]
+      exe = sh_program
+      timeout = -1
+    elif cmd_exe_program.found()
+      depends += [exe]
+      args = [
+        '/c',
+        '"@0@" "@1@" || exit 77'.format(exe.full_path(), ' '.join(args)),
+      ]
+      exe = cmd_exe_program
+    else
+      warning('Test @0@ cannot be ran reliably, skipping it'.format(test_name))
+      continue
+    endif
+  endif
+
+  test(test_name, exe,
+    args: args,
+    env : test_env,
+    timeout : timeout,
+    suite : suite,
+    depends : depends,
+  )
 endforeach
diff --git a/gobject/tests/meson.build b/gobject/tests/meson.build
index 09f23e8bfb..c551647dba 100644
--- a/gobject/tests/meson.build
+++ b/gobject/tests/meson.build
@@ -122,11 +122,11 @@ if cc.get_id() != 'msvc'
   gobject_tests += {'autoptr' : {}}
 endif
 
-python_tests = [
-  'genmarshal.py',
-  'gobject-query.py',
-  'mkenums.py',
-]
+python_tests = {
+  'genmarshal.py' : {},
+  'gobject-query.py' : {},
+  'mkenums.py' : {},
+}
 
 test_env = environment(common_test_env)
 test_env.set('G_TEST_SRCDIR', meson.current_source_dir())
@@ -162,6 +162,8 @@ foreach test_name, extra_args : gobject_tests
     install_tag: 'tests',
     install: install,
   )
+  args = extra_args.get('args', [])
+  depends = extra_args.get('depends', [])
 
   suite = ['gobject'] + extra_args.get('suite', [])
   timeout = suite.contains('slow') ? test_timeout_slow : test_timeout
@@ -172,15 +174,74 @@ foreach test_name, extra_args : gobject_tests
     timeout = timeout * 10
   endif
 
-  test(test_name, exe, env : test_env, timeout : timeout, suite : suite)
+  if extra_args.get('can_fail', false)
+    if have_sh
+      depends += [exe]
+      args = [
+        '-c',
+        'timeout -k @0@ "@1@" "$@" || exit 77'.format(timeout, exe.full_path()),
+        '--',
+        args
+      ]
+      exe = sh_program
+      timeout = -1
+    elif cmd_exe_program.found()
+      depends += [exe]
+      args = [
+        '/c',
+        '"@0@" "@1@" || exit 77'.format(exe.full_path(), ' '.join(args)),
+      ]
+      exe = cmd_exe_program
+    else
+      warning('Test @0@ cannot be ran reliably, skipping it'.format(test_name))
+      continue
+    endif
+  endif
+
+  test(test_name, exe,
+    args: args,
+    env : test_env,
+    timeout : timeout,
+    suite : suite,
+    depends : depends,
+  )
 endforeach
 
-foreach test_name : python_tests
+foreach test_name, extra_args : python_tests
+  exe = python
+  args = ['-B', files(test_name)]
+  timeout = test_timeout
+
+  if extra_args.get('can_fail', false)
+    if have_sh
+      depends += [exe]
+      args = [
+        '-c',
+        'timeout -k @0@ "@1@" "$@" || exit 77'.format(timeout, exe.full_path()),
+        '--',
+        args
+      ]
+      exe = sh_program
+      timeout = -1
+    elif cmd_exe_program.found()
+      depends += [exe]
+      args = [
+        '/c',
+        '"@0@" "@1@" || exit 77'.format(exe.full_path(), ' '.join(args)),
+      ]
+      exe = cmd_exe_program
+    else
+      warning('Test @0@ cannot be ran reliably, skipping it'.format(test_name))
+      continue
+    endif
+  endif
+
   test(
     test_name,
-    python,
-    args: ['-B', files(test_name)],
+    exe,
+    args: args,
     env: test_env,
+    timeout: timeout,
     suite: ['gobject', 'no-valgrind'],
   )
 
diff --git a/meson.build b/meson.build
index ddcdc028d4..9e597f254a 100644
--- a/meson.build
+++ b/meson.build
@@ -2243,7 +2243,9 @@ endif
 # Determine which user environment-dependent files that we want to install
 have_bash = find_program('bash', required : false).found() # For completion scripts
 bash_comp_dep = dependency('bash-completion', version: '>=2.0', required: false)
-have_sh = find_program('sh', required : false).found() # For glib-gettextize
+sh_program = find_program('sh', required : false) # For glib-gettextize
+cmd_exe_program = find_program('cmd.exe', required : false)
+have_sh = sh_program.found()
 
 # Some installed tests require a custom environment
 env_program = find_program('env', required: installed_tests_enabled)