[gnome-continuous-yocto/gnomeostree-3.28-rocko: 7015/8267] oeqa/{core, selftest}: Add support to validate if a specified test case isn't found

[Emmanuele Bassi <ebassi src gnome org>]

commit 2d50f153b5b76adbd4157d2f69001ed91e9148dc
Author: Aníbal Limón <anibal limon linux intel com>
Date:   Wed Jul 26 10:04:09 2017 -0500

    oeqa/{core,selftest}: Add support to validate if a specified test case isn't found
    
    If some test module/case is specified to run and isn't found the OEQA
    framework didn't notice it, so complete the implementation using
    modules_required and validate for the test case prescense.
    
    Raise an exception when the test module/case required isn't found.
    
    [YOCTO #11645]
    
    (From OE-Core rev: e50b415aaaa1581473f85f0a8afa278b5f95129b)
    
    Signed-off-by: Aníbal Limón <anibal limon linux intel com>
    Signed-off-by: Richard Purdie <richard purdie linuxfoundation org>

 meta/lib/oeqa/core/context.py     |    9 +++++++--
 meta/lib/oeqa/core/exception.py   |    3 +++
 meta/lib/oeqa/core/loader.py      |   26 ++++++++++++++++++++++++++
 meta/lib/oeqa/selftest/context.py |    8 ++++++--
 4 files changed, 42 insertions(+), 4 deletions(-)
---
diff --git a/meta/lib/oeqa/core/context.py b/meta/lib/oeqa/core/context.py
index 2d543ff..422e289 100644
--- a/meta/lib/oeqa/core/context.py
+++ b/meta/lib/oeqa/core/context.py
@@ -10,7 +10,7 @@ import collections
 
 from oeqa.core.loader import OETestLoader
 from oeqa.core.runner import OETestRunner
-from oeqa.core.exception import OEQAMissingManifest
+from oeqa.core.exception import OEQAMissingManifest, OEQATestNotFound
 
 class OETestContext(object):
     loaderClass = OETestLoader
@@ -139,6 +139,7 @@ class OETestContextExecutor(object):
 
         if args.run_tests:
             self.tc_kwargs['load']['modules'] = args.run_tests
+            self.tc_kwargs['load']['modules_required'] = args.run_tests
         else:
             self.tc_kwargs['load']['modules'] = []
 
@@ -151,7 +152,11 @@ class OETestContextExecutor(object):
         self._process_args(logger, args)
 
         self.tc = self._context_class(**self.tc_kwargs['init'])
-        self.tc.loadTests(self.module_paths, **self.tc_kwargs['load'])
+        try:
+            self.tc.loadTests(self.module_paths, **self.tc_kwargs['load'])
+        except OEQATestNotFound as ex:
+            logger.error(ex)
+            sys.exit(1)
 
         if args.list_tests:
             rc = self.tc.listTests(args.list_tests, **self.tc_kwargs['run'])
diff --git a/meta/lib/oeqa/core/exception.py b/meta/lib/oeqa/core/exception.py
index a07961a..732f2ef 100644
--- a/meta/lib/oeqa/core/exception.py
+++ b/meta/lib/oeqa/core/exception.py
@@ -18,3 +18,6 @@ class OEQAMissingManifest(OEQAException):
 
 class OEQAPreRun(OEQAException):
     pass
+
+class OEQATestNotFound(OEQAException):
+    pass
diff --git a/meta/lib/oeqa/core/loader.py b/meta/lib/oeqa/core/loader.py
index e4c218b..332086a 100644
--- a/meta/lib/oeqa/core/loader.py
+++ b/meta/lib/oeqa/core/loader.py
@@ -9,6 +9,7 @@ import inspect
 from oeqa.core.utils.path import findFile
 from oeqa.core.utils.test import getSuiteModules, getCaseID
 
+from oeqa.core.exception import OEQATestNotFound
 from oeqa.core.case import OETestCase
 from oeqa.core.decorator import decoratorClasses, OETestDecorator, \
         OETestFilter, OETestDiscover
@@ -277,6 +278,28 @@ class OETestLoader(unittest.TestLoader):
 
         return self.suiteClass(suite)
 
+    def _required_modules_validation(self):
+        """
+            Search in Test context registry if a required
+            test is found, raise an exception when not found.
+        """
+
+        for module in self.modules_required:
+            found = False
+
+            # The module name is splitted to only compare the
+            # first part of a test case id.
+            comp_len = len(module.split('.'))
+            for case in self.tc._registry['cases']:
+                case_comp = '.'.join(case.split('.')[0:comp_len])
+                if module == case_comp:
+                    found = True
+                    break
+
+            if not found:
+                raise OEQATestNotFound("Not found %s in loaded test cases" % \
+                        module)
+
     def discover(self):
         big_suite = self.suiteClass()
         for path in self.module_paths:
@@ -291,6 +314,9 @@ class OETestLoader(unittest.TestLoader):
         for clss in discover_classes:
             cases = clss.discover(self.tc._registry)
 
+        if self.modules_required:
+            self._required_modules_validation()
+
         return self.suiteClass(cases) if cases else big_suite
 
     def _filterModule(self, module):
diff --git a/meta/lib/oeqa/selftest/context.py b/meta/lib/oeqa/selftest/context.py
index 4575a05..990c761 100644
--- a/meta/lib/oeqa/selftest/context.py
+++ b/meta/lib/oeqa/selftest/context.py
@@ -13,7 +13,7 @@ from random import choice
 import oeqa
 
 from oeqa.core.context import OETestContext, OETestContextExecutor
-from oeqa.core.exception import OEQAPreRun
+from oeqa.core.exception import OEQAPreRun, OEQATestNotFound
 
 from oeqa.utils.commands import runCmd, get_bb_vars, get_test_layer
 
@@ -196,7 +196,11 @@ class OESelftestTestContextExecutor(OETestContextExecutor):
                 self.tc_kwargs['init']['td']['BBPATH'].split(':'))
 
         self.tc = self._context_class(**self.tc_kwargs['init'])
-        self.tc.loadTests(self.module_paths, **self.tc_kwargs['load'])
+        try:
+            self.tc.loadTests(self.module_paths, **self.tc_kwargs['load'])
+        except OEQATestNotFound as ex:
+            logger.error(ex)
+            sys.exit(1)
 
         if args.list_tests:
             rc = self.tc.listTests(args.list_tests, **self.tc_kwargs['run'])