[tracker-miners/wip/jfelder/audio-writeback: 7/8] functional-tests: Rework the writeback test case baseclass

[Sam Thursfield <sthursfield src gnome org>]

commit 7315690fb24eed45b4ee635c73c56e9e6f034a0e
Author: Sam Thursfield <sam afuera me uk>
Date:   Mon Jul 8 16:18:52 2019 +0100

    functional-tests: Rework the writeback test case baseclass
    
    This is in preparation for adding audio writeback test cases.
    
    As well as removing some assumptions about the test data, we now use
    the 'pathlib' module to handle paths. In particular we can properly
    generate URIs from filenames now using the .as_uri() method. This
    does mean we now require Python 3.4 at minimum to run the functional
    tests, but that's a 5 year old release already.

 tests/functional-tests/500-writeback-images.py     | 33 +++++-------
 .../501-writeback-image-details.py                 | 17 +++---
 tests/functional-tests/common/utils/extractor.py   |  2 +-
 .../functional-tests/common/utils/writebacktest.py | 63 ++++++++++------------
 4 files changed, 52 insertions(+), 63 deletions(-)
---
diff --git a/tests/functional-tests/500-writeback-images.py b/tests/functional-tests/500-writeback-images.py
index 662a96e49..25d07718c 100755
--- a/tests/functional-tests/500-writeback-images.py
+++ b/tests/functional-tests/500-writeback-images.py
@@ -27,7 +27,7 @@ import time
 
 from common.utils.extractor import get_tracker_extract_jsonld_output
 from common.utils.helpers import log
-from common.utils.writebacktest import CommonTrackerWritebackTest as CommonTrackerWritebackTest
+from common.utils.writebacktest import CommonTrackerWritebackTest
 import unittest as ut
 
 REASONABLE_TIMEOUT = 5  # Seconds we wait for tracker-writeback to do the work
@@ -50,9 +50,8 @@ class WritebackImagesTest (CommonTrackerWritebackTest):
         the @prop is used.
         """
 
-        # FIXME: filename is actually a URI! :(
-        filename_real = filename[len('file://'):]
-        initial_mtime = os.stat(filename_real).st_mtime
+        path = self.prepare_test_image(self.datadir_path(filename))
+        initial_mtime = path.stat().st_mtime
 
         TEST_VALUE = prop.replace(":", "") + "test"
         SPARQL_TMPL = """
@@ -60,13 +59,13 @@ class WritebackImagesTest (CommonTrackerWritebackTest):
            INSERT { ?u %s '%s' }
            WHERE  { ?u nie:url '%s' }
         """
-        self.tracker.update(SPARQL_TMPL % (prop, filename, prop, prop, TEST_VALUE, filename))
+        self.tracker.update(SPARQL_TMPL % (prop, path.as_uri(), prop, prop, TEST_VALUE, path.as_uri()))
 
-        log("Waiting for change on %s" % filename_real)
-        self.wait_for_file_change(filename_real, initial_mtime)
+        log("Waiting for change on %s" % path)
+        self.wait_for_file_change(path, initial_mtime)
         log("Got the change")
 
-        results = get_tracker_extract_jsonld_output(filename, mimetype)
+        results = get_tracker_extract_jsonld_output(path, mimetype)
         keyDict = expectedKey or prop
         self.assertIn(TEST_VALUE, results[keyDict])
 
@@ -102,12 +101,10 @@ class WritebackImagesTest (CommonTrackerWritebackTest):
     # JPEG test
 
     def test_001_jpeg_title(self):
-        #FILENAME = "test-writeback-monitored/writeback-test-1.jpeg"
-        self.__writeback_test(self.get_test_filename_jpeg(), "image/jpeg", "nie:title")
+        self.__writeback_test("writeback-test-1.jpeg", "image/jpeg", "nie:title")
 
     def test_002_jpeg_description(self):
-        #FILENAME = "test-writeback-monitored/writeback-test-1.jpeg"
-        self.__writeback_test(self.get_test_filename_jpeg(), "image/jpeg", "nie:description")
+        self.__writeback_test("writeback-test-1.jpeg", "image/jpeg", "nie:description")
 
     # def test_003_jpeg_keyword (self):
     #    #FILENAME = "test-writeback-monitored/writeback-test-1.jpeg"
@@ -121,12 +118,10 @@ class WritebackImagesTest (CommonTrackerWritebackTest):
     # TIFF tests
 
     def test_011_tiff_title(self):
-        #FILANAME = "test-writeback-monitored/writeback-test-2.tif"
-        self.__writeback_test(self.get_test_filename_tiff(), "image/tiff", "nie:title")
+        self.__writeback_test("writeback-test-2.tif", "image/tiff", "nie:title")
 
     def test_012_tiff_description(self):
-        FILENAME = "test-writeback-monitored/writeback-test-2.tif"
-        self.__writeback_test(self.get_test_filename_tiff(), "image/tiff", "nie:description")
+        self.__writeback_test("writeback-test-2.tif", "image/tiff", "nie:description")
 
     # def test_013_tiff_keyword (self):
     #    FILENAME = "test-writeback-monitored/writeback-test-2.tif"
@@ -140,12 +135,10 @@ class WritebackImagesTest (CommonTrackerWritebackTest):
     # PNG tests
 
     def test_021_png_title(self):
-        FILENAME = "test-writeback-monitored/writeback-test-4.png"
-        self.__writeback_test(self.get_test_filename_png(), "image/png", "nie:title")
+        self.__writeback_test("writeback-test-4.png", "image/png", "nie:title")
 
     def test_022_png_description(self):
-        FILENAME = "test-writeback-monitored/writeback-test-4.png"
-        self.__writeback_test(self.get_test_filename_png(), "image/png", "nie:description")
+        self.__writeback_test("writeback-test-4.png", "image/png", "nie:description")
 
     # def test_023_png_keyword (self):
     #    FILENAME = "test-writeback-monitored/writeback-test-4.png"
diff --git a/tests/functional-tests/501-writeback-image-details.py 
b/tests/functional-tests/501-writeback-image-details.py
index 3dba70650..9475cb394 100755
--- a/tests/functional-tests/501-writeback-image-details.py
+++ b/tests/functional-tests/501-writeback-image-details.py
@@ -52,6 +52,10 @@ class WritebackKeepDateTest (CommonTrackerWritebackTest):
         """
         NB#217627 - Order if results is different when an image is marked as favorite.
         """
+        jpeg_path = self.prepare_test_image(self.datadir_path('writeback-test-1.jpeg'))
+        tif_path = self.prepare_test_image(self.datadir_path('writeback-test-2.tif'))
+        png_path = self.prepare_test_image(self.datadir_path('writeback-test-4.png'))
+
         query_images = """
           SELECT nie:url(?u) ?contentCreated WHERE {
               ?u a nfo:Visual ;
@@ -64,10 +68,7 @@ class WritebackKeepDateTest (CommonTrackerWritebackTest):
         log("Waiting 2 seconds to ensure there is a noticiable difference in the timestamp")
         time.sleep(2)
 
-        url = self.get_test_filename_jpeg()
-
-        filename = url[len('file://'):]
-        initial_mtime = os.stat(filename).st_mtime
+        initial_mtime = jpeg_path.stat().st_mtime
 
         # This triggers the writeback
         mark_as_favorite = """
@@ -76,14 +77,14 @@ class WritebackKeepDateTest (CommonTrackerWritebackTest):
          } WHERE {
            ?u nie:url <%s> .
          }
-        """ % url
+        """ % jpeg_path.as_uri()
         self.tracker.update(mark_as_favorite)
-        log("Setting favorite in <%s>" % url)
+        log("Setting favorite in <%s>" % jpeg_path.as_uri())
 
-        self.wait_for_file_change(filename, initial_mtime)
+        self.wait_for_file_change(jpeg_path, initial_mtime)
 
         # Check the value is written in the file
-        metadata = get_tracker_extract_jsonld_output(filename, "")
+        metadata = get_tracker_extract_jsonld_output(jpeg_path, "")
 
         tags = metadata.get('nao:hasTag', [])
         tag_names = [tag['nao:prefLabel'] for tag in tags]
diff --git a/tests/functional-tests/common/utils/extractor.py 
b/tests/functional-tests/common/utils/extractor.py
index e58128fa0..015da8ee4 100644
--- a/tests/functional-tests/common/utils/extractor.py
+++ b/tests/functional-tests/common/utils/extractor.py
@@ -40,7 +40,7 @@ def get_tracker_extract_jsonld_output(filename, mime_type=None):
     """
 
     tracker_extract = os.path.join(cfg.TRACKER_EXTRACT_PATH)
-    command = [tracker_extract, '--verbosity=0', '--output-format=json-ld', '--file', filename]
+    command = [tracker_extract, '--verbosity=0', '--output-format=json-ld', '--file', str(filename)]
     if mime_type is not None:
         command.extend(['--mime', mime_type])
 
diff --git a/tests/functional-tests/common/utils/writebacktest.py 
b/tests/functional-tests/common/utils/writebacktest.py
index b1f4f249e..4f128a99e 100644
--- a/tests/functional-tests/common/utils/writebacktest.py
+++ b/tests/functional-tests/common/utils/writebacktest.py
@@ -24,6 +24,7 @@ from common.utils.system import TrackerSystemAbstraction
 import shutil
 import unittest as ut
 import os
+import pathlib
 from common.utils import configuration as cfg
 from common.utils.helpers import log
 import time
@@ -32,8 +33,6 @@ TEST_FILE_JPEG = "writeback-test-1.jpeg"
 TEST_FILE_TIFF = "writeback-test-2.tif"
 TEST_FILE_PNG = "writeback-test-4.png"
 
-NFO_IMAGE = 'http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#Image'
-
 
 class CommonTrackerWritebackTest (ut.TestCase):
     """
@@ -41,14 +40,6 @@ class CommonTrackerWritebackTest (ut.TestCase):
     Start all processes including writeback, miner pointing to WRITEBACK_TMP_DIR
     """
 
-    def __prepare_directories(self):
-        datadir = os.path.join(os.path.dirname(__file__), '..', '..', 'test-writeback-data')
-
-        for testfile in [TEST_FILE_JPEG, TEST_FILE_PNG, TEST_FILE_TIFF]:
-            origin = os.path.join(datadir, testfile)
-            log("Copying %s -> %s" % (origin, self.workdir))
-            shutil.copy(origin, self.workdir)
-
     def setUp(self):
         self.workdir = cfg.create_monitored_test_dir()
 
@@ -66,44 +57,48 @@ class CommonTrackerWritebackTest (ut.TestCase):
             }
         }
 
-        self.__prepare_directories()
-
         self.system = TrackerSystemAbstraction()
         self.system.tracker_writeback_testing_start(CONF_OPTIONS)
 
-        def await_resource_extraction(url):
-            # Make sure a resource has been crawled by the FS miner and by
-            # tracker-extract. The extractor adds nie:contentCreated for
-            # image resources, so know once this property is set the
-            # extraction is complete.
-            self.system.store.await_resource_inserted(NFO_IMAGE, url=url, required_property='nfo:width')
-
-        await_resource_extraction(self.get_test_filename_jpeg())
-        await_resource_extraction(self.get_test_filename_tiff())
-        await_resource_extraction(self.get_test_filename_png())
-
         self.tracker = self.system.store
         self.extractor = self.system.extractor
 
     def tearDown(self):
         self.system.finish()
 
-        for testfile in [TEST_FILE_JPEG, TEST_FILE_PNG, TEST_FILE_TIFF]:
-            os.remove(os.path.join(self.workdir, testfile))
-
+        for test_file in pathlib.Path(self.workdir).iterdir():
+            test_file.unlink()
         cfg.remove_monitored_test_dir(self.workdir)
 
-    def uri(self, filename):
-        return "file://" + os.path.join(self.workdir, filename)
+    def datadir_path(self, filename):
+        """Returns the full path to a writeback test file."""
+        datadir = os.path.join(os.path.dirname(__file__), '..', '..', 'test-writeback-data')
+        return pathlib.Path(os.path.join(datadir, filename))
+
+    def prepare_test_file(self, path, expect_mime_type, expect_property):
+        """Copies a file into the test working directory.
+
+        The function waits until the file has been seen by the Tracker
+        miner before returning.
 
-    def get_test_filename_jpeg(self):
-        return self.uri(TEST_FILE_JPEG)
+        """
+        log("Copying %s -> %s" % (path, self.workdir))
+        shutil.copy(path, self.workdir)
 
-    def get_test_filename_tiff(self):
-        return self.uri(TEST_FILE_TIFF)
+        output_path = pathlib.Path(os.path.join(self.workdir, os.path.basename(path)))
 
-    def get_test_filename_png(self):
-        return self.uri(TEST_FILE_PNG)
+        # Make sure a resource has been crawled by the FS miner and by
+        # tracker-extract. The extractor adds nie:contentCreated for
+        # image resources, so know once this property is set the
+        # extraction is complete.
+        self.system.store.await_resource_inserted(expect_mime_type, url=output_path.as_uri(), 
required_property=expect_property)
+        return output_path
+
+    def prepare_test_audio(self, filename):
+        return self.prepare_test_file(filename, 
'http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#Audio', 'nfo:duration')
+
+    def uri(self, filename):
+        return pathlib.Path(filename).as_uri()
 
     def get_mtime(self, filename):
         return os.stat(filename).st_mtime