[tracker-miners/sam/test-removable-devices: 158/158] functional-tests: Add initial removable media test

[Carlos Garnacho <carlosg src gnome org>]

commit ae8587da83e91e8f6d75e968b8e7071bfab63044
Author: Sam Thursfield <sam afuera me uk>
Date:   Wed Aug 18 14:18:40 2021 +0200

    functional-tests: Add initial removable media test
    
    This is a basic test that removable device indexing works as expected.

 tests/functional-tests/configuration.json.in    |  1 +
 tests/functional-tests/configuration.py         |  1 +
 tests/functional-tests/fixtures.py              | 88 ++++++++++++++++++++++++-
 tests/functional-tests/meson.build              |  2 +
 tests/functional-tests/miner-removable-media.py | 86 ++++++++++++++++++++++++
 5 files changed, 176 insertions(+), 2 deletions(-)
---
diff --git a/tests/functional-tests/configuration.json.in b/tests/functional-tests/configuration.json.in
index 6a45d8eca..0d11fe3fa 100644
--- a/tests/functional-tests/configuration.json.in
+++ b/tests/functional-tests/configuration.json.in
@@ -1,4 +1,5 @@
 {
+    "MOCK_VOLUME_MONITOR_DIR": "@MOCK_VOLUME_MONITOR_DIR@",
     "TEST_CLI_DIR": "@TEST_CLI_DIR@",
     "TEST_CLI_SUBCOMMANDS_DIR": "@TEST_CLI_SUBCOMMANDS_DIR@",
     "TEST_DBUS_DAEMON_CONFIG_FILE": "@TEST_DBUS_DAEMON_CONFIG_FILE@",
diff --git a/tests/functional-tests/configuration.py b/tests/functional-tests/configuration.py
index ec68cc681..983edb371 100644
--- a/tests/functional-tests/configuration.py
+++ b/tests/functional-tests/configuration.py
@@ -45,6 +45,7 @@ TRACKER_EXTRACT_PATH = config['TRACKER_EXTRACT_PATH']
 def test_environment(tmpdir):
     return {
         'DCONF_PROFILE': config['TEST_DCONF_PROFILE'],
+        'GIO_MODULE_DIR': config['MOCK_VOLUME_MONITOR_DIR'],
         'TRACKER_TEST_DOMAIN_ONTOLOGY_RULE': config['TEST_DOMAIN_ONTOLOGY_RULE'],
         'TRACKER_EXTRACTOR_RULES_DIR': config['TEST_EXTRACTOR_RULES_DIR'],
         'TRACKER_EXTRACTORS_DIR': config['TEST_EXTRACTORS_DIR'],
diff --git a/tests/functional-tests/fixtures.py b/tests/functional-tests/fixtures.py
index ab4724d1a..74782636e 100644
--- a/tests/functional-tests/fixtures.py
+++ b/tests/functional-tests/fixtures.py
@@ -103,9 +103,13 @@ class TrackerMinerTest(ut.TestCase):
         }
         return settings
 
-    def setUp(self):
+    def environment(self):
         extra_env = cfg.test_environment(self.workdir)
         extra_env['LANG'] = 'en_GB.utf8'
+        return extra_env
+
+    def setUp(self):
+        extra_env = self.environment()
 
         self.sandbox = trackertestutils.helpers.TrackerDBusSandbox(
             session_bus_config_file=cfg.TEST_DBUS_DAEMON_CONFIG_FILE, extra_env=extra_env)
@@ -150,7 +154,10 @@ class TrackerMinerTest(ut.TestCase):
             self.fail("Resource <%s> should not exist" % urn)
 
     def await_document_inserted(self, path, content=None):
-        """Wraps await_insert() context manager."""
+        """Wraps await_insert() context manager.
+
+        Use this if you are triggering insertion inside the context manager"""
+
         if isinstance(path, pathlib.Path):
             url = path.as_uri()
         else:
@@ -167,6 +174,27 @@ class TrackerMinerTest(ut.TestCase):
 
         return self.tracker.await_insert(DOCUMENTS_GRAPH, '; '.join(expected), timeout=cfg.AWAIT_TIMEOUT)
 
+    def ensure_document_inserted(self, path, content=None):
+        """Block until document is inserted.
+
+        Use this if insertion may already have happened."""
+
+        if isinstance(path, pathlib.Path):
+            url = path.as_uri()
+        else:
+            url = self.uri(path)
+
+        expected = [
+            'a nfo:Document',
+            f'nie:isStoredAs <{url}>',
+        ]
+
+        if content:
+            content_escaped = Tracker.sparql_escape_string(content)
+            expected += [f'nie:plainTextContent "{content_escaped}"']
+
+        return self.tracker.ensure_resource(DOCUMENTS_GRAPH, ';'.join(expected), timeout=cfg.AWAIT_TIMEOUT)
+
     def await_insert_dir(self, path):
         if isinstance(path, pathlib.Path):
             url = path.as_uri()
@@ -265,6 +293,62 @@ class TrackerMinerFTSTest (TrackerMinerTest):
         return int(result[0][0])
 
 
+class TrackerMinerRemovableMediaTest(TrackerMinerTest):
+    """
+    Fixture to test removable device handling in tracker-miner-fs.
+    """
+
+    MOCK_VOLUME_MONITOR_DBUS_NAME = 'org.freedesktop.Tracker3.MockVolumeMonitor'
+    MOCK_VOLUME_MONITOR_OBJECT_PATH = '/org/freedesktop/Tracker3/MockVolumeMonitor'
+    MOCK_VOLUME_MONITOR_IFACE = 'org.freedesktop.Tracker3.MockVolumeMonitor'
+
+    def config(self):
+        settings = super(TrackerMinerRemovableMediaTest, self).config()
+        settings['org.freedesktop.Tracker3.Miner.Files']['index-removable-devices'] = 
GLib.Variant.new_boolean(True)
+        return settings
+
+    def environment(self):
+        extra_env = super(TrackerMinerRemovableMediaTest, self).environment()
+        extra_env['GIO_USE_VOLUME_MONITOR'] = 'mockvolumemonitor'
+        return extra_env
+
+    def add_removable_device(self, path):
+        conn = self.sandbox.get_session_bus_connection()
+        timeout = cfg.AWAIT_TIMEOUT * 1000
+        cancellable = None
+        conn.call_sync(self.MOCK_VOLUME_MONITOR_DBUS_NAME,
+                       self.MOCK_VOLUME_MONITOR_OBJECT_PATH,
+                       self.MOCK_VOLUME_MONITOR_IFACE,
+                       'AddMount',
+                       GLib.Variant('(s)', [self.uri(path)]),
+                       None, Gio.DBusCallFlags.NONE,
+                       timeout, cancellable)
+
+    def remove_removable_device(self, path):
+        conn = self.sandbox.get_session_bus_connection()
+        timeout = cfg.AWAIT_TIMEOUT * 1000
+        cancellable = None
+        conn.call_sync(self.MOCK_VOLUME_MONITOR_DBUS_NAME,
+                       self.MOCK_VOLUME_MONITOR_OBJECT_PATH,
+                       self.MOCK_VOLUME_MONITOR_IFACE,
+                       'RemoveMount',
+                       GLib.Variant('(s)', [self.uri(path)]),
+                       None, Gio.DBusCallFlags.NONE,
+                       timeout, cancellable)
+
+    def await_device_removed(self, device_uri):
+        result = self.tracker.query("""
+            SELECT tracker:id(?u) {
+                <%s> a nfo:FileDataObject ;
+                    nie:interpretedAs/nie:rootElementOf ?u .
+            }""" % device_uri)
+        resource_id = int(result[0][0])
+        return self.tracker.await_property_update(
+            FILESYSTEM_GRAPH, resource_id,
+            "tracker:available true",
+            "tracker:available false")
+
+
 def get_tracker_extract_output(extra_env, filename, output_format='json-ld', mime_type=None):
     """
     Runs `tracker-extract --file` to extract metadata from a file.
diff --git a/tests/functional-tests/meson.build b/tests/functional-tests/meson.build
index b6f2eb1ba..cb4ce00fd 100644
--- a/tests/functional-tests/meson.build
+++ b/tests/functional-tests/meson.build
@@ -9,6 +9,7 @@ config_json_full_path = meson.current_build_dir() / 'configuration.json'
 dconf_profile_full_path = meson.current_source_dir() / 'trackertest'
 tracker_extractors_dir = meson.current_build_dir() / '..' / '..' / 'src' / 'tracker-extract'
 
+testconf.set('MOCK_VOLUME_MONITOR_DIR', meson.current_build_dir() / 'mockvolumemonitor')
 testconf.set('TEST_CLI_DIR', tracker_uninstalled_cli_dir)
 testconf.set('TEST_CLI_SUBCOMMANDS_DIR', tracker_uninstalled_cli_subcommands_dir)
 testconf.set('TEST_DBUS_DAEMON_CONFIG_FILE', build_root / 'tests' / 'test-bus.conf')
@@ -119,6 +120,7 @@ endif
 
 functional_tests = [
   'miner-basic',
+  'miner-removable-media',
   'miner-resource-removal',
   'fts-basic',
   'fts-file-operations',
diff --git a/tests/functional-tests/miner-removable-media.py b/tests/functional-tests/miner-removable-media.py
new file mode 100755
index 000000000..1d301071f
--- /dev/null
+++ b/tests/functional-tests/miner-removable-media.py
@@ -0,0 +1,86 @@
+# Copyright (C) 2021, Codethink Ltd
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+# Boston, MA  02110-1301, USA.
+#
+# Author: Sam Thursfield <sam afuera me uk>
+
+
+import logging
+import pathlib
+
+import fixtures
+
+log = logging.getLogger(__name__)
+
+
+class MinerRemovableMediaTest(fixtures.TrackerMinerRemovableMediaTest):
+    """Tests for tracker-miner-fs with index-removable-devices feature."""
+
+    def setUp(self):
+        super(MinerRemovableMediaTest, self).setUp()
+
+        self.device_path = pathlib.Path(self.workdir).joinpath('removable-device-1')
+        self.device_path.mkdir()
+
+    def __get_text_documents(self):
+        return self.tracker.query("""
+          SELECT DISTINCT ?url WHERE {
+              ?u a nfo:TextDocument ;
+                 nie:isStoredAs/nie:url ?url.
+          }
+          """)
+
+    def data_source_available(self, uri):
+        """Check tracker:available set on the datasource containing `uri`"""
+        result = self.tracker.query("""
+          SELECT tracker:available(?folder) WHERE {
+              <%s> nie:dataSource ?folder
+          }""" % uri)
+        return True if result[0][0] == "true" else False
+
+    def create_test_data(self):
+        files = ["file1.txt", "dir1/file2.txt"]
+
+        for f in files:
+            path = self.device_path.joinpath(f)
+            path.parent.mkdir(parents=True, exist_ok=True)
+            path.write_text("This file exists.")
+
+        return files
+
+    def test_add_remove_device(self):
+        """Device should be indexed by Tracker when connected."""
+
+        files = self.create_test_data()
+
+        self.add_removable_device(self.device_path)
+
+        for f in files:
+            path = self.device_path.joinpath(f)
+            self.ensure_document_inserted(path)
+            assert self.data_source_available(path.as_uri())
+
+        with self.await_device_removed(self.device_path.as_uri()):
+            self.remove_removable_device(self.device_path)
+
+        for f in files:
+            self.ensure_document_inserted(path)
+            assert not self.data_source_available(path.as_uri()), \
+                "Path %s should be marked unavailable" % path.as_uri()
+
+
+if __name__ == "__main__":
+    fixtures.tracker_test_main()