[tracker-miners/wip/carlosg/miner-files-queues: 6/10] libtracker-miner: Add control for the "high water" hint in TrackerMinerFS

[Carlos Garnacho <carlosg src gnome org>]

commit cf43cf9cfcfcc0056df65862f2d3cae631734766
Author: Carlos Garnacho <carlosg gnome org>
Date:   Thu Dec 16 14:11:03 2021 +0100

    libtracker-miner: Add control for the "high water" hint in TrackerMinerFS
    
    If there are too many files already queued (worth 2 batches), set the hint
    on. This greatly reduces the peak memory used by tracker-miner-fs-3, esp. on
    large filesystems, since that peak used to come from GFileInfos being queued
    up waiting for extraction.

 src/libtracker-miner/tracker-miner-fs.c | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
---
diff --git a/src/libtracker-miner/tracker-miner-fs.c b/src/libtracker-miner/tracker-miner-fs.c
index aaada76e7..024af7348 100644
--- a/src/libtracker-miner/tracker-miner-fs.c
+++ b/src/libtracker-miner/tracker-miner-fs.c
@@ -975,6 +975,19 @@ process_stop (TrackerMinerFS *fs)
        fs->priv->been_crawled = TRUE;
 }
 
+static void
+check_notifier_high_water (TrackerMinerFS *fs)
+{
+       gboolean high_water;
+
+       /* If there is more than worth 2 batches left processing, we can tell
+        * the notifier to stop a bit.
+        */
+       high_water = (tracker_priority_queue_get_length (fs->priv->items) >
+                     2 * fs->priv->sparql_buffer_limit);
+       tracker_file_notifier_set_high_water (fs->priv->file_notifier, high_water);
+}
+
 static void
 sparql_buffer_flush_cb (GObject      *object,
                         GAsyncResult *result,
@@ -1023,6 +1036,7 @@ sparql_buffer_flush_cb (GObject      *object,
                notify_roots_finished (fs);
        }
 
+       check_notifier_high_water (fs);
        item_queue_handlers_set_up (fs);
 
        g_ptr_array_unref (tasks);
@@ -1586,6 +1600,7 @@ miner_fs_queue_event (TrackerMinerFS *fs,
                link = tracker_priority_queue_add (fs->priv->items, event, priority);
                g_hash_table_replace (fs->priv->items_by_file, event->file, link);
                item_queue_handlers_set_up (fs);
+               check_notifier_high_water (fs);
        }
 }