[tracker/experiment-miner-extractor-ipc] Experiment

[Philip Van Hoof <pvanhoof src gnome org>]

commit 707c7aff154f86cfdce423e7ffb22891626221f6
Author: Philip Van Hoof <philip codeminded be>
Date:   Wed Feb 23 13:53:20 2011 +0100

    Experiment

 src/miners/fs/tracker-miner-files.c |  138 +++++++++++++++++++++++++++++++----
 src/miners/fs/tracker-miner-files.h |   33 ++++----
 2 files changed, 141 insertions(+), 30 deletions(-)
---
diff --git a/src/miners/fs/tracker-miner-files.c b/src/miners/fs/tracker-miner-files.c
index ba18202..a3f87d0 100644
--- a/src/miners/fs/tracker-miner-files.c
+++ b/src/miners/fs/tracker-miner-files.c
@@ -100,6 +100,14 @@ typedef struct {
 	gpointer user_data;
 } FastAsyncData;
 
+typedef struct {
+	gchar *uri;
+	gchar *mime_type;
+	GCancellable *cancellable;
+	fast_async_cb callback;
+	ProcessFileData *user_data;
+} ExtractQueueItem;
+
 struct TrackerMinerFilesPrivate {
 	TrackerConfig *config;
 	TrackerStorage *storage;
@@ -133,6 +141,9 @@ struct TrackerMinerFilesPrivate {
 	gboolean mount_points_initialized;
 
 	guint stale_volumes_check_id;
+
+	GPtrArray *extract_queue;
+	guint extract_queue_id;
 };
 
 enum {
@@ -2015,7 +2026,7 @@ extractor_get_embedded_metadata_cb (const gchar *preupdate,
 	}
 
 	/* Notify about the success */
-	tracker_miner_fs_file_notify (TRACKER_MINER_FS (data->miner), data->file, NULL);
+//	tracker_miner_fs_file_notify (TRACKER_MINER_FS (data->miner), data->file, NULL);
 
 	process_file_data_free (data);
 }
@@ -2326,19 +2337,114 @@ get_metadata_fast_async (GDBusConnection *connection,
 }
 
 static void
-extractor_get_embedded_metadata (ProcessFileData *data,
-                                 const gchar     *uri,
-                                 const gchar     *mime_type)
-{
-	get_metadata_fast_async (data->miner->private->connection,
-	                         uri,
-	                         mime_type,
-	                         data->cancellable,
-	                         extractor_get_embedded_metadata_cb,
-	                         data);
+free_queue_item (ExtractQueueItem *item)
+{
+	g_free (item->uri);
+	g_free (item->mime_type);
+	//g_object_unref (item->cancellable);
+	g_slice_free (ExtractQueueItem, item);
+}
+
+static void
+flush_extract_queue_shared (TrackerMinerFiles *miner)
+{
+	guint i;
+	GPtrArray *queue = miner->private->extract_queue;
+	GDBusConnection *connection = miner->private->connection;
+
+	/* TODO: turn this loop into a single array request */
+	//g_warning ("\n\n\n%d items\n\n\n", queue->len);
+	for (i = 0; i < queue->len; i++) {
+		ExtractQueueItem *item = g_ptr_array_index (queue, i);
+
+		get_metadata_fast_async (connection,
+		                         item->uri,
+		                         item->mime_type,
+		                         item->cancellable,
+		                         item->callback,
+		                         item->user_data);
+	}
+
+	g_ptr_array_remove_range (queue, 0, queue->len);
+}
+
+static gboolean
+flush_extract_queue_idle (gpointer user_data)
+{
+	flush_extract_queue_shared (user_data);
+	return FALSE;
+}
+
+static void
+flush_extract_queue_destroy (gpointer user_data)
+{
+	TrackerMinerFiles *miner = user_data;
+	miner->private->extract_queue_id = 0;
+	g_object_unref (miner);
+}
+
+void
+tracker_miner_files_flush_extract_queue (TrackerMinerFiles *miner)
+{
+	flush_extract_queue_shared (miner);
+}
+
+static void
+get_metadata_fast_queue_async (TrackerMinerFiles *miner,
+                               const gchar       *uri,
+                               const gchar       *mime_type,
+                               GCancellable      *cancellable,
+                               fast_async_cb      callback,
+                               ProcessFileData   *user_data)
+{
+	ExtractQueueItem *item;
+
+	if (!miner->private->extract_queue) {
+		miner->private->extract_queue = g_ptr_array_new_with_free_func ((GDestroyNotify) free_queue_item);
+	}
+
+	item = g_slice_new (ExtractQueueItem);
+	item->uri = g_strdup (uri);
+	item->mime_type = g_strdup (mime_type);
+	item->cancellable = cancellable;
+	item->callback = callback;
+	item->user_data = user_data;
+
+	g_ptr_array_add (miner->private->extract_queue, item);
+
+	if (miner->private->extract_queue->len > 5) {
+		flush_extract_queue_shared (miner);
+	} else {
+		if (!miner->private->extract_queue_id) {
+			/* Automatic flush each second */
+			miner->private->extract_queue_id = g_timeout_add_seconds_full (G_PRIORITY_DEFAULT, 10,
+			                                                               flush_extract_queue_idle,
+			                                                               g_object_ref (miner),
+			                                                               flush_extract_queue_destroy);
+		}
+	}
+
+	// fake one here
+	tracker_miner_fs_file_notify (miner, user_data->file, NULL);
+}
+
+static void
+extractor_get_embedded_metadata (TrackerMinerFiles *miner,
+                                 GCancellable      *cancellable,
+                                 const gchar       *uri,
+                                 const gchar       *mime_type,
+                                 gpointer           user_data)
+{
+	get_metadata_fast_queue_async (miner,
+	                               uri,
+	                               mime_type,
+	                               cancellable,
+	                               extractor_get_embedded_metadata_cb,
+	                               user_data);
 
-	g_signal_connect (data->cancellable, "cancelled",
-	                  G_CALLBACK (extractor_get_embedded_metadata_cancel), data);
+	g_signal_connect (cancellable, "cancelled",
+	                  G_CALLBACK (extractor_get_embedded_metadata_cancel),
+	                  user_data);
 }
 
 static void
@@ -2428,7 +2534,11 @@ process_file_cb (GObject      *object,
 	miner_files_add_to_datasource (data->miner, file, sparql);
 
 	/* Next step, getting embedded metadata */
-	extractor_get_embedded_metadata (data, uri, mime_type);
+	extractor_get_embedded_metadata (data->miner,
+	                                 data->cancellable,
+	                                 uri,
+	                                 mime_type,
+	                                 data);
 
 	g_object_unref (file_info);
 	g_free (uri);
diff --git a/src/miners/fs/tracker-miner-files.h b/src/miners/fs/tracker-miner-files.h
index fc651fb..280f7fb 100644
--- a/src/miners/fs/tracker-miner-files.h
+++ b/src/miners/fs/tracker-miner-files.h
@@ -48,24 +48,25 @@ struct TrackerMinerFilesClass {
 
 GType         tracker_miner_files_get_type                 (void) G_GNUC_CONST;
 
-TrackerMiner *tracker_miner_files_new                      (TrackerConfig  *config,
-                                                            GError        **error);
+TrackerMiner *tracker_miner_files_new                      (TrackerConfig     *config,
+                                                            GError           **error);
 
 /* Convenience functions for --eligible tracker-miner-fs cmdline */
-gboolean      tracker_miner_files_check_file               (GFile          *file,
-                                                            GSList         *ignored_file_paths,
-                                                            GSList         *ignored_file_patterns);
-gboolean      tracker_miner_files_check_directory          (GFile          *file,
-                                                            GSList         *index_recursive_directories,
-                                                            GSList         *index_single_directories,
-                                                            GSList         *ignored_directory_paths,
-                                                            GSList         *ignored_directory_patterns);
-gboolean      tracker_miner_files_check_directory_contents (GFile          *parent,
-                                                            GList          *children,
-                                                            GSList         *ignored_content);
-gboolean      tracker_miner_files_monitor_directory        (GFile          *file,
-                                                            gboolean        enable_monitors,
-                                                            GSList         *directories_to_check);
+gboolean      tracker_miner_files_check_file               (GFile             *file,
+                                                            GSList            *ignored_file_paths,
+                                                            GSList            *ignored_file_patterns);
+gboolean      tracker_miner_files_check_directory          (GFile             *file,
+                                                            GSList            *index_recursive_directories,
+                                                            GSList            *index_single_directories,
+                                                            GSList            *ignored_directory_paths,
+                                                            GSList            *ignored_directory_patterns);
+gboolean      tracker_miner_files_check_directory_contents (GFile             *parent,
+                                                            GList             *children,
+                                                            GSList            *ignored_content);
+gboolean      tracker_miner_files_monitor_directory        (GFile             *file,
+                                                            gboolean           enable_monitors,
+                                                            GSList            *directories_to_check);
+void          tracker_miner_files_flush_extract_queue      (TrackerMinerFiles *miner);
 
 G_END_DECLS