[gnome-photos] change-monitor, change-event: Be smarter when resolving new item IDs

[Debarshi Ray <debarshir src gnome org>]

commit 0f73a9a231ef46d7a9d57d447c46cbd94a56143f
Author: Debarshi Ray <debarshir gnome org>
Date:   Sun Jan 5 21:43:24 2014 +0100

    change-monitor, change-event: Be smarter when resolving new item IDs
    
    Instead of immediately firing a separate SPARQL query to resolve each
    incoming ID, we can batch them together and issue a single query per
    batch. This should reduce the workload on the database.
    
    Original patch from Cosimo Cecchi for gnome-documents.
    
    Fixes: https://bugzilla.gnome.org/721402

 src/photos-tracker-change-event.c   |   49 ++++++--
 src/photos-tracker-change-event.h   |   14 ++-
 src/photos-tracker-change-monitor.c |  220 ++++++++++++++++++++++++++---------
 3 files changed, 212 insertions(+), 71 deletions(-)
---
diff --git a/src/photos-tracker-change-event.c b/src/photos-tracker-change-event.c
index d071df2..8d36f2d 100644
--- a/src/photos-tracker-change-event.c
+++ b/src/photos-tracker-change-event.c
@@ -1,6 +1,6 @@
 /*
  * Photos - access, organize and share your photos on GNOME
- * Copyright © 2012 Red Hat, Inc.
+ * Copyright © 2012, 2014 Red Hat, Inc.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -31,7 +31,10 @@
 struct _PhotosTrackerChangeEvent
 {
   PhotosTrackerChangeEventType type;
+  gchar *predicate;
   gchar *urn;
+  gint32 predicate_id;
+  gint32 urn_id;
 };
 
 
@@ -41,28 +44,25 @@ static const gchar *RDF_TYPE = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type";
 void
 photos_tracker_change_event_free (PhotosTrackerChangeEvent *self)
 {
+  g_free (self->predicate);
   g_free (self->urn);
   g_slice_free (PhotosTrackerChangeEvent, self);
 }
 
 
 PhotosTrackerChangeEvent *
-photos_tracker_change_event_new (const gchar *urn, const gchar *predicate, gboolean is_delete)
+photos_tracker_change_event_new (gint32 urn_id, gint32 predicate_id, gboolean is_delete)
 {
   PhotosTrackerChangeEvent *self;
 
   self = g_slice_new0 (PhotosTrackerChangeEvent);
-  self->urn = g_strdup (urn);
+  self->urn_id = urn_id;
+  self->predicate_id = predicate_id;
 
-  if (g_strcmp0 (predicate, RDF_TYPE) == 0)
-    {
-      if (is_delete)
-        self->type = PHOTOS_TRACKER_CHANGE_EVENT_DELETED;
-      else
-        self->type = PHOTOS_TRACKER_CHANGE_EVENT_CREATED;
-    }
+  if (is_delete)
+    self->type = PHOTOS_TRACKER_CHANGE_EVENT_DELETED;
   else
-    self->type = PHOTOS_TRACKER_CHANGE_EVENT_CHANGED;
+    self->type = PHOTOS_TRACKER_CHANGE_EVENT_CREATED;
 
   return self;
 }
@@ -75,6 +75,13 @@ photos_tracker_change_event_get_type (PhotosTrackerChangeEvent *self)
 }
 
 
+gint32
+photos_tracker_change_event_get_predicate_id (PhotosTrackerChangeEvent *self)
+{
+  return self->predicate_id;
+}
+
+
 const gchar *
 photos_tracker_change_event_get_urn (PhotosTrackerChangeEvent *self)
 {
@@ -82,9 +89,29 @@ photos_tracker_change_event_get_urn (PhotosTrackerChangeEvent *self)
 }
 
 
+gint32
+photos_tracker_change_event_get_urn_id (PhotosTrackerChangeEvent *self)
+{
+  return self->urn_id;
+}
+
+
 void
 photos_tracker_change_event_merge (PhotosTrackerChangeEvent *self, PhotosTrackerChangeEvent *event)
 {
   if (event->type == PHOTOS_TRACKER_CHANGE_EVENT_DELETED || event->type == 
PHOTOS_TRACKER_CHANGE_EVENT_CREATED)
     self->type = event->type;
 }
+
+
+void
+photos_tracker_change_event_set_resolved_values (PhotosTrackerChangeEvent *self,
+                                                 const gchar *urn,
+                                                 const gchar *predicate)
+{
+  self->urn = g_strdup (urn);
+  self->predicate = g_strdup (predicate);
+
+  if (g_strcmp0 (predicate, RDF_TYPE) != 0)
+    self->type = PHOTOS_TRACKER_CHANGE_EVENT_CHANGED;
+}
diff --git a/src/photos-tracker-change-event.h b/src/photos-tracker-change-event.h
index 387893b..2ac9bd3 100644
--- a/src/photos-tracker-change-event.h
+++ b/src/photos-tracker-change-event.h
@@ -1,6 +1,6 @@
 /*
  * Photos - access, organize and share your photos on GNOME
- * Copyright © 2012 Red Hat, Inc.
+ * Copyright © 2012, 2014 Red Hat, Inc.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -38,19 +38,27 @@ typedef enum
 
 typedef struct _PhotosTrackerChangeEvent PhotosTrackerChangeEvent;
 
-PhotosTrackerChangeEvent  *photos_tracker_change_event_new        (const gchar *urn,
-                                                                   const gchar *predicate,
+PhotosTrackerChangeEvent  *photos_tracker_change_event_new        (gint32 urn_id,
+                                                                   gint32 predicate_id,
                                                                    gboolean is_delete);
 
 void                       photos_tracker_change_event_free       (PhotosTrackerChangeEvent *self);
 
 PhotosTrackerChangeEventType photos_tracker_change_event_get_type (PhotosTrackerChangeEvent *self);
 
+gint32                     photos_tracker_change_event_get_predicate_id (PhotosTrackerChangeEvent *self);
+
 const gchar               *photos_tracker_change_event_get_urn    (PhotosTrackerChangeEvent *self);
 
+gint32                     photos_tracker_change_event_get_urn_id (PhotosTrackerChangeEvent *self);
+
 void                       photos_tracker_change_event_merge      (PhotosTrackerChangeEvent *self,
                                                                    PhotosTrackerChangeEvent *event);
 
+void                       photos_tracker_change_event_set_resolved_values (PhotosTrackerChangeEvent *self,
+                                                                            const gchar *urn,
+                                                                            const gchar *predicate);
+
 G_END_DECLS
 
 #endif /* PHOTOS_TRACKER_CHANGE_EVENT_H */
diff --git a/src/photos-tracker-change-monitor.c b/src/photos-tracker-change-monitor.c
index 3bca59d..f1c13ee 100644
--- a/src/photos-tracker-change-monitor.c
+++ b/src/photos-tracker-change-monitor.c
@@ -1,6 +1,6 @@
 /*
  * Photos - access, organize and share your photos on GNOME
- * Copyright © 2012, 2013 Red Hat, Inc.
+ * Copyright © 2012, 2013, 2014 Red Hat, Inc.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -38,9 +38,12 @@
 struct _PhotosTrackerChangeMonitorPrivate
 {
   GHashTable *pending;
+  GHashTable *unresolved_ids;
+  GQueue *pending_events;
   PhotosTrackerQueue *queue;
   TrackerResources *resource_service;
   guint outstanding_ops;
+  guint pending_events_id;
 };
 
 enum
@@ -59,13 +62,21 @@ G_DEFINE_TYPE_WITH_CODE (PhotosTrackerChangeMonitor, photos_tracker_change_monit
                          G_IMPLEMENT_INTERFACE (G_TYPE_INITABLE, 
photos_tracker_change_monitor_initable_iface_init));
 
 
+enum
+{
+  CHANGE_MONITOR_TIMEOUT = 500, /* ms */
+  CHANGE_MONITOR_MAX_ITEMS = 500
+};
+
+
 typedef struct _PhotosTrackerChangeMonitorQueryData PhotosTrackerChangeMonitorQueryData;
 typedef struct _TrackerResourcesEvent TrackerResourcesEvent;
 
 struct _PhotosTrackerChangeMonitorQueryData
 {
   PhotosTrackerChangeMonitor *self;
-  gboolean is_delete;
+  GHashTable *id_table;
+  GQueue *events;
 };
 
 struct _TrackerResourcesEvent
@@ -83,8 +94,14 @@ photos_tracker_change_monitor_query_data_copy (PhotosTrackerChangeMonitorQueryDa
   PhotosTrackerChangeMonitorQueryData *copy;
 
   copy = g_slice_new0 (PhotosTrackerChangeMonitorQueryData);
+
   copy->self = g_object_ref (data->self);
-  copy->is_delete = data->is_delete;
+
+  copy->id_table = data->id_table;
+  data->id_table = NULL;
+
+  copy->events = data->events;
+  data->events = NULL;
 
   return copy;
 }
@@ -94,54 +111,76 @@ static void
 photos_tracker_change_monitor_query_data_free (PhotosTrackerChangeMonitorQueryData *data)
 {
   g_clear_object (&data->self);
+
+  if (data->id_table != NULL)
+    g_hash_table_unref (data->id_table);
+
+  if (data->events != NULL)
+    g_queue_free (data->events);
+
   g_slice_free (PhotosTrackerChangeMonitorQueryData, data);
 }
 
 
 static PhotosTrackerChangeMonitorQueryData *
-photos_tracker_change_monitor_query_data_new (PhotosTrackerChangeMonitor *self, gboolean is_delete)
+photos_tracker_change_monitor_query_data_new (PhotosTrackerChangeMonitor *self,
+                                              GHashTable *id_table,
+                                              GQueue *events)
 {
   PhotosTrackerChangeMonitorQueryData *data;
 
   data = g_slice_new0 (PhotosTrackerChangeMonitorQueryData);
   data->self = g_object_ref (self);
-  data->is_delete = is_delete;
+  data->id_table = id_table;
+  data->events = events;
 
   return data;
 }
 
 
 static void
-photos_tracker_change_monitor_add_event (PhotosTrackerChangeMonitor *self,
-                                         const gchar *subject,
-                                         const gchar *predicate,
-                                         gboolean is_delete)
+photos_tracker_change_monitor_add_event (PhotosTrackerChangeMonitor *self, PhotosTrackerChangeEvent 
*change_event)
 {
   PhotosTrackerChangeMonitorPrivate *priv = self->priv;
-  PhotosTrackerChangeEvent *event;
-  PhotosTrackerChangeEvent *old_event;
+  PhotosTrackerChangeEvent *old_change_event;
+  const gchar *urn;
 
-  event = photos_tracker_change_event_new (subject, predicate, is_delete);
-  old_event = (PhotosTrackerChangeEvent *) g_hash_table_lookup (priv->pending, subject);
+  urn = photos_tracker_change_event_get_urn (change_event);
+  old_change_event = (PhotosTrackerChangeEvent *) g_hash_table_lookup (priv->pending, urn);
 
-  if (old_event != NULL)
+  if (old_change_event != NULL)
     {
-      photos_tracker_change_event_merge (old_event, event);
-      photos_tracker_change_event_free (event);
+      photos_tracker_change_event_merge (old_change_event, change_event);
+      photos_tracker_change_event_free (change_event);
     }
   else
-    g_hash_table_insert (priv->pending, (gpointer) g_strdup (subject), (gpointer) event);
+    g_hash_table_insert (priv->pending, g_strdup (urn), change_event);
 }
 
 
 static void
-photos_tracker_change_monitor_update_collector (PhotosTrackerChangeMonitor *self)
+photos_tracker_change_monitor_send_events (PhotosTrackerChangeMonitor *self, GHashTable *id_table, GQueue 
*events)
 {
   PhotosTrackerChangeMonitorPrivate *priv = self->priv;
+  GList *l;
+
+  for (l = events->head; l != NULL; l = l->next)
+    {
+      PhotosTrackerChangeEvent *change_event = (PhotosTrackerChangeEvent *) l->data;
+      const gchar *predicate;
+      const gchar *urn;
+      gint32 predicate_id;
+      gint32 urn_id;
+
+      predicate_id = photos_tracker_change_event_get_predicate_id (change_event);
+      urn_id = photos_tracker_change_event_get_urn_id (change_event);
 
-  priv->outstanding_ops--;
-  if (priv->outstanding_ops != 0)
-    return;
+      predicate = (gchar *) g_hash_table_lookup (id_table, GINT_TO_POINTER (predicate_id));
+      urn = (gchar *) g_hash_table_lookup (id_table, GINT_TO_POINTER (urn_id));
+
+      photos_tracker_change_event_set_resolved_values (change_event, urn, predicate);
+      photos_tracker_change_monitor_add_event (self, change_event); /* steals change_event */
+    }
 
   g_signal_emit (self, signals[CHANGES_PENDING], 0, priv->pending);
   g_hash_table_remove_all (priv->pending);
@@ -152,22 +191,37 @@ static void
 photos_tracker_change_monitor_cursor_next (GObject *source_object, GAsyncResult *res, gpointer user_data)
 {
   PhotosTrackerChangeMonitorQueryData *data = (PhotosTrackerChangeMonitorQueryData *) user_data;
+  PhotosTrackerChangeMonitor *self = data->self;
   TrackerSparqlCursor *cursor = TRACKER_SPARQL_CURSOR (source_object);
+  GError *error;
+  GHashTableIter iter;
   gboolean valid;
 
-  valid = tracker_sparql_cursor_next_finish (cursor, res, NULL);
+  error = NULL;
+  valid = tracker_sparql_cursor_next_finish (cursor, res, &error);
+  if (error != NULL)
+    {
+      g_warning ("Unable to resolve item URNs for graph changes: %s", error->message);
+      g_error_free (error);
+    }
+
   if (valid)
     {
-      const gchar *predicate;
-      const gchar *subject;
+      guint idx;
 
-      subject = tracker_sparql_cursor_get_string (cursor, 0, NULL);
-      predicate = tracker_sparql_cursor_get_string (cursor, 1, NULL);
-      if (subject != NULL && predicate != NULL)
-        photos_tracker_change_monitor_add_event (data->self, subject, predicate, data->is_delete);
-    }
+      idx = 0;
+      g_hash_table_iter_init (&iter, data->id_table);
+      while (g_hash_table_iter_next (&iter, NULL, NULL))
+        {
+          const gchar *str;
 
-  photos_tracker_change_monitor_update_collector (data->self);
+          str = tracker_sparql_cursor_get_string (cursor, idx, NULL);
+          g_hash_table_iter_replace (&iter, g_strdup (str));
+          idx++;
+        }
+
+      photos_tracker_change_monitor_send_events (self, data->id_table, data->events);
+    }
 
   tracker_sparql_cursor_close (cursor);
   photos_tracker_change_monitor_query_data_free (data);
@@ -186,6 +240,7 @@ photos_tracker_change_monitor_query_executed (GObject *source_object, GAsyncResu
   cursor = tracker_sparql_connection_query_finish (connection, res, &error);
   if (error != NULL)
     {
+      g_warning ("Unable to resolve item URNs for graph changes: %s", error->message);
       g_error_free (error);
       return;
     }
@@ -198,27 +253,71 @@ photos_tracker_change_monitor_query_executed (GObject *source_object, GAsyncResu
 }
 
 
-static void
-photos_tracker_change_monitor_update_iterator (PhotosTrackerChangeMonitor *self,
-                                               const TrackerResourcesEvent *event,
-                                               gboolean is_delete)
+static gboolean
+photos_tracker_change_monitor_process_events (PhotosTrackerChangeMonitor *self)
 {
+  PhotosTrackerChangeMonitorPrivate *priv = self->priv;
+  GHashTable *id_table;
+  GHashTableIter iter;
+  GQueue *events;
+  GString *sparql;
   PhotosTrackerChangeMonitorQueryData *data;
-  gchar *sparql;
+  gpointer id;
+
+  events = priv->pending_events;
+  priv->pending_events = g_queue_new ();
+
+  id_table = priv->unresolved_ids;
+  priv->unresolved_ids = g_hash_table_new_full (g_direct_hash, g_direct_equal, NULL, g_free);
+
+  priv->pending_events_id = 0;
 
-  sparql = g_strdup_printf ("SELECT tracker:uri(%" G_GINT32_FORMAT ") tracker:uri(%" G_GINT32_FORMAT ") {}",
-                            event->second,
-                            event->third);
+  sparql = g_string_new ("SELECT");
 
-  data = photos_tracker_change_monitor_query_data_new (self, is_delete);
+  g_hash_table_iter_init (&iter, id_table);
+  while (g_hash_table_iter_next (&iter, &id, NULL))
+    g_string_append_printf (sparql, " tracker:uri(%d)", GPOINTER_TO_INT (id));
 
-  photos_tracker_queue_select (self->priv->queue,
-                               sparql,
+  g_string_append (sparql, " {}");
+
+  data = photos_tracker_change_monitor_query_data_new (self, id_table, events);
+  photos_tracker_queue_select (priv->queue,
+                               sparql->str,
                                NULL,
                                photos_tracker_change_monitor_query_executed,
                                data,
                                (GDestroyNotify) photos_tracker_change_monitor_query_data_free);
-  g_free (sparql);
+
+  g_string_free (sparql, TRUE);
+  return G_SOURCE_REMOVE;
+}
+
+
+static void
+photos_tracker_change_monitor_add_pending_event (PhotosTrackerChangeMonitor *self,
+                                                 const TrackerResourcesEvent *event,
+                                                 gboolean is_delete)
+{
+  PhotosTrackerChangeMonitorPrivate *priv = self->priv;
+  PhotosTrackerChangeEvent *change_event;
+
+  if (priv->pending_events_id != 0)
+    g_source_remove (priv->pending_events_id);
+
+  g_hash_table_insert (priv->unresolved_ids, GINT_TO_POINTER (event->second), NULL);
+  g_hash_table_insert (priv->unresolved_ids, GINT_TO_POINTER (event->third), NULL);
+
+  change_event = photos_tracker_change_event_new (event->second, event->third, is_delete);
+  g_queue_push_tail (priv->pending_events, change_event);
+
+  if (priv->pending_events->length >= CHANGE_MONITOR_MAX_ITEMS)
+    photos_tracker_change_monitor_process_events (self);
+  else
+    priv->pending_events_id = g_timeout_add_full (G_PRIORITY_DEFAULT,
+                                                  CHANGE_MONITOR_TIMEOUT,
+                                                  (GSourceFunc) photos_tracker_change_monitor_process_events,
+                                                  g_object_ref (self),
+                                                  g_object_unref);
 }
 
 
@@ -230,7 +329,6 @@ photos_tracker_change_monitor_graph_updated (TrackerResources *resource_service,
                                              gpointer user_data)
 {
   PhotosTrackerChangeMonitor *self = PHOTOS_TRACKER_CHANGE_MONITOR (user_data);
-  PhotosTrackerChangeMonitorPrivate *priv = self->priv;
   const TrackerResourcesEvent *events;
   gsize i;
   gsize n_elements;
@@ -239,19 +337,13 @@ photos_tracker_change_monitor_graph_updated (TrackerResources *resource_service,
                                                                       &n_elements,
                                                                       sizeof (TrackerResourcesEvent));
   for (i = 0; i < n_elements; i++)
-    {
-      priv->outstanding_ops++;
-      photos_tracker_change_monitor_update_iterator (self, &events[i], TRUE);
-    }
+    photos_tracker_change_monitor_add_pending_event (self, &events[i], TRUE);
 
   events = (const TrackerResourcesEvent *) g_variant_get_fixed_array (insert_events,
                                                                       &n_elements,
                                                                       sizeof (TrackerResourcesEvent));
   for (i = 0; i < n_elements; i++)
-    {
-      priv->outstanding_ops++;
-      photos_tracker_change_monitor_update_iterator (self, &events[i], FALSE);
-    }
+    photos_tracker_change_monitor_add_pending_event (self, &events[i], FALSE);
 }
 
 
@@ -281,12 +373,6 @@ photos_tracker_change_monitor_dispose (GObject *object)
   PhotosTrackerChangeMonitor *self = PHOTOS_TRACKER_CHANGE_MONITOR (object);
   PhotosTrackerChangeMonitorPrivate *priv = self->priv;
 
-  if (priv->pending != NULL)
-    {
-      g_hash_table_unref (priv->pending);
-      priv->pending = NULL;
-    }
-
   g_clear_object (&priv->queue);
   g_clear_object (&priv->resource_service);
 
@@ -295,6 +381,21 @@ photos_tracker_change_monitor_dispose (GObject *object)
 
 
 static void
+photos_tracker_change_monitor_finalize (GObject *object)
+{
+  PhotosTrackerChangeMonitor *self = PHOTOS_TRACKER_CHANGE_MONITOR (object);
+  PhotosTrackerChangeMonitorPrivate *priv = self->priv;
+
+  g_hash_table_unref (priv->pending);
+  g_hash_table_unref (priv->unresolved_ids);
+
+  g_queue_free_full (priv->pending_events, (GDestroyNotify) photos_tracker_change_event_free);
+
+  G_OBJECT_CLASS (photos_tracker_change_monitor_parent_class)->finalize (object);
+}
+
+
+static void
 photos_tracker_change_monitor_init (PhotosTrackerChangeMonitor *self)
 {
   PhotosTrackerChangeMonitorPrivate *priv;
@@ -306,6 +407,10 @@ photos_tracker_change_monitor_init (PhotosTrackerChangeMonitor *self)
                                          g_str_equal,
                                          g_free,
                                          (GDestroyNotify) photos_tracker_change_event_free);
+
+  priv->unresolved_ids = g_hash_table_new_full (g_direct_hash, g_direct_equal, NULL, g_free);
+
+  priv->pending_events = g_queue_new ();
 }
 
 
@@ -316,6 +421,7 @@ photos_tracker_change_monitor_class_init (PhotosTrackerChangeMonitorClass *class
 
   object_class->constructor = photos_tracker_change_monitor_constructor;
   object_class->dispose = photos_tracker_change_monitor_dispose;
+  object_class->finalize = photos_tracker_change_monitor_finalize;
 
   signals[CHANGES_PENDING] = g_signal_new ("changes-pending",
                                            G_TYPE_FROM_CLASS (class),