[tracker-miners/sam/file-processed-signal: 4/9] miners: Add FilesProcessed signal to D-Bus interface
- From: Sam Thursfield <sthursfield src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [tracker-miners/sam/file-processed-signal: 4/9] miners: Add FilesProcessed signal to D-Bus interface
- Date: Tue, 7 Jul 2020 17:21:33 +0000 (UTC)
commit 1a96521e5991529da07b837c4897cef8314e7829
Author: Sam Thursfield <sam afuera me uk>
Date: Tue Feb 25 01:08:19 2020 +0100
miners: Add FilesProcessed signal to D-Bus interface
This new signal provides more detailed status info from miners, so that
CLI and apps can show the status and reuslts of processing specific files.
Internally, this is exposed as ::file-processed. We batch this signal
before sending it over D-Bus for performance reasons.
The miner-fs ::finished-root signal is removed as it's apparently
unused, and it overlaps with this one.
.../org.freedesktop.Tracker3.Miner.xml | 7 ++-
src/libtracker-miner/tracker-decorator.c | 23 +++++++
src/libtracker-miner/tracker-miner-fs.c | 73 ++++++++--------------
src/libtracker-miner/tracker-miner-fs.h | 8 ---
src/libtracker-miner/tracker-miner-object.c | 51 +++++++++++++++
src/libtracker-miner/tracker-miner-object.h | 10 +++
src/libtracker-miner/tracker-miner-proxy.c | 71 ++++++++++++++++++++-
7 files changed, 185 insertions(+), 58 deletions(-)
---
diff --git a/src/libtracker-miner/org.freedesktop.Tracker3.Miner.xml
b/src/libtracker-miner/org.freedesktop.Tracker3.Miner.xml
index 6fe09c846..47c536287 100644
--- a/src/libtracker-miner/org.freedesktop.Tracker3.Miner.xml
+++ b/src/libtracker-miner/org.freedesktop.Tracker3.Miner.xml
@@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
-<node name="/">
+<node name="/" xmlns:doc="http://www.freedesktop.org/dbus/1.0/doc.dtd";>
<interface name="org.freedesktop.Tracker3.Miner">
<annotation name="org.freedesktop.DBus.GLib.CSymbol" value="_tracker_miner_dbus"/>
<method name="Start">
@@ -52,5 +52,10 @@
<arg type="d" name="progress" />
<arg type="i" name="remaining_time" />
</signal>
+ <signal name='FilesProcessed'>
+ <arg type='a(sbs)' name='files'>
+ <doc:doc><doc:summary>Array of (URI, success, message) for each processed
file.</doc:summary></doc:doc>
+ </arg>
+ </signal>
</interface>
</node>
diff --git a/src/libtracker-miner/tracker-decorator.c b/src/libtracker-miner/tracker-decorator.c
index 8a1674722..3004964a1 100644
--- a/src/libtracker-miner/tracker-decorator.c
+++ b/src/libtracker-miner/tracker-decorator.c
@@ -64,6 +64,7 @@ struct _ClassInfo {
struct _SparqlUpdate {
gchar *sparql;
gint id;
+ GFile *file;
};
struct _TrackerDecoratorPrivate {
@@ -358,9 +359,20 @@ decorator_commit_cb (GObject *object,
SparqlUpdate *update;
update = &g_array_index (priv->commit_buffer, SparqlUpdate, i);
+
+ tracker_miner_file_processed (TRACKER_MINER (decorator), update->file, FALSE,
error->message);
decorator_blocklist_add (decorator, update->id);
item_warn (conn, update->id, update->sparql, error);
}
+ } else {
+ /* Notify success */
+ for (i = 0; i < priv->commit_buffer->len; i++) {
+ SparqlUpdate *update;
+
+ update = &g_array_index (priv->commit_buffer, SparqlUpdate, i);
+
+ tracker_miner_file_processed (TRACKER_MINER (decorator), update->file, TRUE, "");
+ }
}
g_clear_pointer (&priv->commit_buffer, g_array_unref);
@@ -373,6 +385,7 @@ static void
sparql_update_clear (SparqlUpdate *update)
{
g_free (update->sparql);
+ g_object_unref (update->file);
}
static GArray *
@@ -521,26 +534,36 @@ decorator_task_done (GObject *object,
TrackerDecoratorInfo *info = user_data;
TrackerDecoratorPrivate *priv;
GError *error = NULL;
+ GFile *file;
gchar *sparql;
priv = decorator->priv;
sparql = g_task_propagate_pointer (G_TASK (result), &error);
if (!sparql) {
+ file = g_file_new_for_uri (info->url);
+
/* Blocklist item */
decorator_blocklist_add (decorator, info->id);
if (error) {
+ tracker_miner_file_processed (TRACKER_MINER (object), file, FALSE, error->message);
+
g_warning ("Task for '%s' finished with error: %s\n",
info->url, error->message);
g_error_free (error);
+ } else {
+ tracker_miner_file_processed (TRACKER_MINER (object), file, FALSE, "no SPARQL was
generated for this item");
}
+
+ g_object_unref (file);
} else {
SparqlUpdate update;
/* Add resulting sparql to buffer and check whether flushing */
update.sparql = sparql;
update.id = info->id;
+ update.file = g_file_new_for_uri (info->url);
if (!priv->sparql_buffer)
priv->sparql_buffer = sparql_buffer_new ();
diff --git a/src/libtracker-miner/tracker-miner-fs.c b/src/libtracker-miner/tracker-miner-fs.c
index ffe3ea835..d679a7bb8 100644
--- a/src/libtracker-miner/tracker-miner-fs.c
+++ b/src/libtracker-miner/tracker-miner-fs.c
@@ -167,7 +167,6 @@ enum {
PROCESS_FILE,
PROCESS_FILE_ATTRIBUTES,
FINISHED,
- FINISHED_ROOT,
REMOVE_FILE,
REMOVE_CHILDREN,
MOVE_FILE,
@@ -448,31 +447,6 @@ tracker_miner_fs_class_init (TrackerMinerFSClass *klass)
G_TYPE_UINT,
G_TYPE_UINT);
- /**
- * TrackerMinerFS::finished-root:
- * @miner_fs: the #TrackerMinerFS
- * @file: a #GFile
- *
- * The ::finished-crawl signal is emitted when @miner_fs has
- * finished finding all resources that need to be indexed
- * with the root location of @file. At this point, it's likely
- * many are still in the queue to be added to the database,
- * but this gives some indication that a location is
- * processed.
- *
- * Since: 1.2
- **/
- signals[FINISHED_ROOT] =
- g_signal_new ("finished-root",
- G_TYPE_FROM_CLASS (object_class),
- G_SIGNAL_RUN_LAST,
- G_STRUCT_OFFSET (TrackerMinerFSClass, finished_root),
- NULL, NULL,
- NULL,
- G_TYPE_NONE,
- 1,
- G_TYPE_FILE);
-
/**
* TrackerMinerFS::remove-file:
* @miner_fs: the #TrackerMinerFS
@@ -1040,12 +1014,10 @@ notify_roots_finished (TrackerMinerFS *fs,
if (check_queues &&
fs->priv->roots_to_notify &&
g_hash_table_size (fs->priv->roots_to_notify) < 2) {
- /* Technically, if there is only one root, it's
- * pointless to do anything before the FINISHED (not
- * FINISHED_ROOT) signal is emitted. In that
- * situation we calls function first anyway with
- * check_queues=FALSE so we still notify roots. This
- * is really just for efficiency.
+ /* Technically, if there is only one root, it's pointless to do
+ * anything before the FINISHED signal is emitted. In that situation we
+ * calls function first anyway with check_queues=FALSE so we still
+ * notify roots. This is really just for efficiency.
*/
return;
} else if (fs->priv->roots_to_notify == NULL ||
@@ -1068,8 +1040,8 @@ notify_roots_finished (TrackerMinerFS *fs,
continue;
}
- /* Signal root is finished */
- g_signal_emit (fs, signals[FINISHED_ROOT], 0, root);
+ /* Notify completion of the index root */
+ tracker_miner_file_processed (TRACKER_MINER (fs), root, TRUE, NULL);
/* Remove from hash table */
g_hash_table_iter_remove (&iter);
@@ -1180,14 +1152,15 @@ sparql_buffer_task_finished_cb (GObject *object,
task = tracker_sparql_buffer_push_finish (TRACKER_SPARQL_BUFFER (object),
result, &error);
+ task_file = tracker_task_get_file (task);
+
if (error) {
g_critical ("Could not execute sparql: %s", error->message);
+ tracker_miner_file_processed (TRACKER_MINER (fs), task_file, FALSE, error->message);
priv->total_files_notified_error++;
g_error_free (error);
}
- task_file = tracker_task_get_file (task);
-
recursive = GPOINTER_TO_INT (g_object_steal_qdata (G_OBJECT (task_file),
priv->quark_recursive_removal));
tracker_file_notifier_invalidate_file_iri (priv->file_notifier, task_file, recursive);
@@ -1209,6 +1182,14 @@ sparql_buffer_task_finished_cb (GObject *object,
item_queue_handlers_set_up (fs);
}
+ /* Notify completion unless this the root, in which case we wait til
+ * notify_roots_finished() so we know we already notified for all its
+ * children.
+ */
+ if (! tracker_indexing_tree_file_is_root (priv->indexing_tree, task_file)) {
+ tracker_miner_file_processed (TRACKER_MINER (fs), task_file, TRUE, NULL);
+ }
+
tracker_task_unref (task);
}
@@ -1274,7 +1255,7 @@ on_signal_gtask_complete (GObject *source,
uri = g_file_get_uri (file);
if (error) {
- g_message ("Could not process '%s': %s", uri, error->message);
+ tracker_miner_file_processed (TRACKER_MINER (fs), file, FALSE, error->message);
g_error_free (error);
fs->priv->total_files_notified_error++;
@@ -2172,7 +2153,7 @@ file_notifier_directory_finished (TrackerFileNotifier *notifier,
if (directories_found == 0 &&
files_found == 0) {
/* Signal now because we have nothing to index */
- g_signal_emit (fs, signals[FINISHED_ROOT], 0, directory);
+ tracker_miner_file_processed (TRACKER_MINER (fs), directory, TRUE, NULL);
} else {
/* Add root to list we want to be notified about when
* finished indexing! */
@@ -2313,7 +2294,6 @@ tracker_miner_fs_check_file (TrackerMinerFS *fs,
{
gboolean should_process = TRUE;
QueueEvent *event;
- gchar *uri;
g_return_if_fail (TRACKER_IS_MINER_FS (fs));
g_return_if_fail (G_IS_FILE (file));
@@ -2322,24 +2302,21 @@ tracker_miner_fs_check_file (TrackerMinerFS *fs,
should_process = should_check_file (fs, file, FALSE);
}
- uri = g_file_get_uri (file);
-
- TRACKER_NOTE (MINER_FS_EVENTS,
- g_message ("%s:'%s' (FILE) (requested by application)",
- should_process ? "Found " : "Ignored",
- uri));
-
if (should_process) {
if (check_parents && !check_file_parents (fs, file)) {
+ tracker_miner_file_processed (TRACKER_MINER (fs), file, FALSE,
+ "parent directory not eligible for indexing");
+
return;
}
event = queue_event_new (TRACKER_MINER_FS_EVENT_UPDATED, file);
trace_eq_event (event);
miner_fs_queue_event (fs, event, priority);
+ } else {
+ tracker_miner_file_processed (TRACKER_MINER (fs), file, FALSE,
+ "file is ignored by configuration");
}
-
- g_free (uri);
}
/**
diff --git a/src/libtracker-miner/tracker-miner-fs.h b/src/libtracker-miner/tracker-miner-fs.h
index b53a00c17..2386f4b51 100644
--- a/src/libtracker-miner/tracker-miner-fs.h
+++ b/src/libtracker-miner/tracker-miner-fs.h
@@ -78,8 +78,6 @@ struct _TrackerMinerFS {
* @finished: Called when all processing has been performed.
* @process_file_attributes: Called when the metadata associated with
* a file's attributes changes, for example, the mtime.
- * @finished_root: Called when all resources on a particular root URI
- * have been processed.
* @remove_file: Called when a file is removed.
* @remove_children: Called when children have been removed.
* @move_file: Called when a file has moved.
@@ -103,12 +101,6 @@ typedef struct {
gboolean (* process_file_attributes) (TrackerMinerFS *fs,
GFile *file,
GTask *task);
- void (* finished_root) (TrackerMinerFS *fs,
- GFile *root,
- gint directories_found,
- gint directories_ignored,
- gint files_found,
- gint files_ignored);
gchar * (* remove_file) (TrackerMinerFS *fs,
GFile *file);
gchar * (* remove_children) (TrackerMinerFS *fs,
diff --git a/src/libtracker-miner/tracker-miner-object.c b/src/libtracker-miner/tracker-miner-object.c
index 733b54f2a..e4ee6f56c 100644
--- a/src/libtracker-miner/tracker-miner-object.c
+++ b/src/libtracker-miner/tracker-miner-object.c
@@ -84,6 +84,7 @@ enum {
PAUSED,
RESUMED,
PROGRESS,
+ FILE_PROCESSED,
LAST_SIGNAL
};
@@ -231,6 +232,32 @@ tracker_miner_class_init (TrackerMinerClass *klass)
G_TYPE_DOUBLE,
G_TYPE_INT);
+ /**
+ * TrackerMiner::file-processed:
+ * @miner: the #TrackerMiner
+ * @uri: URI of the file that was processed
+ * @status: %FALSE if there was a problem processing this file, %TRUE otherwise
+ * @error: a #GError detailing the error, if any
+ *
+ * each time a file is processed, this signal is emitted and gives status
+ * information. It is useful for notifying the user of any error that
+ * occurred processing a given file. Use of #TrackerNotifier instead if you
+ * want the metadata from the file, not the status info.
+ *
+ * Since: 3.0
+ **/
+ signals[FILE_PROCESSED] =
+ g_signal_new ("file-processed",
+ G_OBJECT_CLASS_TYPE (object_class),
+ G_SIGNAL_RUN_LAST,
+ G_STRUCT_OFFSET (TrackerMinerClass, file_processed),
+ NULL, NULL,
+ NULL,
+ G_TYPE_NONE, 3,
+ G_TYPE_STRING,
+ G_TYPE_BOOLEAN,
+ G_TYPE_STRING);
+
g_object_class_install_property (object_class,
PROP_STATUS,
g_param_spec_string ("status",
@@ -622,6 +649,30 @@ tracker_miner_get_connection (TrackerMiner *miner)
return miner->priv->connection;
}
+void
+tracker_miner_file_processed (TrackerMiner *miner,
+ GFile *file,
+ gboolean success,
+ const gchar *message)
+{
+ gchar *uri;
+
+ g_return_if_fail (G_IS_FILE (file));
+
+ uri = g_file_get_uri (file);
+
+ trace ("(Miner:'%s') File %s processed %s.%s%s",
+ G_OBJECT_TYPE_NAME (miner),
+ uri,
+ success ? "successfully": "unsuccessfully",
+ message && message[0] != '\0' ? " Message: " : "",
+ message ? message : "");
+
+ g_signal_emit (miner, signals[FILE_PROCESSED], 0, uri, success, message);
+
+ g_free (uri);
+}
+
static void
miner_finalize (GObject *object)
{
diff --git a/src/libtracker-miner/tracker-miner-object.h b/src/libtracker-miner/tracker-miner-object.h
index bf26b50dd..1555858cb 100644
--- a/src/libtracker-miner/tracker-miner-object.h
+++ b/src/libtracker-miner/tracker-miner-object.h
@@ -132,6 +132,11 @@ typedef struct {
gdouble progress,
gint remaining_time);
+ void (* file_processed) (TrackerMiner *miner,
+ const gchar *uri,
+ gboolean status,
+ GError *error);
+
/* <Private> */
gpointer padding[10];
} TrackerMinerClass;
@@ -180,6 +185,11 @@ gboolean tracker_miner_resume (TrackerMiner
TrackerSparqlConnection *tracker_miner_get_connection (TrackerMiner *miner);
+void tracker_miner_file_processed (TrackerMiner *miner,
+ GFile *file,
+ gboolean success,
+ const gchar *message);
+
G_END_DECLS
#endif /* __LIBTRACKER_MINER_OBJECT_H__ */
diff --git a/src/libtracker-miner/tracker-miner-proxy.c b/src/libtracker-miner/tracker-miner-proxy.c
index 556384f06..fbbecd55d 100644
--- a/src/libtracker-miner/tracker-miner-proxy.c
+++ b/src/libtracker-miner/tracker-miner-proxy.c
@@ -52,6 +52,9 @@ typedef struct {
gchar *dbus_path;
guint registration_id;
GHashTable *pauses;
+
+ GList *files_processed;
+ guint files_processed_timeout_id;
} TrackerMinerProxyPrivate;
typedef struct {
@@ -76,7 +79,7 @@ G_DEFINE_TYPE_WITH_CODE (TrackerMinerProxy, tracker_miner_proxy, G_TYPE_OBJECT,
G_IMPLEMENT_INTERFACE (G_TYPE_INITABLE, tracker_miner_proxy_initable_iface_init))
static const gchar introspection_xml[] =
- "<node>"
+ "<node xmlns:doc=\"http://www.freedesktop.org/dbus/1.0/doc.dtd\";>"
" <interface name='org.freedesktop.Tracker3.Miner'>"
" <method name='Start'>"
" </method>"
@@ -115,11 +118,18 @@ static const gchar introspection_xml[] =
" <arg type='d' name='progress' />"
" <arg type='i' name='remaining_time' />"
" </signal>"
+ " <signal name='FilesProcessed'>"
+ " <arg type='a(sbs)' name='files'>"
+ " <doc:doc><doc:summary>Array of (URI, success, message) for each processed
file.</doc:summary></doc:doc>"
+ " </arg>"
+ " </signal>"
" </interface>"
"</node>";
#define TRACKER_SERVICE "org.freedesktop.Tracker3"
+#define FILES_PROCESSED_INTERVAL_MSEC 200
+
static PauseData *
pause_data_new (const gchar *application,
const gchar *reason,
@@ -729,6 +739,63 @@ miner_progress_cb (TrackerMiner *miner,
emit_dbus_signal (proxy, "Progress", variant);
}
+static gboolean
+emit_files_processed (gpointer user_data)
+{
+ TrackerMinerProxy *proxy;
+ TrackerMinerProxyPrivate *priv;
+ GVariantBuilder builder;
+ GVariant *variant;
+ GList *l;
+
+ proxy = TRACKER_MINER_PROXY (user_data);
+ priv = tracker_miner_proxy_get_instance_private (proxy);
+
+ /* Assemble the individual GVariant values into a single array */
+ g_variant_builder_init (&builder, G_VARIANT_TYPE ("(a(sbs))"));
+ g_variant_builder_open (&builder, G_VARIANT_TYPE ("a(sbs)"));
+ for (l = g_list_reverse (priv->files_processed); l; l = l->next) {
+ g_variant_builder_add_value (&builder, l->data);
+ }
+ g_variant_builder_close (&builder);
+ variant = g_variant_builder_end (&builder);
+
+ /* Emit the signal. Variant reference is sunk here */
+ emit_dbus_signal (proxy, "FilesProcessed", variant);
+
+ priv->files_processed_timeout_id = 0;
+
+ g_list_free (priv->files_processed);
+ priv->files_processed = NULL;
+
+ return G_SOURCE_REMOVE;
+}
+
+static void
+miner_file_processed_cb (TrackerMiner *miner,
+ const gchar *uri,
+ gboolean success,
+ const gchar *message,
+ TrackerMinerProxy *proxy)
+{
+ TrackerMinerProxyPrivate *priv;
+ GVariant *variant;
+
+ priv = tracker_miner_proxy_get_instance_private (proxy);
+
+ /* For performance reasons, we group the internal signals and only emit
+ * over D-Bus a few times a second.
+ */
+ variant = g_variant_new ("(sbs)", uri, success, message ? message : "");
+
+ priv->files_processed = g_list_prepend (priv->files_processed, variant);
+ if (priv->files_processed_timeout_id == 0) {
+ priv->files_processed_timeout_id = g_timeout_add (FILES_PROCESSED_INTERVAL_MSEC,
+ (GSourceFunc) emit_files_processed,
+ proxy);
+ }
+}
+
static gboolean
tracker_miner_proxy_initable_init (GInitable *initable,
GCancellable *cancellable,
@@ -773,6 +840,8 @@ tracker_miner_proxy_initable_init (GInitable *initable,
G_CALLBACK (miner_resumed_cb), proxy);
g_signal_connect (priv->miner, "progress",
G_CALLBACK (miner_progress_cb), proxy);
+ g_signal_connect (priv->miner, "file-processed",
+ G_CALLBACK (miner_file_processed_cb), proxy);
return TRUE;
}
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
