[tracker/external-crawler] libtracker-miner: Added TrackerCrawlFlags to disable stat() calls
- From: Martyn James Russell <mr src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [tracker/external-crawler] libtracker-miner: Added TrackerCrawlFlags to disable stat() calls
- Date: Tue, 1 Jul 2014 13:29:07 +0000 (UTC)
commit b956ba5e06aa470601e712e07a46e285440dc1c2
Author: Martyn Russell <martyn lanedo com>
Date: Tue Jul 1 14:26:22 2014 +0100
libtracker-miner: Added TrackerCrawlFlags to disable stat() calls
This also allows for future improvements like following symlinks, which there
is interest in.
The flags are attached to the TrackerEnumerator so each enumerator
src/libtracker-miner/Makefile.am | 2 +-
src/libtracker-miner/tracker-crawler.c | 25 +++++--
src/libtracker-miner/tracker-crawler.h | 2 +-
src/libtracker-miner/tracker-enumerator.c | 46 +++++++++++++
src/libtracker-miner/tracker-enumerator.h | 83 ++++++++++++++----------
src/libtracker-miner/tracker-file-enumerator.c | 25 +++++++
src/libtracker-miner/tracker-miner-enums.h | 22 ++++++
7 files changed, 161 insertions(+), 44 deletions(-)
---
diff --git a/src/libtracker-miner/Makefile.am b/src/libtracker-miner/Makefile.am
index e65cb76..8556350 100644
--- a/src/libtracker-miner/Makefile.am
+++ b/src/libtracker-miner/Makefile.am
@@ -102,7 +102,7 @@ if !ENABLE_GCOV
# Using enable_gcov instead of have_unit_test because when doing a release
# we disable gcov but NOT the unit tests
libtracker_miner_ TRACKER_API_VERSION@_la_LDFLAGS += \
- -export-symbols-regex
'^tracker_(enumerator|miner|indexing_tree|directory_flags|filter_type|filter_policy|network_type|decorator)_.*'
+ -export-symbols-regex
'^tracker_(enumerator|crawl|miner|indexing_tree|directory_flags|filter_type|filter_policy|network_type|decorator)_.*'
endif
libtracker_miner_ TRACKER_API_VERSION@_la_LIBADD = \
diff --git a/src/libtracker-miner/tracker-crawler.c b/src/libtracker-miner/tracker-crawler.c
index 49cab90..11b98f6 100644
--- a/src/libtracker-miner/tracker-crawler.c
+++ b/src/libtracker-miner/tracker-crawler.c
@@ -21,6 +21,8 @@
#include "tracker-crawler.h"
#include "tracker-file-enumerator.h"
+#include "tracker-miner-enums.h"
+#include "tracker-miner-enum-types.h"
#include "tracker-utils.h"
#define TRACKER_CRAWLER_GET_PRIVATE(obj) (G_TYPE_INSTANCE_GET_PRIVATE ((obj), TRACKER_TYPE_CRAWLER,
TrackerCrawlerPrivate))
@@ -111,7 +113,7 @@ enum {
enum {
PROP_0,
- PROP_ENUMERATOR
+ PROP_ENUMERATOR,
};
static void crawler_get_property (GObject *object,
@@ -436,12 +438,14 @@ directory_processing_data_add_child (DirectoryProcessingData *data,
}
static DirectoryRootInfo *
-directory_root_info_new (GFile *file,
- gint max_depth,
- gchar *file_attributes)
+directory_root_info_new (GFile *file,
+ gint max_depth,
+ gchar *file_attributes,
+ TrackerCrawlFlags flags)
{
DirectoryRootInfo *info;
DirectoryProcessingData *dir_info;
+ gboolean enable_stat;
info = g_slice_new0 (DirectoryRootInfo);
@@ -451,7 +455,9 @@ directory_root_info_new (GFile *file,
info->tree = g_node_new (g_object_ref (file));
- if (file_attributes) {
+ enable_stat = (flags & TRACKER_CRAWL_FLAG_NO_STAT) == 0;
+
+ if (enable_stat && file_attributes) {
GFileInfo *file_info;
file_info = g_file_query_info (file,
@@ -815,14 +821,19 @@ tracker_crawler_start (TrackerCrawler *crawler,
gint max_depth)
{
TrackerCrawlerPrivate *priv;
+ TrackerCrawlFlags flags;
DirectoryRootInfo *info;
+ gboolean enable_stat;
g_return_val_if_fail (TRACKER_IS_CRAWLER (crawler), FALSE);
g_return_val_if_fail (G_IS_FILE (file), FALSE);
priv = crawler->priv;
- if (!g_file_query_exists (file, NULL)) {
+ flags = tracker_enumerator_get_crawl_flags (priv->enumerator);
+ enable_stat = (flags & TRACKER_CRAWL_FLAG_NO_STAT) == 0;
+
+ if (enable_stat && !g_file_query_exists (file, NULL)) {
/* This shouldn't happen, unless the removal/unmount notification
* didn't yet reach the TrackerFileNotifier.
*/
@@ -846,7 +857,7 @@ tracker_crawler_start (TrackerCrawler *crawler,
priv->is_running = TRUE;
priv->is_finished = FALSE;
- info = directory_root_info_new (file, max_depth, priv->file_attributes);
+ info = directory_root_info_new (file, max_depth, priv->file_attributes, flags);
if (!check_directory (crawler, info, file)) {
directory_root_info_free (info);
diff --git a/src/libtracker-miner/tracker-crawler.h b/src/libtracker-miner/tracker-crawler.h
index 5ec992e..5282e54 100644
--- a/src/libtracker-miner/tracker-crawler.h
+++ b/src/libtracker-miner/tracker-crawler.h
@@ -32,7 +32,7 @@
G_BEGIN_DECLS
#define TRACKER_TYPE_CRAWLER (tracker_crawler_get_type ())
-#define TRACKER_CRAWLER(object) (G_TYPE_CHECK_INSTANCE_CAST ((object), TRACKER_TYPE_CRAWLER,
TrackerCrawler))
+#define TRACKER_CRAWLER(object) (G_TYPE_CHECK_INSTANCE_CAST ((object), TRACKER_TYPE_CRAWLER,
TrackerCrawler))
#define TRACKER_CRAWLER_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST ((klass), TRACKER_TYPE_CRAWLER,
TrackerCrawlerClass))
#define TRACKER_IS_CRAWLER(object) (G_TYPE_CHECK_INSTANCE_TYPE ((object), TRACKER_TYPE_CRAWLER))
#define TRACKER_IS_CRAWLER_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE ((klass), TRACKER_TYPE_CRAWLER))
diff --git a/src/libtracker-miner/tracker-enumerator.c b/src/libtracker-miner/tracker-enumerator.c
index cc84849..233ca7f 100644
--- a/src/libtracker-miner/tracker-enumerator.c
+++ b/src/libtracker-miner/tracker-enumerator.c
@@ -81,6 +81,52 @@ tracker_enumerator_default_init (TrackerEnumeratorInterface *iface)
}
/**
+ * tracker_enumerator_get_crawl_flags:
+ * @enumerator: a #TrackerEnumerator
+ *
+ * Returns the #TrackerCrawlFlags used with this @enumerator.
+ *
+ * Since: 1.2
+ **/
+TrackerCrawlFlags
+tracker_enumerator_get_crawl_flags (TrackerEnumerator *enumerator)
+{
+ TrackerEnumeratorIface *iface;
+
+ g_return_val_if_fail (TRACKER_IS_ENUMERATOR (enumerator), TRACKER_CRAWL_FLAG_NONE);
+
+ iface = TRACKER_ENUMERATOR_GET_IFACE (enumerator);
+
+ g_return_val_if_fail (iface->get_crawl_flags != NULL, TRACKER_CRAWL_FLAG_NONE);
+
+ return (* iface->get_crawl_flags) (enumerator);
+}
+
+/**
+ * tracker_enumerator_set_crawl_flags:
+ * @enumerator: a #TrackerEnumerator
+ *
+ * Sets the #TrackerCrawlFlags used with this @enumerator. Each time
+ * tracker_enumerator_get_children() is called, these flags are used.
+ *
+ * Since: 1.2
+ **/
+void
+tracker_enumerator_set_crawl_flags (TrackerEnumerator *enumerator,
+ TrackerCrawlFlags flags)
+{
+ TrackerEnumeratorIface *iface;
+
+ g_return_if_fail (TRACKER_IS_ENUMERATOR (enumerator));
+
+ iface = TRACKER_ENUMERATOR_GET_IFACE (enumerator);
+
+ g_return_if_fail (iface->set_crawl_flags != NULL);
+
+ (* iface->set_crawl_flags) (enumerator, flags);
+}
+
+/**
* tracker_enumerator_get_children:
* @enumerator: a #TrackerEnumerator
* @dir: a #GFile to enumerate
diff --git a/src/libtracker-miner/tracker-enumerator.h b/src/libtracker-miner/tracker-enumerator.h
index 46a05f9..3fc6eb5 100644
--- a/src/libtracker-miner/tracker-enumerator.h
+++ b/src/libtracker-miner/tracker-enumerator.h
@@ -28,6 +28,8 @@
#include <gio/gio.h>
+#include "tracker-miner-enums.h"
+
G_BEGIN_DECLS
#define TRACKER_TYPE_ENUMERATOR (tracker_enumerator_get_type ())
@@ -46,6 +48,10 @@ typedef struct _TrackerEnumeratorIface TrackerEnumeratorIface;
/**
* TrackerEnumeratorIface:
* @g_iface: Parent interface type.
+ * @get_crawl_flags: Called when before enumerator starts to know how
+ * to enumerate.
+ * @set_crawl_flags: Called when setting the flags an enumerator
+ * should use.
* @get_children: Called when the enumerator is synchronously
* retrieving children.
* @get_children_async: Called when the enumerator is asynchronously
@@ -59,23 +65,26 @@ struct _TrackerEnumeratorIface {
GTypeInterface g_iface;
/* Virtual Table */
- GSList * (* get_children) (TrackerEnumerator *enumerator,
- GFile *dir,
- const gchar *attributes,
- GFileQueryInfoFlags flags,
- GCancellable *cancellable,
- GError **error);
- void (* get_children_async) (TrackerEnumerator *enumerator,
- GFile *dir,
- const gchar *attributes,
- GFileQueryInfoFlags flags,
- int io_priority,
- GCancellable *cancellable,
- GAsyncReadyCallback callback,
- gpointer user_data);
- GSList * (* get_children_finish) (TrackerEnumerator *enumerator,
- GAsyncResult *result,
- GError **error);
+ TrackerCrawlFlags (* get_crawl_flags) (TrackerEnumerator *enumerator);
+ void (* set_crawl_flags) (TrackerEnumerator *enumerator,
+ TrackerCrawlFlags flags);
+ GSList * (* get_children) (TrackerEnumerator *enumerator,
+ GFile *dir,
+ const gchar *attributes,
+ GFileQueryInfoFlags flags,
+ GCancellable *cancellable,
+ GError **error);
+ void (* get_children_async) (TrackerEnumerator *enumerator,
+ GFile *dir,
+ const gchar *attributes,
+ GFileQueryInfoFlags flags,
+ int io_priority,
+ GCancellable *cancellable,
+ GAsyncReadyCallback callback,
+ gpointer user_data);
+ GSList * (* get_children_finish) (TrackerEnumerator *enumerator,
+ GAsyncResult *result,
+ GError **error);
/*< private >*/
/* Padding for future expansion */
@@ -89,24 +98,28 @@ struct _TrackerEnumeratorIface {
void (*_tracker_reserved8) (void);
};
-GType tracker_enumerator_get_type (void) G_GNUC_CONST;
-GSList *tracker_enumerator_get_children (TrackerEnumerator *enumerator,
- GFile *dir,
- const gchar *attributes,
- GFileQueryInfoFlags flags,
- GCancellable *cancellable,
- GError **error);
-void tracker_enumerator_get_children_async (TrackerEnumerator *enumerator,
- GFile *dir,
- const gchar *attributes,
- GFileQueryInfoFlags flags,
- int io_priority,
- GCancellable *cancellable,
- GAsyncReadyCallback callback,
- gpointer user_data);
-GSList *tracker_enumerator_get_children_finish (TrackerEnumerator *enumerator,
- GAsyncResult *result,
- GError **error);
+GType tracker_enumerator_get_type (void) G_GNUC_CONST;
+TrackerCrawlFlags tracker_enumerator_get_crawl_flags (TrackerEnumerator *enumerator);
+void tracker_enumerator_set_crawl_flags (TrackerEnumerator *enumerator,
+ TrackerCrawlFlags flags);
+GSList * tracker_enumerator_get_children (TrackerEnumerator *enumerator,
+ GFile *dir,
+ const gchar *attributes,
+ GFileQueryInfoFlags flags,
+ GCancellable *cancellable,
+ GError **error);
+void tracker_enumerator_get_children_async (TrackerEnumerator *enumerator,
+ GFile *dir,
+ const gchar *attributes,
+ GFileQueryInfoFlags flags,
+ int io_priority,
+ GCancellable *cancellable,
+ GAsyncReadyCallback callback,
+ gpointer user_data);
+GSList * tracker_enumerator_get_children_finish (TrackerEnumerator *enumerator,
+ GAsyncResult *result,
+ GError **error);
+
G_END_DECLS
diff --git a/src/libtracker-miner/tracker-file-enumerator.c b/src/libtracker-miner/tracker-file-enumerator.c
index 823e6d3..4a4f421 100644
--- a/src/libtracker-miner/tracker-file-enumerator.c
+++ b/src/libtracker-miner/tracker-file-enumerator.c
@@ -27,6 +27,7 @@ static void tracker_file_enumerator_file_iface_init (TrackerEnumeratorIface *ifa
struct _TrackerFileEnumerator {
GObject parent_instance;
+ TrackerCrawlFlags crawl_flags;
};
typedef struct {
@@ -70,6 +71,28 @@ tracker_file_enumerator_class_init (TrackerFileEnumeratorClass *klass)
static void
tracker_file_enumerator_init (TrackerFileEnumerator *fe)
{
+ fe->crawl_flags = TRACKER_CRAWL_FLAG_NONE;
+}
+
+static TrackerCrawlFlags
+file_enumerator_get_crawl_flags (TrackerEnumerator *enumerator)
+{
+ TrackerFileEnumerator *fe;
+
+ fe = TRACKER_FILE_ENUMERATOR (enumerator);
+
+ return fe->crawl_flags;
+}
+
+static void
+file_enumerator_set_crawl_flags (TrackerEnumerator *enumerator,
+ TrackerCrawlFlags flags)
+{
+ TrackerFileEnumerator *fe;
+
+ fe = TRACKER_FILE_ENUMERATOR (enumerator);
+
+ fe->crawl_flags = flags;
}
static GetChildrenData *
@@ -257,6 +280,8 @@ file_enumerator_get_children_finish (TrackerEnumerator *enumerator,
static void
tracker_file_enumerator_file_iface_init (TrackerEnumeratorIface *iface)
{
+ iface->get_crawl_flags = file_enumerator_get_crawl_flags;
+ iface->set_crawl_flags = file_enumerator_set_crawl_flags;
iface->get_children = file_enumerator_get_children;
iface->get_children_async = file_enumerator_get_children_async;
iface->get_children_finish = file_enumerator_get_children_finish;
diff --git a/src/libtracker-miner/tracker-miner-enums.h b/src/libtracker-miner/tracker-miner-enums.h
index 5e5c9b3..8bd9c55 100644
--- a/src/libtracker-miner/tracker-miner-enums.h
+++ b/src/libtracker-miner/tracker-miner-enums.h
@@ -109,6 +109,28 @@ typedef enum {
TRACKER_NETWORK_TYPE_LAN
} TrackerNetworkType;
+/**
+ * TrackerCrawlFlags:
+ * @TRACKER_CRAWL_FLAG_NONE: No flags.
+ * @TRACKER_CRAWL_FLAG_NO_STAT: For cases where the content being crawled by the
+ * #TrackerEnumerator is not local (e.g. it's on a server somewhere),
+ * use the #TRACKER_CRAWL_FLAG_NO_STAT flag. The default is to use
+ * stat() and assume we're mining a local or mounted file system.
+ * @TRACKER_CRAWL_FLAG_FOLLOW_SYMLINKS: The enumerator should follow
+ * symlinks. This is disabled by default because it's easy for Tracker
+ * to get into recursive loop situations with symlinks. For those
+ * using interesting scenarios like git-annex or software heavily
+ * using symlinks (and properly) this is for you.
+ *
+ * Flags used when crawling directory structures. This can be
+ * dependent on the #TrackerEnumerator employed at the time of course.
+ */
+typedef enum {
+ TRACKER_CRAWL_FLAG_NONE = 0,
+ TRACKER_CRAWL_FLAG_NO_STAT = 1 << 1,
+ TRACKER_CRAWL_FLAG_FOLLOW_SYMLINKS = 1 << 2,
+} TrackerCrawlFlags;
+
G_END_DECLS
#endif /* __TRACKER_MINER_ENUMS_H__ */
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
