[tracker/miner-gdata] Add GData miner
- From: Adrien Bustany <abustany src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [tracker/miner-gdata] Add GData miner
- Date: Mon, 22 Mar 2010 13:05:29 +0000 (UTC)
commit 6ed934b3808b632701973d9f941fbc4008d65c98
Author: Adrien Bustany <abustany gnome org>
Date: Fri Mar 5 20:53:59 2010 -0300
Add GData miner
So far the miner only exploits the PicasaWeb service
configure.ac | 32 +++
data/dbus/Makefile.am | 1 +
...freedesktop.Tracker1.Miner.PicasaWeb.service.in | 3 +
data/miners/Makefile.am | 6 +-
data/miners/tracker-miner-picasaweb.desktop.in | 7 +
src/Makefile.am | 4 +
src/tracker-miner-gdata/Makefile.am | 65 +++++
src/tracker-miner-gdata/query-queue.vala | 56 ++++
src/tracker-miner-gdata/tracker-miner-gdata.vala | 69 +++++
.../tracker-miner-picasaweb.vala | 286 ++++++++++++++++++++
10 files changed, 527 insertions(+), 2 deletions(-)
---
diff --git a/configure.ac b/configure.ac
index 2ddae7d..c6baa12 100644
--- a/configure.ac
+++ b/configure.ac
@@ -159,6 +159,7 @@ GEE_REQUIRED=0.3
ID3LIB_REQUIRED=3.8.3
GNOME_KEYRING_REQUIRED=2.26
LIBGRSS_REQUIRED=0.3
+LIBGDATA_REQUIRED=0.6
# Library Checks
PKG_CHECK_MODULES(GLIB2, [glib-2.0 >= $GLIB_REQUIRED])
@@ -793,6 +794,32 @@ fi
AM_CONDITIONAL(HAVE_GNOME_KEYRING, test "x$have_gnome_keyring" = "xyes")
+##################################################################
+# GData miner
+##################################################################
+
+AC_ARG_ENABLE(miner_gdata,
+ AS_HELP_STRING([--enable-miner-gdata],
+ [enable GData miner [[default=auto]]]),,
+ [enable_miner_gdata=auto])
+
+if test "x$enable_miner_gdata" != "xno"; then
+ PKG_CHECK_MODULES(MINER_GDATA,
+ [ libgdata >= $LIBGDATA_REQUIRED ],
+ [have_miner_gdata_deps=yes],
+ [have_miner_gdata_deps=no])
+ AC_SUBST(MINER_GDATA_LIBS)
+ AC_SUBST(MINER_GDATA_CFLAGS)
+fi
+
+if test "x$enable_miner_gdata" = "xyes"; then
+ if test "x$have_miner_gdata_deps" != "xyes"; then
+ AC_MSG_ERROR([Couldn't find the required dependencies for the gdata miner: libgdata >= $LIBGDATA_REQUIRED.])
+ fi
+fi
+
+AM_CONDITIONAL(HAVE_MINER_GDATA, test "x$have_miner_gdata_deps" = "xyes")
+
####################################################################
# Mail miners
####################################################################
@@ -1717,6 +1744,7 @@ AC_CONFIG_FILES([
src/tracker-extract/Makefile
src/tracker-miner-fs/Makefile
src/tracker-miner-rss/Makefile
+ src/tracker-miner-gdata/Makefile
src/tracker-preferences/Makefile
src/tracker-preferences/tracker-preferences.desktop.in
src/tracker-search-bar/Makefile
@@ -1838,6 +1866,10 @@ Plugins/Extensions:
KMail plugin (data-push): $enable_kmail_miner
Miner RSS: (data-push): $enable_miner_rss
+Extra miners:
+
+ GData miner $have_miner_gdata_deps
+
Writeback:
MP3 writeback: $have_id3lib
diff --git a/data/dbus/Makefile.am b/data/dbus/Makefile.am
index 0aab644..fecc8db 100644
--- a/data/dbus/Makefile.am
+++ b/data/dbus/Makefile.am
@@ -21,6 +21,7 @@ service_in_files = \
org.freedesktop.Tracker1.service.in \
org.freedesktop.Tracker1.Miner.Applications.service.in \
org.freedesktop.Tracker1.Miner.Files.service.in \
+ org.freedesktop.Tracker1.Miner.PicasaWeb.service.in \
org.freedesktop.Tracker1.Extract.service.in
service_DATA = $(service_in_files:.service.in=.service)
diff --git a/data/dbus/org.freedesktop.Tracker1.Miner.PicasaWeb.service.in b/data/dbus/org.freedesktop.Tracker1.Miner.PicasaWeb.service.in
new file mode 100644
index 0000000..520ecdc
--- /dev/null
+++ b/data/dbus/org.freedesktop.Tracker1.Miner.PicasaWeb.service.in
@@ -0,0 +1,3 @@
+[D-BUS Service]
+Name=org.freedesktop.Tracker1.Miner.PicasaWeb
+Exec= libexecdir@/tracker-miner-gdata
diff --git a/data/miners/Makefile.am b/data/miners/Makefile.am
index b435f45..261ed80 100644
--- a/data/miners/Makefile.am
+++ b/data/miners/Makefile.am
@@ -2,7 +2,8 @@ include $(top_srcdir)/Makefile.decl
desktop_in_files = \
tracker-miner-applications.desktop.in \
- tracker-miner-files.desktop.in
+ tracker-miner-files.desktop.in \
+ tracker-miner-picasaweb.desktop.in
if USING_MINER_RSS
desktop_in_files += tracker-miner-rss.desktop.in
@@ -12,7 +13,8 @@ tracker_minersdir = $(datadir)/tracker/miners
tracker_miners_DATA = \
tracker-miner-applications.desktop \
- tracker-miner-files.desktop
+ tracker-miner-files.desktop \
+ tracker-miner-picasaweb.desktop
if USING_MINER_RSS
tracker_miners_DATA += tracker-miner-rss.desktop
diff --git a/data/miners/tracker-miner-picasaweb.desktop.in b/data/miners/tracker-miner-picasaweb.desktop.in
new file mode 100644
index 0000000..c115942
--- /dev/null
+++ b/data/miners/tracker-miner-picasaweb.desktop.in
@@ -0,0 +1,7 @@
+[Desktop Entry]
+Encoding=UTF-8
+_Name=PicasaWeb
+_Comment=PicasaWeb miner
+DBusName=org.freedesktop.Tracker1.Miner.PicasaWeb
+DBusPath=/org/freedesktop/Tracker1/Miner/PicasaWeb
+AuthenticationMethod=UserPass
diff --git a/src/Makefile.am b/src/Makefile.am
index 3823389..1a37ff2 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -42,3 +42,7 @@ endif
if USING_MINER_RSS
SUBDIRS += tracker-miner-rss
endif
+
+if HAVE_MINER_GDATA
+SUBDIRS += tracker-miner-gdata
+endif
diff --git a/src/tracker-miner-gdata/Makefile.am b/src/tracker-miner-gdata/Makefile.am
new file mode 100644
index 0000000..d1732b0
--- /dev/null
+++ b/src/tracker-miner-gdata/Makefile.am
@@ -0,0 +1,65 @@
+include $(top_srcdir)/Makefile.decl
+
+INCLUDES = \
+ -Wall \
+ -DSHAREDIR=\""$(datadir)"\" \
+ -DPKGLIBDIR=\""$(libdir)/tracker"\" \
+ -DLOCALEDIR=\""$(localedir)"\" \
+ -DLIBEXEC_PATH=\""$(libexecdir)"\" \
+ -DG_LOG_DOMAIN=\"Tracker\" \
+ -DTRACKER_COMPILATION \
+ -I$(top_srcdir)/src \
+ $(WARN_CFLAGS) \
+ $(GMODULE_CFLAGS) \
+ $(PANGO_CFLAGS) \
+ $(DBUS_CFLAGS) \
+ $(MINER_GDATA_CFLAGS) \
+ $(GCOV_CFLAGS)
+
+VALAFLAGS = \
+ --pkg gio-2.0 \
+ --pkg posix \
+ --pkg libgdata \
+ --thread
+
+libexec_PROGRAMS = tracker-miner-gdata
+
+tracker_miner_gdata_VALASOURCES = \
+ tracker-miner-gdata.vala \
+ tracker-miner-picasaweb.vala \
+ query-queue.vala \
+ $(top_srcdir)/src/libtracker-client/tracker-client-0.7.vapi
+
+tracker_miner_gdata_SOURCES = \
+ $(tracker_miner_gdata_VALASOURCES:.vala=.c)
+
+tracker_miner_gdata_LDADD = \
+ $(top_builddir)/src/libtracker-miner/libtracker-miner- TRACKER_API_VERSION@.la \
+ $(top_builddir)/src/libtracker-client/libtracker-client- TRACKER_API_VERSION@.la \
+ $(DBUS_LIBS) \
+ $(GMODULE_LIBS) \
+ $(GTHREAD_LIBS) \
+ $(GIO_LIBS) \
+ $(GCOV_LIBS) \
+ $(GLIB2_LIBS) \
+ $(MINER_GDATA_LIBS) \
+ -lz \
+ -lm
+
+vapi_sources = \
+ $(top_srcdir)/src/libtracker-miner/tracker-miner- TRACKER_API_VERSION@.vapi
+
+tracker-miner-gdata.vala.stamp: $(tracker_miner_gdata_VALASOURCES) $(vapi_sources)
+ $(AM_V_GEN)$(VALAC) $(GCOV_VALAFLAGS) -C $(VALAFLAGS) $^
+ touch $@
+
+
+BUILT_SOURCES = tracker-miner-gdata.vala.stamp
+
+MAINTAINERCLEANFILES = \
+ tracker-miner-gdata.vala.stamp \
+ $(tracker_miner_gdata_VALASOURCES:.vala=.c)
+
+EXTRA_DIST = \
+ $(tracker_miner_gdata_VALASOURCES) \
+ tracker-miner-gdata.vala.stamp
diff --git a/src/tracker-miner-gdata/query-queue.vala b/src/tracker-miner-gdata/query-queue.vala
new file mode 100644
index 0000000..7afeeb7
--- /dev/null
+++ b/src/tracker-miner-gdata/query-queue.vala
@@ -0,0 +1,56 @@
+public class QueryQueue : GLib.Object {
+ /* Holds the pending sparql updates and monitors them */
+ private HashTable<uint, string> queue;
+ private uint cookie;
+
+ private Mutex flush_mutex;
+
+ private Tracker.Miner miner;
+
+ public QueryQueue (Tracker.Miner parent) {
+ miner = parent;
+
+ queue = new HashTable<uint, string> (direct_hash, direct_equal);
+ cookie = 0;
+
+ flush_mutex = new Mutex ();
+ }
+
+ public async void append (string query) {
+ uint current_cookie = cookie ++;
+ queue.insert (current_cookie, query);
+
+ try {
+ yield miner.execute_batch_update (query);
+ } catch (Error tracker_error) {
+ warning ("BatchUpdate query failed: %s", tracker_error.message);
+ }
+
+ queue.remove (current_cookie);
+ }
+
+ /* BLOCKING flush */
+ public void flush () {
+ if (!flush_mutex.trylock ()) {
+ message ("There's already a flush taking place");
+ return;
+ }
+
+ if (queue.size () > 0) {
+ MainLoop wait_loop;
+ try {
+ wait_loop = new MainLoop (null, false);
+ miner.commit (null, () => { wait_loop.quit (); });
+ wait_loop.run ();
+ } catch (Error tracker_error) {
+ warning ("Commit query failed: %s", tracker_error.message);
+ }
+ }
+
+ flush_mutex.unlock ();
+ }
+
+ public uint size () {
+ return queue.size ();
+ }
+}
diff --git a/src/tracker-miner-gdata/tracker-miner-gdata.vala b/src/tracker-miner-gdata/tracker-miner-gdata.vala
new file mode 100644
index 0000000..f9dd050
--- /dev/null
+++ b/src/tracker-miner-gdata/tracker-miner-gdata.vala
@@ -0,0 +1,69 @@
+/*
+ * Copyright (C) 2010, Adrien Bustany <abustany gnome org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+namespace Tracker {
+
+public class MinerGData {
+ private static MainLoop main_loop;
+
+ private static bool in_loop = false;
+ private static void signal_handler (int signo) {
+ if (in_loop) {
+ Posix.exit (Posix.EXIT_FAILURE);
+ }
+
+ switch (signo) {
+ case Posix.SIGINT:
+ case Posix.SIGTERM:
+ in_loop = true;
+ main_loop.quit ();
+ break;
+ }
+ }
+
+ private static void init_signals () {
+#if G_OS_WIN32
+#else
+ Posix.sigaction_t act = Posix.sigaction_t ();
+ Posix.sigset_t empty_mask = Posix.sigset_t ();
+ Posix.sigemptyset (empty_mask);
+ act.sa_handler = signal_handler;
+ act.sa_mask = empty_mask;
+ act.sa_flags = 0;
+
+ Posix.sigaction (Posix.SIGTERM, act, null);
+ Posix.sigaction (Posix.SIGINT, act, null);
+#endif
+ }
+
+ public static void main (string[] args) {
+ Environment.set_application_name ("GData tracker miner");
+ MinerPicasaweb picasaweb_miner;
+ picasaweb_miner = Object.new (typeof (MinerPicasaweb)) as MinerPicasaweb;
+
+ init_signals ();
+
+ main_loop = new MainLoop (null, false);
+ main_loop.run ();
+
+ picasaweb_miner.shutdown ();
+ }
+}
+
+} // End namespace Tracker
diff --git a/src/tracker-miner-gdata/tracker-miner-picasaweb.vala b/src/tracker-miner-gdata/tracker-miner-picasaweb.vala
new file mode 100644
index 0000000..585433e
--- /dev/null
+++ b/src/tracker-miner-gdata/tracker-miner-picasaweb.vala
@@ -0,0 +1,286 @@
+/*
+ * Copyright (C) 2010, Adrien Bustany <abustany gnome org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+namespace Tracker {
+
+public class MinerPicasaweb : Tracker.MinerWeb {
+ private static const string API_KEY = "ABQIAAAAISU_TtyUrVg0N9RGhb8pTxRI3KVolA1NVmS0lLG3RQFyztcosBSHs5rHiQHsmL41EbX1SehKnf1rrw";
+ private static const string DATASOURCE_URN = "urn:682654ec-b59d-4e99-9b0f-7559fc9c5d42";
+ private static const string MINER_NAME = "PicasaWeb";
+ private static const string SERVICE_DESCRIPTION = "Tracker miner for PicasaWeb";
+ private static const uint PULL_INTERVAL = 5*60; /* seconds */
+ private uint pull_timeout_handle;
+
+ private GData.PicasaWebService service = null;
+ private QueryQueue query_queue;
+
+ construct {
+ set ("name", MINER_NAME);
+ set ("associated", false);
+
+ service = new GData.PicasaWebService (API_KEY);
+
+ query_queue = new QueryQueue (this);
+
+ this.notify["associated"].connect (association_status_changed);
+
+ }
+
+ private void association_status_changed (Object source, ParamSpec pspec) {
+ bool associated;
+
+ get ("associated", out associated);
+
+ if (associated) {
+ if (pull_timeout_handle != 0)
+ return;
+
+ message ("Miner is now associated. Initiating periodic pull.");
+ pull_timeout_handle = Timeout.add_seconds (PULL_INTERVAL, pull_timeout_cb);
+ Idle.add ( () => { pull_timeout_cb (); return false; });
+ } else {
+ if (pull_timeout_handle == 0)
+ return;
+
+ Source.remove (pull_timeout_handle);
+ }
+ }
+
+ private bool pull_timeout_cb () {
+ service.query_all_albums_async (null, service.get_username (), null, albums_cb, pull_finished_cb);
+ return true;
+ }
+
+ private void albums_cb (GData.Entry entry, uint entry_key, uint entry_count) {
+ GData.Feed feed_files;
+ GData.PicasaWebAlbum album = entry as GData.PicasaWebAlbum;
+ GData.PicasaWebFile current_file;
+ string current_file_url;
+ SparqlBuilder builder;
+ List<string> album_files_urls;
+ TimeVal tv = TimeVal ();
+
+ try {
+ feed_files = service.query_files (album, null, null, null);
+ } catch (Error service_error) {
+ warning ("Couldn't get files for album %s: %s", album.title, service_error.message);
+ return;
+ }
+
+ album_files_urls = new List<string> ();
+
+ foreach (GData.Entry current_entry in feed_files.get_entries ()) {
+ current_file = current_entry as GData.PicasaWebFile;
+ current_file_url = ((GData.MediaContent)current_file.get_contents ().first ().data).get_uri ();
+
+ album_files_urls.prepend (current_file_url);
+
+ builder = new SparqlBuilder.update ();
+
+ builder.insert_open (current_file_url);
+
+ builder.subject ("_:picture");
+ builder.predicate ("a");
+ builder.object ("nmm:Photo");
+
+ builder.predicate ("a");
+ builder.object ("nfo:MediaFileListEntry");
+
+ builder.predicate ("a");
+ builder.object ("nfo:RemoteDataObject");
+
+ builder.predicate ("nie:url");
+ builder.object_string (current_file_url);
+
+ builder.predicate ("dc:title");
+ if (current_file.get_caption () != null) {
+ builder.object_string (current_file.get_caption ());
+ } else {
+ builder.object_string (current_file.get_title ());
+ }
+
+ current_file.get_timestamp (tv);
+ builder.predicate ("nie:contentCreated");
+ builder.object_string (tv.to_iso8601 ());
+
+ current_file.get_edited (tv);
+ builder.predicate ("nie:contentLastModified");
+ builder.object_string (tv.to_iso8601 ());
+
+ builder.predicate ("nfo:width");
+ builder.object_int64 ((int64)current_file.get_width ());
+
+ builder.predicate ("nfo:height");
+ builder.object_int64 ((int64)current_file.get_height ());
+
+ if (current_file.get_tags () != null) {
+ foreach (string tag_label in current_file.get_tags ().split (",")) {
+ builder.predicate ("nao:hasTag");
+ builder.object_blank_open ();
+ builder.predicate ("nao:prefLabel");
+ builder.object_string (tag_label);
+ builder.object_blank_close ();
+ }
+ }
+
+ builder.insert_close ();
+
+ query_queue.append (builder.get_result ());
+ }
+
+ builder = new SparqlBuilder.update ();
+
+ builder.insert_open (DATASOURCE_URN);
+
+ builder.subject ("_:album");
+ builder.predicate ("a");
+ builder.object ("nmm:ImageList");
+
+ builder.predicate ("dc:title");
+ builder.object_string (album.get_title ());
+
+ if (album.get_summary () != null) {
+ builder.predicate ("rdfs:comment");
+ builder.object_string (album.get_summary ());
+ }
+
+ if (album.get_tags () != null) {
+ foreach (string tag_label in album.get_tags ().split (",")) {
+ builder.predicate ("nao:hasTag");
+ builder.object_blank_open ();
+ builder.predicate ("nao:prefLabel");
+ builder.object_string (tag_label);
+ builder.object_blank_close ();
+ }
+ }
+
+ builder.predicate ("nfo:entryCounter");
+ builder.object_int64 ((int64)album.get_num_photos ());
+
+ album.get_timestamp (tv);
+ builder.predicate ("nie:contentCreated");
+ builder.object_string (tv.to_iso8601 ());
+
+ album.get_edited (tv);
+ builder.predicate ("nie:contentLastModified");
+ builder.object_string (tv.to_iso8601 ());
+
+ foreach (string current_file_url in album_files_urls) {
+ builder.predicate ("nfo:hasMediaFileListEntry");
+ builder.object_blank_open ();
+ builder.predicate ("a");
+ builder.object ("nmm:Photo");
+ builder.predicate ("a");
+ builder.object ("nfo:MediaFileListEntry");
+ builder.predicate ("a");
+ builder.object ("nfo:RemoteDataObject");
+ builder.predicate ("nie:url");
+ builder.object_string (current_file_url);
+ builder.object_blank_close ();
+ }
+
+ builder.insert_close ();
+
+ query_queue.append (builder.get_result ());
+ }
+
+ private void pull_finished_cb () {
+ query_queue.flush ();
+ }
+
+ public void shutdown () {
+ set ("associated", false);
+ }
+
+ public override void authenticate () throws MinerWebError {
+ PasswordProvider password_provider;
+ string username;
+ string password;
+ bool verified = false;
+
+ password_provider = PasswordProvider.get ();
+
+ set ("associated", false);
+
+ try {
+ password = password_provider.get_password (MINER_NAME, out username);
+ } catch (Error e) {
+ if (e is PasswordProviderError.NOTFOUND) {
+ throw new MinerWebError.NO_CREDENTIALS ("Miner is not associated");
+ }
+ throw new MinerWebError.KEYRING (e.message);
+ }
+
+ try {
+ verified = service.authenticate (username, password, null);
+ } catch (Error service_error) {
+ throw new MinerWebError.SERVICE (service_error.message);
+ }
+
+ if (!verified) {
+ throw new MinerWebError.WRONG_CREDENTIALS ("Wrong username and/or password");
+ }
+
+ message ("Authentication successful");
+ set ("associated", true);
+ }
+
+ public override void dissociate () throws MinerWebError {
+ var password_provider = PasswordProvider.get ();
+
+ try {
+ password_provider.forget_password (MINER_NAME);
+ } catch (Error e) {
+ if (e is PasswordProviderError.SERVICE) {
+ throw new MinerWebError.KEYRING (e.message);
+ }
+
+ critical ("Internal error: %s", e.message);
+ return;
+ }
+
+ set ("associated", false);
+ }
+
+ public override void associate (HashTable<string, string> association_data) throws Tracker.MinerWebError {
+ assert (association_data.lookup ("username") != null && association_data.lookup ("password") != null);
+
+ var password_provider = PasswordProvider.get ();
+
+ try {
+ password_provider.store_password (MINER_NAME,
+ SERVICE_DESCRIPTION,
+ association_data.lookup ("username"),
+ association_data.lookup ("password"));
+ } catch (Error e) {
+ if (e is PasswordProviderError.SERVICE) {
+ throw new MinerWebError.KEYRING (e.message);
+ }
+
+ critical ("Internal error: %s", e.message);
+ return;
+ }
+ }
+
+ public GLib.HashTable get_association_data () throws Tracker.MinerWebError {
+ return new HashTable<string, string>(str_hash, str_equal);
+ }
+}
+
+} // End namespace Tracker
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]