[tracker/miner-flickr-review] Add Flickr miner
- From: Adrien Bustany <abustany src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [tracker/miner-flickr-review] Add Flickr miner
- Date: Wed, 24 Mar 2010 14:54:26 +0000 (UTC)
commit 4c61005a8007515fc276fa042e6901c54e76485f
Author: Adrien Bustany <abustany gnome org>
Date: Wed Mar 10 09:44:56 2010 -0300
Add Flickr miner
This commit adds a Flickr miner, using the TrackerMinerWeb class. The Flickr
miner can import your photos and albums, as well as their tags, from Flickr.
configure.ac | 32 +
data/dbus/Makefile.am | 5 +
...rg.freedesktop.Tracker1.Miner.Flickr.service.in | 3 +
data/icons/scalable/Makefile.am | 11 +-
data/icons/scalable/tracker-miner-flickr.svg | 206 +++++++
data/miners/Makefile.am | 7 +
data/miners/tracker-miner-flickr.desktop.in.in | 8 +
src/Makefile.am | 4 +
src/tracker-miner-flickr/Makefile.am | 64 ++
src/tracker-miner-flickr/query-queue.vala | 75 +++
src/tracker-miner-flickr/tracker-miner-flickr.vala | 607 ++++++++++++++++++++
11 files changed, 1021 insertions(+), 1 deletions(-)
---
diff --git a/configure.ac b/configure.ac
index 5e0de56..d99b76a 100644
--- a/configure.ac
+++ b/configure.ac
@@ -159,6 +159,7 @@ GEE_REQUIRED=0.3
ID3LIB_REQUIRED=3.8.3
GNOME_KEYRING_REQUIRED=2.26
LIBGRSS_REQUIRED=0.3
+REST_REQUIRED=0.6
# Library Checks
PKG_CHECK_MODULES(GLIB2, [glib-2.0 >= $GLIB_REQUIRED])
@@ -819,6 +820,32 @@ fi
AM_CONDITIONAL(HAVE_GNOME_KEYRING, test "x$have_gnome_keyring" = "xyes")
+##################################################################
+# Flickr miner
+##################################################################
+
+AC_ARG_ENABLE(miner_flickr,
+ AS_HELP_STRING([--miner-flickr],
+ [enable Flickr miner [[default=auto]]]),,
+ [enable_miner_flickr=auto])
+
+if test "x$enable_miner_flickr" != "xno"; then
+ PKG_CHECK_MODULES(MINER_FLICKR,
+ [ rest-0.6 >= $REST_REQUIRED ],
+ [have_miner_flickr_deps=yes],
+ [have_miner_flickr_deps=no])
+ AC_SUBST(MINER_FLICKR_LIBS)
+ AC_SUBST(MINER_FLICKR_CFLAGS)
+fi
+
+if test "x$enable_miner_flickr" = "xyes"; then
+ if test "x$have_miner_flickr_deps" != "xyes"; then
+ AC_MSG_ERROR([Couldn't find the required dependencies for the Flickr miner: rest-0.6 >= $REST_REQUIRED.])
+ fi
+fi
+
+AM_CONDITIONAL(HAVE_MINER_FLICKR, test "x$have_miner_flickr_deps" = "xyes")
+
####################################################################
# Mail miners
####################################################################
@@ -1741,6 +1768,7 @@ AC_CONFIG_FILES([
src/tracker-extract/Makefile
src/tracker-miner-fs/Makefile
src/tracker-miner-rss/Makefile
+ src/tracker-miner-flickr/Makefile
src/tracker-preferences/Makefile
src/tracker-preferences/tracker-preferences.desktop.in
src/tracker-search-bar/Makefile
@@ -1866,6 +1894,10 @@ Plugins:
Nautilus: (tagging widget) $have_nautilus_extension
Deskbar: $have_deskbar_applet
+Extra miners:
+
+ Flickr miner $have_miner_flickr_deps
+
Writeback:
MP3: $have_id3lib
diff --git a/data/dbus/Makefile.am b/data/dbus/Makefile.am
index 0aab644..91e2505 100644
--- a/data/dbus/Makefile.am
+++ b/data/dbus/Makefile.am
@@ -22,6 +22,11 @@ service_in_files = \
org.freedesktop.Tracker1.Miner.Applications.service.in \
org.freedesktop.Tracker1.Miner.Files.service.in \
org.freedesktop.Tracker1.Extract.service.in
+
+if HAVE_MINER_FLICKR
+service_in_files += org.freedesktop.Tracker1.Miner.Flickr.service.in
+endif
+
service_DATA = $(service_in_files:.service.in=.service)
%.service: %.service.in
diff --git a/data/dbus/org.freedesktop.Tracker1.Miner.Flickr.service.in b/data/dbus/org.freedesktop.Tracker1.Miner.Flickr.service.in
new file mode 100644
index 0000000..b06fe93
--- /dev/null
+++ b/data/dbus/org.freedesktop.Tracker1.Miner.Flickr.service.in
@@ -0,0 +1,3 @@
+[D-BUS Service]
+Name=org.freedesktop.Tracker1.Miner.Flickr
+Exec= libexecdir@/tracker-miner-flickr
diff --git a/data/icons/scalable/Makefile.am b/data/icons/scalable/Makefile.am
index 5583008..a8310e5 100644
--- a/data/icons/scalable/Makefile.am
+++ b/data/icons/scalable/Makefile.am
@@ -3,4 +3,13 @@ include $(top_srcdir)/Makefile.decl
icondir = $(datadir)/icons/hicolor/scalable/apps
icon_DATA = tracker.svg
-EXTRA_DIST = $(icon_DATA)
+minericonsdir = $(datadir)/tracker/icons
+minericons_DATA =
+
+if HAVE_MINER_FLICKR
+minericons_DATA += tracker-miner-flickr.svg
+endif
+
+EXTRA_DIST = \
+ $(icon_DATA) \
+ $(minericons_DATA)
diff --git a/data/icons/scalable/tracker-miner-flickr.svg b/data/icons/scalable/tracker-miner-flickr.svg
new file mode 100644
index 0000000..bf5f830
--- /dev/null
+++ b/data/icons/scalable/tracker-miner-flickr.svg
@@ -0,0 +1,206 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+<svg
+ xmlns:dc="http://purl.org/dc/elements/1.1/";
+ xmlns:cc="http://creativecommons.org/ns#";
+ xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#";
+ xmlns:svg="http://www.w3.org/2000/svg";
+ xmlns="http://www.w3.org/2000/svg";
+ xmlns:xlink="http://www.w3.org/1999/xlink";
+ xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd";
+ xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape";
+ width="48"
+ height="48"
+ id="svg2"
+ sodipodi:version="0.32"
+ inkscape:version="0.46"
+ version="1.0"
+ sodipodi:docname="flickr48.svg"
+ inkscape:output_extension="org.inkscape.output.svg.inkscape"
+ inkscape:export-filename="/home/cube/Dropbox/Inkscape-Files/flickr/flickr48.png"
+ inkscape:export-xdpi="90"
+ inkscape:export-ydpi="90">
+ <defs
+ id="defs4">
+ <linearGradient
+ id="linearGradient6275">
+ <stop
+ id="stop6277"
+ offset="0"
+ style="stop-color:#f45d80;stop-opacity:1;" />
+ <stop
+ id="stop6279"
+ offset="1"
+ style="stop-color:#d40f3d;stop-opacity:1;" />
+ </linearGradient>
+ <linearGradient
+ id="linearGradient6261">
+ <stop
+ id="stop6263"
+ offset="0"
+ style="stop-color:#c5a0df;stop-opacity:1;" />
+ <stop
+ id="stop6265"
+ offset="1"
+ style="stop-color:#3e78b6;stop-opacity:1;" />
+ </linearGradient>
+ <inkscape:perspective
+ sodipodi:type="inkscape:persp3d"
+ inkscape:vp_x="0 : 526.18109 : 1"
+ inkscape:vp_y="0 : 1000 : 0"
+ inkscape:vp_z="744.09448 : 526.18109 : 1"
+ inkscape:persp3d-origin="372.04724 : 350.78739 : 1"
+ id="perspective10" />
+ <radialGradient
+ inkscape:collect="always"
+ xlink:href="#linearGradient6261"
+ id="radialGradient6259"
+ cx="7.3078551"
+ cy="18.809063"
+ fx="7.3078551"
+ fy="18.809063"
+ r="12.897959"
+ gradientUnits="userSpaceOnUse" />
+ <radialGradient
+ inkscape:collect="always"
+ xlink:href="#linearGradient6275"
+ id="radialGradient6273"
+ cx="6.4923115"
+ cy="17.852686"
+ fx="6.4923115"
+ fy="17.852686"
+ r="12.897959"
+ gradientUnits="userSpaceOnUse" />
+ <filter
+ inkscape:collect="always"
+ id="filter6287"
+ x="-0.13891452"
+ width="1.277829"
+ y="-0.83348718"
+ height="2.6669744">
+ <feGaussianBlur
+ inkscape:collect="always"
+ stdDeviation="1.0753131"
+ id="feGaussianBlur6289" />
+ </filter>
+ </defs>
+ <sodipodi:namedview
+ id="base"
+ pagecolor="#ffffff"
+ bordercolor="#666666"
+ borderopacity="1.0"
+ gridtolerance="10000"
+ guidetolerance="10"
+ objecttolerance="10"
+ inkscape:pageopacity="0.0"
+ inkscape:pageshadow="2"
+ inkscape:zoom="8.1666667"
+ inkscape:cx="24"
+ inkscape:cy="24"
+ inkscape:document-units="px"
+ inkscape:current-layer="layer1"
+ showgrid="true"
+ inkscape:window-width="1280"
+ inkscape:window-height="974"
+ inkscape:window-x="-8"
+ inkscape:window-y="-8"
+ inkscape:snap-nodes="false"
+ inkscape:snap-bbox="true">
+ <inkscape:grid
+ type="xygrid"
+ id="grid2383"
+ visible="true"
+ enabled="true" />
+ </sodipodi:namedview>
+ <metadata
+ id="metadata7">
+ <rdf:RDF>
+ <cc:Work
+ rdf:about="">
+ <dc:format>image/svg+xml</dc:format>
+ <dc:type
+ rdf:resource="http://purl.org/dc/dcmitype/StillImage"; />
+ <dc:creator>
+ <cc:Agent>
+ <dc:title>Jakub Szypulka</dc:title>
+ </cc:Agent>
+ </dc:creator>
+ <dc:publisher>
+ <cc:Agent>
+ <dc:title>Jakub Szypulka</dc:title>
+ </cc:Agent>
+ </dc:publisher>
+ <dc:rights>
+ <cc:Agent>
+ <dc:title>Jakub Szypulka</dc:title>
+ </cc:Agent>
+ </dc:rights>
+ </cc:Work>
+ </rdf:RDF>
+ </metadata>
+ <g
+ inkscape:label="Layer 1"
+ inkscape:groupmode="layer"
+ id="layer1">
+ <path
+ sodipodi:type="arc"
+ style="opacity:1;fill:url(#radialGradient6259);fill-opacity:1;fill-rule:evenodd;stroke:#204a87;stroke-width:1.17254185999999990px;stroke-linecap:butt;stroke-linejoin:miter;marker:none;marker-start:none;marker-mid:none;marker-end:none;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;visibility:visible;display:inline;overflow:visible;enable-background:accumulate"
+ id="path5475"
+ sodipodi:cx="10.877922"
+ sodipodi:cy="23.407793"
+ sodipodi:rx="12.311688"
+ sodipodi:ry="12.311688"
+ d="M 23.18961,23.407793 A 12.311688,12.311688 0 1 1 -1.4337664,23.407793 A 12.311688,12.311688 0 1 1 23.18961,23.407793 z"
+ transform="matrix(0.8528481,0,0,0.8528481,2.7227857,3.0367088)" />
+ <path
+ transform="matrix(0.8528481,0,0,0.8528481,25.722785,3.0367083)"
+ d="M 23.18961,23.407793 A 12.311688,12.311688 0 1 1 -1.4337664,23.407793 A 12.311688,12.311688 0 1 1 23.18961,23.407793 z"
+ sodipodi:ry="12.311688"
+ sodipodi:rx="12.311688"
+ sodipodi:cy="23.407793"
+ sodipodi:cx="10.877922"
+ id="path5477"
+ style="opacity:1;fill:url(#radialGradient6273);fill-opacity:1;fill-rule:evenodd;stroke:#a40026;stroke-width:1.17254185999999994px;stroke-linecap:butt;stroke-linejoin:miter;marker:none;marker-start:none;marker-mid:none;marker-end:none;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;visibility:visible;display:inline;overflow:visible;enable-background:accumulate"
+ sodipodi:type="arc" />
+ <path
+ transform="matrix(0.7716245,0,0,0.7716245,3.6063293,4.9379737)"
+ d="M 23.18961,23.407793 A 12.311688,12.311688 0 1 1 -1.4337664,23.407793 A 12.311688,12.311688 0 1 1 23.18961,23.407793 z"
+ sodipodi:ry="12.311688"
+ sodipodi:rx="12.311688"
+ sodipodi:cy="23.407793"
+ sodipodi:cx="10.877922"
+ id="path5479"
+ style="opacity:0.4;fill:none;fill-opacity:1;fill-rule:evenodd;stroke:#ffffff;stroke-width:1.29596722000000010px;stroke-linecap:butt;stroke-linejoin:miter;marker:none;marker-start:none;marker-mid:none;marker-end:none;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;visibility:visible;display:inline;overflow:visible;enable-background:accumulate"
+ sodipodi:type="arc" />
+ <path
+ sodipodi:type="arc"
+ style="opacity:0.4;fill:none;fill-opacity:1;fill-rule:evenodd;stroke:#ffffff;stroke-width:1.29596722px;stroke-linecap:butt;stroke-linejoin:miter;marker:none;marker-start:none;marker-mid:none;marker-end:none;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;visibility:visible;display:inline;overflow:visible;enable-background:accumulate"
+ id="path6251"
+ sodipodi:cx="10.877922"
+ sodipodi:cy="23.407793"
+ sodipodi:rx="12.311688"
+ sodipodi:ry="12.311688"
+ d="M 23.18961,23.407793 A 12.311688,12.311688 0 1 1 -1.4337664,23.407793 A 12.311688,12.311688 0 1 1 23.18961,23.407793 z"
+ transform="matrix(0.7716245,0,0,0.7716245,26.606329,4.9379737)" />
+ <path
+ sodipodi:type="arc"
+ style="opacity:0.3;fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:0.90345997px;stroke-linecap:butt;stroke-linejoin:miter;marker:none;marker-start:none;marker-mid:none;marker-end:none;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;visibility:visible;display:inline;overflow:visible;filter:url(#filter6287);enable-background:accumulate"
+ id="path6281"
+ sodipodi:cx="14.449541"
+ sodipodi:cy="35.821102"
+ sodipodi:rx="9.288991"
+ sodipodi:ry="1.5481651"
+ d="M 23.738532,35.821102 A 9.288991,1.5481651 0 1 1 5.1605501,35.821102 A 9.288991,1.5481651 0 1 1 23.738532,35.821102 z"
+ transform="matrix(1.1659259,0,0,1.1190295,-4.6773701,-6.3524276)" />
+ <path
+ transform="matrix(1.2735802,0,0,1.1190295,16.427625,-6.3524276)"
+ d="M 23.738532,35.821102 A 9.288991,1.5481651 0 1 1 5.1605501,35.821102 A 9.288991,1.5481651 0 1 1 23.738532,35.821102 z"
+ sodipodi:ry="1.5481651"
+ sodipodi:rx="9.288991"
+ sodipodi:cy="35.821102"
+ sodipodi:cx="14.449541"
+ id="path6291"
+ style="opacity:0.3;fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:0.90345997px;stroke-linecap:butt;stroke-linejoin:miter;marker:none;marker-start:none;marker-mid:none;marker-end:none;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;visibility:visible;display:inline;overflow:visible;filter:url(#filter6287);enable-background:accumulate"
+ sodipodi:type="arc" />
+ </g>
+</svg>
diff --git a/data/miners/Makefile.am b/data/miners/Makefile.am
index b435f45..8aa0a1b 100644
--- a/data/miners/Makefile.am
+++ b/data/miners/Makefile.am
@@ -18,7 +18,14 @@ if USING_MINER_RSS
tracker_miners_DATA += tracker-miner-rss.desktop
endif
+if HAVE_MINER_FLICKR
+tracker_miners_DATA += tracker-miner-flickr.desktop
+endif
+
@INTLTOOL_DESKTOP_RULE@
+%.desktop.in: %.desktop.in.in
+ @sed -e "s|@datadir[ ]|$(datadir)|" $< > $@
+
EXTRA_DIST = $(desktop_in_files)
CLEANFILES = $(tracker_miners_DATA)
diff --git a/data/miners/tracker-miner-flickr.desktop.in.in b/data/miners/tracker-miner-flickr.desktop.in.in
new file mode 100644
index 0000000..a9f1ae9
--- /dev/null
+++ b/data/miners/tracker-miner-flickr.desktop.in.in
@@ -0,0 +1,8 @@
+[Desktop Entry]
+Encoding=UTF-8
+_Name=Flickr
+_Comment=Index your Flickr photo albums
+Icon= datadir@/tracker/icons/tracker-miner-flickr.svg
+DBusName=org.freedesktop.Tracker1.Miner.Flickr
+DBusPath=/org/freedesktop/Tracker1/Miner/Flickr
+AuthenticationMethod=Token
diff --git a/src/Makefile.am b/src/Makefile.am
index bbc1d09..00102c3 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -46,3 +46,7 @@ endif
if USING_MINER_RSS
SUBDIRS += tracker-miner-rss
endif
+
+if HAVE_MINER_FLICKR
+SUBDIRS += tracker-miner-flickr
+endif
diff --git a/src/tracker-miner-flickr/Makefile.am b/src/tracker-miner-flickr/Makefile.am
new file mode 100644
index 0000000..54cabcc
--- /dev/null
+++ b/src/tracker-miner-flickr/Makefile.am
@@ -0,0 +1,64 @@
+include $(top_srcdir)/Makefile.decl
+
+INCLUDES = \
+ -Wall \
+ -DSHAREDIR=\""$(datadir)"\" \
+ -DPKGLIBDIR=\""$(libdir)/tracker"\" \
+ -DLOCALEDIR=\""$(localedir)"\" \
+ -DLIBEXEC_PATH=\""$(libexecdir)"\" \
+ -DG_LOG_DOMAIN=\"Tracker\" \
+ -DTRACKER_COMPILATION \
+ -I$(top_srcdir)/src \
+ $(WARN_CFLAGS) \
+ $(GMODULE_CFLAGS) \
+ $(PANGO_CFLAGS) \
+ $(DBUS_CFLAGS) \
+ $(MINER_FLICKR_CFLAGS) \
+ $(GCOV_CFLAGS)
+
+VALAFLAGS = \
+ --pkg gio-2.0 \
+ --pkg rest \
+ --pkg posix \
+ --thread
+
+libexec_PROGRAMS = tracker-miner-flickr
+
+tracker_miner_flickr_VALASOURCES = \
+ tracker-miner-flickr.vala \
+ query-queue.vala \
+ $(top_srcdir)/src/libtracker-client/tracker-client-0.7.vapi
+
+tracker_miner_flickr_SOURCES = \
+ $(tracker_miner_flickr_VALASOURCES:.vala=.c)
+
+tracker_miner_flickr_LDADD = \
+ $(top_builddir)/src/libtracker-miner/libtracker-miner- TRACKER_API_VERSION@.la \
+ $(top_builddir)/src/libtracker-client/libtracker-client- TRACKER_API_VERSION@.la \
+ $(DBUS_LIBS) \
+ $(GMODULE_LIBS) \
+ $(GTHREAD_LIBS) \
+ $(GIO_LIBS) \
+ $(GCOV_LIBS) \
+ $(GLIB2_LIBS) \
+ $(MINER_FLICKR_LIBS) \
+ -lz \
+ -lm
+
+vapi_sources = \
+ $(top_srcdir)/src/libtracker-miner/tracker-miner- TRACKER_API_VERSION@.vapi
+
+tracker-miner-flickr.vala.stamp: $(tracker_miner_flickr_VALASOURCES) $(vapi_sources)
+ $(AM_V_GEN)$(VALAC) $(GCOV_VALAFLAGS) -C $(VALAFLAGS) $^
+ touch $@
+
+
+BUILT_SOURCES = tracker-miner-flickr.vala.stamp
+
+MAINTAINERCLEANFILES = \
+ tracker-miner-flickr.vala.stamp \
+ $(tracker_miner_flickr_VALASOURCES:.vala=.c)
+
+EXTRA_DIST = \
+ $(tracker_miner_flickr_VALASOURCES) \
+ tracker-miner-flickr.vala.stamp
diff --git a/src/tracker-miner-flickr/query-queue.vala b/src/tracker-miner-flickr/query-queue.vala
new file mode 100644
index 0000000..b6104d1
--- /dev/null
+++ b/src/tracker-miner-flickr/query-queue.vala
@@ -0,0 +1,75 @@
+/*
+ * Copyright (C) 2010, Adrien Bustany <abustany gnome org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+public class QueryQueue : GLib.Object {
+ /* Holds the pending sparql updates and monitors them */
+ private HashTable<uint, string> queue;
+ private uint cookie;
+
+ private Mutex flush_mutex;
+
+ private Tracker.Miner miner;
+
+ public QueryQueue (Tracker.Miner parent) {
+ miner = parent;
+
+ queue = new HashTable<uint, string> (direct_hash, direct_equal);
+ cookie = 0;
+
+ flush_mutex = new Mutex ();
+ }
+
+ public async void append (string query) {
+ uint current_cookie = cookie ++;
+ queue.insert (current_cookie, query);
+
+ try {
+ yield miner.execute_batch_update (query);
+ } catch (Error tracker_error) {
+ warning ("BatchUpdate query failed: %s", tracker_error.message);
+ }
+
+ queue.remove (current_cookie);
+ }
+
+ /* BLOCKING flush */
+ public void flush () {
+ if (!flush_mutex.trylock ()) {
+ message ("There's already a flush taking place");
+ return;
+ }
+
+ if (queue.size () > 0) {
+ MainLoop wait_loop;
+ try {
+ wait_loop = new MainLoop (null, false);
+ miner.commit (null, () => { wait_loop.quit (); });
+ wait_loop.run ();
+ } catch (Error tracker_error) {
+ warning ("Commit query failed: %s", tracker_error.message);
+ }
+ }
+
+ flush_mutex.unlock ();
+ }
+
+ public uint size () {
+ return queue.size ();
+ }
+}
diff --git a/src/tracker-miner-flickr/tracker-miner-flickr.vala b/src/tracker-miner-flickr/tracker-miner-flickr.vala
new file mode 100644
index 0000000..acf5d5c
--- /dev/null
+++ b/src/tracker-miner-flickr/tracker-miner-flickr.vala
@@ -0,0 +1,607 @@
+/*
+ * Copyright (C) 2010, Adrien Bustany <abustany gnome org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+namespace Tracker {
+
+private errordomain RestCallError {
+ INVALID_RESPONSE, /* Malformed XML */
+ CALL_ERROR /* Call failed */
+}
+
+public class MinerFlickr : Tracker.MinerWeb {
+ private static const string MINER_NAME = "Flickr";
+ private static const string MINER_DESCRIPTION = "Tracker miner for Flickr";
+ private static const string API_KEY = "7983269709fa3158c752e3e4d6b3b9e5";
+ private static const string SHARED_SECRET = "c0316d1cb4b15e2d";
+ private static const string DATASOURCE_URN = "urn:2208f9fc-3c5b-4e40-ade4-45a0d7b0cf6f";
+ private static const string FLICKR_AUTH_URL = "http://api.flickr.com/services/auth/";;
+ private static const string FLICKR_REST_URL = "http://api.flickr.com/services/rest/";;
+ private static const string FLICKR_PHOTOSET_URL = "http://www.flickr.com/photos/%s/sets/%s";;
+ private static const string FLICKR_PHOTO_URL = "http://farm%s.static.flickr.com/%s/%s_%s.jpg";;
+
+ /* Values taken from the EXIF spec */
+ private enum ExifTag {
+ CAMERA = 271,
+ FLASH = 37385,
+ FNUMBER = 33437,
+ FOCAL_LENGTH = 37386,
+ ISO_SPEED = 2,
+ METERING_MODE = 37383,
+ WHITE_BALANCE = 5
+ }
+
+ private enum ExifMeteringMode {
+ AVERAGE = 1,
+ CENTER_WEIGHTED_AVERAGE,
+ SPOT,
+ MULTISPOT,
+ PATTERN,
+ PARTIAL
+ }
+
+ private enum ExifWhiteBalance {
+ AUTO = 0,
+ MANUAL
+ }
+
+ private static const uint PULL_INTERVAL = 5*60; /* seconds */
+ private uint pull_timeout_handle;
+
+ private QueryQueue query_queue;
+ private static MainLoop main_loop;
+
+ private Rest.Proxy rest;
+
+ /* Only used during association phase */
+ private string frob;
+ /* Used to sign calls */
+ private string auth_token;
+ /* Used to form some urls */
+ private string user_id;
+
+ construct {
+ set ("name", MINER_NAME);
+ set ("associated", false);
+ set ("status", "Idle");
+ set ("progress", 1.0);
+
+ rest = new Rest.Proxy (FLICKR_REST_URL, false);
+
+ query_queue = new QueryQueue (this);
+
+ this.notify["association"].connect (association_status_changed);
+ }
+
+ public void shutdown () {
+ set ("associated", false);
+ }
+
+ private void association_status_changed (Object source, ParamSpec pspec) {
+ bool associated;
+
+ get ("associated", out associated);
+
+ if (associated) {
+ if (pull_timeout_handle != 0)
+ return;
+
+ message ("Miner is now associated. Initiating periodic pull.");
+ pull_timeout_handle = Timeout.add_seconds (PULL_INTERVAL, pull_timeout_cb);
+ Idle.add ( () => { pull_timeout_cb (); return false; });
+ } else {
+ if (pull_timeout_handle == 0)
+ return;
+
+ Source.remove (pull_timeout_handle);
+ }
+ }
+
+ private bool pull_timeout_cb () {
+ init_pull ();
+ return true;
+ }
+
+ private async void init_pull () {
+ Rest.ProxyCall albums_call;
+ Rest.XmlNode photosets_node;
+
+ set ("status", "Refreshing photo albums");
+ set ("progress", 0.0);
+
+ Idle.add (init_pull.callback);
+ yield;
+
+
+ /* First get the list of albums */
+ albums_call = rest.new_call ();
+ albums_call.add_param ("method", "flickr.photosets.getList");
+
+ try {
+ photosets_node = run_call (albums_call);
+ insert_photosets (photosets_node);
+ query_queue.flush ();
+ } catch (Error call_error) {
+ warning ("Could not get photosets list: %s", call_error.message);
+ }
+
+ set ("status", "Idle");
+ set ("progress", 1.0);
+ message ("Pull finished");
+ }
+
+ private void insert_photosets (Rest.XmlNode root_node) {
+ Rest.XmlNode photoset_node;
+ Rest.XmlNode title_node;
+ Rest.XmlNode photos_node;
+ string photoset_url;
+ Rest.ProxyCall photos_call;
+ SparqlBuilder builder;
+ uint n_photosets;
+ uint indexed_photosets = 0;
+
+ photoset_node = root_node.find ("photoset");
+ n_photosets = root_node.children.size ();
+
+ while (photoset_node != null) {
+ photoset_url = FLICKR_PHOTOSET_URL.printf (user_id, photoset_node.get_attr ("id"));
+
+ message ("Getting photos for album %s", photoset_url);
+ builder = new SparqlBuilder.update ();
+ builder.insert_open (photoset_url);
+
+ builder.subject ("_:album");
+ builder.predicate ("a");
+ builder.object ("nfo:MediaList");
+ builder.predicate ("a");
+ builder.object ("nfo:RemoteDataObject");
+ builder.predicate ("nie:url");
+ builder.object_string (photoset_url);
+
+ title_node = photoset_node.find ("title");
+ if (title_node != null) {
+ builder.predicate ("dc:title");
+ builder.object_string (title_node.content);
+ }
+
+ builder.insert_close ();
+
+ query_queue.append (builder.get_result ());
+
+ set ("status", "Refresing album \"%s\"".printf (title_node.content));
+
+ try {
+ photos_call = rest.new_call ();
+ photos_call.add_params ("method", "flickr.photosets.getPhotos",
+ "photoset_id", photoset_node.get_attr ("id"),
+ "media", "photos",
+ "extras", "original_format");
+ photos_node = run_call (photos_call);
+ insert_photos (photos_node);
+ } catch (Error call_error) {
+ warning ("Could not list photos for photoset %s: %s", photoset_url, call_error.message);
+ }
+ photoset_node = photoset_node.next;
+
+ indexed_photosets ++;
+ set ("progress", (1.0*indexed_photosets)/n_photosets);
+ }
+ }
+
+ private void insert_photos (Rest.XmlNode root_node) {
+ Rest.XmlNode photoset_node;
+ string photoset_url;
+ Rest.XmlNode photo_node;
+ string photo_url;
+ SparqlBuilder builder;
+
+ photoset_node = root_node.find ("photoset");
+ if (photoset_node == null || photoset_node.get_attr ("id") == null) {
+ warning ("Malformed response for flickr.photosets.getPhotos");
+ return;
+ }
+
+ photoset_url = FLICKR_PHOTOSET_URL.printf (user_id, photoset_node.get_attr ("id"));
+ message ("Indexing photoset %s", photoset_url);
+
+ photo_node = root_node.find ("photo");
+
+ while (photo_node != null) {
+ photo_url = FLICKR_PHOTO_URL.printf (photo_node.get_attr ("farm"),
+ photo_node.get_attr ("server"),
+ photo_node.get_attr ("id"),
+ photo_node.get_attr ("secret"));
+ builder = new SparqlBuilder.update ();
+ builder.insert_open (photo_url);
+
+ builder.subject ("_:photo");
+ builder.predicate ("a");
+ builder.object ("nmm:Photo");
+ builder.predicate ("a");
+ builder.object ("nfo:RemoteDataObject");
+ builder.predicate ("a");
+ builder.object ("nfo:MediaFileListEntry");
+
+ builder.predicate ("nie:url");
+ builder.object_string (photo_url);
+
+ insert_photo_info (photo_node, builder);
+ insert_exif_data (photo_node, builder);
+
+ builder.insert_close ();
+
+ query_queue.append (builder.get_result ());
+
+ photo_node = photo_node.next;
+ }
+ }
+
+ private void insert_photo_info (Rest.XmlNode photo_node, SparqlBuilder builder) {
+ var info_call = rest.new_call ();
+ Rest.XmlNode root_node;
+ Rest.XmlNode title_node;
+ Rest.XmlNode description_node;
+ Rest.XmlNode tag_node;
+
+ info_call.add_params ("method", "flickr.photos.getInfo",
+ "photo_id", photo_node.get_attr ("id"));
+
+ try {
+ root_node = run_call (info_call);
+ } catch (Error call_error) {
+ warning ("Couldn't get info for photo %s: %s", photo_node.get_attr ("id"), call_error.message);
+ return;
+ }
+
+ title_node = root_node.find ("title");
+ if (title_node != null && title_node.content != null) {
+ builder.predicate ("dc:title");
+ builder.object_string (title_node.content);
+ }
+
+ description_node = root_node.find ("description");
+ if (description_node != null && description_node.content != null) {
+ builder.predicate ("rdfs:comment");
+ builder.object_string (description_node.content);
+ }
+
+ tag_node = root_node.find ("tags").find ("tag");
+
+ while (tag_node != null) {
+ builder.predicate ("nao:hasTag");
+
+ builder.object_blank_open ();
+ builder.predicate ("a");
+ builder.object ("nao:Tag");
+ builder.predicate ("nao:prefLabel");
+ builder.object_string (tag_node.get_attr ("raw"));
+ builder.object_blank_close ();
+
+ tag_node = tag_node.next;
+ }
+ }
+
+ private void insert_exif_data (Rest.XmlNode photo_node, SparqlBuilder builder) {
+ var exif_call = rest.new_call ();
+ Rest.XmlNode root_node;
+ Rest.XmlNode exif_node;
+ string exif_value;
+
+ exif_call.add_params ("method", "flickr.photos.getExif",
+ "photo_id", photo_node.get_attr ("id"));
+
+ try {
+ root_node = run_call (exif_call);
+ } catch (Error call_error) {
+ warning ("Couldn't get EXIF data for photo %s: %s", photo_node.get_attr ("id"), call_error.message);
+ return;
+ }
+
+ exif_node = root_node.find ("exif");
+
+ while (exif_node != null) {
+ exif_value = exif_node.find ("raw").content;
+
+ switch (exif_node.get_attr ("tag").to_int ()) {
+ case ExifTag.CAMERA:
+ builder.predicate ("nmm:camera");
+ builder.object_string (exif_value);
+ break;
+ case ExifTag.FLASH:
+ builder.predicate ("nmm:flash");
+ builder.object (exif_value.to_int ()%2 == 1 ? "nmm:flash-on" : "nmm:flash-off");
+ break;
+ case ExifTag.FNUMBER:
+ builder.predicate ("nmm:fnumber");
+ builder.object_double (ratioToDouble (exif_value));
+ break;
+ case ExifTag.FOCAL_LENGTH:
+ builder.predicate ("nmm:focalLength");
+ builder.object_double (ratioToDouble (exif_value));
+ break;
+ case ExifTag.ISO_SPEED:
+ builder.predicate ("nmm:isoSpeed");
+ builder.object_int64 ((int64)exif_value.to_int ());
+ break;
+ case ExifTag.METERING_MODE:
+ builder.predicate ("nmm:meteringMode");
+ switch (exif_value.to_int ()) {
+ case ExifMeteringMode.AVERAGE:
+ builder.object ("nmm:meteringMode-average");
+ break;
+ case ExifMeteringMode.CENTER_WEIGHTED_AVERAGE:
+ builder.object ("nmm:meteringMode-center-weighted-average");
+ break;
+ case ExifMeteringMode.SPOT:
+ builder.object ("nmm:meteringMode-spot");
+ break;
+ case ExifMeteringMode.MULTISPOT:
+ builder.object ("nmm:meteringMode-multispot");
+ break;
+ case ExifMeteringMode.PATTERN:
+ builder.object ("nmm:meteringMode-pattern");
+ break;
+ case ExifMeteringMode.PARTIAL:
+ builder.object ("nmm:meteringMode-partial");
+ break;
+ default:
+ builder.object ("nmm:meteringMode-other");
+ break;
+ }
+ break;
+ case ExifTag.WHITE_BALANCE:
+ builder.predicate ("nmm:whiteBalance");
+ switch (exif_value.to_int ()) {
+ case ExifWhiteBalance.AUTO:
+ builder.object ("nmm:whiteBalance-auto");
+ break;
+ case ExifWhiteBalance.MANUAL:
+ builder.object ("nmm:whiteBalance-manual");
+ break;
+ }
+ break;
+ default:
+ break;
+ }
+ exif_node = exif_node.next;
+ }
+ }
+
+ private double ratioToDouble (string ratio) {
+ string[] tokens = ratio.split ("/");
+ if (tokens[1].to_int () == 0) {
+ critical ("fracToDouble : divide by zero while parsing ratio '%s'", ratio);
+ return 0;
+ }
+ return (tokens[0].to_int () * 1.0) / (tokens[1].to_int ());
+ }
+
+ private void sign_call (Rest.ProxyCall call) {
+ StringBuilder signature;
+ HashTable<string, string> parameters;
+ List<weak string> parameter_names;
+
+ call.add_param ("api_key", API_KEY);
+ if (auth_token != null)
+ call.add_param ("auth_token", auth_token);
+
+ signature = new StringBuilder (SHARED_SECRET);
+ parameters = call.get_params ();
+
+ parameter_names = parameters.get_keys ().copy ();
+ parameter_names.sort ((CompareFunc)strcmp);
+
+ foreach (string parameter in parameter_names) {
+ signature.append (parameter);
+ signature.append (parameters.lookup (parameter));
+ }
+
+ call.add_param ("api_sig", Checksum.compute_for_string (ChecksumType.MD5, signature.str));
+ }
+
+ private Rest.XmlNode? run_call (Rest.ProxyCall call) throws GLib.Error {
+ Rest.XmlParser parser;
+ Rest.XmlNode root_node;
+
+ sign_call (call);
+
+ try {
+ call.run (null);
+ } catch (Error call_error) {
+ throw call_error;
+ }
+
+ parser = new Rest.XmlParser ();
+ root_node = parser.parse_from_data (call.get_payload (), call.get_payload_length ());
+ if (root_node == null || root_node.name != "rsp") {
+ throw new RestCallError.INVALID_RESPONSE ("Empty payload or root node not \"rsp\"");
+ }
+
+ return root_node;
+ }
+
+ public override HashTable<string, string> get_association_data () throws Tracker.MinerWebError {
+ var association_data = new HashTable<string, string> (str_hash, str_equal);
+ var frob_call = rest.new_call ();
+ Rest.XmlNode root_node;
+ Rest.XmlNode frob_node;
+ string api_signature;
+ string url;
+
+ frob_call.add_param ("method", "flickr.auth.getFrob");
+
+ try {
+ root_node = run_call (frob_call);
+ } catch (Error call_error) {
+ throw new MinerWebError.SERVICE ("Error while getting association data: %s", call_error.message);
+ }
+
+ frob_node = root_node.find ("frob");
+ if (frob_node == null || frob_node.content == null) {
+ throw new MinerWebError.SERVICE ("Malformed XML response while getting frob");
+ }
+
+ this.frob = frob_node.content;
+
+ api_signature = Checksum.compute_for_string (ChecksumType.MD5,
+ SHARED_SECRET + "api_key" + API_KEY + "frob" + this.frob + "permsread");
+ url = FLICKR_AUTH_URL + "?api_key=" + API_KEY + "&perms=read&frob=" + this.frob + "&api_sig=" + api_signature;
+
+ association_data.insert ("url", url);
+
+ return association_data;
+ }
+
+ public override void associate (HashTable<string, string> association_data) throws Tracker.MinerWebError {
+ var password_provider = PasswordProvider.get ();
+ var token_call = rest.new_call ();
+ Rest.XmlNode root_node;
+ Rest.XmlNode token_node;
+ Rest.XmlNode user_node;
+
+ token_call.add_params ("method", "flickr.auth.getToken",
+ "frob", this.frob);
+
+ try {
+ root_node = run_call (token_call);
+ } catch (Error call_error) {
+ throw new MinerWebError.SERVICE ("Unable to get authentication token: %s", call_error.message);
+ }
+
+ token_node = root_node.find ("token");
+ user_node = root_node.find ("user");
+ if (token_node == null || token_node.content == null
+ || user_node == null || user_node.get_attr ("username") == null) {
+ throw new MinerWebError.SERVICE ("Malformed XML response while getting token");
+ }
+
+ try {
+ password_provider.store_password (MINER_NAME,
+ MINER_DESCRIPTION,
+ user_node.get_attr ("username"),
+ token_node.content);
+ } catch (Error e) {
+ if (e is PasswordProviderError.SERVICE) {
+ throw new MinerWebError.KEYRING (e.message);
+ }
+
+ critical ("Internal error: %s", e.message);
+ return;
+ }
+ }
+
+ public override void authenticate () throws MinerWebError {
+ PasswordProvider password_provider;
+ Rest.ProxyCall login_call;
+ Rest.XmlNode root_node;
+ Rest.XmlNode user_node;
+
+ password_provider = PasswordProvider.get ();
+
+ set ("associated", false);
+
+ try {
+ auth_token = password_provider.get_password (MINER_NAME, null);
+ } catch (Error e) {
+ if (e is PasswordProviderError.NOTFOUND) {
+ throw new MinerWebError.NO_CREDENTIALS ("Miner is not associated");
+ }
+ throw new MinerWebError.KEYRING (e.message);
+ }
+
+ login_call = rest.new_call ();
+ login_call.add_param ("method", "flickr.auth.checkToken");
+
+ try {
+ root_node = run_call (login_call);
+ } catch (Error call_error) {
+ throw new MinerWebError.SERVICE ("Cannot verify login: %s", call_error.message);
+ }
+
+ user_node = root_node.find ("user");
+ if (user_node == null || user_node.get_attr ("nsid") == null) {
+ throw new MinerWebError.WRONG_CREDENTIALS ("Stored authentication token is not valid");
+ }
+
+ user_id = user_node.get_attr ("nsid");
+
+ message ("Authentication successful");
+ set ("associated", true);
+ }
+
+ public override void dissociate () throws MinerWebError {
+ var password_provider = PasswordProvider.get ();
+
+ try {
+ password_provider.forget_password (MINER_NAME);
+ } catch (Error e) {
+ if (e is PasswordProviderError.SERVICE) {
+ throw new MinerWebError.KEYRING (e.message);
+ }
+
+ critical ("Internal error: %s", e.message);
+ return;
+ }
+
+ set ("associated", false);
+ }
+
+ private static bool in_loop = false;
+ private static void signal_handler (int signo) {
+ if (in_loop) {
+ Posix.exit (Posix.EXIT_FAILURE);
+ }
+
+ switch (signo) {
+ case Posix.SIGINT:
+ case Posix.SIGTERM:
+ in_loop = true;
+ main_loop.quit ();
+ break;
+ }
+ }
+
+ private static void init_signals () {
+#if G_OS_WIN32
+#else
+ Posix.sigaction_t act = Posix.sigaction_t ();
+ Posix.sigset_t empty_mask = Posix.sigset_t ();
+ Posix.sigemptyset (empty_mask);
+ act.sa_handler = signal_handler;
+ act.sa_mask = empty_mask;
+ act.sa_flags = 0;
+
+ Posix.sigaction (Posix.SIGTERM, act, null);
+ Posix.sigaction (Posix.SIGINT, act, null);
+#endif
+ }
+
+ public static void main (string[] args) {
+ Environment.set_application_name ("Flickr tracker miner");
+ MinerFlickr flickr_miner = Object.new (typeof (MinerFlickr)) as MinerFlickr;
+
+ init_signals ();
+
+ main_loop = new MainLoop (null, false);
+ main_loop.run ();
+
+ flickr_miner.shutdown ();
+ }
+}
+
+} // End namespace Tracker
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
![http://farm%s.static.flickr.com/%s/%s_%s.jpg"](http://farm%s.static.flickr.com/%s/%s_%s.jpg"){kind=link}