[tracker/miner-web: 75/77] libtracker-miner: Add Facebook miner
- From: Adrien Bustany <abustany src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [tracker/miner-web: 75/77] libtracker-miner: Add Facebook miner
- Date: Wed, 3 Mar 2010 12:55:00 +0000 (UTC)
commit 908a1abd3026e877e3e857677522a8a82180cfd8
Author: Adrien Bustany <abustany gnome org>
Date: Fri Nov 6 15:25:19 2009 +0100
libtracker-miner: Add Facebook miner
configure.ac | 55 ++
....freedesktop.Tracker1.Miner.Facebook.service.in | 3 +
data/miners/Makefile.am | 6 +-
data/miners/tracker-miner-facebook.desktop.in | 7 +
src/Makefile.am | 3 +
src/tracker-miner-facebook/Makefile.am | 66 ++
src/tracker-miner-facebook/facebook.vala | 719 ++++++++++++++++++++
7 files changed, 857 insertions(+), 2 deletions(-)
---
diff --git a/configure.ac b/configure.ac
index fbd985d..b30d8a2 100644
--- a/configure.ac
+++ b/configure.ac
@@ -158,6 +158,7 @@ LIBSTREAMANALYZER_REQUIRED=0.7.0
GEE_REQUIRED=0.3
ID3LIB_REQUIRED=3.8.3
GNOME_KEYRING_REQUIRED=2.26
+LIBREST_REQUIRED=0.6.1
# Library Checks
PKG_CHECK_MODULES(GLIB2, [glib-2.0 >= $GLIB_REQUIRED])
@@ -792,6 +793,37 @@ fi
AM_CONDITIONAL(HAVE_GNOME_KEYRING, test "x$have_gnome_keyring" = "xyes")
+##################################################################
+# Enable librest support?
+##################################################################
+
+AC_ARG_ENABLE(librest,
+ AS_HELP_STRING([--enable-librest],
+ [enable librest (necessary for web miners) [[default=auto]]]),,
+ [enable_librest=auto])
+
+if test "x$enable_librest" != "xno"; then
+ PKG_CHECK_MODULES(LIBREST,
+ [ rest-0.6 >= $LIBREST_REQUIRED ],
+ [have_librest=yes],
+ [have_librest=no])
+ AC_SUBST(LIBREST_LIBS)
+ AC_SUBST(LIBREST_CFLAGS)
+
+ if test "x$have_librest" = "xyes"; then
+ AC_DEFINE(HAVE_LIBREST, [], [Define if we have librest for web miners])
+ fi
+fi
+
+if test "x$enable_librest" = "xyes"; then
+ if test "x$have_librest" != "xyes"; then
+ AC_MSG_ERROR([Couldn't find librest >= $LIBREST_REQUIRED.])
+ fi
+fi
+
+AM_CONDITIONAL(HAVE_LIBREST, test "x$have_librest" = "xyes")
+
+
####################################################################
# Mail miners
####################################################################
@@ -1004,6 +1036,27 @@ fi
AM_CONDITIONAL(HAVE_TRACKER_SEARCH_BAR, test "$have_tracker_search_bar" = "yes")
##################################################################
+# Check for dependencies to build tracker facebook miner
+##################################################################
+
+AC_ARG_ENABLE([miner-facebook],
+ AS_HELP_STRING([--enable-miner-facebook],
+ [enable tracker-miner-facebook[[default=no]]]),,
+ [enable_tracker_miner_facebook=no])
+
+if test "x$enable_tracker_miner_facebook" != "xno" ; then
+ if test "x$have_librest" != "xyes" ; then
+ AC_MSG_ERROR([Couldn't find tracker-miner-facebook dependencies ($LIBREST_REQUIRED).])
+ else
+ have_tracker_miner_facebook="yes"
+ fi
+else
+ have_tracker_miner_facebook="no (disabled)"
+fi
+
+AM_CONDITIONAL(HAVE_TRACKER_MINER_FACEBOOK, test "$have_tracker_miner_facebook" = "yes")
+
+##################################################################
# Check for GNOME/GTK dependencies to build tracker search tool
##################################################################
@@ -1691,6 +1744,7 @@ AC_CONFIG_FILES([
src/tracker-control/Makefile
src/tracker-extract/Makefile
src/tracker-miner-fs/Makefile
+ src/tracker-miner-facebook/Makefile
src/tracker-preferences/Makefile
src/tracker-preferences/tracker-preferences.desktop.in
src/tracker-search-bar/Makefile
@@ -1798,6 +1852,7 @@ Metadata Extractors:
Support helix formats (RPM/RM/etc): $have_gstreamer_helix
Support MP3 album art: $selected_for_albumart
Support playlists (w/ Totem): $have_playlist
+ Facebook miner: $have_tracker_miner_facebook
Plugins/Extensions:
diff --git a/data/dbus/org.freedesktop.Tracker1.Miner.Facebook.service.in b/data/dbus/org.freedesktop.Tracker1.Miner.Facebook.service.in
new file mode 100644
index 0000000..cd01265
--- /dev/null
+++ b/data/dbus/org.freedesktop.Tracker1.Miner.Facebook.service.in
@@ -0,0 +1,3 @@
+[D-BUS Service]
+Name=org.freedesktop.Tracker1.Miner.Facebook
+Exec= libexecdir@/tracker-miner-facebook
diff --git a/data/miners/Makefile.am b/data/miners/Makefile.am
index cdb1db4..4ea7bd3 100644
--- a/data/miners/Makefile.am
+++ b/data/miners/Makefile.am
@@ -2,13 +2,15 @@ include $(top_srcdir)/Makefile.decl
desktop_in_files = \
tracker-miner-applications.desktop.in \
- tracker-miner-files.desktop.in
+ tracker-miner-files.desktop.in \
+ tracker-miner-facebook.desktop.in
tracker_minersdir = $(datadir)/tracker/miners
tracker_miners_DATA = \
tracker-miner-applications.desktop \
- tracker-miner-files.desktop
+ tracker-miner-files.desktop \
+ tracker-miner-facebook.desktop
@INTLTOOL_DESKTOP_RULE@
diff --git a/data/miners/tracker-miner-facebook.desktop.in b/data/miners/tracker-miner-facebook.desktop.in
new file mode 100644
index 0000000..0ecf121
--- /dev/null
+++ b/data/miners/tracker-miner-facebook.desktop.in
@@ -0,0 +1,7 @@
+[Desktop Entry]
+Encoding=UTF-8
+_Name=Facebook
+_Comment=Index your data on Facebook
+DBusName=org.freedesktop.Tracker1.Miner.Facebook
+DBusPath=/org/freedesktop/Tracker1/Miner/Facebook
+X-Tracker-Bridge-AuthScheme=Token
diff --git a/src/Makefile.am b/src/Makefile.am
index 2f5ff33..e6ee672 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -39,3 +39,6 @@ if HAVE_TRACKER_EXPLORER
SUBDIRS += tracker-explorer
endif
+if HAVE_TRACKER_MINER_FACEBOOK
+SUBDIRS += tracker-miner-facebook
+endif
diff --git a/src/tracker-miner-facebook/Makefile.am b/src/tracker-miner-facebook/Makefile.am
new file mode 100644
index 0000000..5a67729
--- /dev/null
+++ b/src/tracker-miner-facebook/Makefile.am
@@ -0,0 +1,66 @@
+include $(top_srcdir)/Makefile.decl
+
+AM_CPPFLAGS = \
+ -include $(CONFIG_HEADER)
+
+VALAINCLUDES= \
+ --vapidir $(top_srcdir)/src/libtracker-miner \
+ --pkg posix \
+ --pkg dbus-glib-1 \
+ --pkg tracker-miner-0.7 \
+ --pkg rest \
+ --pkg uuid \
+ --thread
+
+libexec_PROGRAMS=tracker-miner-facebook
+
+tracker_miner_facebook_VALASOURCES= \
+ facebook.vala
+
+tracker_miner_facebook_SOURCES= \
+ $(tracker_miner_facebook_VALASOURCES:.vala=.c)
+
+tracker-miner-facebook.vala.stamp: $(tracker_miner_facebook_VALASOURCES)
+ $(VALAC) -C $(VALAINCLUDES) $(VALAFLAGS) $^
+ touch $@
+
+INCLUDES = \
+ -Wall \
+ -DSHAREDIR=\""$(datadir)"\" \
+ -DPKGLIBDIR=\""$(libdir)/tracker"\" \
+ -DLOCALEDIR=\""$(localedir)"\" \
+ -DLIBEXEC_PATH=\""$(libexecdir)"\" \
+ -DG_LOG_DOMAIN=\"Tracker\" \
+ -DTRACKER_COMPILATION \
+ -I$(top_srcdir)/src \
+ $(WARN_CFLAGS) \
+ $(GMODULE_CFLAGS) \
+ $(GLIB2_CFLAGS) \
+ $(GOBJECT_CFLAGS) \
+ $(GTHREAD_CFLAGS) \
+ $(DBUS_CFLAGS) \
+ $(LIBREST_CFLAGS) \
+ $(UUID_CFLAGS)
+
+tracker_miner_facebook_LDADD= \
+ $(top_builddir)/src/libtracker-miner/libtracker-miner-0.7.la \
+ $(GLIB2_LIBS) \
+ $(GOBJECT_LIBS) \
+ $(GTHREAD_LIBS) \
+ $(DBUS_LIBS) \
+ $(LIBREST_LIBS) \
+ $(UUID_LIBS)
+
+BUILT_SOURCES= \
+ tracker-miner-facebook.vala.stamp
+
+CLEANFILES= $(BUILT_SOURCES)
+
+EXTRA_DIST= \
+ tracker-miner-facebook.vala.stamp \
+ $(tracker_miner_facebook_SOURCES) \
+ $(tracker_miner_facebook_VALASOURCES)
+
+MAINTAINERCLEANFILES= \
+ $(tracker_miner_facebook_SOURCES) \
+ tracker-miner-facebook.vala.stamp
diff --git a/src/tracker-miner-facebook/facebook.vala b/src/tracker-miner-facebook/facebook.vala
new file mode 100644
index 0000000..9c31778
--- /dev/null
+++ b/src/tracker-miner-facebook/facebook.vala
@@ -0,0 +1,719 @@
+using Tracker;
+using Rest;
+
+public class FacebookMiner : Tracker.MinerWeb {
+ private const string SERVICE_NAME = "Facebook";
+ private const string SERVICE_DESCRIPTION = "Authentication token for Facebook miner";
+ private const string API_KEY = "a07366931355e51525938ade2d0df2fb";
+ private const string SHARED_SECRET = "dd34c9d53460953bfd3b5aa87c09b538";
+ private const string FACEBOOK_REST = "https://api.facebook.com/restserver.php";;
+ private const string REST_ERRORMSG = "Error during REST call : %s";
+ private const string MINER_DATASOURCE_URN = "urn:nepomuk:datasource:40d8b787-3de2-46d3-984c-1b021a996ef9";
+
+ private const uint update_interval = 600; // in seconds
+
+ private unowned PasswordProvider password_provider;
+
+ private Proxy rest;
+ private string auth_token;
+ private string session = null;
+ private string secret = SHARED_SECRET;
+
+ private string username = null; // Is actually the numeric user ID
+
+ private uint timeout_handle = 0;
+
+ construct {
+ // Set name for Tracker.Miner
+ set ("name", SERVICE_NAME);
+
+ password_provider = get_password_provider ();
+ }
+
+ public FacebookMiner ()
+ {
+ rest = new Proxy (FACEBOOK_REST, false);
+ }
+
+ // Tracker.Miner functions
+ public override void started ()
+ {
+ message ("Initializing miner...");
+
+ try {
+ authenticate ();
+ } catch (Error e) {
+ warning ("Error while authenticating : %s", e.message);
+ }
+ }
+
+ public override void stopped ()
+ {
+ }
+
+ public override void paused ()
+ {
+ }
+
+ public override void resumed ()
+ {
+ }
+
+ public override void writeback ([CCode (array_length = false, array_null_terminated = true)] string [] subjects)
+ {
+ }
+
+ // Tracker.MinerWeb
+ public override HashTable<string, string> get_association_data () throws MinerWebError
+ {
+ var ret = new HashTable<string, string> (str_hash, str_equal);
+ ProxyCall c = rest.new_call ();
+ c.add_param ("method", "auth.createToken");
+ XmlNode node;
+ try {
+ node = runCall (c);
+ } catch (MinerWebError e) {
+ warning ("Error during REST call : %s", e.message);
+ throw e;
+ }
+
+ if (node.name != "auth_createToken_response") {
+#if DEBUG
+ warning (_("Got answer %s\n", c.get_payload ()));
+ warning (_("Couldn't get authentication token"));
+ //error (new MinerWebError.SERVICE (_("Couldn't get authentication token")));
+#endif
+ return ret;
+ }
+
+ auth_token = node.content;
+ string url = "http://www.facebook.com/login.php?api_key=%s&v=1.0&auth_token=%s".printf (API_KEY, auth_token);
+ ret.insert ("url", url);
+
+ ret.insert ("post_message",
+ _("A last browser window will now open, which will allow you to grant"
+ + "Tracker permanent access to your Facebook account, as well as access"
+ + "to your stream (statuses of your friends etc.). You're not obliged"
+ + "to do so, but if you choose not to grant Tracker these permissions,"
+ + "you'll need to login again every 24 hours, and Tracker will only"
+ + "index your photo albums."));
+ ret.insert ("post_url", "http://www.facebook.com/connect/prompt_permissions.php?api_key=%s&v=1.0&next=http://www.facebook.com/connect/login_success.html&display=popup&ext_perm=read_stream,offline_access".printf (API_KEY));
+ return ret;
+ }
+
+ // This supposes we have a valid auth_token. Else, well, it'll just fail...
+ public override void associate (HashTable<string, string> data) throws MinerWebError
+ {
+ ProxyCall c = rest.new_call ();
+ c.add_param ("method", "auth.getSession");
+ c.add_param ("auth_token", auth_token);
+
+ {
+ XmlNode node = runCall (c);
+
+ if (node.name != "auth_getSession_response") {
+#if DEBUG
+ warning ("Got answer %s\n", c.get_payload ());
+ warning ("Couldn't get session token");
+#endif
+ throw new MinerWebError.SERVICE (_(REST_ERRORMSG), node.find ("error_msg").content);
+ }
+
+ username = node.find ("uid").content;
+ secret = node.find ("secret").content;
+ session = node.find ("session_key").content;
+ }
+
+ try {
+ password_provider.store_password (SERVICE_NAME, SERVICE_DESCRIPTION, session, secret);
+ } catch (Error e) {
+ warning ("Couldn't store credentials in the keyring : %s", e.message);
+ throw new MinerWebError.KEYRING (e.message);
+ }
+ }
+
+ public override void dissociate () throws Tracker.MinerWebError
+ {
+ message ("Dissociate not implemented");
+ }
+
+ public override MinerWebAssociationStatus authenticate () throws MinerWebError
+ {
+ message ("Trying to authenticate");
+ string secret;
+
+ uint association_status = MinerWebAssociationStatus.UNASSOCIATED;
+ try {
+ secret = password_provider.get_password (SERVICE_NAME, out session);
+ } catch (Error e) {
+ if (e is PasswordProviderError.NOTFOUND) {
+ message ("No credentials stored in the keyring");
+ throw new MinerWebError.NO_CREDENTIALS (_("Association needed"));
+ } else {
+ warning ("Couldn't access the keyring : %s", e.message);
+ throw new MinerWebError.KEYRING (e.message);
+ }
+ }
+
+ ProxyCall c = rest.new_call ();
+ c.add_param ("method", "users.getLoggedInUser");
+ XmlNode node;
+ try {
+ node = runCall (c);
+ } catch (MinerWebError e) {
+ throw e;
+ }
+
+ if (node.name != "users_getLoggedInUser_response") {
+#if DEBUG
+ warning ("Couldn't get user info\nGot answer %s\n", c.get_payload ());
+#endif
+ if (node.find ("error_code").content == "102") { // Session key invalid or no longer valid
+ session = null;
+ secret = SHARED_SECRET;
+ message ("Authentication token has expired");
+ throw new MinerWebError.TOKEN_EXPIRED (_("Please associate again"));
+ }
+ } else {
+ username = node.content;
+ association_status = MinerWebAssociationStatus.ASSOCIATED;
+ message ("Authentication successful!");
+ }
+
+ set ("association_status", association_status);
+
+ return (MinerWebAssociationStatus)association_status;
+ }
+
+ public async bool pull ()
+ {
+ uint association_status;
+ get ("association_status", out association_status);
+ // Only accept new work if we're idle
+ if (association_status != MinerWebAssociationStatus.ASSOCIATED && timeout_handle != 0) {
+ timeout_handle = 0;
+ return false;
+ }
+
+
+ // Be smart, only pull what's necessary (we're smart, aren't we ?)
+ string photos_pull_from = "1980-01-01T00:00:00Z,";
+ string stream_pull_from = "1980-01-01T00:00:00Z,";
+
+ try {
+ unowned GLib.PtrArray results = yield execute_sparql ("select ?date where { ?album a nfo:MediaList ; nie:dataSource <%s> ; nie:isStoredAs ?uri ; nie:contentLastModified ?date } ORDER BY DESC(?date) LIMIT 1".printf (MINER_DATASOURCE_URN));
+ unowned string[][] res = (string[][]) results.pdata;
+ if (results.len > 0) {
+ photos_pull_from = res[0][0];
+ }
+
+ results = yield execute_sparql ("select ?date where { ?message a mfo:FeedMessage ; nie:dataSource <%s> ; nie:isStoredAs ?uri ; nmo:receivedDate ?date } ORDER BY DESC(?date) LIMIT 1");
+ res = (string[][]) results.pdata;
+ if (results.len > 0) {
+ stream_pull_from = res[0][0];
+ }
+
+ message ("Pulling photos starting from %s and stream starting from %s", photos_pull_from, stream_pull_from);
+ } catch (Error e) {
+ critical ("Error contacting Tracker : %s", e.message);
+ return true;
+ }
+
+ // Pull photos
+ var c = rest.new_call ();
+ c.add_params ("method", "fql.multiquery",
+ "queries", """{
+ "connections" : "SELECT target_id FROM connection WHERE source_id='%s' AND is_following=1",
+ "friends" : "SELECT uid, name FROM user WHERE uid IN (SELECT target_id FROM #connections)",
+ "photos" : "SELECT pid, src_big, caption, aid, owner, created FROM photo WHERE aid IN (SELECT aid FROM album WHERE owner IN (SELECT target_id FROM #connections)) AND modified > '%2$s'",
+ "albums" : "SELECT aid, name, description, link, owner, modified FROM album WHERE owner IN (SELECT target_id FROM #connections) AND modified > '%2$s'"
+ } """.printf (username, timestamp_from_iso8601 (photos_pull_from)));
+ runCall_async (c, pull_photos_call_cb);
+
+ // Pull streams
+ c = rest.new_call ();
+ c.add_params ("method", "fql.multiquery",
+ "queries", """{
+ "stream":"SELECT post_id, actor_id, target_id, message, attachment, permalink, created_time FROM stream WHERE source_id IN (SELECT target_id FROM connection WHERE source_id='%s' AND is_following=1) AND created_time > '%s'",
+ "actors":"SELECT uid, name, pic_square FROM user WHERE uid IN (SELECT actor_id FROM #stream)"
+ }""".printf (username, timestamp_from_iso8601 (stream_pull_from)));
+ runCall_async (c, pull_stream_call_cb);
+
+ return true;
+ }
+
+ private void pull_photos_call_cb (ProxyCall call, GLib.Error err, Object weak_object)
+ {
+ if (err != null) {
+ warning ("Error while pulling pictures : %s", err.message);
+ error (new MinerWebError.SERVICE (err.message));
+ } else {
+ pull_photos.begin (call);
+ }
+
+ }
+
+ private async void pull_photos (ProxyCall call)
+ {
+ message ("Pulling pictures");
+
+ var parser = new XmlParser ();
+ XmlNode root = parser.parse_from_data (call.get_payload (), call.get_payload_length ());
+
+ weak XmlNode connections_node = null;
+ weak XmlNode friends_node = null;
+ weak XmlNode albums_node = null;
+ weak XmlNode photos_node = null;
+
+ { // Assign the results groups to the right variables
+ XmlNode current = root.find ("fql_result");
+
+ if (current == null) {
+ warning ("Error in request : \n%s", call.get_payload ());
+ //error (new MinerWebError.SERVICE (_(REST_ERRORMSG), root.find ("error_msg").content));
+ return;
+ }
+
+ while (current != null) {
+ switch (current.find ("name").content) {
+ case "connections":
+ connections_node = current;
+ break;
+ case "friends":
+ friends_node = current;
+ break;
+ case "albums":
+ albums_node = current;
+ break;
+ case "photos":
+ photos_node = current;
+ break;
+ default:
+ break;
+ }
+
+ current = current.next;
+ }
+ }
+
+ // Maps the friend's uid to their uri
+ var friend_urn = new HashTable<string, string> (str_hash, str_equal);
+ { // List contacts
+ weak XmlNode current_friend = friends_node.find ("fql_result_set").find ("user");
+
+ while (current_friend != null) {
+ friend_urn.insert (current_friend.find ("uid").content, yield get_contact (current_friend.find ("name").content));
+
+ current_friend = current_friend.next;
+ }
+ }
+
+ // Maps the album's aid to their uri
+ var album_urn = new HashTable<string, string> (str_hash, str_equal);
+ { // List albums
+ weak XmlNode current_album = albums_node.find ("fql_result_set").find ("album");
+
+ while (current_album != null) {
+ string current_album_uri = current_album.find ("link").content;
+ string current_album_urn = yield get_resource ("nfo:MediaList", current_album_uri);
+
+ album_urn.insert (current_album.find ("aid").content, current_album_urn);
+
+ try {
+ string name = current_album.find ("name").content;
+ if (name != null) {
+ message ("Will index album %s", name);
+ yield execute_batch_update ("insert into <%s> {<%s> rdfs:label \"%s\"}"
+ .printf (MINER_DATASOURCE_URN,
+ current_album_urn,
+ escape_string (name)));
+ }
+
+ string comment = current_album.find ("description").content;
+ if (comment != null) {
+ yield execute_batch_update ("insert into <%s> {<%s> rdfs:comment \"%s\"}"
+ .printf (MINER_DATASOURCE_URN,
+ current_album_urn,
+ escape_string (comment)));
+ }
+
+ yield execute_batch_update ("insert into <%s> {<%s> nie:contentLastModified '%s' ; nco:creator <%s>"
+ .printf(MINER_DATASOURCE_URN,
+ current_album_urn,
+ timestamp_to_iso8601 (current_album.find ("modified").content),
+ friend_urn.lookup (current_album.find ("owner").content)));
+ yield commit ();
+ } catch (Error update_album_error) {
+ critical ("Error while contacting Tracker : %s", update_album_error.message);
+ //error (new MinerWebError.TRACKER (e.message));
+ return;
+ }
+
+ current_album = current_album.next;
+ }
+ }
+
+ { // And finally, list photos (remember, that's why whe're here for)
+ weak XmlNode current_photo = photos_node.find ("fql_result_set").find ("photo");
+
+ while (current_photo != null) {
+ string uri = current_photo.find ("src_big").content;
+ if (uri == null) {
+ uri = current_photo.find ("src").content;
+ }
+
+ if (uri == null) {
+ current_photo = current_photo.next;
+ continue;
+ }
+ string urn = yield get_resource ("nmm:Photo", uri);
+
+ message ("Indexing photo %s with urn %s", uri, urn);
+
+ try {
+ string caption = current_photo.find ("caption").content;
+ if (caption != null) {
+ yield execute_batch_update ("insert into <%s> {<%s> rdfs:label \"%s\"}"
+ .printf (MINER_DATASOURCE_URN,
+ urn,
+ escape_string (caption)));
+ }
+
+ string date = timestamp_to_iso8601 (current_photo.find ("created").content);
+ yield execute_batch_update ("insert into <%s> {<%s> nie:contentCreated '%s'}"
+ .printf (MINER_DATASOURCE_URN,
+ urn,
+ date));
+
+ if (album_urn.lookup (current_photo.find ("aid").content) != null) {
+ yield execute_batch_update ("insert into <%s> {<%s> a nfo:MediaFileListEntry . <%s> nfo:mediaListEntry <%1$s>}"
+ .printf (MINER_DATASOURCE_URN,
+ urn,
+ album_urn.lookup (current_photo.find ("aid").content)));
+ } else {
+ warning ("Unknown album for picture %s", uri);
+ }
+
+ yield execute_batch_update ("insert into <%s> {<%s> nco:creator <%s>}"
+ .printf (MINER_DATASOURCE_URN,
+ urn,
+ friend_urn.lookup (current_photo.find ("owner").content)));
+ yield commit ();
+ } catch (Error update_photo_error) {
+ critical ("Error while contacting Tracker : %s", update_photo_error.message);
+ //error (new MinerWebError.TRACKER (e.message));
+ return;
+ }
+
+ current_photo = current_photo.next;
+ }
+ }
+
+ message ("Photos pulled");
+ }
+
+ private void pull_stream_call_cb (ProxyCall call, GLib.Error err, Object weak_object)
+ {
+ if (err != null) {
+ warning ("Error while pulling pictures : %s", err.message);
+ error (new MinerWebError.SERVICE (err.message));
+ return;
+ } else {
+ pull_stream.begin (call);
+ }
+ }
+
+ private async void pull_stream (ProxyCall call)
+ {
+
+ var parser = new XmlParser ();
+ XmlNode root = parser.parse_from_data (call.get_payload (), call.get_payload_length ());
+ XmlNode stream_result;
+ XmlNode actors_result;
+ {
+ XmlNode result_1 = root.find ("fql_result");
+ if (result_1 == null) {
+ warning ("Error in request : \n%s", call.get_payload ());
+ //error (new MinerWebError.SERVICE (_(REST_ERRORMSG), root.find ("error_msg").content));
+ return;
+ }
+
+ if (result_1.find ("name").content == "stream") {
+ stream_result = result_1;
+ actors_result = result_1.next;
+ } else {
+ stream_result = result_1.next;
+ actors_result = result_1;
+ }
+ }
+
+ // Maps Facebook ids to nco urns
+ var authors = new HashTable<string, string>(str_hash, str_equal);
+
+ XmlNode current = actors_result.find ("fql_result_set").find ("user");
+ while (current != null) {
+ string facebook_id = current.find ("uid").content;
+ var author = current.find ("name").content;
+ try {
+ var authorUid = yield get_contact (author);
+
+ var avatarUrl = current.find ("pic_square").content;
+ var avatarUid = yield get_resource ("nfo:Image", avatarUrl);
+ yield execute_update ("insert into <%s> {<%s> nco:photo <%s>}"
+ .printf (MINER_DATASOURCE_URN,
+ authorUid,
+ avatarUid));
+
+ authors.insert (facebook_id, authorUid);
+ break;
+ } catch (Error insert_contact_error) {
+ critical ("Error contacting Tracker : %s", insert_contact_error.message);
+ //error (new MinerWebError.TRACKER (e.message));
+ }
+ current = current.next;
+ }
+
+ // Pull the stream
+ current = stream_result.find ("fql_result_set").find ("stream_post");
+ while (current != null) {
+ try {
+ string uri = current.find ("permalink").content;
+ string urn = yield get_resource ("mfo:FeedMessage", uri);
+
+ string date = timestamp_to_iso8601 (current.find ("created_time").content);
+ yield execute_batch_update ("insert into <%s> {<%s> nmo:from <%s> ; nmo:receivedDate \"%s\"}"
+ .printf (MINER_DATASOURCE_URN,
+ urn,
+ authors.lookup (current.find ("actor_id").content),
+ date));
+
+ if (current.find ("message").content != null) {
+ yield execute_batch_update ("insert into <%s> {<%s> nmo:plainTextMessageContent \"%s\"}"
+ .printf (MINER_DATASOURCE_URN,
+ urn,
+ escape_string (current.find ("message").content)));
+ }
+
+ // Deal with any attachment we might have
+ // We are obliged to first list the pictures, then do a query to get their src_big attribute
+ if (current.find ("attachment").children.size () > 0) {
+ // Maps the photo pid to the nmo:Message urn
+ var pictures = new List<string> ();
+
+ XmlNode current_media = current.find ("stream_media");
+ while (current_media != null) {
+ switch (current_media.find ("type").content) {
+ case "photo":
+ pictures.append (current_media.find ("photo").find ("pid").content);
+ break;
+ default:
+ warning ("Media type %s not handled yet", current_media.find ("type").content);
+ break;
+ }
+ current_media = current_media.next;
+ }
+
+ if (pictures != null) {
+ weak List<string> current_pic = pictures.first ();
+ var query = new StringBuilder ("SELECT src_big FROM photo WHERE pid = '%s'".printf (current_pic.data));
+
+ while ((current_pic = current_pic.next) != null) {
+ query.append_printf (" OR pid = '%s'", current_pic.data);
+ }
+
+ var c = rest.new_call ();
+ c.add_params ("method", "fql.query",
+ "query", query.str);
+
+ // here we can block, since we don't bother anyone
+ XmlNode photos_node;
+ try {
+ photos_node = runCall (c);
+ } catch (MinerWebError e) {
+ warning ("REST call failed!");
+ return;
+ }
+
+ XmlNode current_photo = photos_node.find ("photo");
+ while (current_photo != null) {
+ var picture_uri = current_photo.find ("src_big").content;
+ var enclosure_urn = yield get_resource ("nmm:Photo", picture_uri);
+
+ yield execute_batch_update ("insert into <%s> {<%s> a mfo:Enclosure ; mfo:remoteLink <%s> . <%s> mfo:enclosureList <%1$s>}"
+ .printf (MINER_DATASOURCE_URN,
+ enclosure_urn,
+ picture_uri,
+ urn));
+
+ // The rest of the photo indexing will be done by pull_photos
+
+ current_photo = current_photo.next;
+ }
+ }
+ }
+
+ yield commit ();
+ } catch (Error update_message_error) {
+ critical ("Error while inserting data into Tracker : %s", update_message_error.message);
+ //error (new MinerWebError.TRACKER (e.message));
+ }
+ current = current.next;
+ }
+ message ("Stream pulled");
+ }
+
+ // Private functions
+
+ private XmlNode? runCall (ProxyCall c) throws MinerWebError
+ {
+ signCall (c);
+
+ try {
+ c.run (null);
+ } catch (Error e) {
+ warning ("Error in REST call : %s\n", e.message);
+ //error (new MinerWebError.SERVICE (_(REST_ERRORMSG), e.message));
+ throw new MinerWebError.SERVICE (e.message);
+ }
+
+ var parser = new XmlParser ();
+ weak XmlNode ret = parser.parse_from_data (c.get_payload (), c.get_payload_length ());
+ return ret;
+ }
+
+
+ private void runCall_async (ProxyCall c, ProxyCallAsyncCallback callback)
+ {
+ signCall (c);
+
+ try {
+ c.run_async (callback, this);
+ } catch (Error e) {
+ warning ("Error in REST call : %s\n", e.message);
+ //error (new MinerWebError.SERVICE (_(REST_ERRORMSG), e.message));
+ throw new MinerWebError.SERVICE (e.message);
+ }
+ }
+
+ // Add version, api_key, call_id and sig
+ private void signCall (ProxyCall c)
+ {
+ var time = TimeVal ();
+ var callid = "%ld".printf(1000000*(time.tv_sec) + time.tv_usec);
+ c.add_params ("v", "1.0",
+ "api_key", API_KEY,
+ "call_id", callid,
+ null);
+
+ if (session != null)
+ c.add_param ("session_key", session);
+
+ string sig = "";
+ var par = c.get_params ();
+ List<weak string> keys = par.get_keys ().copy ();
+ keys.sort ((CompareFunc)strcmp);
+
+ for (int i = 0 ; i < keys.length () ; ++i) {
+ sig += keys.nth_data (i) + "=" + par.lookup (keys.nth_data (i));
+ }
+ sig += secret;
+
+ sig = Checksum.compute_for_string (ChecksumType.MD5, sig);
+
+ c.add_param ("sig", sig);
+ }
+
+ // tries to find a resource with the given url as a nie:isStoredAs property. If none is foud, creates one (return value is urn)
+ private async string? get_resource (string klass, string stored_as)
+ {
+ unowned GLib.PtrArray results;
+ try {
+ results = yield execute_sparql ("select ?r where { ?r a %s . ?r nie:isStoredAs \"%s\" }".printf (klass, stored_as));
+ } catch (Error e) {
+ critical ("Couldn't contact Tracker : %s", e.message);
+ //error (new MinerWebError.TRACKER (e.message));
+ return null;
+ }
+
+ unowned string [][] res = (string[][])results.pdata;
+ switch (results.len) {
+ case 0:
+ string urn = "urn:uuid:%s".printf (uuid_generate_string ());
+ yield execute_update ("insert {<%s> a nfo:RemoteDataObject . <%s> a %s ; nie:isStoredAs <%1$s> ; nie:dataSource <%s>}"
+ .printf (stored_as,
+ urn,
+ klass,
+ MINER_DATASOURCE_URN));
+ return urn;
+ case 1:
+ return res[0][0];
+ default:
+ warning ("More than one object of type %s and with nie:isStoredAs '%s'. Returning the first one", klass, stored_as);
+ return res[0][0];
+ }
+ }
+
+ // tries to find a nco:Contact with the given nco:fullname. If none is foud, creates one (return value is urn)
+ private async string? get_contact (string fullname)
+ {
+ unowned GLib.PtrArray results;
+ string escaped_fullname = escape_string (fullname);
+ try {
+ results = yield execute_sparql ("select ?r where { ?r a nco:Contact . ?r nco:fullname \"%s\" }".printf (escaped_fullname));
+ } catch (Error e) {
+ critical ("Couldn't contact Tracker : %s", e.message);
+ //error (new MinerWebError.TRACKER (e.message));
+ return null;
+ }
+
+ unowned string[][] res = (string[][])results.pdata;
+ switch (results.len) {
+ case 0:
+ string urn = "urn:uuid:%s".printf (uuid_generate_string ());
+ yield execute_update ("insert {<%s> a nco:Contact ; nco:fullname \"%s\" ; nie:dataSource <%s>}".printf (urn, escaped_fullname, MINER_DATASOURCE_URN));
+ return urn;
+ case 1:
+ return res[0][0];
+ default:
+ warning ("More than one nco:Contact with nco:fullname '%s'. Returning the first one", fullname);
+ return res[0][0];
+ }
+ }
+
+ private string escape_string(string str) {
+ return str.escape ("").replace ("'", "\\'").replace ("\"", "\\\"");
+ }
+
+
+ private string timestamp_to_iso8601 (string timestamp)
+ {
+ GLib.TimeVal timeval = GLib.TimeVal ();
+ timeval.tv_sec = timestamp.to_long ();
+ return timeval.to_iso8601 ();
+ }
+
+ private string? timestamp_from_iso8601 (string isodate)
+ {
+ GLib.TimeVal timeval = GLib.TimeVal ();
+ if (!timeval.from_iso8601 (isodate))
+ return null;
+
+ return "%ld".printf (timeval.tv_sec);
+ }
+
+}
+
+MainLoop loop;
+
+void main ()
+{
+ loop = new MainLoop (null, false);
+
+ Environment.set_application_name ("FacebookMiner");
+
+ var miner = new FacebookMiner ();
+ miner.start ();
+ message ("Miner started");
+
+ loop.run ();
+}
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
