GNOME.org
 

[tracker/miner-web: 31/48] Facebook miner: Remove direct DBus call and make more async

commit f1e9fe9c729599bd7526a480fd3602af218d27d8
Author: Adrien Bustany <madcat mymadcat com>
Date:   Sat Jan 30 19:33:26 2010 -0300

    Facebook miner: Remove direct DBus call and make more async

 src/tracker-miner-facebook/facebook.vala |  150 +++++++++++++++++-------------
 1 files changed, 86 insertions(+), 64 deletions(-)
---
diff --git a/src/tracker-miner-facebook/facebook.vala b/src/tracker-miner-facebook/facebook.vala
index a32ad9a..f905d46 100644
--- a/src/tracker-miner-facebook/facebook.vala
+++ b/src/tracker-miner-facebook/facebook.vala
@@ -12,8 +12,6 @@ public class FacebookMiner : Tracker.MinerWeb {
 
 	private const uint update_interval = 600; // in seconds
 
-	dynamic DBus.Object tracker;
-
 	private unowned PasswordProvider password_provider;
 
 	private Proxy rest;
@@ -69,6 +67,8 @@ public class FacebookMiner : Tracker.MinerWeb {
 	// Tracker.Miner functions
 	public override void started ()
 	{
+		message ("Initializing miner...");
+
 		try {
 			authenticate ();
 		} catch (Error e) {
@@ -168,6 +168,7 @@ public class FacebookMiner : Tracker.MinerWeb {
 
 	public override MinerWebAssociationStatus authenticate () throws MinerWebError
 	{
+		message ("Trying to authenticate");
 		string secret;
 
 		uint association_status = MinerWebAssociationStatus.UNASSOCIATED;
@@ -175,6 +176,7 @@ public class FacebookMiner : Tracker.MinerWeb {
 			secret = password_provider.get_password (SERVICE_NAME, out session);
 		} catch (Error e) {
 			if (e is PasswordProviderError.NOTFOUND) {
+				message ("No credentials stored in the keyring");
 				throw new MinerWebError.NO_CREDENTIALS (_("Association needed"));
 			} else {
 				warning ("Couldn't access the keyring : %s", e.message);
@@ -198,11 +200,13 @@ public class FacebookMiner : Tracker.MinerWeb {
 			if (node.find ("error_code").content == "102") { // Session key invalid or no longer valid
 				session = null;
 				secret = SHARED_SECRET;
+				message ("Authentication token has expired");
 				throw new MinerWebError.TOKEN_EXPIRED (_("Please associate again"));
 			}
 		} else {
 			username = node.content;
 			association_status = MinerWebAssociationStatus.ASSOCIATED;
+			message ("Authentication successful!");
 		}
 
 		set ("association_status", association_status);
@@ -210,7 +214,7 @@ public class FacebookMiner : Tracker.MinerWeb {
 		return (MinerWebAssociationStatus)association_status;
 	}
 
-	public bool pull ()
+	public async bool pull ()
 	{
 		uint association_status;
 		get ("association_status", out association_status);
@@ -227,14 +231,16 @@ public class FacebookMiner : Tracker.MinerWeb {
 
 		try {
 			// FIXME : REGEX is not really the right solution here
-			string[][] results = tracker.SparqlQuery ("select  ?date where { ?album a nfo:MediaList . ?album nie:isStoredAs ?uri . ?album nie:contentLastModified ?date . FILTER (REGEX(str(?uri), 'www.facebook.com')) } ORDER BY DESC(?date) LIMIT 1");
-			if (results.length > 0) {
-				photos_pull_from = results[0][0];
+			unowned GLib.PtrArray results = yield execute_sparql ("select  ?date where { ?album a nfo:MediaList ; nie:dataSource <%s> ; nie:isStoredAs ?uri ; nie:contentLastModified ?date } ORDER BY DESC(?date) LIMIT 1".printf (MINER_DATASOURCE_URN));
+			unowned string[][] res = (string[][]) results.pdata;
+			if (results.len > 0) {
+				photos_pull_from = res[0][0];
 			}
 
-			results = tracker.SparqlQuery ("select  ?date where { ?message a mfo:FeedMessage . ?message nie:isStoredAs ?uri . ?message nmo:receivedDate ?date . FILTER (REGEX(str(?uri), 'www.facebook.com')) } ORDER BY DESC(?date) LIMIT 1");
-			if (results.length > 0) {
-				stream_pull_from = results[0][0];
+			results = yield execute_sparql ("select  ?date where { ?message a mfo:FeedMessage ; nie:dataSource <%s> ; nie:isStoredAs ?uri ; nmo:receivedDate ?date } ORDER BY DESC(?date) LIMIT 1");
+			res = (string[][]) results.pdata;
+			if (results.len > 0) {
+				stream_pull_from = res[0][0];
 			}
 
 			message ("Pulling photos starting from %s and stream starting from %s", photos_pull_from, stream_pull_from);
@@ -252,7 +258,7 @@ public class FacebookMiner : Tracker.MinerWeb {
 									"photos" : "SELECT pid, src_big, caption, aid, owner, created FROM photo WHERE aid IN (SELECT aid FROM album WHERE owner IN (SELECT target_id FROM #connections)) AND modified > '%2$s'",
 									"albums" : "SELECT aid, name, description, link, owner, modified FROM album WHERE owner IN (SELECT target_id FROM #connections) AND modified > '%2$s'"
 									} """.printf (username, timestamp_from_iso8601 (photos_pull_from)));
-		runCall_async (c, pull_photos_cb);
+		runCall_async (c, pull_photos_call_cb);
 
 		// Pull streams
 		c = rest.new_call ();
@@ -261,19 +267,26 @@ public class FacebookMiner : Tracker.MinerWeb {
 									"stream":"SELECT post_id, actor_id, target_id, message, attachment, permalink, created_time FROM stream WHERE source_id IN (SELECT target_id FROM connection WHERE source_id='%s' AND is_following=1) AND created_time > '%s'",
 									"actors":"SELECT uid, name, pic_square FROM user WHERE uid IN (SELECT actor_id FROM #stream)"
 									}""".printf (username, timestamp_from_iso8601 (stream_pull_from)));
-		runCall_async (c, pull_stream_cb);
+		runCall_async (c, pull_stream_call_cb);
 
 		return true;
 	}
 
-	private void pull_photos_cb (ProxyCall call, GLib.Error err, Object weak_object)
+	private void pull_photos_call_cb (ProxyCall call, GLib.Error err, Object weak_object)
 	{
-		message ("Pulling pictures");
 		if (err != null) {
 			warning ("Error while pulling pictures : %s", err.message);
-			//error (new MinerWebError.SERVICE (err.message));
+			error (new MinerWebError.SERVICE (err.message));
+		} else {
+			pull_photos.begin (call);
 		}
 
+	}
+
+	private async void pull_photos (ProxyCall call)
+	{
+		message ("Pulling pictures");
+
 		var parser = new XmlParser ();
 		XmlNode root = parser.parse_from_data (call.get_payload (), call.get_payload_length ());
 
@@ -319,7 +332,7 @@ public class FacebookMiner : Tracker.MinerWeb {
 			weak XmlNode current_friend = friends_node.find ("fql_result_set").find ("user");
 
 			while (current_friend != null) {
-				friend_urn.insert (current_friend.find ("uid").content, get_contact (current_friend.find ("name").content));
+				friend_urn.insert (current_friend.find ("uid").content, yield get_contact (current_friend.find ("name").content));
 
 				current_friend = current_friend.next;
 			}
@@ -331,30 +344,30 @@ public class FacebookMiner : Tracker.MinerWeb {
 			weak XmlNode current_album = albums_node.find ("fql_result_set").find ("album");
 
 			while (current_album != null) {
-				string uri = current_album.find ("link").content;
-				string urn = get_resource ("nfo:MediaList", uri);
+				string current_album_uri = current_album.find ("link").content;
+				string current_album_urn = yield get_resource ("nfo:MediaList", current_album_uri);
 
-				album_urn.insert (current_album.find ("aid").content, urn);
+				album_urn.insert (current_album.find ("aid").content, current_album_urn);
 
 				try {
 					string name = current_album.find ("name").content;
 					if (name != null) {
 						message ("Will index album %s", name);
-						tracker.BatchSparqlUpdate ("insert {<%s> rdfs:label \"%s\"}".printf (urn, escape_string (name)));
+						yield execute_batch_update ("insert {<%s> rdfs:label \"%s\"}".printf (current_album_urn, escape_string (name)));
 					}
 
 					string comment = current_album.find ("description").content;
 					if (comment != null) {
-						tracker.BatchSparqlUpdate ("insert {<%s> rdfs:comment \"%s\"}".printf (urn, escape_string (comment)));
+						yield execute_batch_update ("insert {<%s> rdfs:comment \"%s\"}".printf (current_album_urn, escape_string (comment)));
 					}
 
-					tracker.BatchSparqlUpdate ("insert {<%s> nie:contentLastModified '%s' ; nco:creator <%s>"
-					                           .printf(urn,
+					yield execute_batch_update ("insert {<%s> nie:contentLastModified '%s' ; nco:creator <%s>"
+					                           .printf(current_album_urn,
 					                                   timestamp_to_iso8601 (current_album.find ("modified").content),
 					                                   friend_urn.lookup    (current_album.find ("owner").content)));
-					tracker.BatchCommit ();
-				} catch (Error e) {
-					critical ("Error while contacting Tracker : %s", e.message);
+					yield commit ();
+				} catch (Error update_album_error) {
+					critical ("Error while contacting Tracker : %s", update_album_error.message);
 					//error (new MinerWebError.TRACKER (e.message));
 					return;
 				}
@@ -376,31 +389,31 @@ public class FacebookMiner : Tracker.MinerWeb {
 					current_photo = current_photo.next;
 					continue;
 				}
-				string urn = get_resource ("nmm:Photo", uri);
+				string urn = yield get_resource ("nmm:Photo", uri);
 
 				message ("Indexing photo %s with urn %s", uri, urn);
 
 				try {
 					string caption = current_photo.find ("caption").content;
 					if (caption != null) {
-						tracker.BatchSparqlUpdate ("insert {<%s> rdfs:label \"%s\"}".printf (urn, escape_string (caption)));
+						yield execute_batch_update ("insert {<%s> rdfs:label \"%s\"}".printf (urn, escape_string (caption)));
 					}
 
 					string date = timestamp_to_iso8601 (current_photo.find ("created").content);
-					tracker.BatchSparqlUpdate ("insert {<%s> nie:contentCreated '%s'}".printf (urn, date));
+					yield execute_batch_update ("insert {<%s> nie:contentCreated '%s'}".printf (urn, date));
 
 					if (album_urn.lookup (current_photo.find ("aid").content) != null) {
-						tracker.BatchSparqlUpdate ("insert {<%s> a nfo:MediaFileListEntry . <%s> nfo:mediaListEntry <%1$s>}"
+						yield execute_batch_update ("insert {<%s> a nfo:MediaFileListEntry . <%s> nfo:mediaListEntry <%1$s>}"
 						                           .printf (urn,
 						                                    album_urn.lookup (current_photo.find ("aid").content)));
 					} else {
 						warning ("Unknown album for picture %s", uri);
 					}
 
-					tracker.BatchSparqlUpdate ("insert {<%s> nco:creator <%s>}".printf (urn, friend_urn.lookup (current_photo.find ("owner").content)));
-					tracker.BatchCommit ();
-				} catch (Error e) {
-					critical ("Error while contacting Tracker : %s", e.message);
+					yield execute_batch_update ("insert {<%s> nco:creator <%s>}".printf (urn, friend_urn.lookup (current_photo.find ("owner").content)));
+					yield commit ();
+				} catch (Error update_photo_error) {
+					critical ("Error while contacting Tracker : %s", update_photo_error.message);
 					//error (new MinerWebError.TRACKER (e.message));
 					return;
 				}
@@ -412,13 +425,19 @@ public class FacebookMiner : Tracker.MinerWeb {
 		message ("Photos pulled");
 	}
 
-	private void pull_stream_cb (ProxyCall call, GLib.Error? err, Object weak_object)
+	private void pull_stream_call_cb (ProxyCall call, GLib.Error err, Object weak_object)
 	{
 		if (err != null) {
 			warning ("Error while pulling pictures : %s", err.message);
-			//error (new MinerWebError.SERVICE (err.message));
+			error (new MinerWebError.SERVICE (err.message));
 			return;
+		} else {
+			pull_stream.begin (call);
 		}
+	}
+
+	private async void pull_stream (ProxyCall call)
+	{
 
 		var parser = new XmlParser ();
 		XmlNode root = parser.parse_from_data (call.get_payload (), call.get_payload_length ());
@@ -449,16 +468,16 @@ public class FacebookMiner : Tracker.MinerWeb {
 			string facebook_id = current.find ("uid").content;
 			var author = current.find ("name").content;
 			try {
-				var authorUid = get_contact (author);
+				var authorUid = yield get_contact (author);
 
 				var avatarUrl = current.find ("pic_square").content;
-				var avatarUid = get_resource ("nfo:Image", avatarUrl);
-				tracker.SparqlUpdate ("insert {<%s> nco:photo <%s>}".printf (authorUid, avatarUid));
+				var avatarUid = yield get_resource ("nfo:Image", avatarUrl);
+				yield execute_update ("insert {<%s> nco:photo <%s>}".printf (authorUid, avatarUid));
 
 				authors.insert (facebook_id, authorUid);
 				break;
-			} catch (Error e) {
-				critical ("Error contacting Tracker : %s", e.message);
+			} catch (Error insert_contact_error) {
+				critical ("Error contacting Tracker : %s", insert_contact_error.message);
 				//error (new MinerWebError.TRACKER (e.message));
 			}
 			current = current.next;
@@ -469,16 +488,16 @@ public class FacebookMiner : Tracker.MinerWeb {
 		while (current != null) {
 			try {
 				string uri = current.find ("permalink").content;
-				string urn = get_resource ("mfo:FeedMessage", uri);
+				string urn = yield get_resource ("mfo:FeedMessage", uri);
 
 				string date = timestamp_to_iso8601 (current.find ("created_time").content);
-				tracker.BatchSparqlUpdate ("insert {<%s> nmo:from <%s> ; nmo:receivedDate \"%s\"}"
+				yield execute_batch_update ("insert {<%s> nmo:from <%s> ; nmo:receivedDate \"%s\"}"
 				                      .printf (urn,
 				                               authors.lookup (current.find ("actor_id").content),
 				                               date));
 
 				if (current.find ("message").content != null) {
-					tracker.BatchSparqlUpdate ("insert {<%s> nmo:plainTextMessageContent \"%s\"}".printf (urn, escape_string (current.find ("message").content)));
+					yield execute_batch_update ("insert {<%s> nmo:plainTextMessageContent \"%s\"}".printf (urn, escape_string (current.find ("message").content)));
 				}
 
 				// Deal with any attachment we might have
@@ -524,23 +543,23 @@ public class FacebookMiner : Tracker.MinerWeb {
 						XmlNode current_photo = photos_node.find ("photo");
 						while (current_photo != null) {
 							var picture_uri = current_photo.find ("src_big").content;
-							var enclosure_urn = get_resource ("nmm:Photo", picture_uri);
+							var enclosure_urn = yield get_resource ("nmm:Photo", picture_uri);
 
-							tracker.BatchSparqlUpdate ("insert {<%s> a mfo:Enclosure ; mfo:remoteLink <%s> . <%s> mfo:enclosureList <%1$s>}"
+							yield execute_batch_update ("insert {<%s> a mfo:Enclosure ; mfo:remoteLink <%s> . <%s> mfo:enclosureList <%1$s>}"
 							                      .printf (enclosure_urn,
 							                               picture_uri,
 							                               urn));
 
-							// The rest of the photo indexing will be done by pull_photos_cb
+							// The rest of the photo indexing will be done by pull_photos
 
 							current_photo = current_photo.next;
 						}
 					}
 				}
 
-				tracker.BatchCommit ();
-			} catch (Error e) {
-				critical ("Error while inserting data into Tracker : %s", e.message);
+				yield commit ();
+			} catch (Error update_message_error) {
+				critical ("Error while inserting data into Tracker : %s", update_message_error.message);
 				//error (new MinerWebError.TRACKER (e.message));
 			}
 			current = current.next;
@@ -610,57 +629,59 @@ public class FacebookMiner : Tracker.MinerWeb {
 	}
 
 	// tries to find a resource with the given url as a nie:isStoredAs property. If none is foud, creates one (return value is urn)
-	private string? get_resource (string klass, string stored_as)
+	private async string? get_resource (string klass, string stored_as)
 	{
-		string[][] results;
+		unowned GLib.PtrArray results;
 		try {
-			results = tracker.SparqlQuery ("select ?r where { ?r a %s . ?r nie:isStoredAs \"%s\" }".printf (klass, stored_as));
+			results = yield execute_sparql ("select ?r where { ?r a %s . ?r nie:isStoredAs \"%s\" }".printf (klass, stored_as));
 		} catch (Error e) {
 			critical ("Couldn't contact Tracker : %s", e.message);
 			//error (new MinerWebError.TRACKER (e.message));
 			return null;
 		}
 
-		switch (results.length) {
+		unowned string [][] res = (string[][])results.pdata;
+		switch (results.len) {
 			case 0:
 				string urn = "urn:uuid:%s".printf (uuid_generate_string ());
-				tracker.SparqlUpdate ("insert {<%s> a nfo:RemoteDataObject . <%s> a %s ; nie:isStoredAs <%1$s> ; nie:dataSource <%s>}"
+				yield execute_update ("insert {<%s> a nfo:RemoteDataObject . <%s> a %s ; nie:isStoredAs <%1$s> ; nie:dataSource <%s>}"
 				                      .printf (stored_as,
 				                               urn,
 				                               klass,
 				                               MINER_DATASOURCE_URN));
 				return urn;
 			case 1:
-				return results[0][0];
+				return res[0][0];
 			default:
 				warning ("More than one object of type %s and with nie:isStoredAs '%s'. Returning the first one", klass, stored_as);
-				return results[0][0];
+				return res[0][0];
 		}
 	}
 
 	// tries to find a nco:Contact with the given nco:fullname. If none is foud, creates one (return value is urn)
-	private string? get_contact (string fullname)
+	private async string? get_contact (string fullname)
 	{
-		string[][] results;
+		unowned GLib.PtrArray results;
 		string escaped_fullname = escape_string (fullname);
 		try {
-			results = tracker.SparqlQuery ("select ?r where { ?r a nco:Contact . ?r nco:fullname \"%s\" }".printf (escaped_fullname));
+			results = yield execute_sparql ("select ?r where { ?r a nco:Contact . ?r nco:fullname \"%s\" }".printf (escaped_fullname));
 		} catch (Error e) {
 			critical ("Couldn't contact Tracker : %s", e.message);
 			//error (new MinerWebError.TRACKER (e.message));
 			return null;
 		}
 
-		switch (results.length) {
+		unowned string[][] res = (string[][])results.pdata;
+		switch (results.len) {
 			case 0:
 				string urn = "urn:uuid:%s".printf (uuid_generate_string ());
-				tracker.SparqlUpdate ("insert {<%s> a nco:Contact ; nco:fullname \"%s\" ; nie:dataSource <%s>}".printf (urn, escaped_fullname, MINER_DATASOURCE_URN));
+				yield execute_update ("insert {<%s> a nco:Contact ; nco:fullname \"%s\" ; nie:dataSource <%s>}".printf (urn, escaped_fullname, MINER_DATASOURCE_URN));
 				return urn;
 			case 1:
-				return results[0][0];
+				return res[0][0];
 			default:
 				warning ("More than one nco:Contact with nco:fullname '%s'. Returning the first one", fullname);
-				return results[0][0];
+				return res[0][0];
 		}
 	}
 
@@ -693,10 +714,11 @@ void main ()
 {
 	loop = new MainLoop (null, false);
 
-	Environment.set_application_name ("FacebookBrige");
+	Environment.set_application_name ("FacebookMiner");
 
 	var miner = new FacebookMiner ();
 	miner.start ();
+	message ("Miner started");
 
 	loop.run ();
 }
[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]