[tracker/wip/fts4: 20/21] Handle FTS queries more generically

[Carlos Garnacho <carlosg src gnome org>]

commit 98778e71f4b6ed35f83d17ab311b08686b8511c8
Author: Carlos Garnacho <carlos lanedo com>
Date:   Fri Jan 25 18:49:22 2013 +0100

    Handle FTS queries more generically
    
    FTS functions and other SQL functions like COUNT() don't
    mix well, so handle it more generically by joining with the
    FTS table at the outmost level, so those functions are handled
    separately.

 src/libtracker-data/tracker-sparql-expression.vala |   68 ++++++++++++++++----
 src/libtracker-data/tracker-sparql-pattern.vala    |   59 +++++++++++------
 2 files changed, 93 insertions(+), 34 deletions(-)
---
diff --git a/src/libtracker-data/tracker-sparql-expression.vala b/src/libtracker-data/tracker-sparql-expression.vala
index 1ca4112..5ea69fb 100644
--- a/src/libtracker-data/tracker-sparql-expression.vala
+++ b/src/libtracker-data/tracker-sparql-expression.vala
@@ -26,6 +26,8 @@ class Tracker.Sparql.Expression : Object {
 	const string FTS_NS = "http://www.tracker-project.org/ontologies/fts#";;
 	const string TRACKER_NS = "http://www.tracker-project.org/ontologies/tracker#";;
 
+	string? fts_sql;
+
 	public Expression (Query query) {
 		this.query = query;
 	}
@@ -188,6 +190,22 @@ class Tracker.Sparql.Expression : Object {
 			}
 		}
 
+		if (pattern.fts_subject != null) {
+			if (variable == null) {
+				// FTS matches still need aliases as the outer MATCH query
+				// will fetch futher values from the joined select
+				variable = context.get_variable ("var%d".printf (variable_index + 1));
+			}
+
+			if (fts_sql == null) {
+				pattern.fts_variables += variable.sql_expression;
+				sql.append_printf (" AS %s", variable.sql_expression);
+			} else {
+				pattern.fts_variables += fts_sql;
+				pattern.queries_fts_data = true;
+			}
+		}
+
 		if (expect_close_parens) {
 			expect (SparqlTokenType.CLOSE_PARENS);
 		}
@@ -204,6 +222,8 @@ class Tracker.Sparql.Expression : Object {
 			((SelectContext) context).variable_names += "var%d".printf (variable_index + 1);
 		}
 
+		fts_sql = null;
+
 		return type;
 	}
 
@@ -647,36 +667,56 @@ class Tracker.Sparql.Expression : Object {
 		} else if (uri == FTS_NS + "offsets") {
 			bool is_var;
 			string v = pattern.parse_var_or_term (null, out is_var);
+			var variable = context.get_variable (v);
 
-			sql.append ("tracker_offsets(offsets(\"fts\"),fts_property_names())");
-
+			sql.append (variable.sql_expression);
+			fts_sql = "tracker_offsets(offsets(\"fts\"),fts_property_names())";
 			return PropertyType.STRING;
 		} else if (uri == FTS_NS + "snippet") {
 			bool is_var;
 
 			string v = pattern.parse_var_or_term (null, out is_var);
+			var variable = context.get_variable (v);
+			var fts = new StringBuilder ();
 
-			sql.append_printf ("snippet(\"fts\"");
+			fts.append_printf ("snippet(\"fts\"");
 
-			/* "start match" text */
+			// "start match" text
 			if (accept (SparqlTokenType.COMMA)) {
-			      sql.append (", ");
-			      translate_expression_as_string (sql);
+				fts.append (", ");
+				translate_expression_as_string (fts);
 
-			      /* "end match" text */
-			      expect (SparqlTokenType.COMMA);
-			      sql.append (", ");
-			      translate_expression_as_string (sql);
+				// "end match" text
+				expect (SparqlTokenType.COMMA);
+				fts.append (", ");
+				translate_expression_as_string (fts);
+			} else {
+				fts.append(",'',''");
 			}
 
-			/* "ellipses" text */
+			// "ellipsis" text
 			if (accept (SparqlTokenType.COMMA)) {
-			      sql.append (", ");
-			      translate_expression_as_string (sql);
+				fts.append (", ");
+				translate_expression_as_string (fts);
+			} else {
+				fts.append (", '...'");
 			}
 
-			sql.append (")");
+			// lookup column
+			fts.append (", -1");
 
+			// Approximate number of words in context
+			if (accept (SparqlTokenType.COMMA)) {
+				fts.append (", ");
+				translate_expression_as_string (fts);
+			} else {
+				fts.append (", 5");
+			}
+
+			fts.append (")");
+
+			fts_sql = fts.str;
+			sql.append (variable.sql_expression);
 			return PropertyType.STRING;
 		} else if (uri == TRACKER_NS + "id") {
 			var type = translate_expression (sql);
diff --git a/src/libtracker-data/tracker-sparql-pattern.vala b/src/libtracker-data/tracker-sparql-pattern.vala
index ded3e9a..7dfbcd4 100644
--- a/src/libtracker-data/tracker-sparql-pattern.vala
+++ b/src/libtracker-data/tracker-sparql-pattern.vala
@@ -169,8 +169,10 @@ class Tracker.Sparql.Pattern : Object {
 	bool current_subject_is_var;
 	string current_predicate;
 	bool current_predicate_is_var;
-
+	public Variable? fts_subject;
+	public string[] fts_variables;
 	internal StringBuilder? match_str;
+	public bool queries_fts_data = false;
 
 	public Pattern (Query query) {
 		this.query = query;
@@ -338,6 +340,17 @@ class Tracker.Sparql.Pattern : Object {
 			}
 		}
 
+		if (queries_fts_data && fts_subject != null) {
+			// Ensure there's a docid to match on in FTS queries
+			if (!first) {
+				sql.append (", ");
+			} else {
+				first = false;
+			}
+
+			sql.append ("%s AS docid ".printf (fts_subject.sql_expression));
+		}
+
 		// literals in select expressions need to be bound before literals in the where clause
 		foreach (var binding in where_bindings) {
 			query.bindings.append (binding);
@@ -348,20 +361,9 @@ class Tracker.Sparql.Pattern : Object {
 		}
 
 		// select from results of WHERE clause
-
-		if (match_str != null) {
-			sql.append (" FROM fts JOIN (");
-			sql.append (pattern_sql.str);
-
-			/* Leave parenthesis opened,
-			 * "group/limit/offset goes in
-			 * the inner query
-			 */
-		} else {
-			sql.append (" FROM (");
-			sql.append (pattern_sql.str);
-			sql.append (")");
-		}
+		sql.append (" FROM (");
+		sql.append (pattern_sql.str);
+		sql.append (")");
 
 		set_location (after_where);
 
@@ -443,14 +445,30 @@ class Tracker.Sparql.Pattern : Object {
 			query.bindings.append (binding);
 		}
 
-		if (match_str != null) {
-			sql.append (") AS ranks USING (docid)");
-			sql.append_printf (" WHERE fts %s", match_str.str);
+		if (queries_fts_data && match_str != null && fts_subject != null) {
+			var str = new StringBuilder ("SELECT ");
+			first = true;
+
+			foreach (var fts_var in fts_variables) {
+				if (!first) {
+					str.append (", ");
+				} else {
+					first = false;
+				}
+
+				str.append (fts_var);
+			}
+
+			str.append (" FROM fts JOIN (");
+			sql.prepend (str.str);
+			sql.append_printf (") AS ranks USING (docid) WHERE fts %s".printf (match_str.str));
 		}
+
 		context = context.parent_context;
 
 		result.type = type;
 		match_str = null;
+		fts_subject = null;
 
 		return result;
 	}
@@ -1312,6 +1330,7 @@ class Tracker.Sparql.Pattern : Object {
 					db_table = "fts";
 					share_table = false;
 					is_fts_match = true;
+					fts_subject = context.get_variable (current_subject);
 				} else {
 					throw new Sparql.Error.UNKNOWN_PROPERTY ("Unknown property `%s'".printf (current_predicate));
 				}
@@ -1436,7 +1455,7 @@ class Tracker.Sparql.Pattern : Object {
 				binding.table = table;
 				binding.type = subject_type;
 				if (is_fts_match) {
-					binding.sql_db_column_name = "rowid";
+					binding.sql_db_column_name = "docid";
 				} else {
 					binding.sql_db_column_name = "ID";
 				}
@@ -1494,7 +1513,7 @@ class Tracker.Sparql.Pattern : Object {
 				binding.sql_db_column_name = "fts";
 				triple_context.bindings.append (binding);
 
-				sql.append_printf ("\"%s\".\"docid\", ",
+				sql.append_printf ("\"%s\".\"docid\" AS \"ID\", ",
 				                   binding.table.sql_query_tablename);
 				sql.append_printf ("tracker_rank(matchinfo(\"%s\".\"fts\", 'cl'),fts_column_weights()) " +
 				                   "AS \"%s_u_rank\", ",