[tracker/ontology-cope] libtracker-data: Coping with modest ontology changes in journal replay
- From: Philip Van Hoof <pvanhoof src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [tracker/ontology-cope] libtracker-data: Coping with modest ontology changes in journal replay
- Date: Thu, 18 Mar 2010 17:35:03 +0000 (UTC)
commit e25b8c0d4f1873d935c1070d8197d05279e7c535
Author: Philip Van Hoof <philip codeminded be>
Date: Thu Mar 18 18:33:49 2010 +0100
libtracker-data: Coping with modest ontology changes in journal replay
src/libtracker-data/tracker-data-manager.c | 237 +++++++++++++++-------------
src/libtracker-data/tracker-data-manager.h | 17 ++
src/libtracker-data/tracker-data-update.c | 181 ++++++++++++++++++++--
3 files changed, 312 insertions(+), 123 deletions(-)
---
diff --git a/src/libtracker-data/tracker-data-manager.c b/src/libtracker-data/tracker-data-manager.c
index 24d922c..6ee3526 100644
--- a/src/libtracker-data/tracker-data-manager.c
+++ b/src/libtracker-data/tracker-data-manager.c
@@ -73,16 +73,16 @@ static gchar *ontologies_dir;
static gboolean initialized;
static gboolean in_journal_replay;
-static void
-load_ontology_statement (const gchar *ontology_file,
- gint subject_id,
- const gchar *subject,
- const gchar *predicate,
- const gchar *object,
- gint *max_id,
- gboolean is_new,
- GHashTable *classes,
- GHashTable *properties)
+void
+tracker_data_ontology_load_statement (const gchar *ontology_file,
+ gint subject_id,
+ const gchar *subject,
+ const gchar *predicate,
+ const gchar *object,
+ gint *max_id,
+ gboolean is_new,
+ GHashTable *classes,
+ GHashTable *properties)
{
if (g_strcmp0 (predicate, RDF_TYPE) == 0) {
if (g_strcmp0 (object, RDFS_CLASS) == 0) {
@@ -406,8 +406,8 @@ load_ontology_file_from_path (const gchar *ontology_file,
predicate = tracker_turtle_reader_get_predicate (reader);
object = tracker_turtle_reader_get_object (reader);
- load_ontology_statement (ontology_file, 0, subject, predicate, object,
- max_id, is_new, NULL, NULL);
+ tracker_data_ontology_load_statement (ontology_file, 0, subject, predicate, object,
+ max_id, is_new, NULL, NULL);
}
g_object_unref (reader);
@@ -553,8 +553,8 @@ load_ontology_from_journal (GHashTable **classes_out,
subject = g_hash_table_lookup (id_uri_map, GINT_TO_POINTER (subject_id));
predicate = g_hash_table_lookup (id_uri_map, GINT_TO_POINTER (predicate_id));
- load_ontology_statement ("journal", subject_id, subject, predicate,
- object, NULL, FALSE, classes, properties);
+ tracker_data_ontology_load_statement ("journal", subject_id, subject, predicate,
+ object, NULL, FALSE, classes, properties);
}
}
@@ -564,133 +564,149 @@ load_ontology_from_journal (GHashTable **classes_out,
g_hash_table_unref (id_uri_map);
}
-
-static void
-load_turtle_file (const gchar* path,
- gboolean is_new,
- gboolean ignore_nao_last_modified)
+void
+tracker_data_ontology_process_statement (const gchar *graph,
+ const gchar *subject,
+ const gchar *predicate,
+ const gchar *object,
+ gboolean is_uri,
+ gboolean is_new,
+ gboolean ignore_nao_last_modified)
{
GError *error = NULL;
- TrackerTurtleReader* reader;
-
- reader = tracker_turtle_reader_new (path, &error);
-
- if (error != NULL) {
- g_critical ("%s", error->message);
- g_error_free (error);
- return;
- }
-
- while (tracker_turtle_reader_next (reader, &error)) {
-
- const gchar *graph = tracker_turtle_reader_get_graph (reader);
- const gchar *subject = tracker_turtle_reader_get_subject (reader);
- const gchar *predicate = tracker_turtle_reader_get_predicate (reader);
- const gchar *object = tracker_turtle_reader_get_object (reader);
-
- if (g_strcmp0 (predicate, RDF_TYPE) == 0) {
- if (g_strcmp0 (object, RDFS_CLASS) == 0) {
- TrackerClass *class;
-
- class = tracker_ontologies_get_class_by_uri (subject);
-
- if (class && tracker_class_get_is_new (class) != is_new) {
- continue;
- }
- } else if (g_strcmp0 (object, RDF_PROPERTY) == 0) {
- TrackerProperty *prop;
-
- prop = tracker_ontologies_get_property_by_uri (subject);
- if (prop && tracker_property_get_is_new (prop) != is_new) {
- continue;
- }
- } else if (g_strcmp0 (object, TRACKER_PREFIX "Namespace") == 0) {
- TrackerNamespace *namespace;
-
- namespace = tracker_ontologies_get_namespace_by_uri (subject);
-
- if (namespace && tracker_namespace_get_is_new (namespace) != is_new) {
- continue;
- }
- } else if (g_strcmp0 (object, TRACKER_PREFIX "Ontology") == 0) {
- TrackerOntology *ontology;
-
- ontology = tracker_ontologies_get_ontology_by_uri (subject);
-
- if (ontology && tracker_ontology_get_is_new (ontology) != is_new) {
- continue;
- }
- }
- } else if (g_strcmp0 (predicate, RDFS_SUB_CLASS_OF) == 0) {
+ if (g_strcmp0 (predicate, RDF_TYPE) == 0) {
+ if (g_strcmp0 (object, RDFS_CLASS) == 0) {
TrackerClass *class;
class = tracker_ontologies_get_class_by_uri (subject);
if (class && tracker_class_get_is_new (class) != is_new) {
- continue;
+ return;
}
- } else if (g_strcmp0 (predicate, RDFS_SUB_PROPERTY_OF) == 0 ||
- g_strcmp0 (predicate, RDFS_DOMAIN) == 0 ||
- g_strcmp0 (predicate, RDFS_RANGE) == 0 ||
- g_strcmp0 (predicate, NRL_MAX_CARDINALITY) == 0 ||
- g_strcmp0 (predicate, TRACKER_PREFIX "indexed") == 0 ||
- g_strcmp0 (predicate, TRACKER_PREFIX "transient") == 0 ||
- g_strcmp0 (predicate, TRACKER_PREFIX "isAnnotation") == 0 ||
- g_strcmp0 (predicate, TRACKER_PREFIX "fulltextIndexed") == 0) {
+ } else if (g_strcmp0 (object, RDF_PROPERTY) == 0) {
TrackerProperty *prop;
prop = tracker_ontologies_get_property_by_uri (subject);
if (prop && tracker_property_get_is_new (prop) != is_new) {
- continue;
+ return;
}
- } else if (g_strcmp0 (predicate, TRACKER_PREFIX "prefix") == 0) {
+ } else if (g_strcmp0 (object, TRACKER_PREFIX "Namespace") == 0) {
TrackerNamespace *namespace;
namespace = tracker_ontologies_get_namespace_by_uri (subject);
if (namespace && tracker_namespace_get_is_new (namespace) != is_new) {
- continue;
+ return;
}
- } else if (g_strcmp0 (predicate, NAO_LAST_MODIFIED) == 0) {
+ } else if (g_strcmp0 (object, TRACKER_PREFIX "Ontology") == 0) {
TrackerOntology *ontology;
ontology = tracker_ontologies_get_ontology_by_uri (subject);
if (ontology && tracker_ontology_get_is_new (ontology) != is_new) {
- continue;
+ return;
}
+ }
+ } else if (g_strcmp0 (predicate, RDFS_SUB_CLASS_OF) == 0) {
+ TrackerClass *class;
- if (ignore_nao_last_modified) {
- continue;
- }
+ class = tracker_ontologies_get_class_by_uri (subject);
+
+ if (class && tracker_class_get_is_new (class) != is_new) {
+ return;
}
+ } else if (g_strcmp0 (predicate, RDFS_SUB_PROPERTY_OF) == 0 ||
+ g_strcmp0 (predicate, RDFS_DOMAIN) == 0 ||
+ g_strcmp0 (predicate, RDFS_RANGE) == 0 ||
+ g_strcmp0 (predicate, NRL_MAX_CARDINALITY) == 0 ||
+ g_strcmp0 (predicate, TRACKER_PREFIX "indexed") == 0 ||
+ g_strcmp0 (predicate, TRACKER_PREFIX "transient") == 0 ||
+ g_strcmp0 (predicate, TRACKER_PREFIX "isAnnotation") == 0 ||
+ g_strcmp0 (predicate, TRACKER_PREFIX "fulltextIndexed") == 0) {
+ TrackerProperty *prop;
+
+ prop = tracker_ontologies_get_property_by_uri (subject);
+
+ if (prop && tracker_property_get_is_new (prop) != is_new) {
+ return;
+ }
+ } else if (g_strcmp0 (predicate, TRACKER_PREFIX "prefix") == 0) {
+ TrackerNamespace *namespace;
- if (tracker_turtle_reader_get_object_is_uri (reader)) {
- tracker_data_insert_statement_with_uri (graph, subject,
- predicate, object,
- &error);
+ namespace = tracker_ontologies_get_namespace_by_uri (subject);
- if (error != NULL) {
- g_critical ("%s", error->message);
- g_error_free (error);
- return;
- }
+ if (namespace && tracker_namespace_get_is_new (namespace) != is_new) {
+ return;
+ }
+ } else if (g_strcmp0 (predicate, NAO_LAST_MODIFIED) == 0) {
+ TrackerOntology *ontology;
- } else {
- tracker_data_insert_statement_with_string (graph, subject,
- predicate, object,
- &error);
+ ontology = tracker_ontologies_get_ontology_by_uri (subject);
- if (error != NULL) {
- g_critical ("%s", error->message);
- g_error_free (error);
- return;
- }
+ if (ontology && tracker_ontology_get_is_new (ontology) != is_new) {
+ return;
+ }
+
+ if (ignore_nao_last_modified) {
+ return;
}
}
+ if (is_uri) {
+ tracker_data_insert_statement_with_uri (graph, subject,
+ predicate, object,
+ &error);
+
+ if (error != NULL) {
+ g_critical ("%s", error->message);
+ g_error_free (error);
+ return;
+ }
+
+ } else {
+ tracker_data_insert_statement_with_string (graph, subject,
+ predicate, object,
+ &error);
+
+ if (error != NULL) {
+ g_critical ("%s", error->message);
+ g_error_free (error);
+ return;
+ }
+ }
+}
+
+static void
+load_turtle_file (const gchar* path,
+ gboolean is_new,
+ gboolean ignore_nao_last_modified)
+{
+ GError *error = NULL;
+ TrackerTurtleReader* reader;
+
+ reader = tracker_turtle_reader_new (path, &error);
+
+ if (error != NULL) {
+ g_critical ("%s", error->message);
+ g_error_free (error);
+ return;
+ }
+
+ while (tracker_turtle_reader_next (reader, &error)) {
+
+ const gchar *graph = tracker_turtle_reader_get_graph (reader);
+ const gchar *subject = tracker_turtle_reader_get_subject (reader);
+ const gchar *predicate = tracker_turtle_reader_get_predicate (reader);
+ const gchar *object = tracker_turtle_reader_get_object (reader);
+
+ tracker_data_ontology_process_statement (graph, subject, predicate, object,
+ tracker_turtle_reader_get_object_is_uri (reader),
+ is_new, ignore_nao_last_modified);
+
+ }
+
g_object_unref (reader);
}
@@ -1340,8 +1356,8 @@ create_fts_table (TrackerDBInterface *iface)
g_free (query);
}
-static void
-import_ontology_into_db (gboolean is_new)
+void
+tracker_data_ontology_import_into_db (gboolean is_new)
{
TrackerDBInterface *iface;
@@ -1519,7 +1535,7 @@ tracker_data_manager_init (TrackerDBManagerFlags flags,
load_ontology_from_journal (&classes, &properties);
tracker_data_begin_db_transaction_for_replay (tracker_db_journal_reader_get_time ());
- import_ontology_into_db (FALSE);
+ tracker_data_ontology_import_into_db (FALSE);
tracker_data_commit_db_transaction ();
tracker_db_journal_reader_shutdown ();
@@ -1530,6 +1546,7 @@ tracker_data_manager_init (TrackerDBManagerFlags flags,
/* open journal for writing */
tracker_db_journal_init (NULL, FALSE);
+
check_ontology = TRUE;
g_hash_table_unref (classes);
@@ -1561,7 +1578,7 @@ tracker_data_manager_init (TrackerDBManagerFlags flags,
tracker_data_begin_db_transaction ();
tracker_db_journal_start_transaction (time (NULL));
- import_ontology_into_db (FALSE);
+ tracker_data_ontology_import_into_db (FALSE);
/* store ontology in database */
for (l = sorted; l; l = l->next) {
@@ -1684,7 +1701,7 @@ tracker_data_manager_init (TrackerDBManagerFlags flags,
}
if (to_reload) {
- import_ontology_into_db (TRUE);
+ tracker_data_ontology_import_into_db (TRUE);
for (l = to_reload; l; l = l->next) {
const gchar *ontology_file = l->data;
import_ontology_file (ontology_file, TRUE, test_schema != NULL);
diff --git a/src/libtracker-data/tracker-data-manager.h b/src/libtracker-data/tracker-data-manager.h
index 5be4eb2..7b89b4f 100644
--- a/src/libtracker-data/tracker-data-manager.h
+++ b/src/libtracker-data/tracker-data-manager.h
@@ -45,6 +45,23 @@ void tracker_data_manager_shutdown (void);
gint64 tracker_data_manager_get_db_option_int64 (const gchar *option);
void tracker_data_manager_set_db_option_int64 (const gchar *option,
gint64 value);
+void tracker_data_ontology_load_statement (const gchar *ontology_file,
+ gint subject_id,
+ const gchar *subject,
+ const gchar *predicate,
+ const gchar *object,
+ gint *max_id,
+ gboolean is_new,
+ GHashTable *classes,
+ GHashTable *properties);
+void tracker_data_ontology_import_into_db (gboolean is_new);
+void tracker_data_ontology_process_statement (const gchar *graph,
+ const gchar *subject,
+ const gchar *predicate,
+ const gchar *object,
+ gboolean is_uri,
+ gboolean is_new,
+ gboolean ignore_nao_last_modified);
G_END_DECLS
diff --git a/src/libtracker-data/tracker-data-update.c b/src/libtracker-data/tracker-data-update.c
index cf023f9..8a79c83 100644
--- a/src/libtracker-data/tracker-data-update.c
+++ b/src/libtracker-data/tracker-data-update.c
@@ -2220,6 +2220,70 @@ tracker_data_sync (void)
tracker_db_journal_fsync ();
}
+static gchar *
+query_resource_by_id (gint id)
+{
+ TrackerDBCursor *cursor;
+ TrackerDBInterface *iface;
+ TrackerDBStatement *stmt;
+ gchar *uri;
+
+ g_return_val_if_fail (id > 0, NULL);
+
+ iface = tracker_db_manager_get_db_interface ();
+
+ stmt = tracker_db_interface_create_statement (iface,
+ "SELECT Uri FROM Resource WHERE ID = ?");
+ tracker_db_statement_bind_int (stmt, 0, id);
+ cursor = tracker_db_statement_start_cursor (stmt, NULL);
+ g_object_unref (stmt);
+
+ tracker_db_cursor_iter_next (cursor);
+ uri = g_strdup (tracker_db_cursor_get_string (cursor, 0));
+ g_object_unref (cursor);
+
+ return uri;
+}
+
+typedef struct {
+ gchar *graph;
+ gchar *subject;
+ gchar *predicate;
+ gchar *object;
+ gboolean is_uri;
+} QueuedStatement;
+
+static void
+free_queued_statement (QueuedStatement *queued)
+{
+ g_free (queued->subject);
+ g_free (queued->predicate);
+ g_free (queued->object);
+ g_free (queued->graph);
+ g_free (queued);
+}
+
+static GList*
+queue_statement (GList *queue,
+ const gchar *graph,
+ const gchar *subject,
+ const gchar *predicate,
+ const gchar *object,
+ gboolean is_uri)
+{
+ QueuedStatement *queued = g_new (QueuedStatement, 1);
+
+ queued->subject = g_strdup (subject);
+ queued->predicate = g_strdup (predicate);
+ queued->object = g_strdup (object);
+ queued->is_uri = is_uri;
+ queued->graph = graph ? g_strdup (graph) : NULL;
+
+ queue = g_list_append (queue, queued);
+
+ return queue;
+}
+
void
tracker_data_replay_journal (GHashTable *classes,
GHashTable *properties)
@@ -2227,6 +2291,8 @@ tracker_data_replay_journal (GHashTable *classes,
GError *journal_error = NULL;
static TrackerProperty *rdf_type = NULL;
gint last_operation_type = 0;
+ gboolean in_ontology = FALSE;
+ GList *ontology_queue = NULL;
tracker_data_begin_db_transaction_for_replay (0);
@@ -2253,6 +2319,10 @@ tracker_data_replay_journal (GHashTable *classes,
tracker_db_journal_reader_get_resource (&id, &uri);
+ if (in_ontology) {
+ continue;
+ }
+
class = g_hash_table_lookup (classes, GINT_TO_POINTER (id));
if (!class)
property = g_hash_table_lookup (properties, GINT_TO_POINTER (id));
@@ -2276,19 +2346,72 @@ tracker_data_replay_journal (GHashTable *classes,
g_error_free (new_error);
}
+ } else if (type == TRACKER_DB_JOURNAL_START_ONTOLOGY_TRANSACTION) {
+
+ in_ontology = TRUE;
+
} else if (type == TRACKER_DB_JOURNAL_START_TRANSACTION) {
resource_time = tracker_db_journal_reader_get_time ();
} else if (type == TRACKER_DB_JOURNAL_END_TRANSACTION) {
- GError *new_error = NULL;
- tracker_data_update_buffer_might_flush (&new_error);
- if (new_error) {
- g_warning ("Journal replay error: '%s'", new_error->message);
- g_clear_error (&new_error);
+ if (in_ontology) {
+ GList *l;
+
+ tracker_data_ontology_import_into_db (FALSE);
+
+ for (l = ontology_queue; l; l = l->next) {
+ QueuedStatement *queued = ontology_queue->data;
+
+ tracker_data_ontology_process_statement (queued->graph,
+ queued->subject,
+ queued->predicate,
+ queued->object,
+ queued->is_uri,
+ FALSE, TRUE);
+
+ free_queued_statement (queued);
+ }
+ g_list_free (ontology_queue);
+ ontology_queue = NULL;
+
+ } else {
+ GError *new_error = NULL;
+ tracker_data_update_buffer_might_flush (&new_error);
+ if (new_error) {
+ g_warning ("Journal replay error: '%s'", new_error->message);
+ g_clear_error (&new_error);
+ }
}
+ in_ontology = FALSE;
} else if (type == TRACKER_DB_JOURNAL_INSERT_STATEMENT) {
GError *new_error = NULL;
TrackerProperty *property;
+ tracker_db_journal_reader_get_statement (&graph_id, &subject_id, &predicate_id, &object);
+
+ if (in_ontology) {
+ gchar *graph, *subject, *predicate;
+
+ if (graph_id > 0) {
+ graph = query_resource_by_id (graph_id);
+ } else {
+ graph = NULL;
+ }
+
+ subject = query_resource_by_id (subject_id);
+ predicate = query_resource_by_id (predicate_id);
+
+ tracker_data_ontology_load_statement ("journal", subject_id, subject, predicate,
+ object, NULL, FALSE, classes, properties);
+
+ ontology_queue = queue_statement (ontology_queue, graph, subject, predicate, object, FALSE);
+
+ g_free (graph);
+ g_free (subject);
+ g_free (predicate);
+
+ continue;
+ }
+
if (last_operation_type == -1) {
tracker_data_update_buffer_flush (&new_error);
if (new_error) {
@@ -2298,8 +2421,6 @@ tracker_data_replay_journal (GHashTable *classes,
}
last_operation_type = 1;
- tracker_db_journal_reader_get_statement (&graph_id, &subject_id, &predicate_id, &object);
-
property = g_hash_table_lookup (properties, GINT_TO_POINTER (predicate_id));
if (property) {
@@ -2321,6 +2442,34 @@ tracker_data_replay_journal (GHashTable *classes,
TrackerClass *class = NULL;
TrackerProperty *property;
+ tracker_db_journal_reader_get_statement_id (&graph_id, &subject_id, &predicate_id, &object_id);
+
+ if (in_ontology) {
+ gchar *graph, *subject, *predicate, *object_n;
+
+ if (graph_id > 0) {
+ graph = query_resource_by_id (graph_id);
+ } else {
+ graph = NULL;
+ }
+
+ subject = query_resource_by_id (subject_id);
+ predicate = query_resource_by_id (predicate_id);
+ object_n = query_resource_by_id (object_id);
+
+ tracker_data_ontology_load_statement ("journal", subject_id, subject, predicate,
+ object_n, NULL, FALSE, classes, properties);
+
+ ontology_queue = queue_statement (ontology_queue, graph, subject, predicate, object_n, TRUE);
+
+ g_free (graph);
+ g_free (subject);
+ g_free (object_n);
+ g_free (predicate);
+
+ continue;
+ }
+
if (last_operation_type == -1) {
tracker_data_update_buffer_flush (&new_error);
if (new_error) {
@@ -2330,8 +2479,6 @@ tracker_data_replay_journal (GHashTable *classes,
}
last_operation_type = 1;
- tracker_db_journal_reader_get_statement_id (&graph_id, &subject_id, &predicate_id, &object_id);
-
property = g_hash_table_lookup (properties, GINT_TO_POINTER (predicate_id));
if (property) {
@@ -2368,6 +2515,12 @@ tracker_data_replay_journal (GHashTable *classes,
GError *new_error = NULL;
TrackerProperty *property;
+ tracker_db_journal_reader_get_statement (&graph_id, &subject_id, &predicate_id, &object);
+
+ if (in_ontology) {
+ continue;
+ }
+
if (last_operation_type == 1) {
tracker_data_update_buffer_flush (&new_error);
if (new_error) {
@@ -2377,8 +2530,6 @@ tracker_data_replay_journal (GHashTable *classes,
}
last_operation_type = -1;
- tracker_db_journal_reader_get_statement (&graph_id, &subject_id, &predicate_id, &object);
-
resource_buffer_switch (NULL, graph_id, NULL, subject_id);
property = g_hash_table_lookup (properties, GINT_TO_POINTER (predicate_id));
@@ -2413,6 +2564,12 @@ tracker_data_replay_journal (GHashTable *classes,
TrackerClass *class = NULL;
TrackerProperty *property;
+ tracker_db_journal_reader_get_statement_id (&graph_id, &subject_id, &predicate_id, &object_id);
+
+ if (in_ontology) {
+ continue;
+ }
+
if (last_operation_type == 1) {
tracker_data_update_buffer_flush (&new_error);
if (new_error) {
@@ -2422,8 +2579,6 @@ tracker_data_replay_journal (GHashTable *classes,
}
last_operation_type = -1;
- tracker_db_journal_reader_get_statement_id (&graph_id, &subject_id, &predicate_id, &object_id);
-
property = g_hash_table_lookup (properties, GINT_TO_POINTER (predicate_id));
if (property) {
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
