tracker r1961 - in branches/indexer-split: . src/trackerd

[mr svn gnome org]

Author: mr
Date: Tue Jul 29 15:57:40 2008
New Revision: 1961
URL: http://svn.gnome.org/viewvc/tracker?rev=1961&view=rev

Log:


Modified:
   branches/indexer-split/ChangeLog
   branches/indexer-split/src/trackerd/tracker-index-manager.c
   branches/indexer-split/src/trackerd/tracker-index-manager.h

Modified: branches/indexer-split/src/trackerd/tracker-index-manager.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-index-manager.c	(original)
+++ branches/indexer-split/src/trackerd/tracker-index-manager.c	Tue Jul 29 15:57:40 2008
@@ -31,97 +31,107 @@
 #define MIN_BUCKET_DEFAULT 10
 #define MAX_BUCKET_DEFAULT 20
 
-#define TRACKER_INDEX_FILE_INDEX_DB_FILENAME         "file-index.db"
-#define TRACKER_INDEX_EMAIL_INDEX_DB_FILENAME        "email-index.db"
-#define TRACKER_INDEX_FILE_UPDATE_INDEX_DB_FILENAME  "file-update-index.db"
+#define TRACKER_INDEX_FILE_INDEX_FILENAME         "file-index.db"
+#define TRACKER_INDEX_EMAIL_INDEX_FILENAME        "email-index.db"
+#define TRACKER_INDEX_FILE_UPDATE_INDEX_FILENAME  "file-update-index.db"
 
 #define MAX_INDEX_FILE_SIZE 2000000000
 
-static gboolean      initialized;
-static TrackerIndex *indexes[] = { NULL, NULL, NULL };
-static gint          index_manager_min_bucket;
-static gint          index_manager_max_bucket;
-static gchar        *index_manager_data_dir;
-
-static const gchar *
-get_index_name (TrackerIndexType index) 
-{
-        const gchar *name;
-
-        switch (index) {
-        case TRACKER_INDEX_TYPE_FILES:
-                name = TRACKER_INDEX_FILE_INDEX_DB_FILENAME;
-                break;
-        case TRACKER_INDEX_TYPE_EMAILS:
-                name = TRACKER_INDEX_EMAIL_INDEX_DB_FILENAME;
-                break;
-        case TRACKER_INDEX_TYPE_FILES_UPDATE:
-                name = TRACKER_INDEX_FILE_UPDATE_INDEX_DB_FILENAME;
-                break;
-        }
-
-        return name;
-}
+typedef struct {
+        TrackerIndexType  type;
+	TrackerIndex     *index;
+        const gchar      *file;
+        const gchar      *name;
+        gchar            *abs_filename;
+} TrackerIndexDefinition;
+
+static gboolean               initialized;
+
+static TrackerIndexDefinition indexes[] = {
+        { TRACKER_INDEX_TYPE_FILES, 
+	  NULL,
+          TRACKER_INDEX_FILE_INDEX_FILENAME,
+          "file-index",
+          NULL },
+        { TRACKER_INDEX_TYPE_EMAILS, 
+	  NULL,
+          TRACKER_INDEX_EMAIL_INDEX_FILENAME,
+          "email-index",
+          NULL },
+        { TRACKER_INDEX_TYPE_FILES_UPDATE, 
+	  NULL,
+          TRACKER_INDEX_FILE_UPDATE_INDEX_FILENAME,
+          "file-update-index",
+          NULL }
+};
 
 static gboolean
-initialize_indexes (TrackerIndexManagerFlags flags)
+has_tmp_merge_files (TrackerIndexType type)
 {
-	gchar *final_index_name;
+	GFile           *file;
+	GFileEnumerator *enumerator;
+	GFileInfo       *info;
+	GError          *error = NULL;
+	const gchar     *prefix;
+	const gchar     *data_dir;
+	gchar           *dirname;
+	gboolean         found;
 
-	/* Create index files */
-	final_index_name = g_build_filename (index_manager_data_dir,
-                                             "file-index-final", 
-                                             NULL);
+	dirname = g_path_get_dirname (indexes[type].abs_filename);
+	file = g_file_new_for_path (dirname);
 
-	if (g_file_test (final_index_name, G_FILE_TEST_EXISTS) && 
-	    (flags & TRACKER_INDEX_MANAGER_FORCE_REINDEX)) {
-		    g_unlink (final_index_name);
-	}
+	enumerator = g_file_enumerate_children (file,
+						G_FILE_ATTRIBUTE_STANDARD_NAME ","
+						G_FILE_ATTRIBUTE_STANDARD_TYPE,
+						G_PRIORITY_DEFAULT,
+						NULL, 
+						&error);
+
+	if (error) {
+		g_warning ("Could not check for temporary index files in "
+			   "directory:'%s', %s",
+			   dirname,
+			   error->message);
 
-	if (g_file_test (final_index_name, G_FILE_TEST_EXISTS) && 
-	    !tracker_index_manager_has_tmp_merge_files (TRACKER_INDEX_TYPE_FILES)) {
-		gchar *file_index_name;
+		g_error_free (error);
+		g_free (dirname);
+		g_object_unref (file);
 
-		file_index_name = g_build_filename (index_manager_data_dir, 
-						    TRACKER_INDEX_FILE_INDEX_DB_FILENAME, 
-						    NULL);
-	
-		g_message ("Overwriting '%s' with '%s'", 
-			   file_index_name, 
-			   final_index_name);	
-		g_rename (final_index_name, file_index_name);
-		g_free (file_index_name);
+		return FALSE;
 	}
 	
-	g_free (final_index_name);
-	
-	final_index_name = g_build_filename (index_manager_data_dir, 
-					     "email-index-final", 
-					     NULL);
-
-	if (g_file_test (final_index_name, G_FILE_TEST_EXISTS) && 
-	    (flags & TRACKER_INDEX_MANAGER_FORCE_REINDEX)) {
-		    g_unlink (final_index_name);
+	if (type == TRACKER_INDEX_TYPE_FILES) {
+		prefix = "file-index.tmp.";
+	} else {
+		prefix = "email-index.tmp.";
 	}
 
-	if (g_file_test (final_index_name, G_FILE_TEST_EXISTS) && 
-	    !tracker_index_manager_has_tmp_merge_files (TRACKER_INDEX_TYPE_EMAILS)) {
-		gchar *file_index_name;
+	found = FALSE;
 
-		file_index_name = g_build_filename (index_manager_data_dir, 
-						    TRACKER_INDEX_EMAIL_INDEX_DB_FILENAME, 
-						    NULL);
-	
-		g_message ("Overwriting '%s' with '%s'", 
-			   file_index_name, 
-			   final_index_name);	
-		g_rename (final_index_name, file_index_name);
-		g_free (file_index_name);
+	for (info = g_file_enumerator_next_file (enumerator, NULL, &error);
+	     info && !error && !found;
+	     info = g_file_enumerator_next_file (enumerator, NULL, &error)) {
+		/* Check each file has or hasn't got the prefix */
+		if (g_str_has_prefix (g_file_info_get_name (info), prefix)) {
+			found = TRUE;
+		}
+
+		g_object_unref (info);
 	}
-	
-	g_free (final_index_name);
 
-	return TRUE;
+	if (error) {
+		g_warning ("Could not get file information for temporary "
+			   "index files in directory:'%s', %s",
+			   data_dir,
+			   error->message);
+		g_error_free (error);
+	}
+		
+	g_object_unref (enumerator);
+	g_object_unref (file);
+	g_free (dirname);
+
+	return found;
 }
 
 gboolean
@@ -130,18 +140,92 @@
                             gint                     min_bucket, 
                             gint                     max_bucket)
 {
+	gchar    *final_index_filename;
+	gboolean  need_reindex = FALSE;
+	guint     i;
+
+	g_return_val_if_fail (data_dir != NULL, FALSE);
+	g_return_val_if_fail (min_bucket >= 0, FALSE);
+	g_return_val_if_fail (max_bucket >= min_bucket, FALSE);
+
         if (initialized) {
                 return TRUE;
         }
 
-        index_manager_data_dir = g_strdup (data_dir);
+	g_message ("Checking index directories exist");
 
-        index_manager_min_bucket = min_bucket;
-        index_manager_max_bucket = max_bucket;
+	g_mkdir_with_parents (data_dir, 00755);
+
+	g_message ("Checking index files exist");
+
+	for (i = 0; i < G_N_ELEMENTS (indexes); i++) {
+		indexes[i].abs_filename = g_build_filename (data_dir, indexes[i].file, NULL);
+
+		if (need_reindex) {
+			continue;
+		}
+
+		if (!g_file_test (indexes[i].abs_filename, G_FILE_TEST_EXISTS)) { 
+			g_message ("Could not find index file:'%s'", indexes[i].abs_filename);
+			g_message ("One or more index files are missing, a reindex will be forced");
+			need_reindex = TRUE;
+		}
+	}
+	
+	g_message ("Merging old temporary indexes");
+	
+	/* Files */
+	i = TRACKER_INDEX_TYPE_FILES;
+	final_index_filename = g_build_filename (data_dir, "file-index-final", NULL);
+	
+	if (g_file_test (final_index_filename, G_FILE_TEST_EXISTS) && 
+	    !has_tmp_merge_files (i)) {
+		g_message ("  Overwriting '%s' with '%s'", 
+			   indexes[i].abs_filename, 
+			   final_index_filename);	
+		
+		g_rename (final_index_filename, indexes[i].abs_filename);
+	}
+	
+	g_free (final_index_filename);
+	
+	/* Emails */
+	i = TRACKER_INDEX_TYPE_EMAILS;
+	final_index_filename = g_build_filename (data_dir, "email-index-final", NULL);
+	
+	if (g_file_test (final_index_filename, G_FILE_TEST_EXISTS) && 
+	    !has_tmp_merge_files (i)) {
+		g_message ("  Overwriting '%s' with '%s'", 
+			   indexes[i].abs_filename, 
+			   final_index_filename);	
+		
+		g_rename (final_index_filename, indexes[i].abs_filename);
+	}
+	
+	g_free (final_index_filename);
+
+	/* Now we have cleaned up merge files, see if we are supposed
+	 * to be reindexing.
+	 */ 
+	if (flags & TRACKER_INDEX_MANAGER_FORCE_REINDEX || need_reindex) {
+		g_message ("Cleaning up index files for reindex");
+
+		for (i = 0; i < G_N_ELEMENTS (indexes); i++) {
+			g_unlink (indexes[i].abs_filename);
+		}
+	}
+
+	g_message ("Creating index files, this may take a few moments...");
+	
+	for (i = 0; i < G_N_ELEMENTS (indexes); i++) {
+		indexes[i].index = tracker_index_new (indexes[i].abs_filename,
+						      min_bucket, 
+						      max_bucket);
+	}
 
         initialized = TRUE;
 
-        return initialize_indexes (flags);
+	return TRUE;
 }
 
 void
@@ -153,12 +237,12 @@
                 return;
         }
         
-        g_free (index_manager_data_dir);
-        index_manager_data_dir = NULL;
-
 	for (i = 0; i < G_N_ELEMENTS (indexes); i++) {
-		g_object_unref (indexes[i]);
-		indexes[i] = NULL;
+		g_object_unref (indexes[i].index);
+		indexes[i].index = NULL;
+
+		g_free (indexes[i].abs_filename);
+		indexes[i].abs_filename = NULL;
 	}
 
         initialized = FALSE;
@@ -167,115 +251,31 @@
 TrackerIndex * 
 tracker_index_manager_get_index (TrackerIndexType type)
 {
-        gchar *filename;
-
-	if (indexes[type]) {
-		return indexes[type];
-	}
-
-        filename = tracker_index_manager_get_filename (type);
-        indexes[type] = tracker_index_new (filename,
-					   index_manager_min_bucket, 
-					   index_manager_max_bucket);
-        g_free (filename);
-
-        return indexes[type];
+	return indexes[type].index;
 }
 
-gchar *
-tracker_index_manager_get_filename (TrackerIndexType index)
+const gchar *
+tracker_index_manager_get_filename (TrackerIndexType type)
 {
-        return g_build_filename (index_manager_data_dir, 
-                                 get_index_name (index), 
-                                 NULL);
+        return indexes[type].abs_filename;
 }
 
 gboolean
 tracker_index_manager_are_indexes_too_big (void)
 {
-	gchar       *filename;
-        gboolean     too_big;
+        gboolean too_big;
+	guint    i;
 
-	filename = g_build_filename (index_manager_data_dir, TRACKER_INDEX_FILE_INDEX_DB_FILENAME, NULL);
-	too_big = tracker_file_get_size (filename) > MAX_INDEX_FILE_SIZE;
-        g_free (filename);
-        
-        if (too_big) {
-		g_critical ("File index database is too big, discontinuing indexing");
-		return TRUE;	
-	}
+	g_return_val_if_fail (initialized == TRUE, FALSE);
 
-	filename = g_build_filename (index_manager_data_dir, TRACKER_INDEX_EMAIL_INDEX_DB_FILENAME, NULL);
-	too_big = tracker_file_get_size (filename) > MAX_INDEX_FILE_SIZE;
-	g_free (filename);
+	for (i = 0, too_big = FALSE; i < G_N_ELEMENTS (indexes) && !too_big; i++) {
+		too_big = tracker_file_get_size (indexes[i].abs_filename) > MAX_INDEX_FILE_SIZE;
+	}
         
         if (too_big) {
-		g_critical ("Email index database is too big, discontinuing indexing");
+		g_critical ("One or more index files are too big, indexing disabled");
 		return TRUE;	
 	}
 
 	return FALSE;
 }
-
-gboolean
-tracker_index_manager_has_tmp_merge_files (TrackerIndexType type)
-{
-	GFile           *file;
-	GFileEnumerator *enumerator;
-	GFileInfo       *info;
-	GError          *error = NULL;
-	const gchar     *prefix;
-	const gchar     *data_dir;
-	gboolean         found;
-
-	file = g_file_new_for_path (index_manager_data_dir);
-
-	enumerator = g_file_enumerate_children (file,
-						G_FILE_ATTRIBUTE_STANDARD_NAME ","
-						G_FILE_ATTRIBUTE_STANDARD_TYPE,
-						G_PRIORITY_DEFAULT,
-						NULL, 
-						&error);
-
-	if (error) {
-		g_warning ("Could not check for temporary index files in "
-			   "directory:'%s', %s",
-			   data_dir,
-			   error->message);
-		g_error_free (error);
-		g_object_unref (file);
-		return FALSE;
-	}
-	
-	if (type == TRACKER_INDEX_TYPE_FILES) {
-		prefix = "file-index.tmp.";
-	} else {
-		prefix = "email-index.tmp.";
-	}
-
-	found = FALSE;
-
-	for (info = g_file_enumerator_next_file (enumerator, NULL, &error);
-	     info && !error && !found;
-	     info = g_file_enumerator_next_file (enumerator, NULL, &error)) {
-		/* Check each file has or hasn't got the prefix */
-		if (g_str_has_prefix (g_file_info_get_name (info), prefix)) {
-			found = TRUE;
-		}
-
-		g_object_unref (info);
-	}
-
-	if (error) {
-		g_warning ("Could not get file information for temporary "
-			   "index files in directory:'%s', %s",
-			   data_dir,
-			   error->message);
-		g_error_free (error);
-	}
-		
-	g_object_unref (enumerator);
-	g_object_unref (file);
-
-	return found;
-}

Modified: branches/indexer-split/src/trackerd/tracker-index-manager.h
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-index-manager.h	(original)
+++ branches/indexer-split/src/trackerd/tracker-index-manager.h	Tue Jul 29 15:57:40 2008
@@ -38,15 +38,14 @@
 	TRACKER_INDEX_MANAGER_FORCE_REINDEX    = 1 << 1,
 } TrackerIndexManagerFlags;
 
-gboolean      tracker_index_manager_init                (TrackerIndexManagerFlags flags,
-							 const gchar             *data_dir,
-                                                         gint                     min_bucket,
-                                                         gint                     max_bucket);
+gboolean      tracker_index_manager_init                (TrackerIndexManagerFlags  flags,
+							 const gchar              *data_dir,
+							 gint                      min_bucket,
+							 gint                      max_bucket);
 void          tracker_index_manager_shutdown            (void);
-gchar *       tracker_index_manager_get_filename        (TrackerIndexType  index);
-TrackerIndex *tracker_index_manager_get_index           (TrackerIndexType  index);
+const gchar * tracker_index_manager_get_filename        (TrackerIndexType          index);
+TrackerIndex *tracker_index_manager_get_index           (TrackerIndexType          index);
 gboolean      tracker_index_manager_are_indexes_too_big (void);
-gboolean      tracker_index_manager_has_tmp_merge_files (TrackerIndexType  type);
 
 G_END_DECLS