GNOME.org
 

[tracker/journal-compression-review2: 4/16] libtracker-data, libtracker-db: Reimplement Backup() and Restore() to use tar

commit 650c9e865b4dbad611dddbbda70ac4dd281afd81
Author: Philip Van Hoof <philip codeminded be>
Date:   Fri May 7 13:31:04 2010 +0200

    libtracker-data, libtracker-db: Reimplement Backup() and Restore() to use tar
    
    The original implementation of Backup() and Restore() used a simple file copy
    of the journal. When we have rotated journals we of course can't use only the
    active journal for Backup(). And for Restore() we want to put the rotation state
    back the way it was when the backup was made too. So the best option I could
    come up with was to use a simple tar.

 configure.ac                                      |    7 +
 src/libtracker-common/tracker-os-dependant-unix.c |    9 +-
 src/libtracker-common/tracker-os-dependant.h      |    1 +
 src/libtracker-data/tracker-data-backup.c         |  281 +++++++++++++++++++--
 src/libtracker-db/tracker-db-journal.c            |    9 +-
 src/libtracker-db/tracker-db-journal.h            |    1 +
 src/libtracker-db/tracker-db-manager.c            |   94 +++++++-
 7 files changed, 366 insertions(+), 36 deletions(-)
---
diff --git a/configure.ac b/configure.ac
index f3bb067..8c0b334 100644
--- a/configure.ac
+++ b/configure.ac
@@ -223,6 +223,13 @@ fi
 
 AM_CONDITIONAL(HAVE_ENCA, test "$have_enca" = "yes")
 
+# We need tar for Backup and Restore support
+AC_PATH_PROG(TAR, tar, tar)
+if test -z $TAR; then
+   AC_MSG_ERROR([Could not find 'tar'])
+fi
+AC_DEFINE_UNQUOTED(TAR, "$TAR", [The tar program])
+
 AC_PATH_PROG(VALAC, valac, valac)
 AC_SUBST(VALAC)
 
diff --git a/src/libtracker-common/tracker-os-dependant-unix.c b/src/libtracker-common/tracker-os-dependant-unix.c
index 266aa5d..e218490 100644
--- a/src/libtracker-common/tracker-os-dependant-unix.c
+++ b/src/libtracker-common/tracker-os-dependant-unix.c
@@ -50,6 +50,7 @@ gboolean
 tracker_spawn (gchar **argv,
                gint    timeout,
                gchar **tmp_stdout,
+               gchar **tmp_stderr,
                gint   *exit_status)
 {
 	GError      *error = NULL;
@@ -60,8 +61,10 @@ tracker_spawn (gchar **argv,
 	g_return_val_if_fail (argv[0] != NULL, FALSE);
 	g_return_val_if_fail (timeout >= 0, FALSE);
 
-	flags = G_SPAWN_SEARCH_PATH |
-		G_SPAWN_STDERR_TO_DEV_NULL;
+	flags = G_SPAWN_SEARCH_PATH;
+
+	if (tmp_stderr == NULL)
+		flags |= G_SPAWN_STDERR_TO_DEV_NULL;
 
 	if (!tmp_stdout) {
 		flags = flags | G_SPAWN_STDOUT_TO_DEV_NULL;
@@ -74,7 +77,7 @@ tracker_spawn (gchar **argv,
 	                       tracker_spawn_child_func,
 	                       GINT_TO_POINTER (timeout),
 	                       tmp_stdout,
-	                       NULL,
+	                       tmp_stderr,
 	                       exit_status,
 	                       &error);
 
diff --git a/src/libtracker-common/tracker-os-dependant.h b/src/libtracker-common/tracker-os-dependant.h
index f5b467d..e92ea45 100644
--- a/src/libtracker-common/tracker-os-dependant.h
+++ b/src/libtracker-common/tracker-os-dependant.h
@@ -34,6 +34,7 @@ G_BEGIN_DECLS
 gboolean tracker_spawn                     (gchar       **argv,
                                             gint          timeout,
                                             gchar       **tmp_stdout,
+                                            gchar       **tmp_stderr,
                                             gint         *exit_status);
 gboolean tracker_spawn_async_with_channels (const gchar **argv,
                                             gint          timeout,
diff --git a/src/libtracker-data/tracker-data-backup.c b/src/libtracker-data/tracker-data-backup.c
index 3c5c2f5..f08f4d0 100644
--- a/src/libtracker-data/tracker-data-backup.c
+++ b/src/libtracker-data/tracker-data-backup.c
@@ -19,12 +19,15 @@
  */
 #include "config.h"
 
+#include <string.h>
+
 #include <glib.h>
 #include <glib/gstdio.h>
 
 #include <libtracker-db/tracker-db-manager.h>
 #include <libtracker-db/tracker-db-journal.h>
 #include <libtracker-data/tracker-data-manager.h>
+#include <libtracker-common/tracker-os-dependant.h>
 
 #include "tracker-data-backup.h"
 
@@ -36,6 +39,17 @@ typedef struct {
 	GError *error;
 } BackupSaveInfo;
 
+typedef struct {
+	GPid pid;
+	guint stdout_watch_id;
+	guint stderr_watch_id;
+	GIOChannel *stdin_channel;
+	GIOChannel *stdout_channel;
+	GIOChannel *stderr_channel;
+	gpointer data;
+	GString *lines;
+} ProcessContext;
+
 GQuark
 tracker_data_backup_error_quark (void)
 {
@@ -63,22 +77,139 @@ free_backup_save_info (BackupSaveInfo *info)
 }
 
 static void
-on_journal_copied (GObject *source_object,
-                   GAsyncResult *res,
-                   gpointer user_data)
+on_journal_copied (BackupSaveInfo *info, GError *error)
 {
-	BackupSaveInfo *info = user_data;
-	GError *error = NULL;
-
-	g_file_copy_finish (info->journal, res, &error);
-
 	if (info->callback) {
 		info->callback (error, info->user_data);
 	}
 
 	free_backup_save_info (info);
+}
+
+
+
+static void
+process_context_destroy (ProcessContext *context, GError *error)
+{
+	on_journal_copied (context->data, error);
+
+	if (context->lines) {
+		g_string_free (context->lines, TRUE);
+	}
+
+	if (context->stdin_channel) {
+		g_io_channel_shutdown (context->stdin_channel, FALSE, NULL);
+		g_io_channel_unref (context->stdin_channel);
+		context->stdin_channel = NULL;
+	}
+
+	if (context->stdout_watch_id != 0) {
+		g_source_remove (context->stdout_watch_id);
+		context->stdout_watch_id = 0;
+	}
+
+	if (context->stdout_channel) {
+		g_io_channel_shutdown (context->stdout_channel, FALSE, NULL);
+		g_io_channel_unref (context->stdout_channel);
+		context->stdout_channel = NULL;
+	}
 
-	g_clear_error (&error);
+	if (context->stderr_watch_id != 0) {
+		g_source_remove (context->stderr_watch_id);
+		context->stderr_watch_id = 0;
+	}
+
+	if (context->stderr_channel) {
+		g_io_channel_shutdown (context->stderr_channel, FALSE, NULL);
+		g_io_channel_unref (context->stderr_channel);
+		context->stderr_channel = NULL;
+	}
+
+	if (context->pid != 0) {
+		g_spawn_close_pid (context->pid);
+		context->pid = 0;
+	}
+
+	g_free (context);
+}
+
+static gboolean
+read_line_of_tar_output (GIOChannel  *channel,
+                         GIOCondition condition,
+                         gpointer     user_data)
+{
+	ProcessContext *context;
+
+	context = user_data;
+
+	if (condition & G_IO_ERR || condition & G_IO_HUP) {
+		return FALSE;
+	}
+
+	/* TODO: progress support */
+	return TRUE;
+}
+
+static gboolean
+read_error_of_tar_output (GIOChannel  *channel,
+                          GIOCondition condition,
+                          gpointer     user_data)
+{
+	ProcessContext *context;
+	GIOStatus status;
+	gchar *line;
+
+	context = user_data;
+	status = G_IO_STATUS_NORMAL;
+
+	if (condition & G_IO_IN || condition & G_IO_PRI) {
+		do {
+			GError *error = NULL;
+
+			status = g_io_channel_read_line (channel, &line, NULL, NULL, &error);
+
+			if (status == G_IO_STATUS_NORMAL) {
+				if (context->lines == NULL)
+					context->lines = g_string_new (NULL);
+				g_string_append (context->lines, line);
+				g_free (line);
+			} else if (error) {
+				g_warning ("%s", error->message);
+				g_error_free (error);
+			}
+		} while (status == G_IO_STATUS_NORMAL);
+
+		if (status == G_IO_STATUS_EOF ||
+		    status == G_IO_STATUS_ERROR) {
+			return FALSE;
+		}
+	}
+
+	if (condition & G_IO_ERR || condition & G_IO_HUP) {
+		return FALSE;
+	}
+
+	return TRUE;
+}
+
+static void
+process_context_child_watch_cb (GPid     pid,
+                                gint     status,
+                                gpointer user_data)
+{
+	ProcessContext *context;
+	GError *error = NULL;
+
+	g_debug ("Process '%d' exited with code %d", pid, status);
+
+	context = (ProcessContext *) user_data;
+
+	if (context->lines) {
+		g_set_error (&error, TRACKER_DATA_BACKUP_ERROR, 0, 
+		             "%s", context->lines->str);
+	}
+
+	process_context_destroy (context, error);
 }
 
 void
@@ -88,6 +219,16 @@ tracker_data_backup_save (GFile *destination,
                           GDestroyNotify destroy)
 {
 	BackupSaveInfo *info;
+	ProcessContext *context;
+	gchar **argv;
+	gchar *path, *directory;
+	GDir *journal_dir;
+	GFile *parent;
+	GIOChannel *stdin_channel, *stdout_channel, *stderr_channel;
+	GPid pid;
+	GPtrArray *files;
+	const gchar *f_name;
+	guint i;
 
 	info = g_new0 (BackupSaveInfo, 1);
 	info->destination = g_object_ref (destination);
@@ -96,23 +237,86 @@ tracker_data_backup_save (GFile *destination,
 	info->user_data = user_data;
 	info->destroy = destroy;
 
-	/* It's fine to copy this asynchronous: the journal replay code can or 
+	parent = g_file_get_parent (info->journal);
+	directory = g_file_get_path (parent);
+	g_object_unref (parent);
+	path = g_file_get_path (destination);
+
+	journal_dir = g_dir_open (directory, 0, NULL);
+	f_name = g_dir_read_name (journal_dir);
+	files = g_ptr_array_new ();
+
+	while (f_name) {
+		if (f_name) {
+			if (!g_str_has_prefix (f_name, TRACKER_DB_JOURNAL_FILENAME ".")) {
+				f_name = g_dir_read_name (journal_dir);
+				continue;
+			}
+			g_ptr_array_add (files, g_strdup (f_name));
+		}
+		f_name = g_dir_read_name (journal_dir);
+	}
+
+	g_dir_close (journal_dir);
+
+	argv = g_new0 (gchar*, files->len + 7);
+
+	argv[0] = g_strdup (TAR);
+	argv[1] = g_strdup ("-zcf");
+	argv[2] = path;
+	argv[3] = g_strdup ("-C");
+	argv[4] = directory;
+	argv[5] = g_strdup (TRACKER_DB_JOURNAL_FILENAME);
+
+	for (i = 0; i < files->len; i++) {
+		argv[i+6] = g_ptr_array_index (files, i);
+	}
+
+	/* It's fine to untar this asynchronous: the journal replay code can or
 	 * should cope with unfinished entries at the end of the file, while
 	 * restoring a backup made this way. */
 
-	g_file_copy_async (info->journal, info->destination,
-	                   G_FILE_COPY_OVERWRITE,
-	                   G_PRIORITY_HIGH,
-	                   NULL, NULL, NULL,
-	                   on_journal_copied,
-	                   info);
+	if (!tracker_spawn_async_with_channels ((const gchar **) argv,
+	                                        0, &pid,
+	                                        &stdin_channel,
+	                                        &stdout_channel,
+	                                        &stderr_channel)) {
+		GError *error = NULL;
+		g_set_error (&error, TRACKER_DATA_BACKUP_ERROR, 0, 
+		             "Error starting tar program");
+		on_journal_copied (info, error);
+		g_strfreev (argv);
+		g_error_free (error);
+		return;
+	}
+
+	context = g_new0 (ProcessContext, 1);
+	context->lines = NULL;
+	context->data = info;
+	context->pid = pid;
+	context->stdin_channel = stdin_channel;
+	context->stderr_channel = stderr_channel;
+	context->stdout_watch_id = g_io_add_watch (stdout_channel,
+	                                           G_IO_IN | G_IO_PRI | G_IO_ERR | G_IO_HUP,
+	                                           read_line_of_tar_output,
+	                                           context);
+	context->stderr_watch_id = g_io_add_watch (stderr_channel,
+	                                           G_IO_IN | G_IO_PRI | G_IO_ERR | G_IO_HUP,
+	                                           read_error_of_tar_output,
+	                                           context);
+
+	g_child_watch_add (context->pid, process_context_child_watch_cb, context);
+
+	g_strfreev (argv);
+
+	g_debug ("Process '%d' spawned for command:'%s %s %s'",
+	         pid, argv[0], argv[1], argv[2]);
+
 }
 
 static gboolean
-on_restore_done (gpointer user_data)
+on_restore_done (BackupSaveInfo *info)
 {
-	BackupSaveInfo *info = user_data;
-
 	if (info->callback) {
 		info->callback (info->error, info->user_data);
 	}
@@ -145,17 +349,48 @@ tracker_data_backup_restore (GFile *journal,
 		gboolean is_first;
 		gsize chunk_size = 0;
 		gboolean do_rotating = FALSE;
+		GFile *parent = g_file_get_parent (info->destination);
+		gchar *tmp_stdout = NULL;
+		gchar *tmp_stderr = NULL;
+		gchar **argv;
+		gint exit_status;
 
 		tracker_db_manager_move_to_temp ();
 		tracker_data_manager_shutdown ();
 
+		argv = g_new0 (char*, 6);
+
+		argv[0] = g_strdup (TAR);
+		argv[1] = g_strdup ("-zxf");
+		argv[2] = g_file_get_path (info->journal);
+		argv[3] = g_strdup ("-C");
+		argv[4] = g_file_get_path (parent);
+
+		g_object_unref (parent);
+
 		/* Synchronous: we don't want the mainloop to run while copying the
 		 * journal, as nobody should be writing anything at this point */
 
-		g_file_copy (info->journal, info->destination,
-		             G_FILE_COPY_OVERWRITE,
-		             NULL, NULL, NULL,
-		             &info->error);
+		if (!tracker_spawn (argv, 0, &tmp_stdout, &tmp_stderr, &exit_status)) {
+			g_set_error (&info->error, TRACKER_DATA_BACKUP_ERROR, 0, 
+			             "Error starting tar program");
+		}
+
+		if (!info->error && tmp_stderr) {
+			if (strlen (tmp_stderr) > 0) {
+				g_set_error (&info->error, TRACKER_DATA_BACKUP_ERROR, 0, 
+				             "%s", tmp_stderr);
+			}
+		}
+
+		if (!info->error && exit_status != 0) {
+			g_set_error (&info->error, TRACKER_DATA_BACKUP_ERROR, 0, 
+			             "Unknown error, tar exited with exit status %d", exit_status);
+		}
+
+		g_free (tmp_stderr);
+		g_free (tmp_stdout);
+		g_strfreev (argv);
 
 		tracker_db_manager_init_locations ();
 		tracker_db_journal_get_rotating (&do_rotating, &chunk_size);
diff --git a/src/libtracker-db/tracker-db-journal.c b/src/libtracker-db/tracker-db-journal.c
index 8fc34f5..5db144d 100644
--- a/src/libtracker-db/tracker-db-journal.c
+++ b/src/libtracker-db/tracker-db-journal.c
@@ -43,7 +43,6 @@
 
 #include "tracker-db-journal.h"
 
-#define JOURNAL_FILENAME  "tracker-store.journal"
 #define MIN_BLOCK_SIZE    1024
 
 /*
@@ -291,7 +290,7 @@ tracker_db_journal_init (const gchar *filename,
 		writer.journal_filename = g_build_filename (g_get_user_data_dir (),
 		                                            "tracker",
 		                                            "data",
-		                                            JOURNAL_FILENAME,
+		                                            TRACKER_DB_JOURNAL_FILENAME,
 		                                            NULL);
 	}
 
@@ -655,7 +654,7 @@ tracker_db_journal_rotate (void)
 
 			if (f_name) {
 
-				if (!g_str_has_prefix (f_name, "tracker-store.journal.")) {
+				if (!g_str_has_prefix (f_name, TRACKER_DB_JOURNAL_FILENAME ".")) {
 					f_name = g_dir_read_name (journal_dir);
 					continue;
 				}
@@ -666,7 +665,7 @@ tracker_db_journal_rotate (void)
 					cur = atoi (ptr);
 					max = MAX (cur, max);
 				}
-			}
+			} 
 
 			f_name = g_dir_read_name (journal_dir);
 		}
@@ -775,7 +774,7 @@ tracker_db_journal_reader_init (const gchar *filename)
 		filename_used = g_build_filename (g_get_user_data_dir (),
 		                                  "tracker",
 		                                  "data",
-		                                  JOURNAL_FILENAME,
+		                                  TRACKER_DB_JOURNAL_FILENAME,
 		                                  NULL);
 	}
 
diff --git a/src/libtracker-db/tracker-db-journal.h b/src/libtracker-db/tracker-db-journal.h
index 4ef0ca5..fefd515 100644
--- a/src/libtracker-db/tracker-db-journal.h
+++ b/src/libtracker-db/tracker-db-journal.h
@@ -29,6 +29,7 @@ G_BEGIN_DECLS
 
 #define TRACKER_DB_JOURNAL_ERROR_DOMAIN "TrackerDBJournal"
 #define TRACKER_DB_JOURNAL_ERROR        tracker_db_journal_error_quark()
+#define TRACKER_DB_JOURNAL_FILENAME     "tracker-store.journal"
 
 typedef enum {
 	TRACKER_DB_JOURNAL_START,
diff --git a/src/libtracker-db/tracker-db-manager.c b/src/libtracker-db/tracker-db-manager.c
index 82fd4c9..8d468a7 100644
--- a/src/libtracker-db/tracker-db-manager.c
+++ b/src/libtracker-db/tracker-db-manager.c
@@ -459,15 +459,13 @@ db_manager_remove_all (gboolean rm_journal)
 				gchar *fullpath;
 
 				if (f_name) {
-					if (!g_str_has_prefix (f_name, "tracker-store.journal.")) {
+					if (!g_str_has_prefix (f_name, TRACKER_DB_JOURNAL_FILENAME ".")) {
 						f_name = g_dir_read_name (journal_dir);
 						continue;
 					}
-					
 				}
 
 				fullpath = g_build_filename (directory, f_name, NULL);
-
 				file = g_file_new_for_path (fullpath);
 				g_file_delete (file, NULL, NULL);
 				g_object_unref (file);
@@ -1087,6 +1085,9 @@ tracker_db_manager_move_to_temp (void)
 {
 	guint i;
 	gchar *cpath, *new_filename;
+	gchar *directory;
+	GDir *journal_dir;
+	const gchar *f_name;
 
 	g_return_if_fail (initialized != FALSE);
 
@@ -1101,6 +1102,32 @@ tracker_db_manager_move_to_temp (void)
 	}
 
 	cpath = g_strdup (tracker_db_journal_get_filename ());
+
+	directory = g_path_get_dirname (cpath);
+	journal_dir = g_dir_open (directory, 0, NULL);
+	f_name = g_dir_read_name (journal_dir);
+
+	while (f_name) {
+		gchar *fullpath;
+
+		if (f_name) {
+			if (!g_str_has_prefix (f_name, TRACKER_DB_JOURNAL_FILENAME ".")) {
+				f_name = g_dir_read_name (journal_dir);
+				continue;
+			}
+		}
+
+		fullpath = g_build_filename (directory, f_name, NULL);
+		new_filename = g_strdup_printf ("%s.tmp", fullpath);
+		g_rename (fullpath, new_filename);
+		g_free (new_filename);
+		g_free (fullpath);
+		f_name = g_dir_read_name (journal_dir);
+	}
+
+	g_dir_close (journal_dir);
+	g_free (directory);
+
 	new_filename = g_strdup_printf ("%s.tmp", cpath);
 	g_message ("  Renaming journal:'%s' -> '%s'",
 	           cpath, new_filename);
@@ -1115,6 +1142,9 @@ tracker_db_manager_restore_from_temp (void)
 {
 	guint i;
 	gchar *cpath, *new_filename;
+	gchar *directory;
+	GDir *journal_dir;
+	const gchar *f_name;
 
 	g_return_if_fail (locations_initialized != FALSE);
 
@@ -1133,8 +1163,37 @@ tracker_db_manager_restore_from_temp (void)
 	g_message ("  Renaming journal:'%s' -> '%s'",
 	           cpath, new_filename);
 	g_rename (cpath, new_filename);
-	g_free (cpath);
 	g_free (new_filename);
+
+	directory = g_path_get_dirname (cpath);
+	journal_dir = g_dir_open (directory, 0, NULL);
+	f_name = g_dir_read_name (journal_dir);
+
+	while (f_name) {
+		gchar *fullpath, *ptr;
+
+		if (f_name) {
+			if (!g_str_has_suffix (f_name, ".tmp")) {
+				f_name = g_dir_read_name (journal_dir);
+				continue;
+			}
+		}
+
+		fullpath = g_build_filename (directory, f_name, NULL);
+		new_filename = g_strdup (fullpath);
+		ptr = strstr (new_filename, ".tmp");
+		if (ptr) {
+			*ptr = '\0';
+			g_rename (fullpath, new_filename);
+		}
+		g_free (new_filename);
+		g_free (fullpath);
+		f_name = g_dir_read_name (journal_dir);
+	}
+
+	g_dir_close (journal_dir);
+	g_free (directory);
+	g_free (cpath);
 }
 
 void
@@ -1142,6 +1201,9 @@ tracker_db_manager_remove_temp (void)
 {
 	guint i;
 	gchar *cpath, *new_filename;
+	gchar *directory;
+	GDir *journal_dir;
+	const gchar *f_name;
 
 	g_return_if_fail (locations_initialized != FALSE);
 
@@ -1160,8 +1222,30 @@ tracker_db_manager_remove_temp (void)
 	g_message ("  Removing temp journal:'%s'",
 	           new_filename);
 	g_unlink (new_filename);
-	g_free (cpath);
 	g_free (new_filename);
+
+	directory = g_path_get_dirname (cpath);
+	journal_dir = g_dir_open (directory, 0, NULL);
+	f_name = g_dir_read_name (journal_dir);
+
+	while (f_name) {
+		if (f_name) {
+			if (!g_str_has_suffix (f_name, ".tmp")) {
+				f_name = g_dir_read_name (journal_dir);
+				continue;
+			}
+		}
+
+		new_filename = g_build_filename (directory, f_name, NULL);
+		g_unlink (new_filename);
+		g_free (new_filename);
+
+		f_name = g_dir_read_name (journal_dir);
+	}
+
+	g_dir_close (journal_dir);
+	g_free (directory);
+	g_free (cpath);
 }
 
 void
[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]