[evolution-data-server] Bug 343904 - [POP3] Local cache keeps stored orphaned files

[Milan Crha <mcrha src gnome org>]

commit af6938510c2744928d0613095706247bae3204a9
Author: Milan Crha <mcrha redhat com>
Date:   Wed Jun 21 10:30:19 2017 +0200

    Bug 343904 - [POP3] Local cache keeps stored orphaned files

 src/camel/camel-data-cache.c                   |   91 ++++++++++++++++++++++++
 src/camel/camel-data-cache.h                   |   22 ++++++
 src/camel/providers/pop3/camel-pop3-folder.c   |   89 +++++++++++++++++++++++-
 src/camel/providers/pop3/camel-pop3-settings.c |   52 +++++++++++++-
 src/camel/providers/pop3/camel-pop3-settings.h |    5 ++
 5 files changed, 257 insertions(+), 2 deletions(-)
---
diff --git a/src/camel/camel-data-cache.c b/src/camel/camel-data-cache.c
index 818ed60..3be6943 100644
--- a/src/camel/camel-data-cache.c
+++ b/src/camel/camel-data-cache.c
@@ -665,3 +665,94 @@ camel_data_cache_clear (CamelDataCache *cdc,
        g_dir_close (dir);
        g_free (base_dir);
 }
+
+static void
+data_cache_foreach_remove (CamelDataCache *cdc,
+                          const gchar *path,
+                          CamelDataCacheRemoveFunc func,
+                          gpointer user_data)
+{
+       GDir *dir;
+       const gchar *dname;
+       struct stat st;
+       GIOStream *stream;
+
+       dir = g_dir_open (path, 0, NULL);
+       if (!dir)
+               return;
+
+       while ((dname = g_dir_read_name (dir))) {
+               gchar *filename;
+
+               filename = g_build_filename (path, dname, NULL);
+
+               if (g_stat (filename, &st) == 0
+                   && S_ISREG (st.st_mode)
+                   && func (cdc, filename, user_data)) {
+                       g_unlink (filename);
+                       stream = camel_object_bag_get (cdc->priv->busy_bag, filename);
+                       if (stream) {
+                               camel_object_bag_remove (cdc->priv->busy_bag, stream);
+                               g_object_unref (stream);
+                       }
+               }
+
+               g_free (filename);
+       }
+
+       g_dir_close (dir);
+}
+
+/**
+ * camel_data_cache_foreach_remove:
+ * @cdc: a #CamelDataCache
+ * @path: Path to the (sub) cache the items exist in
+ * @func: (scope call) (closure user_data): a callback to call for each found file in the cache
+ * @user_data: user data passed to @func
+ *
+ * Traverses the @cdc sub-cache identified by @path and calls @func for each found file.
+ * If the @func returns %TRUE, then the file is removed, if %FALSE, it's kept in the cache.
+ *
+ * Since: 3.26
+ **/
+void
+camel_data_cache_foreach_remove (CamelDataCache *cdc,
+                                const gchar *path,
+                                CamelDataCacheRemoveFunc func,
+                                gpointer user_data)
+{
+       gchar *base_dir;
+       GDir *dir;
+       const gchar *dname;
+       struct stat st;
+
+       g_return_if_fail (CAMEL_IS_DATA_CACHE (cdc));
+       g_return_if_fail (path != NULL);
+       g_return_if_fail (func != NULL);
+
+       base_dir = g_build_filename (cdc->priv->path, path, NULL);
+
+       dir = g_dir_open (base_dir, 0, NULL);
+       if (dir == NULL) {
+               g_free (base_dir);
+               return;
+       }
+
+       while ((dname = g_dir_read_name (dir))) {
+               gchar *dpath;
+
+               dpath = g_build_filename (base_dir, dname, NULL);
+
+               if (g_stat (dpath, &st) == 0
+                   && S_ISDIR (st.st_mode)
+                   && !g_str_equal (dname, ".")
+                   && !g_str_equal (dname, "..")) {
+                       data_cache_foreach_remove (cdc, dpath, func, user_data);
+               }
+
+               g_free (dpath);
+       }
+
+       g_dir_close (dir);
+       g_free (base_dir);
+}
diff --git a/src/camel/camel-data-cache.h b/src/camel/camel-data-cache.h
index e5725f2..2654baf 100644
--- a/src/camel/camel-data-cache.h
+++ b/src/camel/camel-data-cache.h
@@ -52,6 +52,24 @@ typedef struct _CamelDataCache CamelDataCache;
 typedef struct _CamelDataCacheClass CamelDataCacheClass;
 typedef struct _CamelDataCachePrivate CamelDataCachePrivate;
 
+/**
+ * CamelDataCacheRemoveFunc:
+ * @cdc: a #CamelDataCache
+ * @filename: a file name found in the cache
+ * @user_data: user data passed to camel_data_cache_foreach_remove()
+ *
+ * A callback called for each found file in the cache, used
+ * by camel_data_cache_foreach_remove(). The @filename corresponds
+ * to the result of camel_data_cache_get_filename().
+ *
+ * Returns: %TRUE, to delete the file, %FALSE to keep in in the cache
+ *
+ * Since: 3.26
+ **/
+typedef gboolean (* CamelDataCacheRemoveFunc)  (CamelDataCache *cdc,
+                                                const gchar *filename,
+                                                gpointer user_data);
+
 struct _CamelDataCache {
        GObject parent;
        CamelDataCachePrivate *priv;
@@ -97,6 +115,10 @@ gchar *             camel_data_cache_get_filename   (CamelDataCache *cdc,
                                                 const gchar *key);
 void           camel_data_cache_clear          (CamelDataCache *cdc,
                                                 const gchar *path);
+void           camel_data_cache_foreach_remove (CamelDataCache *cdc,
+                                                const gchar *path,
+                                                CamelDataCacheRemoveFunc func,
+                                                gpointer user_data);
 
 G_END_DECLS
 
diff --git a/src/camel/providers/pop3/camel-pop3-folder.c b/src/camel/providers/pop3/camel-pop3-folder.c
index 1db295b..9998625 100644
--- a/src/camel/providers/pop3/camel-pop3-folder.c
+++ b/src/camel/providers/pop3/camel-pop3-folder.c
@@ -768,6 +768,89 @@ pop3_folder_refresh_info_sync (CamelFolder *folder,
        return success;
 }
 
+static guint32
+pop3_folder_get_current_time_mark (void)
+{
+       GDate date;
+
+       g_date_clear (&date, 1);
+       g_date_set_time_t (&date, time (NULL));
+
+       return g_date_get_julian (&date);
+}
+
+static gboolean
+pop3_folder_cache_foreach_remove_cb (CamelDataCache *cache,
+                                    const gchar *filename,
+                                    gpointer user_data)
+{
+       GHashTable *filenames = user_data;
+
+       g_return_val_if_fail (filenames != NULL, FALSE);
+
+       return !g_hash_table_contains (filenames, filename);
+}
+
+static void
+pop3_folder_maybe_expunge_cache (CamelPOP3Folder *pop3_folder)
+{
+       CamelService *service;
+       CamelSettings *settings;
+       CamelDataCache *pop3_cache;
+       GHashTable *filenames;
+       guint32 last_cache_expunge, current_time;
+       gint ii;
+
+       g_return_if_fail (CAMEL_IS_POP3_FOLDER (pop3_folder));
+
+       service = CAMEL_SERVICE (camel_folder_get_parent_store (CAMEL_FOLDER (pop3_folder)));
+       g_return_if_fail (CAMEL_IS_SERVICE (service));
+
+       /* Expunge local cache only if online, which should guarantee that
+          the 'pop3_folder->uids' is properly populated. */
+       if (camel_service_get_connection_status (service) != CAMEL_SERVICE_CONNECTED)
+               return;
+
+       pop3_cache = camel_pop3_store_ref_cache (CAMEL_POP3_STORE (service));
+       g_return_if_fail (CAMEL_IS_DATA_CACHE (pop3_cache));
+
+       settings = camel_service_ref_settings (service);
+
+       last_cache_expunge = camel_pop3_settings_get_last_cache_expunge (CAMEL_POP3_SETTINGS (settings));
+       current_time = pop3_folder_get_current_time_mark ();
+
+       if (last_cache_expunge + 7 > current_time && current_time >= last_cache_expunge) {
+               d (printf ("%s: No need to expunge cache yet; last did %d, now is %d\n", G_STRFUNC, 
last_cache_expunge, current_time));
+               g_clear_object (&pop3_cache);
+               g_clear_object (&settings);
+               return;
+       }
+
+       d (printf ("%s: Going to expunge cache; last did %d, now is %d\n", G_STRFUNC, last_cache_expunge, 
current_time));
+       camel_pop3_settings_set_last_cache_expunge (CAMEL_POP3_SETTINGS (settings), current_time);
+       g_clear_object (&settings);
+
+       filenames = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
+
+       for (ii = 0; ii < pop3_folder->uids->len; ii++) {
+               CamelPOP3FolderInfo *fi = pop3_folder->uids->pdata[ii];
+               gchar *filename;
+
+               if (!fi || !fi->uid)
+                       continue;
+
+               filename = camel_data_cache_get_filename (pop3_cache, "cache", fi->uid);
+               if (filename)
+                       g_hash_table_insert (filenames, filename, NULL);
+       }
+
+       d (printf ("%s: Recognized %d downloaded messages\n", G_STRFUNC, g_hash_table_size (filenames)));
+       camel_data_cache_foreach_remove (pop3_cache, "cache", pop3_folder_cache_foreach_remove_cb, filenames);
+
+       g_hash_table_destroy (filenames);
+       g_clear_object (&pop3_cache);
+}
+
 static gboolean
 pop3_folder_synchronize_sync (CamelFolder *folder,
                               gboolean expunge,
@@ -825,8 +908,10 @@ pop3_folder_synchronize_sync (CamelFolder *folder,
                return FALSE;
        }
 
-       if (!expunge || (keep_on_server && !delete_expunged))
+       if (!expunge || (keep_on_server && !delete_expunged)) {
+               pop3_folder_maybe_expunge_cache (pop3_folder);
                return TRUE;
+       }
 
        if (!is_online) {
                g_set_error (
@@ -911,6 +996,8 @@ pop3_folder_synchronize_sync (CamelFolder *folder,
        g_clear_object (&pop3_cache);
        g_clear_object (&pop3_engine);
 
+       pop3_folder_maybe_expunge_cache (pop3_folder);
+
        camel_operation_pop_message (cancellable);
 
        return camel_pop3_store_expunge (pop3_store, cancellable, error);
diff --git a/src/camel/providers/pop3/camel-pop3-settings.c b/src/camel/providers/pop3/camel-pop3-settings.c
index e4e048f..f315e09 100644
--- a/src/camel/providers/pop3/camel-pop3-settings.c
+++ b/src/camel/providers/pop3/camel-pop3-settings.c
@@ -27,6 +27,7 @@ struct _CamelPOP3SettingsPrivate {
        gboolean disable_extensions;
        gboolean keep_on_server;
        gboolean auto_fetch;
+       guint32 last_cache_expunge;
 };
 
 enum {
@@ -40,7 +41,8 @@ enum {
        PROP_PORT,
        PROP_SECURITY_METHOD,
        PROP_USER,
-       PROP_AUTO_FETCH
+       PROP_AUTO_FETCH,
+       PROP_LAST_CACHE_EXPUNGE
 };
 
 G_DEFINE_TYPE_WITH_CODE (
@@ -93,6 +95,12 @@ pop3_settings_set_property (GObject *object,
                                g_value_get_boolean (value));
                        return;
 
+               case PROP_LAST_CACHE_EXPUNGE:
+                       camel_pop3_settings_set_last_cache_expunge (
+                               CAMEL_POP3_SETTINGS (object),
+                               g_value_get_uint (value));
+                       return;
+
                case PROP_PORT:
                        camel_network_settings_set_port (
                                CAMEL_NETWORK_SETTINGS (object),
@@ -170,6 +178,13 @@ pop3_settings_get_property (GObject *object,
                                CAMEL_POP3_SETTINGS (object)));
                        return;
 
+               case PROP_LAST_CACHE_EXPUNGE:
+                       g_value_set_uint (
+                               value,
+                               camel_pop3_settings_get_last_cache_expunge (
+                               CAMEL_POP3_SETTINGS (object)));
+                       return;
+
                case PROP_PORT:
                        g_value_set_uint (
                                value,
@@ -279,6 +294,20 @@ camel_pop3_settings_class_init (CamelPOP3SettingsClass *class)
 
        g_object_class_install_property (
                object_class,
+               PROP_LAST_CACHE_EXPUNGE,
+               g_param_spec_uint (
+                       "last-cache-expunge",
+                       "Last Cache Expunge",
+                       "Date as Julian value, when the cache had been checked for orphaned files the last 
time",
+                       0,
+                       G_MAXUINT32,
+                       0,
+                       G_PARAM_READWRITE |
+                       G_PARAM_CONSTRUCT |
+                       G_PARAM_STATIC_STRINGS));
+
+       g_object_class_install_property (
+               object_class,
                PROP_AUTO_FETCH,
                g_param_spec_boolean (
                        "auto-fetch",
@@ -541,3 +570,24 @@ camel_pop3_settings_set_auto_fetch (CamelPOP3Settings *settings,
        g_object_notify (G_OBJECT (settings), "auto-fetch");
 }
 
+guint32
+camel_pop3_settings_get_last_cache_expunge (CamelPOP3Settings *settings)
+{
+       g_return_val_if_fail (CAMEL_IS_POP3_SETTINGS (settings), 0);
+
+       return settings->priv->last_cache_expunge;
+}
+
+void
+camel_pop3_settings_set_last_cache_expunge (CamelPOP3Settings *settings,
+                                           guint32 last_cache_expunge)
+{
+       g_return_if_fail (CAMEL_IS_POP3_SETTINGS (settings));
+
+       if (settings->priv->last_cache_expunge == last_cache_expunge)
+               return;
+
+       settings->priv->last_cache_expunge = last_cache_expunge;
+
+       g_object_notify (G_OBJECT (settings), "last-cache-expunge");
+}
diff --git a/src/camel/providers/pop3/camel-pop3-settings.h b/src/camel/providers/pop3/camel-pop3-settings.h
index d69b71b..fe30dca 100644
--- a/src/camel/providers/pop3/camel-pop3-settings.h
+++ b/src/camel/providers/pop3/camel-pop3-settings.h
@@ -83,6 +83,11 @@ gboolean     camel_pop3_settings_get_auto_fetch
 void           camel_pop3_settings_set_auto_fetch
                                                (CamelPOP3Settings *settings,
                                                 gboolean auto_fetch);
+guint32                camel_pop3_settings_get_last_cache_expunge
+                                               (CamelPOP3Settings *settings);
+void           camel_pop3_settings_set_last_cache_expunge
+                                               (CamelPOP3Settings *settings,
+                                                guint32 last_cache_expunge);
 
 G_END_DECLS