[ostree/wip/delta2] wip/delta: Implement fallback fetching

[Colin Walters <walters src gnome org>]

commit a4427823b84d9c77c646d9ca5ccf3d803d2e8a59
Author: Colin Walters <walters verbum org>
Date:   Sun Apr 20 13:40:08 2014 -0400

    wip/delta: Implement fallback fetching
    
    It's really not worth putting huge objects into a delta part alone -
    we're just wasting space.  Add a fallback array to the superblock
    which says to fetch individual objects.
    
    However, it's likely that huge objects contain other data inside them,
    such as GResources.  These are prime candidates for future rollsum
    deltas.

 src/libostree/ostree-repo-pull.c                   |   75 ++++++++++++++++
 .../ostree-repo-static-delta-compilation.c         |   93 +++++++++++++++++++-
 src/libostree/ostree-repo-static-delta-core.c      |    9 ++
 tests/pull-test.sh                                 |   27 ++++++
 4 files changed, 202 insertions(+), 2 deletions(-)
---
diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c
index cccda89..1183a98 100644
--- a/src/libostree/ostree-repo-pull.c
+++ b/src/libostree/ostree-repo-pull.c
@@ -1168,6 +1168,65 @@ request_static_delta_superblock_sync (OtPullData  *pull_data,
 }
 
 static gboolean
+process_one_static_delta_fallback (OtPullData   *pull_data,
+                                   GVariant     *fallback_object,
+                                   GCancellable *cancellable,
+                                   GError      **error)
+{
+  gboolean ret = FALSE;
+  gs_unref_variant GVariant *csum_v = NULL;
+  gs_free char *checksum = NULL;
+  guint8 objtype_y;
+  OstreeObjectType objtype;
+  gboolean is_stored;
+  guint64 compressed_size, uncompressed_size;
+
+  g_variant_get (fallback_object, "(y aytt)",
+                 &objtype_y, &csum_v, &compressed_size, &uncompressed_size);
+  if (!ostree_validate_structureof_objtype (objtype_y, error))
+    goto out;
+  if (!ostree_validate_structureof_csum_v (csum_v, error))
+    goto out;
+
+  objtype = (OstreeObjectType)objtype_y;
+  checksum = ostree_checksum_from_bytes_v (csum_v);
+
+  if (!ostree_repo_has_object (pull_data->repo, objtype, checksum,
+                               &is_stored,
+                               cancellable, error))
+    goto out;
+
+  if (!is_stored)
+    { 
+      if (OSTREE_OBJECT_TYPE_IS_META (objtype))
+        {
+          if (!g_hash_table_lookup (pull_data->requested_metadata, checksum))
+            {
+              gboolean do_fetch_detached;
+              g_hash_table_insert (pull_data->requested_metadata, checksum, checksum);
+              
+              do_fetch_detached = (objtype == OSTREE_OBJECT_TYPE_COMMIT);
+              enqueue_one_object_request (pull_data, checksum, objtype, do_fetch_detached);
+              checksum = NULL;  /* Transfer ownership */
+            }
+        }
+      else
+        {
+          if (!g_hash_table_lookup (pull_data->requested_content, checksum))
+            {
+              g_hash_table_insert (pull_data->requested_content, checksum, checksum);
+              enqueue_one_object_request (pull_data, checksum, OSTREE_OBJECT_TYPE_FILE, FALSE);
+              checksum = NULL;  /* Transfer ownership */
+            }
+        }
+    }
+
+  ret = TRUE;
+ out:
+  return ret;
+}
+
+static gboolean
 process_one_static_delta (OtPullData   *pull_data,
                           const char   *from_revision,
                           const char   *to_revision,
@@ -1177,10 +1236,26 @@ process_one_static_delta (OtPullData   *pull_data,
 {
   gboolean ret = FALSE;
   gs_unref_variant GVariant *headers = NULL;
+  gs_unref_variant GVariant *fallback_objects = NULL;
   guint i, n;
 
   /* Parsing OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT */
   headers = g_variant_get_child_value (delta_superblock, 3);
+  fallback_objects = g_variant_get_child_value (delta_superblock, 4);
+
+  /* First process the fallbacks */
+  n = g_variant_n_children (fallback_objects);
+  for (i = 0; i < n; i++)
+    {
+      gs_unref_variant GVariant *fallback_object =
+        g_variant_get_child_value (fallback_objects, i);
+
+      if (!process_one_static_delta_fallback (pull_data,
+                                              fallback_object,
+                                              cancellable, error))
+        goto out;
+    }
+
   n = g_variant_n_children (headers);
   for (i = 0; i < n; i++)
     {
diff --git a/src/libostree/ostree-repo-static-delta-compilation.c 
b/src/libostree/ostree-repo-static-delta-compilation.c
index 9f1bb59..1e5ef41 100644
--- a/src/libostree/ostree-repo-static-delta-compilation.c
+++ b/src/libostree/ostree-repo-static-delta-compilation.c
@@ -38,6 +38,7 @@ typedef struct {
 
 typedef struct {
   GPtrArray *parts;
+  GPtrArray *fallback_objects;
 } OstreeStaticDeltaBuilder;
 
 static void
@@ -172,6 +173,19 @@ generate_delta_lowlatency (OstreeRepo                       *repo,
                                            cancellable, error))
         goto out;
 
+      /* Fall back to plain HTTP-based fetch for large objects;
+       * in the future we should try an rsync-style rolling checksum
+       * against a previous version, if any.
+       */
+      if (content_size > OSTREE_STATIC_DELTA_PART_MAX_SIZE_BYTES)
+        {
+          g_printerr ("fallback for %s\n",
+                      ostree_object_to_string (checksum, objtype));
+          g_ptr_array_add (builder->fallback_objects, 
+                           ostree_object_name_serialize (checksum, objtype));
+          continue;
+        }
+
       /* Ensure we have at least one object per delta, even if a given
        * object is larger.
        */
@@ -224,6 +238,71 @@ generate_delta_lowlatency (OstreeRepo                       *repo,
   return ret;
 }
 
+static gboolean
+get_fallback_headers (OstreeRepo               *self,
+                      OstreeStaticDeltaBuilder *builder,
+                      GVariant                **out_headers,
+                      GCancellable             *cancellable,
+                      GError                  **error)
+{
+  gboolean ret = FALSE;
+  guint i;
+  gs_unref_variant GVariant *ret_headers = NULL;
+  gs_unref_variant_builder GVariantBuilder *fallback_builder = NULL;
+
+  fallback_builder = g_variant_builder_new (G_VARIANT_TYPE ("a" OSTREE_STATIC_DELTA_FALLBACK_FORMAT));
+
+  for (i = 0; i < builder->fallback_objects->len; i++)
+    {
+      GVariant *serialized = builder->fallback_objects->pdata[i];
+      const char *checksum;
+      OstreeObjectType objtype;
+      guint64 compressed_size;
+      guint64 uncompressed_size;
+
+      ostree_object_name_deserialize (serialized, &checksum, &objtype);
+
+      if (OSTREE_OBJECT_TYPE_IS_META (objtype))
+        {
+          if (!ostree_repo_load_object_stream (self, objtype, checksum,
+                                               NULL, &uncompressed_size,
+                                               cancellable, error))
+            goto out;
+          compressed_size = uncompressed_size;
+        }
+      else
+        {
+          gs_unref_object GFileInfo *file_info = NULL;
+
+          if (!ostree_repo_query_object_storage_size (self, OSTREE_OBJECT_TYPE_FILE,
+                                                      checksum,
+                                                      &compressed_size,
+                                                      cancellable, error))
+            goto out;
+
+          if (!ostree_repo_load_file (self, checksum,
+                                      NULL, &file_info, NULL,
+                                      cancellable, error))
+            goto out;
+
+          uncompressed_size = g_file_info_get_size (file_info);
+        }
+
+      g_variant_builder_add_value (fallback_builder,
+                                   g_variant_new ("(y aytt)",
+                                                  objtype,
+                                                  ostree_checksum_to_bytes_v (checksum),
+                                                  compressed_size, uncompressed_size));
+    }
+
+  ret_headers = g_variant_ref_sink (g_variant_builder_end (fallback_builder));
+
+  ret = TRUE;
+  gs_transfer_out_value (out_headers, &ret_headers);
+ out:
+  return ret;
+}
+
 /**
  * ostree_repo_static_delta_generate:
  * @self: Repo
@@ -259,8 +338,10 @@ ostree_repo_static_delta_generate (OstreeRepo                   *self,
   gs_unref_object GFile *descriptor_path = NULL;
   gs_unref_object GFile *descriptor_dir = NULL;
   gs_unref_variant GVariant *tmp_metadata = NULL;
+  gs_unref_variant GVariant *fallback_headers = NULL;
 
   builder.parts = g_ptr_array_new_with_free_func ((GDestroyNotify)ostree_static_delta_part_builder_unref);
+  builder.fallback_objects = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
 
   /* Ignore optimization flags */
   if (!generate_delta_lowlatency (self, from, to, &builder,
@@ -372,13 +453,20 @@ ostree_repo_static_delta_generate (OstreeRepo                   *self,
       metadata_source = tmp_metadata;
     }
 
+  if (!get_fallback_headers (self, &builder, &fallback_headers,
+                             cancellable, error))
+    goto out;
+
   {
     GDateTime *now = g_date_time_new_now_utc ();
-    delta_descriptor = g_variant_new ("(@(a(ss)a(say))taya" OSTREE_STATIC_DELTA_META_ENTRY_FORMAT ")",
+    delta_descriptor = g_variant_new ("(@(a(ss)a(say))tay"
+                                      "a" OSTREE_STATIC_DELTA_META_ENTRY_FORMAT
+                                      "@a" OSTREE_STATIC_DELTA_FALLBACK_FORMAT ")",
                                       metadata_source,
                                       GUINT64_TO_BE (g_date_time_to_unix (now)),
                                       g_variant_builder_new (G_VARIANT_TYPE ("ay")),
-                                      part_headers);
+                                      part_headers,
+                                      fallback_headers);
     g_date_time_unref (now);
   }
 
@@ -388,5 +476,6 @@ ostree_repo_static_delta_generate (OstreeRepo                   *self,
   ret = TRUE;
  out:
   g_clear_pointer (&builder.parts, g_ptr_array_unref);
+  g_clear_pointer (&builder.fallback_objects, g_ptr_array_unref);
   return ret;
 }
diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c
index d86ac78..7eb211b 100644
--- a/src/libostree/ostree-repo-static-delta-core.c
+++ b/src/libostree/ostree-repo-static-delta-core.c
@@ -185,11 +185,20 @@ ostree_repo_static_delta_execute_offline (OstreeRepo                    *self,
   gs_unref_object GFile *meta_file = g_file_get_child (dir, "superblock");
   gs_unref_variant GVariant *meta = NULL;
   gs_unref_variant GVariant *headers = NULL;
+  gs_unref_variant GVariant *fallback = NULL;
 
   if (!ot_util_variant_map (meta_file, G_VARIANT_TYPE (OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT),
                             FALSE, &meta, error))
     goto out;
 
+  fallback = g_variant_get_child_value (meta, 4);
+  if (g_variant_n_children (fallback) > 0)
+    {
+      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                   "Cannot execute delta offline: contains nonempty http fallback entries");
+      goto out;
+    }
+
   headers = g_variant_get_child_value (meta, 3);
   n = g_variant_n_children (headers);
   for (i = 0; i < n; i++)
diff --git a/tests/pull-test.sh b/tests/pull-test.sh
index ea091e7..3829e4c 100755
--- a/tests/pull-test.sh
+++ b/tests/pull-test.sh
@@ -86,3 +86,30 @@ assert_file_has_content baz/cow "modified file for static deltas"
 assert_not_has_file baz/saucer
 
 echo "ok static delta"
+
+cd ${test_tmpdir}
+rm main-files -rf
+ostree --repo=ostree-srv/gnomerepo checkout main main-files
+cd main-files
+# Make a file larger than 16M for testing
+dd if=/dev/zero of=test-bigfile count=1 seek=42678
+echo "further modified file for static deltas" > baz/cow
+ostree --repo=${test_tmpdir}/ostree-srv/gnomerepo commit -b main -s '2nd static delta test'
+cd ..
+rm main-files -rf
+ostree --repo=ostree-srv/gnomerepo static-delta main
+
+cd ${test_tmpdir}
+${CMD_PREFIX} ostree --repo=repo pull origin main
+${CMD_PREFIX} ostree --repo=repo fsck
+
+rm checkout-origin-main -rf
+$OSTREE checkout origin:main checkout-origin-main
+cd checkout-origin-main
+assert_has_file test-bigfile
+stat --format=%s test-bigfile > bigfile-size
+assert_file_has_content bigfile-size 21851648
+assert_file_has_content baz/cow "further modified file for static deltas"
+assert_not_has_file baz/saucer
+
+echo "ok static delta 2"