Add "pull --localcache-repo"

This is a lot like `git clone --reference`, but we chose "localcache" as the
term "reference" is already used.

The main use case I'm targeting this for is the Fedora Atomic Host installer
case where we embed the repo content in the installer, but we may want to
kickstart and download newer content. There, while we want to get a newer ref,
we can still use the local repo as an object cache, since we have it sitting
there in memory anyways.

Another case is where one has a host ostree (say e.g. Fedora Atomic
Workstation), and one wants to create a local archive mirror of FAH. Then one
can use `pull --reference /ostree/repo` and pull the common objects (e.g.
contents of `bash.rpm` etc.)

Closes: https://github.com/ostreedev/ostree/issues/975

Closes: #982
Approved by: jlebon

src/libostree/ostree-repo-pull.c

@@ -64,6 +64,7 @@ typedef struct {
   GPtrArray     *meta_mirrorlist;    /* List of base URIs for fetching metadata */
   GPtrArray     *content_mirrorlist; /* List of base URIs for fetching content */
   OstreeRepo   *remote_repo_local;
+  GPtrArray    *localcache_repos; /* Array<OstreeRepo> */
 
   GMainContext    *main_context;
   GCancellable *cancellable;
@@ -118,6 +119,9 @@ typedef struct {
   guint             n_fetched_deltapart_fallbacks;
   guint             n_fetched_metadata;
   guint             n_fetched_content;
+  /* Objects from pull --localcache-repo */
+  guint             n_fetched_localcache_metadata;
+  guint             n_fetched_localcache_content;
 
   int               maxdepth;
   guint64           start_time;
@@ -236,6 +240,8 @@ update_progress (gpointer user_data)
                              "scanned-metadata", "u", n_scanned_metadata,
                              "bytes-transferred", "t", bytes_transferred,
                              "start-time", "t", start_time,
+                             "metadata-fetched-localcache", "u", pull_data->n_fetched_localcache_metadata,
+                             "content-fetched-localcache", "u", pull_data->n_fetched_localcache_content,
                              /* Deltas */
                              "fetched-delta-parts",
                                   "u", pull_data->n_fetched_deltaparts,
@@ -606,16 +612,15 @@ validate_bareuseronly_mode (OtPullData *pull_data,
  */
 static gboolean
 import_one_local_content_object (OtPullData *pull_data,
+                                 OstreeRepo *src_repo,
                                  const char *checksum,
                                  GCancellable *cancellable,
                                  GError    **error)
 {
-  g_assert (pull_data->remote_repo_local);
-
   const gboolean trusted = !pull_data->is_untrusted;
   if (trusted && !pull_data->is_bareuseronly_files)
     {
-      if (!ostree_repo_import_object_from_with_trust (pull_data->repo, pull_data->remote_repo_local,
+      if (!ostree_repo_import_object_from_with_trust (pull_data->repo, src_repo,
                                                       OSTREE_OBJECT_TYPE_FILE, checksum,
                                                       trusted,
                                                       cancellable, error))
@@ -630,7 +635,7 @@ import_one_local_content_object (OtPullData *pull_data,
       g_autoptr(GFileInfo) content_finfo = NULL;
       g_autoptr(GVariant) content_xattrs = NULL;
 
-      if (!ostree_repo_load_file (pull_data->remote_repo_local, checksum,
+      if (!ostree_repo_load_file (src_repo, checksum,
                                   &content_input, &content_finfo, &content_xattrs,
                                   cancellable, error))
         return FALSE;
@@ -709,16 +714,43 @@ scan_dirtree_object (OtPullData   *pull_data,
       /* Is this a local repo? */
       if (pull_data->remote_repo_local)
         {
-          if (!import_one_local_content_object (pull_data, file_checksum, cancellable, error))
+          if (!import_one_local_content_object (pull_data, pull_data->remote_repo_local,
+                                                file_checksum, cancellable, error))
             return FALSE;
+
+          /* Note early loop continue */
+          continue;
         }
-      else
+
+      /* We're doing HTTP, but see if we have the object in a local cache first */
+      gboolean did_import_from_cache_repo = FALSE;
+      if (pull_data->localcache_repos)
         {
-          /* In this case we're doing HTTP pulls */
-          g_hash_table_add (pull_data->requested_content, file_checksum);
-          enqueue_one_object_request (pull_data, file_checksum, OSTREE_OBJECT_TYPE_FILE, path, FALSE, FALSE);
-          file_checksum = NULL;  /* Transfer ownership */
+          for (guint j = 0; j < pull_data->localcache_repos->len; j++)
+            {
+              OstreeRepo *localcache_repo = pull_data->localcache_repos->pdata[j];
+              gboolean localcache_repo_has_obj;
+
+              if (!ostree_repo_has_object (localcache_repo, OSTREE_OBJECT_TYPE_FILE, file_checksum,
+                                           &localcache_repo_has_obj, cancellable, error))
+                return FALSE;
+              if (!localcache_repo_has_obj)
+                continue;
+              if (!import_one_local_content_object (pull_data, localcache_repo, file_checksum,
+                                                    cancellable, error))
+                return FALSE;
+              did_import_from_cache_repo = TRUE;
+              pull_data->n_fetched_localcache_content++;
+              break;
+            }
         }
+      if (did_import_from_cache_repo)
+        continue; /* Note early continue */
+
+      /* Not available locally, queue a HTTP request */
+      g_hash_table_add (pull_data->requested_content, file_checksum);
+      enqueue_one_object_request (pull_data, file_checksum, OSTREE_OBJECT_TYPE_FILE, path, FALSE, FALSE);
+      file_checksum = NULL;  /* Transfer ownership */
     }
 
   g_autoptr(GVariant) dirs_variant = g_variant_get_child_value (tree, 1);
@@ -1527,6 +1559,33 @@ scan_one_metadata_object_c (OtPullData         *pull_data,
       is_stored = TRUE;
       is_requested = TRUE;
     }
+  /* Do we have any localcache repos? */
+  else if (!is_stored && pull_data->localcache_repos)
+    {
+      for (guint i = 0; i < pull_data->localcache_repos->len; i++)
+        {
+          OstreeRepo *refd_repo = pull_data->localcache_repos->pdata[i];
+          gboolean localcache_repo_has_obj;
+
+          if (!ostree_repo_has_object (refd_repo, objtype, tmp_checksum,
+                                       &localcache_repo_has_obj, cancellable, error))
+            return FALSE;
+          if (!localcache_repo_has_obj)
+            continue;
+          if (!ostree_repo_import_object_from_with_trust (pull_data->repo, refd_repo,
+                                                          objtype, tmp_checksum,
+                                                          !pull_data->is_untrusted,
+                                                          cancellable, error))
+            return FALSE;
+          /* See comment above */
+          if (objtype == OSTREE_OBJECT_TYPE_COMMIT)
+            g_hash_table_add (pull_data->fetched_detached_metadata, g_strdup (tmp_checksum));
+          is_stored = TRUE;
+          is_requested = TRUE;
+          pull_data->n_fetched_localcache_metadata++;
+          break;
+        }
+    }
 
   if (!is_stored && !is_requested)
     {
@@ -2865,6 +2924,7 @@ initiate_request (OtPullData                 *pull_data,
  *   * inherit-transaction (b): Don't initiate, finish or abort a transaction, useful to do multiple pulls in one transaction.
  *   * http-headers (a(ss)): Additional headers to add to all HTTP requests
  *   * update-frequency (u): Frequency to call the async progress callback in milliseconds, if any; only values higher than 0 are valid
+ *   * localcache-repos (as): File paths for local repos to use as caches when doing remote fetches
  */
 gboolean
 ostree_repo_pull_with_options (OstreeRepo             *self,
@@ -2903,6 +2963,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
   gboolean inherit_transaction = FALSE;
   g_autoptr(GHashTable) updated_requested_refs_to_fetch = NULL;  /* (element-type OstreeCollectionRef utf8) */
   int i;
+  g_autofree char **opt_localcache_repos = NULL;
 
   if (options)
     {
@@ -2929,6 +2990,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
       (void) g_variant_lookup (options, "inherit-transaction", "b", &inherit_transaction);
       (void) g_variant_lookup (options, "http-headers", "@a(ss)", &pull_data->extra_headers);
       (void) g_variant_lookup (options, "update-frequency", "u", &update_frequency);
+      (void) g_variant_lookup (options, "localcache-repos", "^a&s", &opt_localcache_repos);
     }
 
   g_return_val_if_fail (pull_data->maxdepth >= -1, FALSE);
@@ -2993,6 +3055,20 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
                                                              (GDestroyNotify)fetch_object_data_free);
   pull_data->pending_fetch_deltaparts = g_hash_table_new_full (NULL, NULL, (GDestroyNotify)fetch_static_delta_data_free, NULL);
 
+  if (opt_localcache_repos && *opt_localcache_repos)
+    {
+      pull_data->localcache_repos = g_ptr_array_new_with_free_func (g_object_unref);
+      for (char **it = opt_localcache_repos; it && *it; it++)
+        {
+          const char *localcache_path = *it;
+          g_autoptr(GFile) localcache_file = g_file_new_for_path (localcache_path);
+          g_autoptr(OstreeRepo) cacherepo = ostree_repo_new (localcache_file);
+          if (!ostree_repo_open (cacherepo, cancellable, error))
+            goto out;
+          g_ptr_array_add (pull_data->localcache_repos, g_steal_pointer (&cacherepo));
+        }
+    }
+
   if (dir_to_pull != NULL || dirs_to_pull != NULL)
     {
       pull_data->dirs = g_ptr_array_new_with_free_func (g_free);
@@ -3722,6 +3798,11 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
       else
         g_string_append_printf (buf, "%u metadata, %u content objects fetched",
                                 pull_data->n_fetched_metadata, pull_data->n_fetched_content);
+      if (pull_data->n_fetched_localcache_metadata ||
+          pull_data->n_fetched_localcache_content)
+        g_string_append_printf (buf, " (%u meta, %u content local)",
+                                pull_data->n_fetched_localcache_metadata,
+                                pull_data->n_fetched_localcache_content);
 
       g_string_append_printf (buf, "; %" G_GUINT64_FORMAT " %s transferred in %u seconds",
                               (guint64)(bytes_transferred / shift),
@@ -3769,6 +3850,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
   g_clear_object (&pull_data->fetcher);
   g_clear_pointer (&pull_data->extra_headers, (GDestroyNotify)g_variant_unref);
   g_clear_object (&pull_data->cancellable);
+  g_clear_pointer (&pull_data->localcache_repos, (GDestroyNotify)g_ptr_array_unref);
   g_clear_object (&pull_data->remote_repo_local);
   g_free (pull_data->remote_name);
   g_clear_pointer (&pull_data->meta_mirrorlist, (GDestroyNotify) g_ptr_array_unref);

src/ostree/ot-builtin-pull.c

@@ -41,6 +41,7 @@ static char* opt_cache_dir;
 static int opt_depth = 0;
 static int opt_frequency = 0;
 static char* opt_url;
+static char** opt_localcache_repos;
 
 static GOptionEntry options[] = {
    { "commit-metadata-only", 0, 0, G_OPTION_ARG_NONE, &opt_commit_only, "Fetch only the commit metadata", NULL },
@@ -57,6 +58,7 @@ static GOptionEntry options[] = {
    { "url", 0, 0, G_OPTION_ARG_STRING, &opt_url, "Pull objects from this URL instead of the one from the remote config", NULL },
    { "http-header", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_http_headers, "Add NAME=VALUE as HTTP header to all requests", "NAME=VALUE" },
    { "update-frequency", 0, 0, G_OPTION_ARG_INT, &opt_frequency, "Sets the update frequency, in milliseconds (0=1000ms) (default: 0)", "FREQUENCY" },
+   { "localcache-repo", 'L', 0, G_OPTION_ARG_FILENAME_ARRAY, &opt_localcache_repos, "Add REPO as local cache source for objects during this pull", "REPO" },
    { NULL }
  };
 
@@ -281,6 +283,9 @@ ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError **
     if (override_commit_ids)
       g_variant_builder_add (&builder, "{s@v}", "override-commit-ids",
                              g_variant_new_variant (g_variant_new_strv ((const char*const*)override_commit_ids->pdata, override_commit_ids->len)));
+    if (opt_localcache_repos)
+      g_variant_builder_add (&builder, "{s@v}", "localcache-repos",
+                             g_variant_new_variant (g_variant_new_strv ((const char*const*)opt_localcache_repos, -1)));
 
     if (opt_http_headers)
       {