repo: Store transaction file, use it to optimize for pull resumes

If pull is interrupted, we may have downloaded an arbitrary subset of
the requested objects.  Previously, we handled this by scanning for
all objects each time.

However, there's an easy optimization - this patch creates a lock file
in the repo.  If we don't see that file when starting a pull, we know
we don't need to stat() every file; presence of a dirtree object for
example implies the existence of everything it references.

src/libostree/ostree-repo.c

@@ -54,6 +54,8 @@ struct OstreeRepo {
   GFile *remote_cache_dir;
   GFile *config_file;
 
+  GFile *transaction_lock_path;
+
   GMutex cache_lock;
   GPtrArray *cached_meta_indexes;
   GPtrArray *cached_content_indexes;
@@ -106,6 +108,9 @@ ostree_repo_finalize (GObject *object)
   g_clear_object (&self->uncompressed_objects_dir);
   g_clear_object (&self->remote_cache_dir);
   g_clear_object (&self->config_file);
+
+  g_clear_object (&self->transaction_lock_path);
+
   if (self->loose_object_devino_hash)
     g_hash_table_destroy (self->loose_object_devino_hash);
   if (self->updated_uncompressed_dirs)
@@ -1357,14 +1362,34 @@ devino_cache_lookup (OstreeRepo           *self,
 gboolean
 ostree_repo_prepare_transaction (OstreeRepo     *self,
                                  gboolean        enable_commit_hardlink_scan,
+                                 gboolean       *out_transaction_resume,
                                  GCancellable   *cancellable,
                                  GError        **error)
 {
   gboolean ret = FALSE;
+  gboolean ret_transaction_resume = FALSE;
+  gs_free char *transaction_str = NULL;
 
   g_return_val_if_fail (self->in_transaction == FALSE, FALSE);
 
+  if (self->transaction_lock_path == NULL)
+    self->transaction_lock_path = g_file_resolve_relative_path (self->repodir, "transaction");
+
+  if (g_file_query_file_type (self->transaction_lock_path, G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, NULL) == G_FILE_TYPE_SYMBOLIC_LINK)
+    ret_transaction_resume = TRUE;
+  else
+    ret_transaction_resume = FALSE;
+
   self->in_transaction = TRUE;
+  if (ret_transaction_resume)
+    {
+      if (!ot_gfile_ensure_unlinked (self->transaction_lock_path, cancellable, error))
+        goto out;
+    }
+  transaction_str = g_strdup_printf ("pid=%llu", (unsigned long long) getpid ());
+  if (!g_file_make_symbolic_link (self->transaction_lock_path, transaction_str,
+                                  cancellable, error))
+    goto out;
 
   if (enable_commit_hardlink_scan)
     {
@@ -1376,6 +1401,8 @@ ostree_repo_prepare_transaction (OstreeRepo     *self,
     }
 
   ret = TRUE;
+  if (out_transaction_resume)
+    *out_transaction_resume = ret_transaction_resume;
  out:
   return ret;
 }
@@ -1389,12 +1416,15 @@ ostree_repo_commit_transaction (OstreeRepo     *self,
 
   g_return_val_if_fail (self->in_transaction == TRUE, FALSE);
 
-  ret = TRUE;
+  if (!ot_gfile_ensure_unlinked (self->transaction_lock_path, cancellable, error))
-  /* out: */
+    goto out;
-  self->in_transaction = FALSE;
+
   if (self->loose_object_devino_hash)
     g_hash_table_remove_all (self->loose_object_devino_hash);
 
+  self->in_transaction = FALSE;
+  ret = TRUE;
+ out:
   return ret;
 }
 

src/libostree/ostree-repo.h

@@ -78,6 +78,7 @@ GFile *       ostree_repo_get_file_object_path (OstreeRepo   *self,
 
 gboolean      ostree_repo_prepare_transaction (OstreeRepo     *self,
                                                gboolean        enable_commit_hardlink_scan,
+                                               gboolean       *out_transaction_resume,
                                                GCancellable   *cancellable,
                                                GError        **error);
 

src/ostree/ostree-pull.c

@@ -108,6 +108,7 @@ typedef struct {
   GMainLoop    *loop;
   GCancellable *cancellable;
 
+  gboolean      transaction_resuming;
   volatile gint n_scanned_metadata;
   guint         outstanding_uri_requests;
 
@@ -834,23 +835,26 @@ scan_one_metadata_object (OtPullData         *pull_data,
     }
   else if (is_stored)
     {
-      switch (objtype)
+      if (pull_data->transaction_resuming || is_requested)
         {
-        case OSTREE_OBJECT_TYPE_COMMIT:
+          switch (objtype)
-          if (!scan_commit_object (pull_data, tmp_checksum, recursion_depth,
+            {
-                                   pull_data->cancellable, error))
+            case OSTREE_OBJECT_TYPE_COMMIT:
-            goto out;
+              if (!scan_commit_object (pull_data, tmp_checksum, recursion_depth,
-          break;
+                                       pull_data->cancellable, error))
-        case OSTREE_OBJECT_TYPE_DIR_META:
+                goto out;
-          break;
+              break;
-        case OSTREE_OBJECT_TYPE_DIR_TREE:
+            case OSTREE_OBJECT_TYPE_DIR_META:
-          if (!scan_dirtree_object (pull_data, tmp_checksum, recursion_depth,
+              break;
-                                    pull_data->cancellable, error))
+            case OSTREE_OBJECT_TYPE_DIR_TREE:
-            goto out;
+              if (!scan_dirtree_object (pull_data, tmp_checksum, recursion_depth,
-          break;
+                                        pull_data->cancellable, error))
-        case OSTREE_OBJECT_TYPE_FILE:
+                goto out;
-          g_assert_not_reached ();
+              break;
-          break;
+            case OSTREE_OBJECT_TYPE_FILE:
+              g_assert_not_reached ();
+              break;
+            }
         }
       g_hash_table_insert (pull_data->scanned_metadata, g_variant_ref (object), object);
       g_atomic_int_inc (&pull_data->n_scanned_metadata);
@@ -1347,7 +1351,8 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error)
         }
     }
 
-  if (!ostree_repo_prepare_transaction (pull_data->repo, FALSE, NULL, error))
+  if (!ostree_repo_prepare_transaction (pull_data->repo, FALSE, &pull_data->transaction_resuming,
+                                        cancellable, error))
     goto out;
 
   pull_data->metadata_objects_to_fetch = ot_waitable_queue_new ();

src/ostree/ot-builtin-commit.c

@@ -351,7 +351,7 @@ ostree_builtin_commit (int argc, char **argv, GFile *repo_path, GError **error)
         goto out;
     }
 
-  if (!ostree_repo_prepare_transaction (repo, TRUE, cancellable, error))
+  if (!ostree_repo_prepare_transaction (repo, TRUE, NULL, cancellable, error))
     goto out;
 
   in_transaction = TRUE;

src/ostree/ot-builtin-pull-local.c

@@ -170,6 +170,7 @@ ostree_builtin_pull_local (int argc, char **argv, GFile *repo_path, GError **err
   int i;
   GHashTableIter hash_iter;
   gpointer key, value;
+  gboolean transaction_resuming = FALSE;
   gs_unref_hashtable GHashTable *objects = NULL;
   gs_unref_object GFile *src_f = NULL;
   gs_unref_object GFile *src_repo_dir = NULL;
@@ -247,6 +248,10 @@ ostree_builtin_pull_local (int argc, char **argv, GFile *repo_path, GError **err
         }
     }
 
+  if (!ostree_repo_prepare_transaction (data->dest_repo, FALSE, &transaction_resuming,
+                                        cancellable, error))
+    goto out;
+
   g_print ("Enumerating objects...\n");
 
   source_objects = ostree_traverse_new_reachable ();
@@ -275,9 +280,6 @@ ostree_builtin_pull_local (int argc, char **argv, GFile *repo_path, GError **err
         }
     }
 
-  if (!ostree_repo_prepare_transaction (data->dest_repo, FALSE, cancellable, error))
-    goto out;
-
   data->n_objects_to_check = g_hash_table_size (source_objects);
   g_hash_table_iter_init (&hash_iter, source_objects);
   while (g_hash_table_iter_next (&hash_iter, &key, &value))