From 56f8584fe4034c2cc018f8d48139a97498e34d2f Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 8 Jul 2013 09:05:34 -0400 Subject: [PATCH] repo: Store transaction file, use it to optimize for pull resumes If pull is interrupted, we may have downloaded an arbitrary subset of the requested objects. Previously, we handled this by scanning for all objects each time. However, there's an easy optimization - this patch creates a lock file in the repo. If we don't see that file when starting a pull, we know we don't need to stat() every file; presence of a dirtree object for example implies the existence of everything it references. --- src/libostree/ostree-repo.c | 36 +++++++++++++++++++++++--- src/libostree/ostree-repo.h | 1 + src/ostree/ostree-pull.c | 41 +++++++++++++++++------------- src/ostree/ot-builtin-commit.c | 2 +- src/ostree/ot-builtin-pull-local.c | 8 +++--- 5 files changed, 63 insertions(+), 25 deletions(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 0975e72a..016d3616 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -54,6 +54,8 @@ struct OstreeRepo { GFile *remote_cache_dir; GFile *config_file; + GFile *transaction_lock_path; + GMutex cache_lock; GPtrArray *cached_meta_indexes; GPtrArray *cached_content_indexes; @@ -106,6 +108,9 @@ ostree_repo_finalize (GObject *object) g_clear_object (&self->uncompressed_objects_dir); g_clear_object (&self->remote_cache_dir); g_clear_object (&self->config_file); + + g_clear_object (&self->transaction_lock_path); + if (self->loose_object_devino_hash) g_hash_table_destroy (self->loose_object_devino_hash); if (self->updated_uncompressed_dirs) @@ -1357,14 +1362,34 @@ devino_cache_lookup (OstreeRepo *self, gboolean ostree_repo_prepare_transaction (OstreeRepo *self, gboolean enable_commit_hardlink_scan, + gboolean *out_transaction_resume, GCancellable *cancellable, GError **error) { gboolean ret = FALSE; + gboolean ret_transaction_resume = FALSE; + gs_free char *transaction_str = NULL; g_return_val_if_fail (self->in_transaction == FALSE, FALSE); + if (self->transaction_lock_path == NULL) + self->transaction_lock_path = g_file_resolve_relative_path (self->repodir, "transaction"); + + if (g_file_query_file_type (self->transaction_lock_path, G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, NULL) == G_FILE_TYPE_SYMBOLIC_LINK) + ret_transaction_resume = TRUE; + else + ret_transaction_resume = FALSE; + self->in_transaction = TRUE; + if (ret_transaction_resume) + { + if (!ot_gfile_ensure_unlinked (self->transaction_lock_path, cancellable, error)) + goto out; + } + transaction_str = g_strdup_printf ("pid=%llu", (unsigned long long) getpid ()); + if (!g_file_make_symbolic_link (self->transaction_lock_path, transaction_str, + cancellable, error)) + goto out; if (enable_commit_hardlink_scan) { @@ -1376,6 +1401,8 @@ ostree_repo_prepare_transaction (OstreeRepo *self, } ret = TRUE; + if (out_transaction_resume) + *out_transaction_resume = ret_transaction_resume; out: return ret; } @@ -1389,12 +1416,15 @@ ostree_repo_commit_transaction (OstreeRepo *self, g_return_val_if_fail (self->in_transaction == TRUE, FALSE); - ret = TRUE; - /* out: */ - self->in_transaction = FALSE; + if (!ot_gfile_ensure_unlinked (self->transaction_lock_path, cancellable, error)) + goto out; + if (self->loose_object_devino_hash) g_hash_table_remove_all (self->loose_object_devino_hash); + self->in_transaction = FALSE; + ret = TRUE; + out: return ret; } diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index 69b4dff7..6417b917 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -78,6 +78,7 @@ GFile * ostree_repo_get_file_object_path (OstreeRepo *self, gboolean ostree_repo_prepare_transaction (OstreeRepo *self, gboolean enable_commit_hardlink_scan, + gboolean *out_transaction_resume, GCancellable *cancellable, GError **error); diff --git a/src/ostree/ostree-pull.c b/src/ostree/ostree-pull.c index 16af0720..eb7cc403 100644 --- a/src/ostree/ostree-pull.c +++ b/src/ostree/ostree-pull.c @@ -108,6 +108,7 @@ typedef struct { GMainLoop *loop; GCancellable *cancellable; + gboolean transaction_resuming; volatile gint n_scanned_metadata; guint outstanding_uri_requests; @@ -822,7 +823,7 @@ scan_one_metadata_object (OtPullData *pull_data, if (!ostree_repo_has_object (pull_data->repo, objtype, tmp_checksum, &is_stored, cancellable, error)) goto out; - + if (!is_stored && !is_requested) { char *duped_checksum = g_strdup (tmp_checksum); @@ -834,23 +835,26 @@ scan_one_metadata_object (OtPullData *pull_data, } else if (is_stored) { - switch (objtype) + if (pull_data->transaction_resuming || is_requested) { - case OSTREE_OBJECT_TYPE_COMMIT: - if (!scan_commit_object (pull_data, tmp_checksum, recursion_depth, - pull_data->cancellable, error)) - goto out; - break; - case OSTREE_OBJECT_TYPE_DIR_META: - break; - case OSTREE_OBJECT_TYPE_DIR_TREE: - if (!scan_dirtree_object (pull_data, tmp_checksum, recursion_depth, - pull_data->cancellable, error)) - goto out; - break; - case OSTREE_OBJECT_TYPE_FILE: - g_assert_not_reached (); - break; + switch (objtype) + { + case OSTREE_OBJECT_TYPE_COMMIT: + if (!scan_commit_object (pull_data, tmp_checksum, recursion_depth, + pull_data->cancellable, error)) + goto out; + break; + case OSTREE_OBJECT_TYPE_DIR_META: + break; + case OSTREE_OBJECT_TYPE_DIR_TREE: + if (!scan_dirtree_object (pull_data, tmp_checksum, recursion_depth, + pull_data->cancellable, error)) + goto out; + break; + case OSTREE_OBJECT_TYPE_FILE: + g_assert_not_reached (); + break; + } } g_hash_table_insert (pull_data->scanned_metadata, g_variant_ref (object), object); g_atomic_int_inc (&pull_data->n_scanned_metadata); @@ -1347,7 +1351,8 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error) } } - if (!ostree_repo_prepare_transaction (pull_data->repo, FALSE, NULL, error)) + if (!ostree_repo_prepare_transaction (pull_data->repo, FALSE, &pull_data->transaction_resuming, + cancellable, error)) goto out; pull_data->metadata_objects_to_fetch = ot_waitable_queue_new (); diff --git a/src/ostree/ot-builtin-commit.c b/src/ostree/ot-builtin-commit.c index f01815b7..b79a3a35 100644 --- a/src/ostree/ot-builtin-commit.c +++ b/src/ostree/ot-builtin-commit.c @@ -351,7 +351,7 @@ ostree_builtin_commit (int argc, char **argv, GFile *repo_path, GError **error) goto out; } - if (!ostree_repo_prepare_transaction (repo, TRUE, cancellable, error)) + if (!ostree_repo_prepare_transaction (repo, TRUE, NULL, cancellable, error)) goto out; in_transaction = TRUE; diff --git a/src/ostree/ot-builtin-pull-local.c b/src/ostree/ot-builtin-pull-local.c index edb19b99..471306fb 100644 --- a/src/ostree/ot-builtin-pull-local.c +++ b/src/ostree/ot-builtin-pull-local.c @@ -170,6 +170,7 @@ ostree_builtin_pull_local (int argc, char **argv, GFile *repo_path, GError **err int i; GHashTableIter hash_iter; gpointer key, value; + gboolean transaction_resuming = FALSE; gs_unref_hashtable GHashTable *objects = NULL; gs_unref_object GFile *src_f = NULL; gs_unref_object GFile *src_repo_dir = NULL; @@ -247,6 +248,10 @@ ostree_builtin_pull_local (int argc, char **argv, GFile *repo_path, GError **err } } + if (!ostree_repo_prepare_transaction (data->dest_repo, FALSE, &transaction_resuming, + cancellable, error)) + goto out; + g_print ("Enumerating objects...\n"); source_objects = ostree_traverse_new_reachable (); @@ -275,9 +280,6 @@ ostree_builtin_pull_local (int argc, char **argv, GFile *repo_path, GError **err } } - if (!ostree_repo_prepare_transaction (data->dest_repo, FALSE, cancellable, error)) - goto out; - data->n_objects_to_check = g_hash_table_size (source_objects); g_hash_table_iter_init (&hash_iter, source_objects); while (g_hash_table_iter_next (&hash_iter, &key, &value))