From 255b7d13090ee8d3957288e7d831d376292753ff Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 16 Apr 2012 17:14:43 -0400 Subject: [PATCH] core: Pull pack files individually --- src/libostree/ostree-repo.c | 14 +- src/ostree/ostree-pull.c | 465 +++++++++++++++++++++++------------- 2 files changed, 305 insertions(+), 174 deletions(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 92948712..a871498b 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -2163,7 +2163,8 @@ ostree_repo_get_cached_remote_pack_data (OstreeRepo *self, } /** - * Add file @cached_path into the cache for given @remote_name. + * Add file @cached_path into the cache for given @remote_name. If + * @cached_path is %NULL, delete the cached pack data (if any). * * * This unlinks @cached_path. @@ -2186,8 +2187,15 @@ ostree_repo_take_cached_remote_pack_data (OstreeRepo *self, goto out; target_path = get_pack_data_path (cachedir, is_meta, pack_checksum); - if (!ot_gfile_rename (cached_path, target_path, cancellable, error)) - goto out; + if (cached_path) + { + if (!ot_gfile_rename (cached_path, target_path, cancellable, error)) + goto out; + } + else + { + (void) ot_gfile_unlink (target_path, cancellable, NULL); + } ret = TRUE; out: diff --git a/src/ostree/ostree-pull.c b/src/ostree/ostree-pull.c index 7a48b114..84d43ec6 100644 --- a/src/ostree/ostree-pull.c +++ b/src/ostree/ostree-pull.c @@ -295,6 +295,38 @@ fetch_one_pack_file (OtPullData *pull_data, return ret; } +static gboolean +find_object_in_one_remote_pack (OtPullData *pull_data, + GVariant *csum_bytes_v, + OstreeObjectType objtype, + const char *pack_checksum, + gboolean *out_exists, + guint64 *out_offset, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + guint64 ret_offset; + gboolean ret_exists; + ot_lvariant GVariant *mapped_pack = NULL; + + if (!ostree_repo_map_cached_remote_pack_index (pull_data->repo, pull_data->remote_name, + pack_checksum, OSTREE_OBJECT_TYPE_IS_META (objtype), + &mapped_pack, + cancellable, error)) + goto out; + + ret_exists = ostree_pack_index_search (mapped_pack, csum_bytes_v, objtype, &ret_offset); + + ret = TRUE; + if (out_exists) + *out_exists = ret_exists; + if (out_offset) + *out_offset = ret_offset; + out: + return ret; +} + static gboolean find_object_in_remote_packs (OtPullData *pull_data, const char *checksum, @@ -305,14 +337,13 @@ find_object_in_remote_packs (OtPullData *pull_data, GError **error) { gboolean ret = FALSE; - guint64 offset; + guint64 ret_offset; guint i; GPtrArray *iter; - ot_lvariant GVariant *mapped_pack = NULL; - ot_lvariant GVariant *csum_bytes = NULL; + ot_lvariant GVariant *csum_bytes_v = NULL; ot_lfree char *ret_pack_checksum = NULL; - csum_bytes = ostree_checksum_to_bytes_v (checksum); + csum_bytes_v = ostree_checksum_to_bytes_v (checksum); if (OSTREE_OBJECT_TYPE_IS_META (objtype)) iter = pull_data->cached_meta_pack_indexes; @@ -321,15 +352,14 @@ find_object_in_remote_packs (OtPullData *pull_data, for (i = 0; i < iter->len; i++) { const char *pack_checksum = iter->pdata[i]; + gboolean exists; - ot_clear_gvariant (&mapped_pack); - if (!ostree_repo_map_cached_remote_pack_index (pull_data->repo, pull_data->remote_name, - pack_checksum, OSTREE_OBJECT_TYPE_IS_META (objtype), - &mapped_pack, - cancellable, error)) + if (!find_object_in_one_remote_pack (pull_data, csum_bytes_v, objtype, + pack_checksum, &exists, &ret_offset, + cancellable, error)) goto out; - if (ostree_pack_index_search (mapped_pack, csum_bytes, objtype, &offset)) + if (exists) { ret_pack_checksum = g_strdup (pack_checksum); break; @@ -339,7 +369,7 @@ find_object_in_remote_packs (OtPullData *pull_data, ret = TRUE; ot_transfer_out_value (out_pack_checksum, &ret_pack_checksum); if (out_offset) - *out_offset = offset; + *out_offset = ret_offset; out: return ret; } @@ -473,21 +503,20 @@ fetch_loose_object (OtPullData *pull_data, } static gboolean -find_object_ensure_packs (OtPullData *pull_data, - const char *checksum, - OstreeObjectType objtype, - gboolean *out_is_stored, - GFile **out_remote_pack_path, - guint64 *out_remote_pack_offset, - GCancellable *cancellable, - GError **error) +find_object_ensure_indexes (OtPullData *pull_data, + const char *checksum, + OstreeObjectType objtype, + gboolean *out_is_stored, + char **out_remote_pack_checksum, + guint64 *out_remote_pack_offset, + GCancellable *cancellable, + GError **error) { gboolean ret = FALSE; gboolean ret_is_stored; ot_lobj GFile *stored_path = NULL; ot_lfree char *local_pack_checksum = NULL; - ot_lfree char *remote_pack_checksum = NULL; - ot_lobj GFile *ret_remote_pack_path = NULL; + ot_lfree char *ret_remote_pack_checksum = NULL; if (!ostree_repo_find_object (pull_data->repo, objtype, checksum, &stored_path, &local_pack_checksum, NULL, @@ -495,6 +524,7 @@ find_object_ensure_packs (OtPullData *pull_data, goto out; ret_is_stored = (stored_path != NULL || local_pack_checksum != NULL); + if (!ret_is_stored) { if (!pull_data->fetched_packs) @@ -508,10 +538,41 @@ find_object_ensure_packs (OtPullData *pull_data, } if (!find_object_in_remote_packs (pull_data, checksum, objtype, - &remote_pack_checksum, out_remote_pack_offset, + &ret_remote_pack_checksum, out_remote_pack_offset, cancellable, error)) goto out; + } + ret = TRUE; + if (out_is_stored) + *out_is_stored = ret_is_stored; + ot_transfer_out_value (out_remote_pack_checksum, &ret_remote_pack_checksum); + out: + return ret; +} + +static gboolean +find_object_ensure_pack_data (OtPullData *pull_data, + const char *checksum, + OstreeObjectType objtype, + gboolean *out_is_stored, + GFile **out_remote_pack_path, + guint64 *out_remote_pack_offset, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + gboolean ret_is_stored; + ot_lfree char *remote_pack_checksum = NULL; + ot_lobj GFile *ret_remote_pack_path = NULL; + + if (!find_object_ensure_indexes (pull_data, checksum, objtype, &ret_is_stored, + &remote_pack_checksum, out_remote_pack_offset, + cancellable, error)) + goto out; + + if (!ret_is_stored) + { if (remote_pack_checksum) { if (!fetch_one_pack_file (pull_data, remote_pack_checksum, OSTREE_OBJECT_TYPE_IS_META (objtype), @@ -550,9 +611,9 @@ fetch_and_store_metadata (OtPullData *pull_data, g_assert (OSTREE_OBJECT_TYPE_IS_META (objtype)); - if (!find_object_ensure_packs (pull_data, checksum, objtype, - &is_stored, &remote_pack_path, &pack_offset, - cancellable, error)) + if (!find_object_ensure_pack_data (pull_data, checksum, objtype, + &is_stored, &remote_pack_path, &pack_offset, + cancellable, error)) goto out; if (remote_pack_path != NULL) @@ -606,76 +667,6 @@ fetch_and_store_metadata (OtPullData *pull_data, return ret; } -static gboolean -fetch_and_store_file (OtPullData *pull_data, - const char *checksum, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - gboolean is_stored; - guint64 pack_offset; - ot_lobj GFile *remote_pack_path = NULL; - ot_lobj GFile *temp_path = NULL; - ot_lvariant GVariant *pack_entry = NULL; - ot_lobj GInputStream *input = NULL; - ot_lobj GFileInfo *file_info = NULL; - ot_lvariant GVariant *xattrs = NULL; - GMappedFile *pack_map = NULL; - - if (!find_object_ensure_packs (pull_data, checksum, OSTREE_OBJECT_TYPE_FILE, - &is_stored, &remote_pack_path, &pack_offset, - cancellable, error)) - goto out; - - if (remote_pack_path != NULL) - { - g_assert (!is_stored); - - pack_map = g_mapped_file_new (ot_gfile_get_path_cached (remote_pack_path), FALSE, error); - if (!pack_map) - goto out; - - if (!ostree_read_pack_entry_raw ((guchar*)g_mapped_file_get_contents (pack_map), - g_mapped_file_get_length (pack_map), - pack_offset, FALSE, FALSE, &pack_entry, - cancellable, error)) - goto out; - - if (!ostree_parse_file_pack_entry (pack_entry, &input, &file_info, &xattrs, - cancellable, error)) - goto out; - - if (!ostree_repo_stage_object (pull_data->repo, OSTREE_OBJECT_TYPE_FILE, checksum, - file_info, xattrs, input, - cancellable, error)) - goto out; - } - else if (!is_stored) - { - if (!fetch_loose_object (pull_data, checksum, OSTREE_OBJECT_TYPE_FILE, &temp_path, - cancellable, error)) - goto out; - - if (!ostree_content_file_parse (temp_path, FALSE, &input, &file_info, &xattrs, - cancellable, error)) - goto out; - - if (!ostree_repo_stage_object (pull_data->repo, OSTREE_OBJECT_TYPE_FILE, checksum, - file_info, xattrs, input, - cancellable, error)) - goto out; - } - - ret = TRUE; - out: - if (temp_path) - (void) ot_gfile_unlink (temp_path, NULL, NULL); - if (pack_map) - g_mapped_file_unref (pack_map); - return ret; -} - static gboolean fetch_and_store_tree_metadata_recurse (OtPullData *pull_data, const char *rev, @@ -814,91 +805,176 @@ fetch_ref_contents (OtPullData *pull_data, } static gboolean -fetch_files (OtPullData *pull_data, - GCancellable *cancellable, - GError **error) +store_file_from_pack (OtPullData *pull_data, + const char *checksum, + const char *pack_checksum, + GFile *pack_file, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + gboolean exists; + guint64 pack_offset; + ot_lobj GFile *remote_pack_path = NULL; + ot_lobj GFile *temp_path = NULL; + ot_lvariant GVariant *pack_entry = NULL; + ot_lobj GInputStream *input = NULL; + ot_lobj GFileInfo *file_info = NULL; + ot_lvariant GVariant *xattrs = NULL; + ot_lvariant GVariant *csum_bytes_v = NULL; + GMappedFile *pack_map = NULL; + + csum_bytes_v = ostree_checksum_to_bytes_v (checksum); + + if (!find_object_in_one_remote_pack (pull_data, csum_bytes_v, OSTREE_OBJECT_TYPE_FILE, + pack_checksum, &exists, &pack_offset, + cancellable, error)) + goto out; + + g_assert (exists); + + pack_map = g_mapped_file_new (ot_gfile_get_path_cached (pack_file), FALSE, error); + if (!pack_map) + goto out; + + if (!ostree_read_pack_entry_raw ((guchar*)g_mapped_file_get_contents (pack_map), + g_mapped_file_get_length (pack_map), + pack_offset, FALSE, FALSE, &pack_entry, + cancellable, error)) + goto out; + + if (!ostree_parse_file_pack_entry (pack_entry, &input, &file_info, &xattrs, + cancellable, error)) + goto out; + + if (!ostree_repo_stage_object (pull_data->repo, OSTREE_OBJECT_TYPE_FILE, checksum, + file_info, xattrs, input, + cancellable, error)) + goto out; + + ret = TRUE; + out: + if (pack_map) + g_mapped_file_unref (pack_map); + return ret; +} + +static gboolean +fetch_content (OtPullData *pull_data, + GCancellable *cancellable, + GError **error) { gboolean ret = FALSE; GHashTableIter hash_iter; gpointer key, value; + ot_lobj GFile *temp_path = NULL; + ot_lhash GHashTable *data_packs_to_fetch = NULL; + ot_lhash GHashTable *loose_files = NULL; + guint n_objects_to_fetch = 0; + data_packs_to_fetch = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, + (GDestroyNotify) g_ptr_array_unref); + loose_files = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); + g_hash_table_iter_init (&hash_iter, pull_data->file_checksums_to_fetch); while (g_hash_table_iter_next (&hash_iter, &key, &value)) { const char *checksum = key; + GPtrArray *files_to_fetch; + gboolean is_stored; + ot_lfree char *remote_pack_checksum = NULL; - if (!fetch_and_store_file (pull_data, checksum, cancellable, error)) + if (!find_object_ensure_indexes (pull_data, checksum, OSTREE_OBJECT_TYPE_FILE, + &is_stored, &remote_pack_checksum, NULL, + cancellable, error)) goto out; + + if (remote_pack_checksum) + { + files_to_fetch = g_hash_table_lookup (data_packs_to_fetch, remote_pack_checksum); + if (files_to_fetch == NULL) + { + files_to_fetch = g_ptr_array_new_with_free_func (g_free); + g_hash_table_insert (data_packs_to_fetch, remote_pack_checksum, files_to_fetch); + /* transfer ownership */ + remote_pack_checksum = NULL; + } + g_ptr_array_add (files_to_fetch, g_strdup (checksum)); + n_objects_to_fetch++; + } + else if (!is_stored) + { + char *key = g_strdup (checksum); + g_hash_table_insert (loose_files, key, key); + n_objects_to_fetch++; + } } - ret = TRUE; - out: - return ret; -} + if (n_objects_to_fetch > 0) + g_print ("%u content objects to fetch\n", n_objects_to_fetch); -static gboolean -pull_one_commit (OtPullData *pull_data, - const char *rev, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; + if (g_hash_table_size (data_packs_to_fetch) > 0) + g_print ("Fetching %u content packs\n", + g_hash_table_size (data_packs_to_fetch)); - if (!ostree_repo_prepare_transaction (pull_data->repo, NULL, error)) - goto out; - - if (!fetch_and_store_commit_metadata_recurse (pull_data, rev, cancellable, error)) - goto out; - - if (!fetch_files (pull_data, cancellable, error)) - goto out; - - if (!ostree_repo_commit_transaction (pull_data->repo, cancellable, error)) - goto out; - - ret = TRUE; - out: - return ret; -} - -static gboolean -pull_one_ref (OtPullData *pull_data, - const char *branch, - const char *rev, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - ot_lfree char *key = NULL; - ot_lfree char *remote_ref = NULL; - ot_lfree char *baseurl = NULL; - ot_lfree char *original_rev = NULL; - - remote_ref = g_strdup_printf ("%s/%s", pull_data->remote_name, branch); - - if (!ostree_repo_resolve_rev (pull_data->repo, remote_ref, TRUE, &original_rev, error)) - goto out; - - if (original_rev && strcmp (rev, original_rev) == 0) + g_hash_table_iter_init (&hash_iter, data_packs_to_fetch); + while (g_hash_table_iter_next (&hash_iter, &key, &value)) { - g_print ("No changes in %s\n", remote_ref); - } - else - { - if (!ostree_validate_checksum_string (rev, error)) + const char *pack_checksum = key; + GPtrArray *file_checksums = value; + guint i; + ot_lobj GFile *pack_path = NULL; + + if (!fetch_one_pack_file (pull_data, pack_checksum, FALSE, + &pack_path, cancellable, error)) goto out; - if (!pull_one_commit (pull_data, rev, cancellable, error)) + g_print ("Storing %u objects from content pack %s\n", file_checksums->len, + pack_checksum); + for (i = 0; i < file_checksums->len; i++) + { + const char *checksum = file_checksums->pdata[i]; + if (!store_file_from_pack (pull_data, checksum, pack_checksum, pack_path, + cancellable, error)) + goto out; + } + + if (!ostree_repo_take_cached_remote_pack_data (pull_data->repo, pull_data->remote_name, + pack_checksum, FALSE, NULL, + cancellable, error)) goto out; - - if (!ostree_repo_write_ref (pull_data->repo, pull_data->remote_name, branch, rev, error)) + } + + if (g_hash_table_size (loose_files) > 0) + g_print ("Fetching %u loose objects\n", + g_hash_table_size (loose_files)); + + g_hash_table_iter_init (&hash_iter, loose_files); + while (g_hash_table_iter_next (&hash_iter, &key, &value)) + { + const char *checksum = key; + ot_lobj GInputStream *input = NULL; + ot_lobj GFileInfo *file_info = NULL; + ot_lvariant GVariant *xattrs = NULL; + + if (!fetch_loose_object (pull_data, checksum, OSTREE_OBJECT_TYPE_FILE, &temp_path, + cancellable, error)) + goto out; + + if (!ostree_content_file_parse (temp_path, FALSE, &input, &file_info, &xattrs, + cancellable, error)) + goto out; + + if (!ostree_repo_stage_object (pull_data->repo, OSTREE_OBJECT_TYPE_FILE, checksum, + file_info, xattrs, input, + cancellable, error)) goto out; - - g_print ("remote %s is now %s\n", remote_ref, rev); } ret = TRUE; out: + if (temp_path) + (void) ot_gfile_unlink (temp_path, NULL, NULL); return ret; } @@ -969,7 +1045,8 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error) ot_lfree char *path = NULL; ot_lfree char *baseurl = NULL; ot_lfree char *summary_data = NULL; - ot_lhash GHashTable *refs_to_fetch = NULL; + ot_lhash GHashTable *requested_refs_to_fetch = NULL; + ot_lhash GHashTable *updated_refs = NULL; ot_lhash GHashTable *commits_to_fetch = NULL; ot_lfree char *branch_rev = NULL; OtPullData pull_data_real; @@ -1020,7 +1097,8 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error) goto out; } - refs_to_fetch = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, g_free); + requested_refs_to_fetch = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, g_free); + updated_refs = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, g_free); commits_to_fetch = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); if (argc > 2) @@ -1041,7 +1119,7 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error) goto out; /* Transfer ownership of contents */ - g_hash_table_insert (refs_to_fetch, g_strdup (branch), contents); + g_hash_table_insert (requested_refs_to_fetch, g_strdup (branch), contents); } } } @@ -1076,7 +1154,7 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error) if (!fetch_uri_contents_utf8 (pull_data, summary_uri, &summary_data, cancellable, error)) goto out; - if (!parse_ref_summary (summary_data, &refs_to_fetch, error)) + if (!parse_ref_summary (summary_data, &requested_refs_to_fetch, error)) goto out; } else @@ -1094,30 +1172,75 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error) goto out; /* Transfer ownership of contents */ - g_hash_table_insert (refs_to_fetch, g_strdup (branch), contents); + g_hash_table_insert (requested_refs_to_fetch, g_strdup (branch), contents); } } } - g_hash_table_iter_init (&hash_iter, refs_to_fetch); - while (g_hash_table_iter_next (&hash_iter, &key, &value)) - { - const char *ref = key; - const char *sha256 = value; - - if (!pull_one_ref (pull_data, ref, sha256, cancellable, error)) - goto out; - } + if (!ostree_repo_prepare_transaction (pull_data->repo, NULL, error)) + goto out; g_hash_table_iter_init (&hash_iter, commits_to_fetch); while (g_hash_table_iter_next (&hash_iter, &key, &value)) { const char *commit = value; - if (!pull_one_commit (pull_data, commit, cancellable, error)) + if (!fetch_and_store_commit_metadata_recurse (pull_data, commit, cancellable, error)) goto out; } + g_hash_table_iter_init (&hash_iter, requested_refs_to_fetch); + while (g_hash_table_iter_next (&hash_iter, &key, &value)) + { + const char *ref = key; + const char *sha256 = value; + ot_lfree char *key = NULL; + ot_lfree char *remote_ref = NULL; + ot_lfree char *baseurl = NULL; + ot_lfree char *original_rev = NULL; + + remote_ref = g_strdup_printf ("%s/%s", pull_data->remote_name, ref); + + if (!ostree_repo_resolve_rev (pull_data->repo, remote_ref, TRUE, &original_rev, error)) + goto out; + + if (original_rev && strcmp (sha256, original_rev) == 0) + { + g_print ("No changes in %s\n", remote_ref); + } + else + { + if (!ostree_validate_checksum_string (sha256, error)) + goto out; + + if (!fetch_and_store_commit_metadata_recurse (pull_data, sha256, cancellable, error)) + goto out; + + g_hash_table_insert (updated_refs, g_strdup (ref), g_strdup (sha256)); + } + } + + if (!fetch_content (pull_data, cancellable, error)) + goto out; + + if (!ostree_repo_commit_transaction (pull_data->repo, cancellable, error)) + goto out; + + g_hash_table_iter_init (&hash_iter, updated_refs); + while (g_hash_table_iter_next (&hash_iter, &key, &value)) + { + const char *ref = key; + const char *checksum = value; + ot_lfree char *remote_ref = NULL; + + remote_ref = g_strdup_printf ("%s/%s", pull_data->remote_name, ref); + + if (!ostree_repo_write_ref (pull_data->repo, pull_data->remote_name, ref, checksum, error)) + goto out; + + g_print ("remote %s is now %s\n", remote_ref, checksum); + } + if (!ostree_repo_clean_cached_remote_pack_data (pull_data->repo, pull_data->remote_name, cancellable, error)) goto out;