From c6f972406e1f3b939c11252d343bb3e91299f8b9 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 26 Sep 2017 13:58:54 -0400 Subject: [PATCH] lib/pull: Add status for imported objects Followup for recent work in commits: - 8a7a35970928331a028ccddd04590ac300dbd26e - 1a9a473580e9066f5d4a969932d703b5b104b2b7 Keep track of how many objects we imported, and print that for `ostree pull-local` (also do this even if noninteractive, like we did for `pull`). In implementing this at first I used separate variables for import from repo vs import from localcache, but that broke some of the tests that checked those values. It's easier to just merge them; we know from looking at whether or not `remote_repo_local` is set whether or not we were doing a "HTTP pull with localcache" versus a true `pull-local` and can use that when rendering status. Closes: #1219 Approved by: jlebon --- src/libostree/ostree-repo-pull.c | 56 ++++++++++++++++++------------ src/ostree/ot-builtin-pull-local.c | 20 +++++++++-- tests/basic-test.sh | 3 +- tests/pull-test.sh | 2 +- 4 files changed, 54 insertions(+), 27 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 4aff9e09..45e3eae5 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -132,9 +132,9 @@ typedef struct { guint n_fetched_deltapart_fallbacks; guint n_fetched_metadata; guint n_fetched_content; - /* Objects from pull --localcache-repo */ - guint n_fetched_localcache_metadata; - guint n_fetched_localcache_content; + /* Objects imported via hardlink/reflink/copying or --localcache-repo*/ + guint n_imported_metadata; + guint n_imported_content; gboolean timestamp_check; /* Verify commit timestamps */ int maxdepth; @@ -261,8 +261,11 @@ update_progress (gpointer user_data) "scanned-metadata", "u", n_scanned_metadata, "bytes-transferred", "t", bytes_transferred, "start-time", "t", start_time, - "metadata-fetched-localcache", "u", pull_data->n_fetched_localcache_metadata, - "content-fetched-localcache", "u", pull_data->n_fetched_localcache_content, + /* We use these status keys even though we now also + * use these values for filesystem-local pulls. + */ + "metadata-fetched-localcache", "u", pull_data->n_imported_metadata, + "content-fetched-localcache", "u", pull_data->n_imported_content, /* Deltas */ "fetched-delta-parts", "u", pull_data->n_fetched_deltaparts, @@ -696,6 +699,7 @@ on_local_object_imported (GObject *object, goto out; out: + pull_data->n_imported_content++; g_assert_cmpint (pull_data->n_outstanding_content_write_requests, >, 0); pull_data->n_outstanding_content_write_requests--; check_outstanding_requests_handle_error (pull_data, &local_error); @@ -778,7 +782,6 @@ scan_dirtree_object (OtPullData *pull_data, on_local_object_imported, pull_data); g_hash_table_add (pull_data->requested_content, g_steal_pointer (&file_checksum)); did_import_from_cache_repo = TRUE; - pull_data->n_fetched_localcache_content++; break; } } @@ -1793,6 +1796,7 @@ scan_one_metadata_object_c (OtPullData *pull_data, */ if (objtype == OSTREE_OBJECT_TYPE_COMMIT) g_hash_table_add (pull_data->fetched_detached_metadata, g_strdup (tmp_checksum)); + pull_data->n_imported_metadata++; is_stored = TRUE; is_requested = TRUE; } @@ -1824,7 +1828,7 @@ scan_one_metadata_object_c (OtPullData *pull_data, g_hash_table_add (pull_data->fetched_detached_metadata, g_strdup (tmp_checksum)); is_stored = TRUE; is_requested = TRUE; - pull_data->n_fetched_localcache_metadata++; + pull_data->n_imported_metadata++; break; } } @@ -4078,36 +4082,42 @@ ostree_repo_pull_with_options (OstreeRepo *self, end_time = g_get_monotonic_time (); bytes_transferred = _ostree_fetcher_bytes_transferred (pull_data->fetcher); - if (bytes_transferred > 0 && pull_data->progress) + if (pull_data->progress) { - guint shift; g_autoptr(GString) buf = g_string_new (""); /* Ensure the rest of the progress keys are set appropriately. */ update_progress (pull_data); - if (bytes_transferred < 1024) - shift = 1; - else - shift = 1024; - - if (pull_data->n_fetched_deltaparts > 0) + /* See if we did a local-only import */ + if (pull_data->remote_repo_local) + g_string_append_printf (buf, "%u metadata, %u content objects imported", + pull_data->n_imported_metadata, pull_data->n_imported_content); + else if (pull_data->n_fetched_deltaparts > 0) g_string_append_printf (buf, "%u delta parts, %u loose fetched", pull_data->n_fetched_deltaparts, pull_data->n_fetched_metadata + pull_data->n_fetched_content); else g_string_append_printf (buf, "%u metadata, %u content objects fetched", pull_data->n_fetched_metadata, pull_data->n_fetched_content); - if (pull_data->n_fetched_localcache_metadata || - pull_data->n_fetched_localcache_content) + if (!pull_data->remote_repo_local && + (pull_data->n_imported_metadata || pull_data->n_imported_content)) g_string_append_printf (buf, " (%u meta, %u content local)", - pull_data->n_fetched_localcache_metadata, - pull_data->n_fetched_localcache_content); + pull_data->n_imported_metadata, + pull_data->n_imported_content); - g_string_append_printf (buf, "; %" G_GUINT64_FORMAT " %s transferred in %u seconds", - (guint64)(bytes_transferred / shift), - shift == 1 ? "B" : "KiB", - (guint) ((end_time - pull_data->start_time) / G_USEC_PER_SEC)); + if (bytes_transferred > 0) + { + guint shift; + if (bytes_transferred < 1024) + shift = 1; + else + shift = 1024; + g_string_append_printf (buf, "; %" G_GUINT64_FORMAT " %s transferred in %u seconds", + (guint64)(bytes_transferred / shift), + shift == 1 ? "B" : "KiB", + (guint) ((end_time - pull_data->start_time) / G_USEC_PER_SEC)); + } ostree_async_progress_set_status (pull_data->progress, buf->str); } diff --git a/src/ostree/ot-builtin-pull-local.c b/src/ostree/ot-builtin-pull-local.c index c3dc50c9..8d4e8d60 100644 --- a/src/ostree/ot-builtin-pull-local.c +++ b/src/ostree/ot-builtin-pull-local.c @@ -55,6 +55,14 @@ static GOptionEntry options[] = { { NULL } }; +/* See canonical version of this in ot-builtin-pull.c */ +static void +noninteractive_console_progress_changed (OstreeAsyncProgress *progress, + gpointer user_data) +{ + /* We do nothing here - we just want the final status */ +} + gboolean ostree_builtin_pull_local (int argc, char **argv, GCancellable *cancellable, GError **error) { @@ -170,6 +178,8 @@ ostree_builtin_pull_local (int argc, char **argv, GCancellable *cancellable, GEr if (console.is_tty) progress = ostree_async_progress_new_and_connect (ostree_repo_pull_default_console_progress_changed, &console); + else + progress = ostree_async_progress_new_and_connect (noninteractive_console_progress_changed, &console); opts = g_variant_ref_sink (g_variant_builder_end (&builder)); if (!ostree_repo_pull_with_options (repo, src_repo_uri, @@ -178,8 +188,14 @@ ostree_builtin_pull_local (int argc, char **argv, GCancellable *cancellable, GEr cancellable, error)) goto out; - if (progress) - ostree_async_progress_finish (progress); + if (!console.is_tty) + { + g_assert (progress); + const char *status = ostree_async_progress_get_status (progress); + if (status) + g_print ("%s\n", status); + } + ostree_async_progress_finish (progress); } ret = TRUE; diff --git a/tests/basic-test.sh b/tests/basic-test.sh index 482f6979..4df6a079 100644 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -278,7 +278,8 @@ if ! skip_one_without_user_xattrs; then opposite_mode=archive fi ostree_repo_init repo2 --mode=$opposite_mode - ${CMD_PREFIX} ostree --repo=repo2 pull-local repo + ${CMD_PREFIX} ostree --repo=repo2 pull-local repo >out.txt + assert_file_has_content out.txt "[1-9][0-9]* metadata, [1-9][0-9]* content objects imported" test2_commitid=$(${CMD_PREFIX} ostree --repo=repo rev-parse test2) test2_commit_relpath=/objects/${test2_commitid:0:2}/${test2_commitid:2}.commit assert_files_hardlinked repo/${test2_commit_relpath} repo2/${test2_commit_relpath} diff --git a/tests/pull-test.sh b/tests/pull-test.sh index 49209839..f2c45f93 100644 --- a/tests/pull-test.sh +++ b/tests/pull-test.sh @@ -42,7 +42,7 @@ repo_init --no-gpg-verify ${CMD_PREFIX} ostree --repo=repo pull origin main >out.txt assert_file_has_content out.txt "[1-9][0-9]* metadata, [1-9][0-9]* content objects fetched" ${CMD_PREFIX} ostree --repo=repo pull origin:main > out.txt -assert_not_file_has_content out.txt "content objects fetched" +assert_not_file_has_content out.txt "[1-9][0-9]* content objects fetched" ${CMD_PREFIX} ostree --repo=repo fsck echo "ok pull"