diff --git a/apidoc/html/ostree-Content-addressed-object-store.html b/apidoc/html/ostree-Content-addressed-object-store.html index 428d805f..f506463a 100644 --- a/apidoc/html/ostree-Content-addressed-object-store.html +++ b/apidoc/html/ostree-Content-addressed-object-store.html @@ -6122,6 +6122,7 @@ The following are currently defined:

  • refs (as): Array of string refs

  • flags (i): An instance of OstreeRepoPullFlags

  • subdir (s): Pull just this subdirectory

    • +

  • subdirs (as): Pull just these subdirectories

  • override-remote-name (s): If local, add this remote to refspec

  • gpg-verify (b): GPG verify commits

  • gpg-verify-summary (b): GPG verify summary

    • @@ -6131,6 +6132,7 @@ The following are currently defined:

  • override-commit-ids (as): Array of specific commit IDs to fetch for refs

  • dry-run (b): Only print information on what will be downloaded (requires static deltas)

  • override-url (s): Fetch objects from this URL if remote specifies no metalink in options

    • +

  • inherit-transaction (b): Don't initiate, finish or abort a transaction, usefult to do mutliple pulls in one transaction.

    • Parameters

    diff --git a/configure b/configure index 5640891d..9d7df117 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for ostree 2016.11. +# Generated by GNU Autoconf 2.69 for ostree 2016.12. # # Report bugs to . # @@ -590,8 +590,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='ostree' PACKAGE_TARNAME='ostree' -PACKAGE_VERSION='2016.11' -PACKAGE_STRING='ostree 2016.11' +PACKAGE_VERSION='2016.12' +PACKAGE_STRING='ostree 2016.12' PACKAGE_BUGREPORT='walters@verbum.org' PACKAGE_URL='' @@ -1480,7 +1480,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures ostree 2016.11 to adapt to many kinds of systems. +\`configure' configures ostree 2016.12 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1550,7 +1550,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of ostree 2016.11:";; + short | recursive ) echo "Configuration of ostree 2016.12:";; esac cat <<\_ACEOF @@ -1759,7 +1759,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -ostree configure 2016.11 +ostree configure 2016.12 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -2174,7 +2174,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by ostree $as_me 2016.11, which was +It was created by ostree $as_me 2016.12, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -3041,7 +3041,7 @@ fi # Define the identity of the package. PACKAGE='ostree' - VERSION='2016.11' + VERSION='2016.12' # Some tools Automake needs. @@ -17092,7 +17092,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by ostree $as_me 2016.11, which was +This file was extended by ostree $as_me 2016.12, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -17158,7 +17158,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -ostree config.status 2016.11 +ostree config.status 2016.12 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index e69d94e7..7a6fdf13 100644 --- a/configure.ac +++ b/configure.ac @@ -1,6 +1,6 @@ AC_PREREQ([2.63]) dnl If incrementing the version here, remember to update libostree.sym too -AC_INIT([ostree], [2016.11], [walters@verbum.org]) +AC_INIT([ostree], [2016.12], [walters@verbum.org]) AC_CONFIG_HEADER([config.h]) AC_CONFIG_MACRO_DIR([buildutil]) AC_CONFIG_AUX_DIR([build-aux]) diff --git a/src/libostree/libostree.sym b/src/libostree/libostree.sym index 4aa8d6cf..ddb87e7b 100644 --- a/src/libostree/libostree.sym +++ b/src/libostree/libostree.sym @@ -356,6 +356,7 @@ global: /* No new symbols in 2016.9 */ /* No new symbols in 2016.10 */ /* No new symbols in 2016.11 */ +/* No new symbols in 2016.12 */ /* NOTE NOTE NOTE * Versions above here are released. Only add symbols below this line. diff --git a/src/libostree/ostree-fetcher.c b/src/libostree/ostree-fetcher.c index 3ddf2389..c2dc8ea4 100644 --- a/src/libostree/ostree-fetcher.c +++ b/src/libostree/ostree-fetcher.c @@ -65,6 +65,9 @@ typedef struct { /* Also protected by output_stream_set_lock. */ guint64 total_downloaded; + + GError *oob_error; + } ThreadClosure; static void @@ -159,6 +162,8 @@ thread_closure_unref (ThreadClosure *thread_closure) g_clear_pointer (&thread_closure->output_stream_set, g_hash_table_unref); g_mutex_clear (&thread_closure->output_stream_set_lock); + g_clear_pointer (&thread_closure->oob_error, g_error_free); + g_slice_free (ThreadClosure, thread_closure); } } @@ -276,6 +281,29 @@ session_thread_config_flags (ThreadClosure *thread_closure, } } +static void +on_authenticate (SoupSession *session, SoupMessage *msg, SoupAuth *auth, + gboolean retrying, gpointer user_data) +{ + ThreadClosure *thread_closure = user_data; + + if (msg->status_code == SOUP_STATUS_PROXY_UNAUTHORIZED) + { + SoupURI *uri = NULL; + g_object_get (session, SOUP_SESSION_PROXY_URI, &uri, NULL); + if (retrying) + { + g_autofree char *s = soup_uri_to_string (uri, FALSE); + g_set_error (&thread_closure->oob_error, + G_IO_ERROR, G_IO_ERROR_PROXY_AUTH_FAILED, + "Invalid username or password for proxy '%s'", s); + } + else + soup_auth_authenticate (auth, soup_uri_get_user (uri), + soup_uri_get_password (uri)); + } +} + static void session_thread_set_proxy_cb (ThreadClosure *thread_closure, gpointer data) @@ -285,6 +313,17 @@ session_thread_set_proxy_cb (ThreadClosure *thread_closure, g_object_set (thread_closure->session, SOUP_SESSION_PROXY_URI, proxy_uri, NULL); + + /* libsoup won't necessarily pass any embedded username and password to proxy + * requests, so we have to be ready to handle 407 and handle them ourselves. + * See also: https://bugzilla.gnome.org/show_bug.cgi?id=772932 + * */ + if (soup_uri_get_user (proxy_uri) && + soup_uri_get_password (proxy_uri)) + { + g_signal_connect (thread_closure->session, "authenticate", + G_CALLBACK (on_authenticate), thread_closure); + } } #ifdef HAVE_LIBSOUP_CLIENT_CERTS @@ -998,10 +1037,23 @@ on_request_sent (GObject *object, code = G_IO_ERROR_FAILED; } - local_error = g_error_new (G_IO_ERROR, code, - "Server returned status %u: %s", - msg->status_code, - soup_status_get_phrase (msg->status_code)); + { + g_autofree char *errmsg = + g_strdup_printf ("Server returned status %u: %s", + msg->status_code, + soup_status_get_phrase (msg->status_code)); + + /* Let's make OOB errors be the final one since they're probably + * the cause for the error here. */ + if (pending->thread_closure->oob_error) + { + local_error = + g_error_copy (pending->thread_closure->oob_error); + g_prefix_error (&local_error, "%s: ", errmsg); + } + else + local_error = g_error_new_literal (G_IO_ERROR, code, errmsg); + } if (pending->mirrorlist->len > 1) g_prefix_error (&local_error, diff --git a/src/libostree/ostree-repo-checkout.c b/src/libostree/ostree-repo-checkout.c index 23258a47..9b640648 100644 --- a/src/libostree/ostree-repo-checkout.c +++ b/src/libostree/ostree-repo-checkout.c @@ -437,7 +437,7 @@ checkout_one_file_at (OstreeRepo *repo, need_copy = FALSE; } - else if (!is_symlink) + else { gboolean did_hardlink = FALSE; /* Try to do a hardlink first, if it's a regular file. This also @@ -450,7 +450,9 @@ checkout_one_file_at (OstreeRepo *repo, gboolean is_bare = ((current_repo->mode == OSTREE_REPO_MODE_BARE && options->mode == OSTREE_REPO_CHECKOUT_MODE_NONE) || (current_repo->mode == OSTREE_REPO_MODE_BARE_USER - && options->mode == OSTREE_REPO_CHECKOUT_MODE_USER)); + && options->mode == OSTREE_REPO_CHECKOUT_MODE_USER + /* NOTE: bare-user symlinks are not stored as symlinks */ + && !is_symlink)); gboolean current_can_cache = (options->enable_uncompressed_cache && current_repo->enable_uncompressed_cache); gboolean is_archive_z2_with_cache = (current_repo->mode == OSTREE_REPO_MODE_ARCHIVE_Z2 diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 8dfe276f..42e06400 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -2035,8 +2035,9 @@ ostree_repo_read_commit_detached_metadata (OstreeRepo *self, g_autoptr(GVariant) ret_metadata = NULL; _ostree_loose_path (buf, checksum, OSTREE_OBJECT_TYPE_COMMIT_META, self->mode); - - if (!ot_util_variant_map_at (self->objects_dir_fd, buf, + + if (self->commit_stagedir_fd != -1 && + !ot_util_variant_map_at (self->commit_stagedir_fd, buf, G_VARIANT_TYPE ("a{sv}"), OT_VARIANT_MAP_ALLOW_NOENT | OT_VARIANT_MAP_TRUSTED, &ret_metadata, error)) { @@ -2044,6 +2045,21 @@ ostree_repo_read_commit_detached_metadata (OstreeRepo *self, goto out; } + if (ret_metadata == NULL && + !ot_util_variant_map_at (self->objects_dir_fd, buf, + G_VARIANT_TYPE ("a{sv}"), + OT_VARIANT_MAP_ALLOW_NOENT | OT_VARIANT_MAP_TRUSTED, &ret_metadata, error)) + { + g_prefix_error (error, "Unable to read existing detached metadata: "); + goto out; + } + + if (ret_metadata == NULL && self->parent_repo) + return ostree_repo_read_commit_detached_metadata (self->parent_repo, + checksum, + out_metadata, + cancellable, + error); ret = TRUE; ot_transfer_out_value (out_metadata, &ret_metadata); out: @@ -2073,10 +2089,16 @@ ostree_repo_write_commit_detached_metadata (OstreeRepo *self, g_autoptr(GVariant) normalized = NULL; gsize normalized_size = 0; const guint8 *data = NULL; + int dest_dfd; + + if (self->in_transaction) + dest_dfd = self->commit_stagedir_fd; + else + dest_dfd = self->objects_dir_fd; _ostree_loose_path (pathbuf, checksum, OSTREE_OBJECT_TYPE_COMMIT_META, self->mode); - if (!_ostree_repo_ensure_loose_objdir_at (self->objects_dir_fd, checksum, + if (!_ostree_repo_ensure_loose_objdir_at (dest_dfd, checksum, cancellable, error)) return FALSE; @@ -2090,7 +2112,7 @@ ostree_repo_write_commit_detached_metadata (OstreeRepo *self, if (data == NULL) data = (guint8*)""; - if (!glnx_file_replace_contents_at (self->objects_dir_fd, pathbuf, + if (!glnx_file_replace_contents_at (dest_dfd, pathbuf, data, normalized_size, 0, cancellable, error)) { diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 01561df2..52455ed4 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -99,7 +99,7 @@ typedef struct { gboolean is_commit_only; gboolean is_untrusted; - char *dir; + GPtrArray *dirs; gboolean commitpartial_exists; gboolean have_previous_bytes; @@ -117,6 +117,7 @@ typedef struct { typedef struct { OtPullData *pull_data; GVariant *object; + char *path; gboolean is_detached_meta; /* Only relevant when is_detached_meta is TRUE. Controls @@ -133,6 +134,7 @@ typedef struct { typedef struct { guchar csum[OSTREE_SHA256_DIGEST_LEN]; + char *path; OstreeObjectType objtype; guint recursion_depth; } ScanObjectQueueData; @@ -140,16 +142,19 @@ typedef struct { static void queue_scan_one_metadata_object (OtPullData *pull_data, const char *csum, OstreeObjectType objtype, + const char *path, guint recursion_depth); static void queue_scan_one_metadata_object_c (OtPullData *pull_data, const guchar *csum, OstreeObjectType objtype, + const char *path, guint recursion_depth); static gboolean scan_one_metadata_object_c (OtPullData *pull_data, const guchar *csum, OstreeObjectType objtype, + const char *path, guint recursion_depth, GCancellable *cancellable, GError **error); @@ -278,11 +283,13 @@ idle_worker (gpointer user_data) scan_one_metadata_object_c (pull_data, scan_data->csum, scan_data->objtype, + scan_data->path, scan_data->recursion_depth, pull_data->cancellable, &error); check_outstanding_requests_handle_error (pull_data, error); + g_free (scan_data->path); g_free (scan_data); return G_SOURCE_CONTINUE; } @@ -381,12 +388,80 @@ static void enqueue_one_object_request (OtPullData *pull_data, const char *checksum, OstreeObjectType objtype, + const char *path, gboolean is_detached_meta, gboolean object_is_stored); +static gboolean +matches_pull_dir (const char *current_file, + const char *pull_dir, + gboolean current_file_is_dir) +{ + const char *rest; + + if (g_str_has_prefix (pull_dir, current_file)) + { + rest = pull_dir + strlen (current_file); + if (*rest == 0) + { + /* The current file is exactly the same as the specified + pull dir. This matches always, even if the file is not a + directory. */ + return TRUE; + } + + if (*rest == '/') + { + /* The current file is a directory-prefix of the pull_dir. + Match only if this is supposed to be a directory */ + return current_file_is_dir; + } + + /* Matched a non-directory prefix such as /foo being a prefix of /fooo, + no match */ + return FALSE; + } + + if (g_str_has_prefix (current_file, pull_dir)) + { + rest = current_file + strlen (pull_dir); + /* Only match if the prefix match matched the entire directory + component */ + return *rest == '/'; + } + + return FALSE; +} + + +static gboolean +pull_matches_subdir (OtPullData *pull_data, + const char *path, + const char *basename, + gboolean basename_is_dir) +{ + int i; + g_autofree char *file = NULL; + + if (pull_data->dirs == NULL) + return TRUE; + + file = g_strconcat (path, basename, NULL); + + for (i = 0; i < pull_data->dirs->len; i++) + { + const char *pull_dir = g_ptr_array_index (pull_data->dirs, i); + if (matches_pull_dir (file, pull_dir, basename_is_dir)) + return TRUE; + } + + return FALSE; +} + static gboolean scan_dirtree_object (OtPullData *pull_data, const char *checksum, + const char *path, int recursion_depth, GCancellable *cancellable, GError **error) @@ -396,7 +471,6 @@ scan_dirtree_object (OtPullData *pull_data, g_autoptr(GVariant) tree = NULL; g_autoptr(GVariant) files_variant = NULL; g_autoptr(GVariant) dirs_variant = NULL; - char *subdir_target = NULL; const char *dirname = NULL; if (recursion_depth > OSTREE_MAX_RECURSION) @@ -429,10 +503,7 @@ scan_dirtree_object (OtPullData *pull_data, /* Skip files if we're traversing a request only directory, unless it exactly * matches the path */ - if (pull_data->dir && - /* Should always an initial slash, we assert it in scan_dirtree_object */ - pull_data->dir[0] == '/' && - strcmp (pull_data->dir+1, filename) != 0) + if (!pull_matches_subdir (pull_data, path, filename, FALSE)) continue; file_checksum = ostree_checksum_from_bytes_v (csum); @@ -451,32 +522,11 @@ scan_dirtree_object (OtPullData *pull_data, else if (!file_is_stored && !g_hash_table_lookup (pull_data->requested_content, file_checksum)) { g_hash_table_insert (pull_data->requested_content, file_checksum, file_checksum); - enqueue_one_object_request (pull_data, file_checksum, OSTREE_OBJECT_TYPE_FILE, FALSE, FALSE); + enqueue_one_object_request (pull_data, file_checksum, OSTREE_OBJECT_TYPE_FILE, path, FALSE, FALSE); file_checksum = NULL; /* Transfer ownership */ } } - if (pull_data->dir) - { - const char *subpath = NULL; - const char *nextslash = NULL; - g_autofree char *dir_data = NULL; - - g_assert (pull_data->dir[0] == '/'); // assert it starts with / like "/usr/share/rpm" - subpath = pull_data->dir + 1; // refers to name minus / like "usr/share/rpm" - nextslash = strchr (subpath, '/'); //refers to start of next slash like "/share/rpm" - dir_data = pull_data->dir; // keep the original pointer around since strchr() points into it - pull_data->dir = NULL; - - if (nextslash) - { - subdir_target = g_strndup (subpath, nextslash - subpath); // refers to first dir, like "usr" - pull_data->dir = g_strdup (nextslash); // sets dir to new deeper level like "/share/rpm" - } - else // we're as deep as it goes, i.e. subpath = "rpm" - subdir_target = g_strdup (subpath); - } - n = g_variant_n_children (dirs_variant); for (i = 0; i < n; i++) @@ -485,6 +535,7 @@ scan_dirtree_object (OtPullData *pull_data, g_autoptr(GVariant) meta_csum = NULL; const guchar *tree_csum_bytes; const guchar *meta_csum_bytes; + g_autofree char *subpath = NULL; g_variant_get_child (dirs_variant, i, "(&s@ay@ay)", &dirname, &tree_csum, &meta_csum); @@ -492,7 +543,7 @@ scan_dirtree_object (OtPullData *pull_data, if (!ot_util_filename_validate (dirname, error)) goto out; - if (subdir_target && strcmp (subdir_target, dirname) != 0) + if (!pull_matches_subdir (pull_data, path, dirname, TRUE)) continue; tree_csum_bytes = ostree_checksum_bytes_peek_validate (tree_csum, error); @@ -503,10 +554,12 @@ scan_dirtree_object (OtPullData *pull_data, if (meta_csum_bytes == NULL) goto out; + subpath = g_strconcat (path, dirname, "/", NULL); + queue_scan_one_metadata_object_c (pull_data, tree_csum_bytes, - OSTREE_OBJECT_TYPE_DIR_TREE, recursion_depth + 1); + OSTREE_OBJECT_TYPE_DIR_TREE, subpath, recursion_depth + 1); queue_scan_one_metadata_object_c (pull_data, meta_csum_bytes, - OSTREE_OBJECT_TYPE_DIR_META, recursion_depth + 1); + OSTREE_OBJECT_TYPE_DIR_META, subpath, recursion_depth + 1); } ret = TRUE; @@ -583,6 +636,14 @@ lookup_commit_checksum_from_summary (OtPullData *pull_data, return ret; } +static void +fetch_object_data_free (FetchObjectData *fetch_data) +{ + g_variant_unref (fetch_data->object); + g_free (fetch_data->path); + g_free (fetch_data); +} + static void content_fetch_on_write_complete (GObject *object, GAsyncResult *result, @@ -622,8 +683,7 @@ content_fetch_on_write_complete (GObject *object, out: pull_data->n_outstanding_content_write_requests--; check_outstanding_requests_handle_error (pull_data, local_error); - g_variant_unref (fetch_data->object); - g_free (fetch_data); + fetch_object_data_free (fetch_data); } static void @@ -712,10 +772,7 @@ content_fetch_on_complete (GObject *object, pull_data->n_outstanding_content_fetches--; check_outstanding_requests_handle_error (pull_data, local_error); if (free_fetch_data) - { - g_variant_unref (fetch_data->object); - g_free (fetch_data); - } + fetch_object_data_free (fetch_data); } static void @@ -753,12 +810,11 @@ on_metadata_written (GObject *object, goto out; } - queue_scan_one_metadata_object_c (pull_data, csum, objtype, 0); + queue_scan_one_metadata_object_c (pull_data, csum, objtype, fetch_data->path, 0); out: pull_data->n_outstanding_metadata_write_requests--; - g_variant_unref (fetch_data->object); - g_free (fetch_data); + fetch_object_data_free (fetch_data); check_outstanding_requests_handle_error (pull_data, local_error); } @@ -796,7 +852,7 @@ meta_fetch_on_complete (GObject *object, /* There isn't any detached metadata, just fetch the commit */ g_clear_error (&local_error); if (!fetch_data->object_is_stored) - enqueue_one_object_request (pull_data, checksum, objtype, FALSE, FALSE); + enqueue_one_object_request (pull_data, checksum, objtype, fetch_data->path, FALSE, FALSE); } /* When traversing parents, do not fail on a missing commit. @@ -811,7 +867,7 @@ meta_fetch_on_complete (GObject *object, if (pull_data->has_tombstone_commits) { enqueue_one_object_request (pull_data, checksum, OSTREE_OBJECT_TYPE_TOMBSTONE_COMMIT, - FALSE, FALSE); + fetch_data->path, FALSE, FALSE); } } } @@ -844,7 +900,7 @@ meta_fetch_on_complete (GObject *object, goto out; if (!fetch_data->object_is_stored) - enqueue_one_object_request (pull_data, checksum, objtype, FALSE, FALSE); + enqueue_one_object_request (pull_data, checksum, objtype, fetch_data->path, FALSE, FALSE); } else { @@ -874,10 +930,7 @@ meta_fetch_on_complete (GObject *object, pull_data->n_fetched_metadata++; check_outstanding_requests_handle_error (pull_data, local_error); if (free_fetch_data) - { - g_variant_unref (fetch_data->object); - g_free (fetch_data); - } + fetch_object_data_free (fetch_data); } static void @@ -1053,7 +1106,8 @@ scan_commit_object (OtPullData *pull_data, if (parent_csum_bytes != NULL && pull_data->maxdepth == -1) { queue_scan_one_metadata_object_c (pull_data, parent_csum_bytes, - OSTREE_OBJECT_TYPE_COMMIT, recursion_depth + 1); + OSTREE_OBJECT_TYPE_COMMIT, NULL, + recursion_depth + 1); } else if (parent_csum_bytes != NULL && depth > 0) { @@ -1078,7 +1132,9 @@ scan_commit_object (OtPullData *pull_data, g_hash_table_insert (pull_data->commit_to_depth, g_strdup (parent_checksum), GINT_TO_POINTER (parent_depth)); queue_scan_one_metadata_object_c (pull_data, parent_csum_bytes, - OSTREE_OBJECT_TYPE_COMMIT, recursion_depth + 1); + OSTREE_OBJECT_TYPE_COMMIT, + NULL, + recursion_depth + 1); } } @@ -1101,10 +1157,10 @@ scan_commit_object (OtPullData *pull_data, goto out; queue_scan_one_metadata_object_c (pull_data, tree_contents_csum_bytes, - OSTREE_OBJECT_TYPE_DIR_TREE, recursion_depth + 1); + OSTREE_OBJECT_TYPE_DIR_TREE, "/", recursion_depth + 1); queue_scan_one_metadata_object_c (pull_data, tree_meta_csum_bytes, - OSTREE_OBJECT_TYPE_DIR_META, recursion_depth + 1); + OSTREE_OBJECT_TYPE_DIR_META, NULL, recursion_depth + 1); } ret = TRUE; @@ -1116,23 +1172,26 @@ static void queue_scan_one_metadata_object (OtPullData *pull_data, const char *csum, OstreeObjectType objtype, + const char *path, guint recursion_depth) { guchar buf[OSTREE_SHA256_DIGEST_LEN]; ostree_checksum_inplace_to_bytes (csum, buf); - queue_scan_one_metadata_object_c (pull_data, buf, objtype, recursion_depth); + queue_scan_one_metadata_object_c (pull_data, buf, objtype, path, recursion_depth); } static void queue_scan_one_metadata_object_c (OtPullData *pull_data, const guchar *csum, OstreeObjectType objtype, + const char *path, guint recursion_depth) { ScanObjectQueueData *scan_data = g_new0 (ScanObjectQueueData, 1); memcpy (scan_data->csum, csum, sizeof (scan_data->csum)); scan_data->objtype = objtype; + scan_data->path = g_strdup (path); scan_data->recursion_depth = recursion_depth; g_queue_push_tail (&pull_data->scan_object_queue, scan_data); @@ -1143,6 +1202,7 @@ static gboolean scan_one_metadata_object_c (OtPullData *pull_data, const guchar *csum, OstreeObjectType objtype, + const char *path, guint recursion_depth, GCancellable *cancellable, GError **error) @@ -1190,7 +1250,7 @@ scan_one_metadata_object_c (OtPullData *pull_data, g_hash_table_insert (pull_data->requested_metadata, duped_checksum, duped_checksum); do_fetch_detached = (objtype == OSTREE_OBJECT_TYPE_COMMIT); - enqueue_one_object_request (pull_data, tmp_checksum, objtype, do_fetch_detached, FALSE); + enqueue_one_object_request (pull_data, tmp_checksum, objtype, path, do_fetch_detached, FALSE); } else if (objtype == OSTREE_OBJECT_TYPE_COMMIT && pull_data->is_commit_only) { @@ -1207,7 +1267,7 @@ scan_one_metadata_object_c (OtPullData *pull_data, /* For commits, always refetch detached metadata. */ if (objtype == OSTREE_OBJECT_TYPE_COMMIT) - enqueue_one_object_request (pull_data, tmp_checksum, objtype, TRUE, TRUE); + enqueue_one_object_request (pull_data, tmp_checksum, objtype, path, TRUE, TRUE); /* For commits, check whether we only had a partial fetch */ if (!do_scan && objtype == OSTREE_OBJECT_TYPE_COMMIT) @@ -1245,7 +1305,7 @@ scan_one_metadata_object_c (OtPullData *pull_data, case OSTREE_OBJECT_TYPE_DIR_META: break; case OSTREE_OBJECT_TYPE_DIR_TREE: - if (!scan_dirtree_object (pull_data, tmp_checksum, recursion_depth, + if (!scan_dirtree_object (pull_data, tmp_checksum, path, recursion_depth, pull_data->cancellable, error)) goto out; break; @@ -1267,6 +1327,7 @@ static void enqueue_one_object_request (OtPullData *pull_data, const char *checksum, OstreeObjectType objtype, + const char *path, gboolean is_detached_meta, gboolean object_is_stored) { @@ -1308,6 +1369,7 @@ enqueue_one_object_request (OtPullData *pull_data, fetch_data = g_new0 (FetchObjectData, 1); fetch_data->pull_data = pull_data; fetch_data->object = ostree_object_name_serialize (checksum, objtype); + fetch_data->path = g_strdup (path); fetch_data->is_detached_meta = is_detached_meta; fetch_data->object_is_stored = object_is_stored; @@ -1478,7 +1540,7 @@ process_one_static_delta_fallback (OtPullData *pull_data, g_hash_table_insert (pull_data->requested_metadata, checksum, checksum); do_fetch_detached = (objtype == OSTREE_OBJECT_TYPE_COMMIT); - enqueue_one_object_request (pull_data, checksum, objtype, do_fetch_detached, FALSE); + enqueue_one_object_request (pull_data, checksum, objtype, NULL, do_fetch_detached, FALSE); checksum = NULL; /* Transfer ownership */ } } @@ -1487,7 +1549,7 @@ process_one_static_delta_fallback (OtPullData *pull_data, if (!g_hash_table_lookup (pull_data->requested_content, checksum)) { g_hash_table_insert (pull_data->requested_content, checksum, checksum); - enqueue_one_object_request (pull_data, checksum, OSTREE_OBJECT_TYPE_FILE, FALSE, FALSE); + enqueue_one_object_request (pull_data, checksum, OSTREE_OBJECT_TYPE_FILE, NULL, FALSE, FALSE); checksum = NULL; /* Transfer ownership */ } } @@ -2227,6 +2289,7 @@ repo_remote_fetch_summary (OstreeRepo *self, * * refs (as): Array of string refs * * flags (i): An instance of #OstreeRepoPullFlags * * subdir (s): Pull just this subdirectory + * * subdirs (as): Pull just these subdirectories * * override-remote-name (s): If local, add this remote to refspec * * gpg-verify (b): GPG verify commits * * gpg-verify-summary (b): GPG verify summary @@ -2236,6 +2299,7 @@ repo_remote_fetch_summary (OstreeRepo *self, * * override-commit-ids (as): Array of specific commit IDs to fetch for refs * * dry-run (b): Only print information on what will be downloaded (requires static deltas) * * override-url (s): Fetch objects from this URL if remote specifies no metalink in options + * * inherit-transaction (b): Don't initiate, finish or abort a transaction, usefult to do mutliple pulls in one transaction. */ gboolean ostree_repo_pull_with_options (OstreeRepo *self, @@ -2262,6 +2326,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, guint64 end_time; OstreeRepoPullFlags flags = 0; const char *dir_to_pull = NULL; + g_autofree char **dirs_to_pull = NULL; g_autofree char **refs_to_fetch = NULL; char **override_commit_ids = NULL; GSource *update_timeout = NULL; @@ -2273,6 +2338,8 @@ ostree_repo_pull_with_options (OstreeRepo *self, g_autofree char *base_meta_url = NULL; g_autofree char *base_content_url = NULL; gboolean mirroring_into_archive; + gboolean inherit_transaction = FALSE; + int i; if (options) { @@ -2282,6 +2349,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, /* Reduce risk of issues if enum happens to be 64 bit for some reason */ flags = flags_i; (void) g_variant_lookup (options, "subdir", "&s", &dir_to_pull); + (void) g_variant_lookup (options, "subdirs", "^a&s", &dirs_to_pull); (void) g_variant_lookup (options, "override-remote-name", "s", &pull_data->remote_name); opt_gpg_verify_set = g_variant_lookup (options, "gpg-verify", "b", &pull_data->gpg_verify); @@ -2293,6 +2361,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, (void) g_variant_lookup (options, "override-commit-ids", "^a&s", &override_commit_ids); (void) g_variant_lookup (options, "dry-run", "b", &pull_data->dry_run); (void) g_variant_lookup (options, "override-url", "&s", &url_override); + (void) g_variant_lookup (options, "inherit-transaction", "b", &inherit_transaction); } g_return_val_if_fail (pull_data->maxdepth >= -1, FALSE); @@ -2302,6 +2371,9 @@ ostree_repo_pull_with_options (OstreeRepo *self, if (dir_to_pull) g_return_val_if_fail (dir_to_pull[0] == '/', FALSE); + for (i = 0; dirs_to_pull != NULL && dirs_to_pull[i] != NULL; i++) + g_return_val_if_fail (dirs_to_pull[i][0] == '/', FALSE); + g_return_val_if_fail (!(disable_static_deltas && require_static_deltas), FALSE); /* We only do dry runs with static deltas, because we don't really have any * in-advance information for bare fetches. @@ -2340,7 +2412,19 @@ ostree_repo_pull_with_options (OstreeRepo *self, (GDestroyNotify)g_free, NULL); pull_data->requested_metadata = g_hash_table_new_full (g_str_hash, g_str_equal, (GDestroyNotify)g_free, NULL); - pull_data->dir = g_strdup (dir_to_pull); + if (dir_to_pull != NULL || dirs_to_pull != NULL) + { + pull_data->dirs = g_ptr_array_new_with_free_func (g_free); + if (dir_to_pull != NULL) + g_ptr_array_add (pull_data->dirs, g_strdup (dir_to_pull)); + + if (dirs_to_pull != NULL) + { + for (i = 0; dirs_to_pull[i] != NULL; i++) + g_ptr_array_add (pull_data->dirs, g_strdup (dirs_to_pull[i])); + } + } + g_queue_init (&pull_data->scan_object_queue); pull_data->start_time = g_get_monotonic_time (); @@ -2822,7 +2906,9 @@ ostree_repo_pull_with_options (OstreeRepo *self, if (pull_data->fetcher == NULL) goto out; - if (!ostree_repo_prepare_transaction (pull_data->repo, &pull_data->legacy_transaction_resuming, + pull_data->legacy_transaction_resuming = FALSE; + if (!inherit_transaction && + !ostree_repo_prepare_transaction (pull_data->repo, &pull_data->legacy_transaction_resuming, cancellable, error)) goto out; @@ -2833,7 +2919,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, while (g_hash_table_iter_next (&hash_iter, &key, &value)) { const char *commit = value; - queue_scan_one_metadata_object (pull_data, commit, OSTREE_OBJECT_TYPE_COMMIT, 0); + queue_scan_one_metadata_object (pull_data, commit, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0); } g_hash_table_iter_init (&hash_iter, requested_refs_to_fetch); @@ -2866,7 +2952,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, goto out; } g_debug ("no delta superblock for %s-%s", from_revision ? from_revision : "empty", to_revision); - queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, 0); + queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0); } else { @@ -2954,7 +3040,8 @@ ostree_repo_pull_with_options (OstreeRepo *self, } } - if (!ostree_repo_commit_transaction (pull_data->repo, NULL, cancellable, error)) + if (!inherit_transaction && + !ostree_repo_commit_transaction (pull_data->repo, NULL, cancellable, error)) goto out; end_time = g_get_monotonic_time (); @@ -2988,7 +3075,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, } /* iterate over commits fetched and delete any commitpartial files */ - if (!dir_to_pull && !pull_data->is_commit_only) + if (pull_data->dirs == NULL && !pull_data->is_commit_only) { g_hash_table_iter_init (&hash_iter, requested_refs_to_fetch); while (g_hash_table_iter_next (&hash_iter, &key, &value)) @@ -3020,8 +3107,9 @@ ostree_repo_pull_with_options (OstreeRepo *self, g_propagate_error (error, pull_data->cached_async_error); else g_clear_error (&pull_data->cached_async_error); - - ostree_repo_abort_transaction (pull_data->repo, cancellable, NULL); + + if (!inherit_transaction) + ostree_repo_abort_transaction (pull_data->repo, cancellable, NULL); g_main_context_unref (pull_data->main_context); if (update_timeout) g_source_destroy (update_timeout); diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index 65173ffe..a14f6005 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -2002,7 +2002,7 @@ allocate_deployserial (OstreeSysroot *self, g_ptr_array_new_with_free_func (g_object_unref); osdir = ot_gfile_get_child_build_path (self->path, "ostree/deploy", osname, NULL); - + if (!_ostree_sysroot_list_deployment_dirs_for_os (osdir, tmp_current_deployments, cancellable, error)) goto out; @@ -2010,9 +2010,7 @@ allocate_deployserial (OstreeSysroot *self, for (i = 0; i < tmp_current_deployments->len; i++) { OstreeDeployment *deployment = tmp_current_deployments->pdata[i]; - - if (strcmp (ostree_deployment_get_osname (deployment), osname) != 0) - continue; + if (strcmp (ostree_deployment_get_csum (deployment), revision) != 0) continue; diff --git a/src/ostree/ot-builtin-pull.c b/src/ostree/ot-builtin-pull.c index 99b25937..9f48c2e0 100644 --- a/src/ostree/ot-builtin-pull.c +++ b/src/ostree/ot-builtin-pull.c @@ -34,7 +34,7 @@ static gboolean opt_dry_run; static gboolean opt_disable_static_deltas; static gboolean opt_require_static_deltas; static gboolean opt_untrusted; -static char* opt_subpath; +static char** opt_subpaths; static char* opt_cache_dir; static int opt_depth = 0; static char* opt_url; @@ -46,7 +46,7 @@ static GOptionEntry options[] = { { "disable-static-deltas", 0, 0, G_OPTION_ARG_NONE, &opt_disable_static_deltas, "Do not use static deltas", NULL }, { "require-static-deltas", 0, 0, G_OPTION_ARG_NONE, &opt_require_static_deltas, "Require static deltas", NULL }, { "mirror", 0, 0, G_OPTION_ARG_NONE, &opt_mirror, "Write refs suitable for a mirror", NULL }, - { "subpath", 0, 0, G_OPTION_ARG_STRING, &opt_subpath, "Only pull the provided subpath", NULL }, + { "subpath", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_subpaths, "Only pull the provided subpath(s)", NULL }, { "untrusted", 0, 0, G_OPTION_ARG_NONE, &opt_untrusted, "Do not trust (local) sources", NULL }, { "dry-run", 0, 0, G_OPTION_ARG_NONE, &opt_dry_run, "Only print information on what will be downloaded (requires static deltas)", NULL }, { "depth", 0, 0, G_OPTION_ARG_INT, &opt_depth, "Traverse DEPTH parents (-1=infinite) (default: 0)", "DEPTH" }, @@ -216,9 +216,17 @@ ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError ** if (opt_url) g_variant_builder_add (&builder, "{s@v}", "override-url", g_variant_new_variant (g_variant_new_string (opt_url))); - if (opt_subpath) - g_variant_builder_add (&builder, "{s@v}", "subdir", - g_variant_new_variant (g_variant_new_string (opt_subpath))); + if (opt_subpaths && opt_subpaths[0] != NULL) + { + /* Special case the one-element case so that we excercise this + old single-argument version in the tests */ + if (opt_subpaths[1] == NULL) + g_variant_builder_add (&builder, "{s@v}", "subdir", + g_variant_new_variant (g_variant_new_string (opt_subpaths[0]))); + else + g_variant_builder_add (&builder, "{s@v}", "subdirs", + g_variant_new_variant (g_variant_new_strv ((const char *const*) opt_subpaths, -1))); + } g_variant_builder_add (&builder, "{s@v}", "flags", g_variant_new_variant (g_variant_new_int32 (pullflags))); if (refs_to_fetch) diff --git a/tests/test-pull-subpath.sh b/tests/test-pull-subpath.sh index 05f685b9..09145f09 100755 --- a/tests/test-pull-subpath.sh +++ b/tests/test-pull-subpath.sh @@ -36,9 +36,10 @@ mkdir repo ${CMD_PREFIX} ostree --repo=repo init ${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin ${remoteurl} -${CMD_PREFIX} ostree --repo=repo pull --subpath=/baz origin main +${CMD_PREFIX} ostree --repo=repo pull --subpath=/baz/deeper --subpath=/baz/another origin main -${CMD_PREFIX} ostree --repo=repo ls origin:main /baz +${CMD_PREFIX} ostree --repo=repo ls origin:main /baz/deeper +${CMD_PREFIX} ostree --repo=repo ls origin:main /baz/another if ${CMD_PREFIX} ostree --repo=repo ls origin:main /firstfile 2>err.txt; then assert_not_reached fi