From 247866a9bca3557fb4a22a772a034eb89682b6a3 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 29 Jan 2015 21:23:05 -0500 Subject: [PATCH] Add an internal API to stream content objects For future delta work where we do more interesting things than just "tar of new objects", this lays the groundwork for doing streaming writes into content objects. It's also more efficient, as we avoid many intermediate allocations and virtual calls. Just a single `g_output_stream_write_all` for the splice case. Conflicts: src/libostree/ostree-repo-private.h src/libostree/ostree-repo-static-delta-processing.c --- src/libostree/ostree-repo-commit.c | 178 ++++++++++++++++++++-------- src/libostree/ostree-repo-private.h | 26 ++++ 2 files changed, 155 insertions(+), 49 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index c6ac5728..054ed659 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -74,7 +74,9 @@ _ostree_repo_get_tmpobject_path (OstreeRepo *repo, } static GVariant * -create_file_metadata (GFileInfo *file_info, +create_file_metadata (guint32 uid, + guint32 gid, + guint32 mode, GVariant *xattrs) { GVariant *ret_metadata = NULL; @@ -84,9 +86,9 @@ create_file_metadata (GFileInfo *file_info, tmp_xattrs = g_variant_ref_sink (g_variant_new_array (G_VARIANT_TYPE ("(ayay)"), NULL, 0)); ret_metadata = g_variant_new ("(uuu@a(ayay))", - GUINT32_TO_BE (g_file_info_get_attribute_uint32 (file_info, "unix::uid")), - GUINT32_TO_BE (g_file_info_get_attribute_uint32 (file_info, "unix::gid")), - GUINT32_TO_BE (g_file_info_get_attribute_uint32 (file_info, "unix::mode")), + GUINT32_TO_BE (uid), + GUINT32_TO_BE (gid), + GUINT32_TO_BE (mode), xattrs ? xattrs : tmp_xattrs); g_variant_ref_sink (ret_metadata); @@ -95,14 +97,16 @@ create_file_metadata (GFileInfo *file_info, static gboolean write_file_metadata_to_xattr (int fd, - GFileInfo *file_info, + guint32 uid, + guint32 gid, + guint32 mode, GVariant *xattrs, GError **error) { gs_unref_variant GVariant *filemeta = NULL; int res; - filemeta = create_file_metadata (file_info, xattrs); + filemeta = create_file_metadata (uid, gid, mode, xattrs); do res = fsetxattr (fd, "user.ostreemeta", @@ -125,12 +129,13 @@ commit_loose_object_trusted (OstreeRepo *self, const char *checksum, OstreeObjectType objtype, const char *loose_path, - GFile *temp_file, const char *temp_filename, gboolean object_is_symlink, - GFileInfo *file_info, + guint32 uid, + guint32 gid, + guint32 mode, GVariant *xattrs, - GOutputStream *temp_out, + int fd, GCancellable *cancellable, GError **error) { @@ -162,8 +167,7 @@ commit_loose_object_trusted (OstreeRepo *self, * as regular files. */ if (G_UNLIKELY (fchownat (self->tmp_dir_fd, temp_filename, - g_file_info_get_attribute_uint32 (file_info, "unix::uid"), - g_file_info_get_attribute_uint32 (file_info, "unix::gid"), + uid, gid, AT_SYMLINK_NOFOLLOW) == -1)) { gs_set_error_from_errno (error, errno); @@ -179,22 +183,13 @@ commit_loose_object_trusted (OstreeRepo *self, } else { - int fd; int res; struct timespec times[2]; - g_assert (temp_out != NULL); - - fd = g_file_descriptor_based_get_fd ((GFileDescriptorBased*)temp_out); - if (objtype == OSTREE_OBJECT_TYPE_FILE && self->mode == OSTREE_REPO_MODE_BARE) { - g_assert (file_info != NULL); - do - res = fchown (fd, - g_file_info_get_attribute_uint32 (file_info, "unix::uid"), - g_file_info_get_attribute_uint32 (file_info, "unix::gid")); + res = fchown (fd, uid, gid); while (G_UNLIKELY (res == -1 && errno == EINTR)); if (G_UNLIKELY (res == -1)) { @@ -203,7 +198,7 @@ commit_loose_object_trusted (OstreeRepo *self, } do - res = fchmod (fd, g_file_info_get_attribute_uint32 (file_info, "unix::mode")); + res = fchmod (fd, mode); while (G_UNLIKELY (res == -1 && errno == EINTR)); if (G_UNLIKELY (res == -1)) { @@ -220,9 +215,7 @@ commit_loose_object_trusted (OstreeRepo *self, if (objtype == OSTREE_OBJECT_TYPE_FILE && self->mode == OSTREE_REPO_MODE_BARE_USER) { - g_assert (file_info != NULL); - - if (!write_file_metadata_to_xattr (fd, file_info, xattrs, error)) + if (!write_file_metadata_to_xattr (fd, uid, gid, mode, xattrs, error)) goto out; if (!object_is_symlink) @@ -232,7 +225,7 @@ commit_loose_object_trusted (OstreeRepo *self, checkout. To make this work we apply all user bits and the read bits for group/other */ do - res = fchmod (fd, g_file_info_get_attribute_uint32 (file_info, "unix::mode") | 0744); + res = fchmod (fd, mode | 0744); while (G_UNLIKELY (res == -1 && errno == EINTR)); if (G_UNLIKELY (res == -1)) { @@ -274,9 +267,6 @@ commit_loose_object_trusted (OstreeRepo *self, goto out; } } - - if (!g_output_stream_close (temp_out, cancellable, error)) - goto out; } if (!_ostree_repo_ensure_loose_objdir_at (self->objects_dir_fd, loose_path, @@ -467,6 +457,85 @@ fallocate_stream (GFileDescriptorBased *stream, return ret; } +gboolean +_ostree_repo_open_trusted_content_bare (OstreeRepo *self, + const char *checksum, + guint64 content_len, + OstreeRepoTrustedContentBareCommit *out_state, + GOutputStream **out_stream, + gboolean *out_have_object, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + gs_free char *temp_filename = NULL; + gs_unref_object GOutputStream *ret_stream = NULL; + gboolean have_obj; + char loose_objpath[_OSTREE_LOOSE_PATH_MAX]; + + if (!_ostree_repo_has_loose_object (self, checksum, OSTREE_OBJECT_TYPE_FILE, + &have_obj, loose_objpath, + NULL, + cancellable, error)) + goto out; + + if (!have_obj) + { + if (!gs_file_open_in_tmpdir_at (self->tmp_dir_fd, 0644, &temp_filename, &ret_stream, + cancellable, error)) + goto out; + + if (!fallocate_stream ((GFileDescriptorBased*)ret_stream, content_len, + cancellable, error)) + goto out; + } + + ret = TRUE; + if (!have_obj) + { + out_state->temp_filename = temp_filename; + temp_filename = NULL; + out_state->fd = g_file_descriptor_based_get_fd ((GFileDescriptorBased*)ret_stream); + gs_transfer_out_value (out_stream, &ret_stream); + } + *out_have_object = have_obj; + out: + return ret; +} + +gboolean +_ostree_repo_commit_trusted_content_bare (OstreeRepo *self, + const char *checksum, + OstreeRepoTrustedContentBareCommit *state, + guint32 uid, + guint32 gid, + guint32 mode, + GVariant *xattrs, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + char loose_objpath[_OSTREE_LOOSE_PATH_MAX]; + + if (state->fd != -1) + { + _ostree_loose_path (loose_objpath, checksum, OSTREE_OBJECT_TYPE_FILE, self->mode); + + if (!commit_loose_object_trusted (self, checksum, OSTREE_OBJECT_TYPE_FILE, + loose_objpath, + state->temp_filename, + FALSE, uid, gid, mode, + xattrs, state->fd, + cancellable, error)) + goto out; + } + + ret = TRUE; + out: + g_free (state->temp_filename); + return ret; +} + static gboolean write_object (OstreeRepo *self, OstreeObjectType objtype, @@ -482,7 +551,6 @@ write_object (OstreeRepo *self, gboolean do_commit; OstreeRepoMode repo_mode; gs_free char *temp_filename = NULL; - gs_unref_object GFile *temp_file = NULL; gs_unref_object GFile *stored_path = NULL; gs_free guchar *ret_csum = NULL; gs_unref_object OstreeChecksumInputStream *checksum_input = NULL; @@ -583,7 +651,6 @@ write_object (OstreeRepo *self, cancellable, error)) goto out; - temp_file = g_file_get_child (self->tmp_dir, temp_filename); if (g_output_stream_splice (temp_out, file_input, 0, cancellable, error) < 0) goto out; @@ -595,7 +662,6 @@ write_object (OstreeRepo *self, &temp_filename, cancellable, error)) goto out; - temp_file = g_file_get_child (self->tmp_dir, temp_filename); } else if (repo_mode == OSTREE_REPO_MODE_ARCHIVE_Z2) { @@ -610,7 +676,6 @@ write_object (OstreeRepo *self, &temp_filename, &temp_out, cancellable, error)) goto out; - temp_file = g_file_get_child (self->tmp_dir, temp_filename); temp_file_is_regular = TRUE; file_meta = _ostree_zlib_file_header_new (file_info, xattrs); @@ -645,7 +710,6 @@ write_object (OstreeRepo *self, cancellable, error)) goto out; - temp_file = g_file_get_child (self->tmp_dir, temp_filename); if (g_output_stream_splice (temp_out, checksum_input ? (GInputStream*)checksum_input : input, 0, cancellable, error) < 0) @@ -676,16 +740,17 @@ write_object (OstreeRepo *self, g_assert (actual_checksum != NULL); /* Pacify static analysis */ - if (indexable) + if (indexable && temp_file_is_regular) { - gsize archived_size; - gs_unref_object GFileInfo *compressed_info = - g_file_query_info (temp_file, G_FILE_ATTRIBUTE_STANDARD_SIZE, 0, - cancellable, error); - if (!compressed_info) - goto out; - archived_size = g_file_info_get_size (compressed_info); - repo_store_size_entry (self, actual_checksum, unpacked_size, archived_size); + struct stat stbuf; + + if (fstatat (self->tmp_dir_fd, temp_filename, &stbuf, AT_SYMLINK_NOFOLLOW) == -1) + { + gs_set_error_from_errno (error, errno); + goto out; + } + + repo_store_size_entry (self, actual_checksum, unpacked_size, stbuf.st_size); } if (!_ostree_repo_has_loose_object (self, actual_checksum, objtype, @@ -697,11 +762,27 @@ write_object (OstreeRepo *self, if (do_commit) { - if (!commit_loose_object_trusted (self, actual_checksum, - objtype, loose_objpath, - temp_file, temp_filename, - object_is_symlink, file_info, - xattrs, temp_out, + guint32 uid, gid, mode; + int fd = -1; + + if (file_info) + { + uid = g_file_info_get_attribute_uint32 (file_info, "unix::uid"); + gid = g_file_info_get_attribute_uint32 (file_info, "unix::gid"); + mode = g_file_info_get_attribute_uint32 (file_info, "unix::mode"); + } + else + uid = gid = mode = 0; + + if (temp_out) + fd = g_file_descriptor_based_get_fd ((GFileDescriptorBased*)temp_out); + + if (!commit_loose_object_trusted (self, actual_checksum, objtype, + loose_objpath, + temp_filename, + object_is_symlink, + uid, gid, mode, + xattrs, fd, cancellable, error)) goto out; @@ -720,7 +801,6 @@ write_object (OstreeRepo *self, } g_clear_pointer (&temp_filename, g_free); - g_clear_object (&temp_file); } g_mutex_lock (&self->txn_stats_lock); diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index 82d35343..a19108f1 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -194,4 +194,30 @@ _ostree_repo_gpg_verify_file_with_metadata (OstreeRepo *self, GCancellable *cancellable, GError **error); +typedef struct { + int fd; + char *temp_filename; +} OstreeRepoTrustedContentBareCommit; + +gboolean +_ostree_repo_open_trusted_content_bare (OstreeRepo *self, + const char *checksum, + guint64 content_len, + OstreeRepoTrustedContentBareCommit *out_state, + GOutputStream **out_stream, + gboolean *out_have_object, + GCancellable *cancellable, + GError **error); + +gboolean +_ostree_repo_commit_trusted_content_bare (OstreeRepo *self, + const char *checksum, + OstreeRepoTrustedContentBareCommit *state, + guint32 uid, + guint32 gid, + guint32 mode, + GVariant *xattrs, + GCancellable *cancellable, + GError **error); + G_END_DECLS