packaging
/
ostree
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

lib/repo: Add a new private API for bare content writes 

This lowers into the commit core what the static delta code
was doing, and improves the API.

The bigger picture issue is that for writing large files, our current "pull" API
where the caller provides a `GInputStream` is very awkward in some scenarios.
For example, we have a whole "libarchive input stream" that is a ~200 line
GObject that boils down to wrapping `archive_read_data()`.

This came more to a head when I was working on rpm-ostree jigdo since I had to
copy that object.

One step we can take after this is to further split `write_content_object()`
into a "write symlink or archive object" versus "write bare content object"
(it already has a mess of conditionals) and teach the latter case to call
this.

The eventual goal here is to make this API public.

Closes: #1355
Approved by: jlebon
This commit is contained in:
Colin Walters 2017-10-13 21:01:20 -04:00 committed by Atomic Bot
parent bd6a15e7a3
commit 17308e2149
3 changed files with 159 additions and 131 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

128
src/libostree/ostree-repo-commit.c
View File

@ -405,46 +405,116 @@ add_size_index_to_metadata (OstreeRepo *self,
return g_variant_ref_sink (g_variant_builder_end (builder));
}
typedef struct {
gboolean initialized;
GLnxTmpfile tmpf;
char *expected_checksum;
OtChecksum checksum;
guint64 content_len;
guint64 bytes_written;
guint uid;
guint gid;
guint mode;
GVariant *xattrs;
} OstreeRealRepoBareContent;
G_STATIC_ASSERT (sizeof (OstreeRepoBareContent) >= sizeof (OstreeRealRepoBareContent));
/* Create a tmpfile for writing a bare file. Currently just used
* by the static delta code, but will likely later be extended
* to be used also by the dfd_iter commit path.
*/
gboolean
_ostree_repo_open_content_bare (OstreeRepo *self,
const char *checksum,
guint64 content_len,
GLnxTmpfile *out_tmpf,
GCancellable *cancellable,
GError **error)
_ostree_repo_bare_content_open (OstreeRepo *self,
const char *expected_checksum,
guint64 content_len,
guint uid,
guint gid,
guint mode,
GVariant *xattrs,
OstreeRepoBareContent *out_regwrite,
GCancellable *cancellable,
GError **error)
{
return glnx_open_tmpfile_linkable_at (self->tmp_dir_fd, ".", O_WRONLY|O_CLOEXEC,
out_tmpf, error);
OstreeRealRepoBareContent *real = (OstreeRealRepoBareContent*) out_regwrite;
g_assert (!real->initialized);
real->initialized = TRUE;
g_assert (S_ISREG (mode));
if (!glnx_open_tmpfile_linkable_at (self->tmp_dir_fd, ".", O_WRONLY|O_CLOEXEC,
&real->tmpf, error))
return FALSE;
ot_checksum_init (&real->checksum);
real->expected_checksum = g_strdup (expected_checksum);
real->content_len = content_len;
real->bytes_written = 0;
real->uid = uid;
real->gid = gid;
real->mode = mode;
real->xattrs = xattrs ? g_variant_ref (xattrs) : NULL;
/* Initialize the checksum with the header info */
g_autoptr(GFileInfo) finfo = _ostree_mode_uidgid_to_gfileinfo (mode, uid, gid);
g_autoptr(GBytes) header = _ostree_file_header_new (finfo, xattrs);
ot_checksum_update_bytes (&real->checksum, header);
return TRUE;
}
/* Used by static deltas, which have a separate "push" flow for
* regfile objects distinct from the "pull" model used by
* write_content_object().
*/
gboolean
_ostree_repo_commit_trusted_content_bare (OstreeRepo *self,
const char *checksum,
GLnxTmpfile *tmpf,
guint32 uid,
guint32 gid,
guint32 mode,
GVariant *xattrs,
GCancellable *cancellable,
GError **error)
_ostree_repo_bare_content_write (OstreeRepo *repo,
OstreeRepoBareContent *barewrite,
const guint8 *buf,
size_t len,
GCancellable *cancellable,
GError **error)
{
/* I don't think this is necessary, but a similar check was here previously,
* keeping it for extra redundancy.
*/
if (!tmpf->initialized || tmpf->fd == -1)
return TRUE;
OstreeRealRepoBareContent *real = (OstreeRealRepoBareContent*) barewrite;
g_assert (real->initialized);
ot_checksum_update (&real->checksum, buf, len);
if (glnx_loop_write (real->tmpf.fd, buf, len) < 0)
return glnx_throw_errno_prefix (error, "write");
return TRUE;
}
return commit_loose_regfile_object (self, checksum,
tmpf, uid, gid, mode, xattrs,
cancellable, error);
gboolean
_ostree_repo_bare_content_commit (OstreeRepo *self,
OstreeRepoBareContent *barewrite,
char *checksum_buf,
size_t buflen,
GCancellable *cancellable,
GError **error)
{
OstreeRealRepoBareContent *real = (OstreeRealRepoBareContent*) barewrite;
g_assert (real->initialized);
ot_checksum_get_hexdigest (&real->checksum, checksum_buf, buflen);
if (real->expected_checksum &&
!_ostree_compare_object_checksum (OSTREE_OBJECT_TYPE_FILE,
real->expected_checksum, checksum_buf,
error))
return FALSE;
if (!commit_loose_regfile_object (self, checksum_buf,
&real->tmpf, real->uid, real->gid,
real->mode, real->xattrs,
cancellable, error))
return FALSE;
/* Let's have a guarantee that after commit the object is cleaned up */
_ostree_repo_bare_content_cleanup (barewrite);
return TRUE;
}
void
_ostree_repo_bare_content_cleanup (OstreeRepoBareContent *regwrite)
{
OstreeRealRepoBareContent *real = (OstreeRealRepoBareContent*) regwrite;
if (!real->initialized)
return;
glnx_tmpfile_clear (&real->tmpf);
ot_checksum_clear (&real->checksum);
g_clear_pointer (&real->expected_checksum, (GDestroyNotify)g_free);
g_clear_pointer (&real->xattrs, (GDestroyNotify)g_variant_unref);
real->initialized = FALSE;
}
/* Allocate an O_TMPFILE, write everything from @input to it, but

43
src/libostree/ostree-repo-private.h
View File

@ -365,24 +365,41 @@ _ostree_repo_commit_tmpf_final (OstreeRepo *self,
GCancellable *cancellable,
GError **error);
typedef struct {
gboolean initialized;
gpointer opaque0[10];
guint opaque1[10];
} OstreeRepoBareContent;
void _ostree_repo_bare_content_cleanup (OstreeRepoBareContent *regwrite);
G_DEFINE_AUTO_CLEANUP_CLEAR_FUNC(OstreeRepoBareContent, _ostree_repo_bare_content_cleanup)
gboolean
_ostree_repo_open_content_bare (OstreeRepo *self,
const char *checksum,
guint64 content_len,
GLnxTmpfile *out_tmpf,
_ostree_repo_bare_content_open (OstreeRepo *self,
const char *checksum,
guint64 content_len,
guint uid,
guint gid,
guint mode,
GVariant *xattrs,
OstreeRepoBareContent *out_regwrite,
GCancellable *cancellable,
GError **error);
gboolean
_ostree_repo_commit_trusted_content_bare (OstreeRepo *self,
const char *checksum,
GLnxTmpfile *tmpf,
guint32 uid,
guint32 gid,
guint32 mode,
GVariant *xattrs,
GCancellable *cancellable,
GError **error);
_ostree_repo_bare_content_write (OstreeRepo *repo,
OstreeRepoBareContent *barewrite,
const guint8 *buf,
size_t len,
GCancellable *cancellable,
GError **error);
gboolean
_ostree_repo_bare_content_commit (OstreeRepo *self,
OstreeRepoBareContent *barewrite,
char *checksum_buf,
size_t buflen,
GCancellable *cancellable,
GError **error);
gboolean
_ostree_repo_load_file_bare (OstreeRepo *self,

119
src/libostree/ostree-repo-static-delta-processing.c
View File

@ -55,11 +55,9 @@ typedef struct {
GError **async_error;
OstreeObjectType output_objtype;
GLnxTmpfile tmpf;
guint64 content_size;
GOutputStream *content_out;
OtChecksum content_checksum;
char checksum[OSTREE_SHA256_STRING_LEN+1];
OstreeRepoBareContent content_out;
char *read_source_object;
int read_source_fd;
gboolean have_obj;
@ -278,9 +276,7 @@ _ostree_static_delta_part_execute (OstreeRepo *repo,
ret = TRUE;
out:
glnx_tmpfile_clear (&state->tmpf);
g_clear_object (&state->content_out);
ot_checksum_clear (&state->content_checksum);
_ostree_repo_bare_content_cleanup (&state->content_out);
return ret;
}
@ -378,29 +374,6 @@ validate_ofs (StaticDeltaExecutionState *state,
return TRUE;
}
static gboolean
content_out_write (OstreeRepo *repo,
StaticDeltaExecutionState *state,
const guint8* buf,
gsize len,
GCancellable *cancellable,
GError **error)
{
gsize bytes_written;
if (state->content_checksum.initialized)
ot_checksum_update (&state->content_checksum, buf, len);
/* Ignore bytes_written since we discard partial content */
if (!g_output_stream_write_all (state->content_out,
buf, len,
&bytes_written,
cancellable, error))
return FALSE;
return TRUE;
}
static gboolean
do_content_open_generic (OstreeRepo *repo,
StaticDeltaExecutionState *state,
@ -485,33 +458,15 @@ dispatch_bspatch (OstreeRepo *repo,
&stream) < 0)
return FALSE;
if (!content_out_write (repo, state, buf, state->content_size,
cancellable, error))
if (!_ostree_repo_bare_content_write (repo, &state->content_out,
buf, state->content_size,
cancellable, error))
return FALSE;
}
return TRUE;
}
/* Before, we had a distinction between "trusted" and "untrusted" deltas
* which we've decided wasn't a good idea. Now, we always checksum the content.
* Compare with what ostree_checksum_file_from_input() is doing too.
*/
static gboolean
handle_untrusted_content_checksum (OstreeRepo *repo,
StaticDeltaExecutionState *state,
GCancellable *cancellable,
GError **error)
{
g_autoptr(GFileInfo) finfo = _ostree_mode_uidgid_to_gfileinfo (state->mode, state->uid, state->gid);
g_autoptr(GBytes) header = _ostree_file_header_new (finfo, state->xattrs);
ot_checksum_init (&state->content_checksum);
ot_checksum_update_bytes (&state->content_checksum, header);
return TRUE;
}
static gboolean
dispatch_open_splice_and_close (OstreeRepo *repo,
StaticDeltaExecutionState *state,
@ -589,20 +544,18 @@ dispatch_open_splice_and_close (OstreeRepo *repo,
if (!state->have_obj)
{
if (!_ostree_repo_open_content_bare (repo, state->checksum,
if (!_ostree_repo_bare_content_open (repo, state->checksum,
state->content_size,
&state->tmpf,
state->uid, state->gid, state->mode,
state->xattrs,
&state->content_out,
cancellable, error))
goto out;
state->content_out = g_unix_output_stream_new (state->tmpf.fd, FALSE);
if (!handle_untrusted_content_checksum (repo, state, cancellable, error))
goto out;
if (!content_out_write (repo, state,
state->payload_data + content_offset,
state->content_size,
cancellable, error))
if (!_ostree_repo_bare_content_write (repo, &state->content_out,
state->payload_data + content_offset,
state->content_size,
cancellable, error))
goto out;
}
}
@ -691,17 +644,15 @@ dispatch_open (OstreeRepo *repo,
if (!state->have_obj)
{
if (!_ostree_repo_open_content_bare (repo, state->checksum,
if (!_ostree_repo_bare_content_open (repo, state->checksum,
state->content_size,
&state->tmpf,
state->uid, state->gid, state->mode,
state->xattrs,
&state->content_out,
cancellable, error))
return FALSE;
state->content_out = g_unix_output_stream_new (state->tmpf.fd, FALSE);
}
if (!handle_untrusted_content_checksum (repo, state, cancellable, error))
return FALSE;
return TRUE;
}
@ -740,8 +691,9 @@ dispatch_write (OstreeRepo *repo,
if (G_UNLIKELY (bytes_read == 0))
return glnx_throw (error, "Unexpected EOF reading object %s", state->read_source_object);
if (!content_out_write (repo, state, (guint8*)buf, bytes_read,
cancellable, error))
if (!_ostree_repo_bare_content_write (repo, &state->content_out,
(guint8*)buf, bytes_read,
cancellable, error))
return FALSE;
content_size -= bytes_read;
@ -753,8 +705,9 @@ dispatch_write (OstreeRepo *repo,
if (!validate_ofs (state, content_offset, content_size, error))
return FALSE;
if (!content_out_write (repo, state, state->payload_data + content_offset, content_size,
cancellable, error))
if (!_ostree_repo_bare_content_write (repo, &state->content_out,
state->payload_data + content_offset, content_size,
cancellable, error))
return FALSE;
}
}
@ -818,34 +771,22 @@ dispatch_close (OstreeRepo *repo,
{
GLNX_AUTO_PREFIX_ERROR("opcode close", error);
if (state->content_out)
if (state->content_out.initialized)
{
if (!g_output_stream_flush (state->content_out, cancellable, error))
char actual_checksum[OSTREE_SHA256_STRING_LEN+1];
if (!_ostree_repo_bare_content_commit (repo, &state->content_out, actual_checksum,
sizeof (actual_checksum),
cancellable, error))
return FALSE;
if (state->content_checksum.initialized)
{
char actual_checksum[OSTREE_SHA256_STRING_LEN+1];
ot_checksum_get_hexdigest (&state->content_checksum, actual_checksum, sizeof (actual_checksum));
if (strcmp (actual_checksum, state->checksum) != 0)
return glnx_throw (error, "Corrupted object %s (actual checksum is %s)",
state->checksum, actual_checksum);
}
if (!_ostree_repo_commit_trusted_content_bare (repo, state->checksum, &state->tmpf,
state->uid, state->gid, state->mode,
state->xattrs,
cancellable, error))
return FALSE;
g_clear_object (&state->content_out);
g_assert_cmpstr (state->checksum, ==, actual_checksum);
}
if (!dispatch_unset_read_source (repo, state, cancellable, error))
return FALSE;
g_clear_pointer (&state->xattrs, g_variant_unref);
ot_checksum_clear (&state->content_checksum);
_ostree_repo_bare_content_cleanup (&state->content_out);
state->checksum_index++;
state->output_target = NULL;