packaging
/
ostree
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add an internal API to stream content objects 

For future delta work where we do more interesting things than just
"tar of new objects", this lays the groundwork for doing streaming
writes into content objects.

It's also more efficient, as we avoid many intermediate allocations
and virtual calls.  Just a single `g_output_stream_write_all` for the
splice case.

Conflicts:
	src/libostree/ostree-repo-private.h
	src/libostree/ostree-repo-static-delta-processing.c
This commit is contained in:
Colin Walters 2015-01-29 21:23:05 -05:00
parent 65afe1110d
commit 247866a9bc
2 changed files with 155 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

178
src/libostree/ostree-repo-commit.c
View File

@ -74,7 +74,9 @@ _ostree_repo_get_tmpobject_path (OstreeRepo *repo,
}
static GVariant *
create_file_metadata (GFileInfo *file_info,
create_file_metadata (guint32 uid,
guint32 gid,
guint32 mode,
GVariant *xattrs)
{
GVariant *ret_metadata = NULL;
@ -84,9 +86,9 @@ create_file_metadata (GFileInfo *file_info,
tmp_xattrs = g_variant_ref_sink (g_variant_new_array (G_VARIANT_TYPE ("(ayay)"), NULL, 0));
ret_metadata = g_variant_new ("(uuu@a(ayay))",
GUINT32_TO_BE (g_file_info_get_attribute_uint32 (file_info, "unix::uid")),
GUINT32_TO_BE (g_file_info_get_attribute_uint32 (file_info, "unix::gid")),
GUINT32_TO_BE (g_file_info_get_attribute_uint32 (file_info, "unix::mode")),
GUINT32_TO_BE (uid),
GUINT32_TO_BE (gid),
GUINT32_TO_BE (mode),
xattrs ? xattrs : tmp_xattrs);
g_variant_ref_sink (ret_metadata);
@ -95,14 +97,16 @@ create_file_metadata (GFileInfo *file_info,
static gboolean
write_file_metadata_to_xattr (int fd,
GFileInfo *file_info,
guint32 uid,
guint32 gid,
guint32 mode,
GVariant *xattrs,
GError **error)
{
gs_unref_variant GVariant *filemeta = NULL;
int res;
filemeta = create_file_metadata (file_info, xattrs);
filemeta = create_file_metadata (uid, gid, mode, xattrs);
do
res = fsetxattr (fd, "user.ostreemeta",
@ -125,12 +129,13 @@ commit_loose_object_trusted (OstreeRepo *self,
const char *checksum,
OstreeObjectType objtype,
const char *loose_path,
GFile *temp_file,
const char *temp_filename,
gboolean object_is_symlink,
GFileInfo *file_info,
guint32 uid,
guint32 gid,
guint32 mode,
GVariant *xattrs,
GOutputStream *temp_out,
int fd,
GCancellable *cancellable,
GError **error)
{
@ -162,8 +167,7 @@ commit_loose_object_trusted (OstreeRepo *self,
* as regular files.
*/
if (G_UNLIKELY (fchownat (self->tmp_dir_fd, temp_filename,
g_file_info_get_attribute_uint32 (file_info, "unix::uid"),
g_file_info_get_attribute_uint32 (file_info, "unix::gid"),
uid, gid,
AT_SYMLINK_NOFOLLOW) == -1))
{
gs_set_error_from_errno (error, errno);
@ -179,22 +183,13 @@ commit_loose_object_trusted (OstreeRepo *self,
}
else
{
int fd;
int res;
struct timespec times[2];
g_assert (temp_out != NULL);
fd = g_file_descriptor_based_get_fd ((GFileDescriptorBased*)temp_out);
if (objtype == OSTREE_OBJECT_TYPE_FILE && self->mode == OSTREE_REPO_MODE_BARE)
{
g_assert (file_info != NULL);
do
res = fchown (fd,
g_file_info_get_attribute_uint32 (file_info, "unix::uid"),
g_file_info_get_attribute_uint32 (file_info, "unix::gid"));
res = fchown (fd, uid, gid);
while (G_UNLIKELY (res == -1 && errno == EINTR));
if (G_UNLIKELY (res == -1))
{
@ -203,7 +198,7 @@ commit_loose_object_trusted (OstreeRepo *self,
}
do
res = fchmod (fd, g_file_info_get_attribute_uint32 (file_info, "unix::mode"));
res = fchmod (fd, mode);
while (G_UNLIKELY (res == -1 && errno == EINTR));
if (G_UNLIKELY (res == -1))
{
@ -220,9 +215,7 @@ commit_loose_object_trusted (OstreeRepo *self,
if (objtype == OSTREE_OBJECT_TYPE_FILE && self->mode == OSTREE_REPO_MODE_BARE_USER)
{
g_assert (file_info != NULL);
if (!write_file_metadata_to_xattr (fd, file_info, xattrs, error))
if (!write_file_metadata_to_xattr (fd, uid, gid, mode, xattrs, error))
goto out;
if (!object_is_symlink)
@ -232,7 +225,7 @@ commit_loose_object_trusted (OstreeRepo *self,
checkout. To make this work we apply all user bits and the read bits for
group/other */
do
res = fchmod (fd, g_file_info_get_attribute_uint32 (file_info, "unix::mode") | 0744);
res = fchmod (fd, mode | 0744);
while (G_UNLIKELY (res == -1 && errno == EINTR));
if (G_UNLIKELY (res == -1))
{
@ -274,9 +267,6 @@ commit_loose_object_trusted (OstreeRepo *self,
goto out;
}
}
if (!g_output_stream_close (temp_out, cancellable, error))
goto out;
}
if (!_ostree_repo_ensure_loose_objdir_at (self->objects_dir_fd, loose_path,
@ -467,6 +457,85 @@ fallocate_stream (GFileDescriptorBased *stream,
return ret;
}
gboolean
_ostree_repo_open_trusted_content_bare (OstreeRepo *self,
const char *checksum,
guint64 content_len,
OstreeRepoTrustedContentBareCommit *out_state,
GOutputStream **out_stream,
gboolean *out_have_object,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
gs_free char *temp_filename = NULL;
gs_unref_object GOutputStream *ret_stream = NULL;
gboolean have_obj;
char loose_objpath[_OSTREE_LOOSE_PATH_MAX];
if (!_ostree_repo_has_loose_object (self, checksum, OSTREE_OBJECT_TYPE_FILE,
&have_obj, loose_objpath,
NULL,
cancellable, error))
goto out;
if (!have_obj)
{
if (!gs_file_open_in_tmpdir_at (self->tmp_dir_fd, 0644, &temp_filename, &ret_stream,
cancellable, error))
goto out;
if (!fallocate_stream ((GFileDescriptorBased*)ret_stream, content_len,
cancellable, error))
goto out;
}
ret = TRUE;
if (!have_obj)
{
out_state->temp_filename = temp_filename;
temp_filename = NULL;
out_state->fd = g_file_descriptor_based_get_fd ((GFileDescriptorBased*)ret_stream);
gs_transfer_out_value (out_stream, &ret_stream);
}
*out_have_object = have_obj;
out:
return ret;
}
gboolean
_ostree_repo_commit_trusted_content_bare (OstreeRepo *self,
const char *checksum,
OstreeRepoTrustedContentBareCommit *state,
guint32 uid,
guint32 gid,
guint32 mode,
GVariant *xattrs,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
char loose_objpath[_OSTREE_LOOSE_PATH_MAX];
if (state->fd != -1)
{
_ostree_loose_path (loose_objpath, checksum, OSTREE_OBJECT_TYPE_FILE, self->mode);
if (!commit_loose_object_trusted (self, checksum, OSTREE_OBJECT_TYPE_FILE,
loose_objpath,
state->temp_filename,
FALSE, uid, gid, mode,
xattrs, state->fd,
cancellable, error))
goto out;
}
ret = TRUE;
out:
g_free (state->temp_filename);
return ret;
}
static gboolean
write_object (OstreeRepo *self,
OstreeObjectType objtype,
@ -482,7 +551,6 @@ write_object (OstreeRepo *self,
gboolean do_commit;
OstreeRepoMode repo_mode;
gs_free char *temp_filename = NULL;
gs_unref_object GFile *temp_file = NULL;
gs_unref_object GFile *stored_path = NULL;
gs_free guchar *ret_csum = NULL;
gs_unref_object OstreeChecksumInputStream *checksum_input = NULL;
@ -583,7 +651,6 @@ write_object (OstreeRepo *self,
cancellable, error))
goto out;
temp_file = g_file_get_child (self->tmp_dir, temp_filename);
if (g_output_stream_splice (temp_out, file_input, 0,
cancellable, error) < 0)
goto out;
@ -595,7 +662,6 @@ write_object (OstreeRepo *self,
&temp_filename,
cancellable, error))
goto out;
temp_file = g_file_get_child (self->tmp_dir, temp_filename);
}
else if (repo_mode == OSTREE_REPO_MODE_ARCHIVE_Z2)
{
@ -610,7 +676,6 @@ write_object (OstreeRepo *self,
&temp_filename, &temp_out,
cancellable, error))
goto out;
temp_file = g_file_get_child (self->tmp_dir, temp_filename);
temp_file_is_regular = TRUE;
file_meta = _ostree_zlib_file_header_new (file_info, xattrs);
@ -645,7 +710,6 @@ write_object (OstreeRepo *self,
cancellable, error))
goto out;
temp_file = g_file_get_child (self->tmp_dir, temp_filename);
if (g_output_stream_splice (temp_out, checksum_input ? (GInputStream*)checksum_input : input,
0,
cancellable, error) < 0)
@ -676,16 +740,17 @@ write_object (OstreeRepo *self,
g_assert (actual_checksum != NULL); /* Pacify static analysis */
if (indexable)
if (indexable && temp_file_is_regular)
{
gsize archived_size;
gs_unref_object GFileInfo *compressed_info =
g_file_query_info (temp_file, G_FILE_ATTRIBUTE_STANDARD_SIZE, 0,
cancellable, error);
if (!compressed_info)
goto out;
archived_size = g_file_info_get_size (compressed_info);
repo_store_size_entry (self, actual_checksum, unpacked_size, archived_size);
struct stat stbuf;
if (fstatat (self->tmp_dir_fd, temp_filename, &stbuf, AT_SYMLINK_NOFOLLOW) == -1)
{
gs_set_error_from_errno (error, errno);
goto out;
}
repo_store_size_entry (self, actual_checksum, unpacked_size, stbuf.st_size);
}
if (!_ostree_repo_has_loose_object (self, actual_checksum, objtype,
@ -697,11 +762,27 @@ write_object (OstreeRepo *self,
if (do_commit)
{
if (!commit_loose_object_trusted (self, actual_checksum,
objtype, loose_objpath,
temp_file, temp_filename,
object_is_symlink, file_info,
xattrs, temp_out,
guint32 uid, gid, mode;
int fd = -1;
if (file_info)
{
uid = g_file_info_get_attribute_uint32 (file_info, "unix::uid");
gid = g_file_info_get_attribute_uint32 (file_info, "unix::gid");
mode = g_file_info_get_attribute_uint32 (file_info, "unix::mode");
}
else
uid = gid = mode = 0;
if (temp_out)
fd = g_file_descriptor_based_get_fd ((GFileDescriptorBased*)temp_out);
if (!commit_loose_object_trusted (self, actual_checksum, objtype,
loose_objpath,
temp_filename,
object_is_symlink,
uid, gid, mode,
xattrs, fd,
cancellable, error))
goto out;
@ -720,7 +801,6 @@ write_object (OstreeRepo *self,
}
g_clear_pointer (&temp_filename, g_free);
g_clear_object (&temp_file);
}
g_mutex_lock (&self->txn_stats_lock);

26
src/libostree/ostree-repo-private.h
View File

@ -194,4 +194,30 @@ _ostree_repo_gpg_verify_file_with_metadata (OstreeRepo *self,
GCancellable *cancellable,
GError **error);
typedef struct {
int fd;
char *temp_filename;
} OstreeRepoTrustedContentBareCommit;
gboolean
_ostree_repo_open_trusted_content_bare (OstreeRepo *self,
const char *checksum,
guint64 content_len,
OstreeRepoTrustedContentBareCommit *out_state,
GOutputStream **out_stream,
gboolean *out_have_object,
GCancellable *cancellable,
GError **error);
gboolean
_ostree_repo_commit_trusted_content_bare (OstreeRepo *self,
const char *checksum,
OstreeRepoTrustedContentBareCommit *state,
guint32 uid,
guint32 gid,
guint32 mode,
GVariant *xattrs,
GCancellable *cancellable,
GError **error);
G_END_DECLS