packaging
/
ostree
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

core: Optimize commits to raw repositories 

Avoid creating a temporary intermediate file in the case where we have
the total object length available.
This commit is contained in:
Colin Walters 2012-05-04 06:15:47 -04:00
parent 67f61976f4
commit 7ce587da30
7 changed files with 208 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

92
src/libostree/ostree-core.c
View File

@ -340,6 +340,7 @@ ostree_raw_file_to_content_stream (GInputStream *input,
GFileInfo *file_info, GFileInfo *file_info,
GVariant *xattrs, GVariant *xattrs,
GInputStream **out_input, GInputStream **out_input,
guint64 *out_length,
GCancellable *cancellable, GCancellable *cancellable,
GError **error) GError **error)
{ {
@ -377,21 +378,23 @@ ostree_raw_file_to_content_stream (GInputStream *input,
ret = TRUE; ret = TRUE;
ot_transfer_out_value (out_input, &ret_input); ot_transfer_out_value (out_input, &ret_input);
if (out_length)
*out_length = header_size + g_file_info_get_size (file_info);
out: out:
return ret; return ret;
} }
gboolean gboolean
ostree_content_file_parse (GFile *content_path, ostree_content_stream_parse (GInputStream *input,
gboolean trusted, guint64 input_length,
GInputStream **out_input, gboolean trusted,
GFileInfo **out_file_info, GInputStream **out_input,
GVariant **out_xattrs, GFileInfo **out_file_info,
GCancellable *cancellable, GVariant **out_xattrs,
GError **error) GCancellable *cancellable,
GError **error)
{ {
gboolean ret = FALSE; gboolean ret = FALSE;
guint64 length;
guint32 archive_header_size; guint32 archive_header_size;
guchar dummy[4]; guchar dummy[4];
gsize bytes_read; gsize bytes_read;
@ -402,39 +405,27 @@ ostree_content_file_parse (GFile *content_path,
ot_lvariant GVariant *file_header = NULL; ot_lvariant GVariant *file_header = NULL;
ot_lfree guchar *buf = NULL; ot_lfree guchar *buf = NULL;
ret_input = (GInputStream*)g_file_read (content_path, cancellable, error); if (!g_input_stream_read_all (input,
if (!ret_input)
goto out;
content_file_info = g_file_input_stream_query_info ((GFileInputStream*)ret_input,
OSTREE_GIO_FAST_QUERYINFO,
cancellable, error);
if (!content_file_info)
goto out;
length = g_file_info_get_size (content_file_info);
if (!g_input_stream_read_all (ret_input,
&archive_header_size, 4, &bytes_read, &archive_header_size, 4, &bytes_read,
cancellable, error)) cancellable, error))
goto out; goto out;
archive_header_size = GUINT32_FROM_BE (archive_header_size); archive_header_size = GUINT32_FROM_BE (archive_header_size);
if (archive_header_size > length) if (archive_header_size > input_length)
{ {
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"File header size %u exceeds size %" G_GUINT64_FORMAT, "File header size %u exceeds size %" G_GUINT64_FORMAT,
(guint)archive_header_size, length); (guint)archive_header_size, input_length);
goto out; goto out;
} }
/* Skip over padding */ /* Skip over padding */
if (!g_input_stream_read_all (ret_input, if (!g_input_stream_read_all (input,
dummy, 4, &bytes_read, dummy, 4, &bytes_read,
cancellable, error)) cancellable, error))
goto out; goto out;
buf = g_malloc (archive_header_size); buf = g_malloc (archive_header_size);
if (!g_input_stream_read_all (ret_input, buf, archive_header_size, &bytes_read, if (!g_input_stream_read_all (input, buf, archive_header_size, &bytes_read,
cancellable, error)) cancellable, error))
goto out; goto out;
file_header = g_variant_new_from_data (OSTREE_FILE_HEADER_GVARIANT_FORMAT, file_header = g_variant_new_from_data (OSTREE_FILE_HEADER_GVARIANT_FORMAT,
@ -448,14 +439,61 @@ ostree_content_file_parse (GFile *content_path,
error)) error))
goto out; goto out;
if (ret_file_info) if (ret_file_info)
g_file_info_set_size (ret_file_info, length - archive_header_size - 8); g_file_info_set_size (ret_file_info, input_length - archive_header_size - 8);
if (g_file_info_get_file_type (ret_file_info) != G_FILE_TYPE_REGULAR) if (g_file_info_get_file_type (ret_file_info) != G_FILE_TYPE_REGULAR)
{ {
g_clear_object (&ret_input); g_clear_object (&ret_input);
} }
/* Now give the input stream at its current position as return value */ ret = TRUE;
/* Give the input stream at its current position as return value;
* assuming the caller doesn't seek, this should be fine. We might
* want to wrap it though in a non-seekable stream.
**/
g_object_ref (input);
ot_transfer_out_value (out_input, &input);
ot_transfer_out_value (out_file_info, &ret_file_info);
ot_transfer_out_value (out_xattrs, &ret_xattrs);
out:
return ret;
}
gboolean
ostree_content_file_parse (GFile *content_path,
gboolean trusted,
GInputStream **out_input,
GFileInfo **out_file_info,
GVariant **out_xattrs,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
guint64 length;
ot_lobj GInputStream *file_input = NULL;
ot_lobj GInputStream *ret_input = NULL;
ot_lobj GFileInfo *content_file_info = NULL;
ot_lobj GFileInfo *ret_file_info = NULL;
ot_lvariant GVariant *ret_xattrs = NULL;
file_input = (GInputStream*)g_file_read (content_path, cancellable, error);
if (!file_input)
goto out;
content_file_info = g_file_input_stream_query_info ((GFileInputStream*)file_input,
OSTREE_GIO_FAST_QUERYINFO,
cancellable, error);
if (!content_file_info)
goto out;
length = g_file_info_get_size (content_file_info);
if (!ostree_content_stream_parse (file_input, length, trusted,
out_input ? &ret_input : NULL,
&ret_file_info, &ret_xattrs,
cancellable, error))
goto out;
ret = TRUE; ret = TRUE;
ot_transfer_out_value (out_input, &ret_input); ot_transfer_out_value (out_input, &ret_input);

10
src/libostree/ostree-core.h
View File

@ -207,6 +207,15 @@ gboolean ostree_file_header_parse (GVariant *data,
GFileInfo **out_file_info, GFileInfo **out_file_info,
GVariant **out_xattrs, GVariant **out_xattrs,
GError **error); GError **error);
gboolean
ostree_content_stream_parse (GInputStream *input,
guint64 input_length,
gboolean trusted,
GInputStream **out_input,
GFileInfo **out_file_info,
GVariant **out_xattrs,
GCancellable *cancellable,
GError **error);
gboolean ostree_content_file_parse (GFile *content_path, gboolean ostree_content_file_parse (GFile *content_path,
gboolean trusted, gboolean trusted,
@ -226,6 +235,7 @@ gboolean ostree_raw_file_to_content_stream (GInputStream *input,
GFileInfo *file_info, GFileInfo *file_info,
GVariant *xattrs, GVariant *xattrs,
GInputStream **out_input, GInputStream **out_input,
guint64 *out_length,
GCancellable *cancellable, GCancellable *cancellable,
GError **error); GError **error);

146
src/libostree/ostree-repo.c
View File

@ -780,16 +780,6 @@ ostree_repo_get_file_object_path (OstreeRepo *self,
return ostree_repo_get_object_path (self, checksum, OSTREE_OBJECT_TYPE_FILE); return ostree_repo_get_object_path (self, checksum, OSTREE_OBJECT_TYPE_FILE);
} }
static gboolean
stage_object_impl (OstreeRepo *self,
OstreeObjectType objtype,
gboolean store_if_packed,
GInputStream *input,
const char *expected_checksum,
guchar **out_csum,
GCancellable *cancellable,
GError **error);
static gboolean static gboolean
commit_loose_object_trusted (OstreeRepo *self, commit_loose_object_trusted (OstreeRepo *self,
const char *checksum, const char *checksum,
@ -826,11 +816,17 @@ commit_loose_object_trusted (OstreeRepo *self,
return ret; return ret;
} }
typedef enum {
OSTREE_REPO_STAGE_FLAGS_STORE_IF_PACKED = (1<<0),
OSTREE_REPO_STAGE_FLAGS_LENGTH_VALID = (1<<1)
} OstreeRepoStageFlags;
static gboolean static gboolean
stage_object_impl (OstreeRepo *self, stage_object_impl (OstreeRepo *self,
OstreeRepoStageFlags flags,
OstreeObjectType objtype, OstreeObjectType objtype,
gboolean store_if_packed,
GInputStream *input, GInputStream *input,
guint64 file_object_length,
const char *expected_checksum, const char *expected_checksum,
guchar **out_csum, guchar **out_csum,
GCancellable *cancellable, GCancellable *cancellable,
@ -860,7 +856,7 @@ stage_object_impl (OstreeRepo *self,
if (expected_checksum) if (expected_checksum)
{ {
if (!store_if_packed) if (!(flags & OSTREE_REPO_STAGE_FLAGS_STORE_IF_PACKED))
{ {
if (!repo_find_object (self, objtype, expected_checksum, if (!repo_find_object (self, objtype, expected_checksum,
&stored_path, &pack_checksum, &pack_offset, &stored_path, &pack_checksum, &pack_offset,
@ -879,6 +875,7 @@ stage_object_impl (OstreeRepo *self,
if (stored_path == NULL && pack_checksum == NULL) if (stored_path == NULL && pack_checksum == NULL)
{ {
ot_lvariant GVariant *file_header = NULL; ot_lvariant GVariant *file_header = NULL;
gboolean do_stage_bare_file;
if (out_csum) if (out_csum)
{ {
@ -887,14 +884,39 @@ stage_object_impl (OstreeRepo *self,
checksum_input = ostree_checksum_input_stream_new (input, checksum); checksum_input = ostree_checksum_input_stream_new (input, checksum);
} }
if (!ostree_create_temp_file_from_input (priv->tmp_dir, do_stage_bare_file = (flags & OSTREE_REPO_STAGE_FLAGS_LENGTH_VALID)
ostree_object_type_to_string (objtype), NULL, && objtype == OSTREE_OBJECT_TYPE_FILE
NULL, NULL, && priv->mode == OSTREE_REPO_MODE_BARE;
checksum_input ? (GInputStream*)checksum_input : input, if (do_stage_bare_file)
&temp_file, {
cancellable, error)) ot_lobj GInputStream *file_input = NULL;
goto out; ot_lobj GFileInfo *file_info = NULL;
ot_lvariant GVariant *xattrs = NULL;
if (!ostree_content_stream_parse (checksum_input ? (GInputStream*)checksum_input : input,
file_object_length, FALSE,
&file_input, &file_info, &xattrs,
cancellable, error))
goto out;
if (!ostree_create_temp_file_from_input (priv->tmp_dir,
ostree_object_type_to_string (objtype), NULL,
file_info, xattrs, file_input,
&temp_file,
cancellable, error))
goto out;
}
else
{
if (!ostree_create_temp_file_from_input (priv->tmp_dir,
ostree_object_type_to_string (objtype), NULL,
NULL, NULL,
checksum_input ? (GInputStream*)checksum_input : input,
&temp_file,
cancellable, error))
goto out;
}
if (!checksum) if (!checksum)
actual_checksum = expected_checksum; actual_checksum = expected_checksum;
else else
@ -911,7 +933,7 @@ stage_object_impl (OstreeRepo *self,
} }
if (!store_if_packed) if (!(flags & OSTREE_REPO_STAGE_FLAGS_STORE_IF_PACKED))
{ {
gboolean have_obj; gboolean have_obj;
@ -926,7 +948,9 @@ stage_object_impl (OstreeRepo *self,
if (do_commit) if (do_commit)
{ {
if (objtype == OSTREE_OBJECT_TYPE_FILE && priv->mode == OSTREE_REPO_MODE_BARE) /* Only do this if we *didn't* stage a bare file above */
if (!do_stage_bare_file
&& objtype == OSTREE_OBJECT_TYPE_FILE && priv->mode == OSTREE_REPO_MODE_BARE)
{ {
ot_lobj GInputStream *file_input = NULL; ot_lobj GInputStream *file_input = NULL;
ot_lobj GFileInfo *file_info = NULL; ot_lobj GFileInfo *file_info = NULL;
@ -1039,14 +1063,12 @@ stage_metadata_object (OstreeRepo *self,
GError **error) GError **error)
{ {
gboolean ret = FALSE; gboolean ret = FALSE;
ot_lobj GInputStream *mem = NULL; ot_lobj GInputStream *input = NULL;
mem = g_memory_input_stream_new_from_data (g_variant_get_data (variant), input = ot_variant_read (variant);
g_variant_get_size (variant),
NULL);
if (!stage_object_impl (self, type, FALSE, mem, if (!stage_object_impl (self, 0, type, input, 0, NULL,
NULL, out_csum, cancellable, error)) out_csum, cancellable, error))
goto out; goto out;
ret = TRUE; ret = TRUE;
@ -1104,8 +1126,11 @@ ostree_repo_stage_object_trusted (OstreeRepo *self,
GCancellable *cancellable, GCancellable *cancellable,
GError **error) GError **error)
{ {
return stage_object_impl (self, objtype, store_if_packed, int flags = 0;
object_input, checksum, NULL, if (store_if_packed)
flags |= OSTREE_REPO_STAGE_FLAGS_STORE_IF_PACKED;
return stage_object_impl (self, flags, objtype,
object_input, 0, checksum, NULL,
cancellable, error); cancellable, error);
} }
@ -1120,8 +1145,47 @@ ostree_repo_stage_object (OstreeRepo *self,
gboolean ret = FALSE; gboolean ret = FALSE;
ot_lfree guchar *actual_csum = NULL; ot_lfree guchar *actual_csum = NULL;
if (!stage_object_impl (self, objtype, FALSE, if (!stage_object_impl (self, 0, objtype,
object_input, expected_checksum, &actual_csum, object_input, 0, expected_checksum, &actual_csum,
cancellable, error))
goto out;
ret = TRUE;
out:
return ret;
}
gboolean
ostree_repo_stage_file_object_trusted (OstreeRepo *self,
const char *checksum,
gboolean store_if_packed,
GInputStream *object_input,
guint64 length,
GCancellable *cancellable,
GError **error)
{
int flags = OSTREE_REPO_STAGE_FLAGS_LENGTH_VALID;
if (store_if_packed)
flags |= OSTREE_REPO_STAGE_FLAGS_STORE_IF_PACKED;
return stage_object_impl (self, flags, OSTREE_OBJECT_TYPE_FILE,
object_input, length, checksum, NULL,
cancellable, error);
}
gboolean
ostree_repo_stage_file_object (OstreeRepo *self,
const char *expected_checksum,
GInputStream *object_input,
guint64 length,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
int flags = OSTREE_REPO_STAGE_FLAGS_LENGTH_VALID;
ot_lfree guchar *actual_csum = NULL;
if (!stage_object_impl (self, flags, OSTREE_OBJECT_TYPE_FILE,
object_input, length, expected_checksum, &actual_csum,
cancellable, error)) cancellable, error))
goto out; goto out;
@ -2416,6 +2480,7 @@ stage_directory_to_mtree_internal (OstreeRepo *self,
else else
{ {
ot_lobj GInputStream *file_object_input = NULL; ot_lobj GInputStream *file_object_input = NULL;
guint64 file_obj_length;
g_clear_object (&file_input); g_clear_object (&file_input);
if (g_file_info_get_file_type (modified_info) == G_FILE_TYPE_REGULAR) if (g_file_info_get_file_type (modified_info) == G_FILE_TYPE_REGULAR)
@ -2435,12 +2500,14 @@ stage_directory_to_mtree_internal (OstreeRepo *self,
g_free (child_file_csum); g_free (child_file_csum);
child_file_csum = NULL; child_file_csum = NULL;
if (!ostree_raw_file_to_content_stream (file_input, modified_info, xattrs, if (!ostree_raw_file_to_content_stream (file_input,
&file_object_input, cancellable, error)) modified_info, xattrs,
&file_object_input, &file_obj_length,
cancellable, error))
goto out; goto out;
if (!stage_object_impl (self, OSTREE_OBJECT_TYPE_FILE, FALSE, if (!stage_object_impl (self, OSTREE_REPO_STAGE_FLAGS_LENGTH_VALID,
file_object_input, NULL, OSTREE_OBJECT_TYPE_FILE, file_object_input, file_obj_length,
&child_file_csum, cancellable, error)) NULL, &child_file_csum, cancellable, error))
goto out; goto out;
g_free (tmp_checksum); g_free (tmp_checksum);
@ -2619,6 +2686,7 @@ import_libarchive_entry_file (OstreeRepo *self,
gboolean ret = FALSE; gboolean ret = FALSE;
ot_lobj GInputStream *file_object_input = NULL; ot_lobj GInputStream *file_object_input = NULL;
ot_lobj GInputStream *archive_stream = NULL; ot_lobj GInputStream *archive_stream = NULL;
guint64 length;
if (g_cancellable_set_error_if_cancelled (cancellable, error)) if (g_cancellable_set_error_if_cancelled (cancellable, error))
return FALSE; return FALSE;
@ -2627,11 +2695,11 @@ import_libarchive_entry_file (OstreeRepo *self,
archive_stream = ostree_libarchive_input_stream_new (a); archive_stream = ostree_libarchive_input_stream_new (a);
if (!ostree_raw_file_to_content_stream (archive_stream, file_info, NULL, if (!ostree_raw_file_to_content_stream (archive_stream, file_info, NULL,
&file_object_input, cancellable, error)) &file_object_input, &length, cancellable, error))
goto out; goto out;
if (!stage_object_impl (self, OSTREE_OBJECT_TYPE_FILE, FALSE, if (!stage_object_impl (self, OSTREE_REPO_STAGE_FLAGS_LENGTH_VALID, OSTREE_OBJECT_TYPE_FILE,
file_object_input, NULL, out_csum, file_object_input, length, NULL, out_csum,
cancellable, error)) cancellable, error))
goto out; goto out;

15
src/libostree/ostree-repo.h
View File

@ -106,6 +106,13 @@ gboolean ostree_repo_stage_object (OstreeRepo *self,
GCancellable *cancellable, GCancellable *cancellable,
GError **error); GError **error);
gboolean ostree_repo_stage_file_object (OstreeRepo *self,
const char *expected_checksum,
GInputStream *content,
guint64 content_length,
GCancellable *cancellable,
GError **error);
gboolean ostree_repo_stage_object_trusted (OstreeRepo *self, gboolean ostree_repo_stage_object_trusted (OstreeRepo *self,
OstreeObjectType objtype, OstreeObjectType objtype,
const char *checksum, const char *checksum,
@ -114,6 +121,14 @@ gboolean ostree_repo_stage_object_trusted (OstreeRepo *self,
GCancellable *cancellable, GCancellable *cancellable,
GError **error); GError **error);
gboolean ostree_repo_stage_file_object_trusted (OstreeRepo *self,
const char *checksum,
gboolean store_if_packed,
GInputStream *content,
guint64 content_length,
GCancellable *cancellable,
GError **error);
gboolean ostree_repo_resolve_rev (OstreeRepo *self, gboolean ostree_repo_resolve_rev (OstreeRepo *self,
const char *rev, const char *rev,
gboolean allow_noent, gboolean allow_noent,

2
src/ostree/ostree-pull.c
View File

@ -843,7 +843,7 @@ store_file_from_pack (OtPullData *pull_data,
goto out; goto out;
if (!ostree_raw_file_to_content_stream (input, file_info, xattrs, if (!ostree_raw_file_to_content_stream (input, file_info, xattrs,
&file_object_input, cancellable, error)) &file_object_input, NULL, cancellable, error))
goto out; goto out;
if (!ostree_repo_stage_object (pull_data->repo, OSTREE_OBJECT_TYPE_FILE, checksum, if (!ostree_repo_stage_object (pull_data->repo, OSTREE_OBJECT_TYPE_FILE, checksum,

10
src/ostree/ot-builtin-pull-local.c
View File

@ -55,19 +55,21 @@ import_one_object (OtLocalCloneData *data,
if (objtype == OSTREE_OBJECT_TYPE_FILE) if (objtype == OSTREE_OBJECT_TYPE_FILE)
{ {
ot_lobj GInputStream *file_object = NULL; ot_lobj GInputStream *file_object = NULL;
guint64 length;
if (!ostree_repo_load_file (data->src_repo, checksum, if (!ostree_repo_load_file (data->src_repo, checksum,
&input, &file_info, &xattrs, &input, &file_info, &xattrs,
cancellable, error)) cancellable, error))
goto out; goto out;
if (!ostree_raw_file_to_content_stream (input, file_info, xattrs, &file_object, if (!ostree_raw_file_to_content_stream (input, file_info, xattrs,
&file_object, &length,
cancellable, error)) cancellable, error))
goto out; goto out;
if (!ostree_repo_stage_object_trusted (data->dest_repo, OSTREE_OBJECT_TYPE_FILE, if (!ostree_repo_stage_file_object_trusted (data->dest_repo, checksum, FALSE,
checksum, FALSE, file_object, file_object, length,
cancellable, error)) cancellable, error))
goto out; goto out;
} }
else else

8
src/ostree/ot-builtin-unpack.c
View File

@ -105,19 +105,19 @@ unpack_one_object (OstreeRepo *repo,
if (objtype == OSTREE_OBJECT_TYPE_FILE) if (objtype == OSTREE_OBJECT_TYPE_FILE)
{ {
ot_lobj GInputStream *file_object = NULL; ot_lobj GInputStream *file_object = NULL;
guint64 length;
if (!ostree_repo_load_file (repo, checksum, if (!ostree_repo_load_file (repo, checksum,
&input, &file_info, &xattrs, &input, &file_info, &xattrs,
cancellable, error)) cancellable, error))
goto out; goto out;
if (!ostree_raw_file_to_content_stream (input, file_info, xattrs, &file_object, if (!ostree_raw_file_to_content_stream (input, file_info, xattrs, &file_object, &length,
cancellable, error)) cancellable, error))
goto out; goto out;
if (!ostree_repo_stage_object_trusted (repo, OSTREE_OBJECT_TYPE_FILE, if (!ostree_repo_stage_file_object_trusted (repo, checksum, TRUE, file_object, length,
checksum, TRUE, file_object, cancellable, error))
cancellable, error))
goto out; goto out;
} }
else else