packaging
/
ostree
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

core: Drop "staged" state for objects, refactor pull to split metadata/data 

Previously we had the "staged" state to ensure we didn't add a commit
object without the associated dirtree, etc.  However it's
easier/better to just ensure in the pull command that we have all
referenced objects.

Also change pull to download metadata first.  This will allow adding
a progress bar later.
This commit is contained in:
Colin Walters 2012-04-02 19:52:22 -04:00
parent 56089abd43
commit 9c7a47434d
4 changed files with 148 additions and 325 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/libostree/README.md
View File

@ -44,6 +44,8 @@ MILESTONE 2
----------- -----------
* Store checksums as ay * Store checksums as ay
* Drop version/metadata from tree/dirmeta objects * Drop version/metadata from tree/dirmeta objects
* Add index size to superindex, pack size to index
- So pull can calculate how much we need to download
* Split pack files into metadata/data * Split pack files into metadata/data
* Restructure repository so that links can be generated as a cache; * Restructure repository so that links can be generated as a cache;
i.e. objects/raw, pack files are now the canonical i.e. objects/raw, pack files are now the canonical

167
src/libostree/ostree-repo.c
View File

@ -72,8 +72,6 @@ struct _OstreeRepoPrivate {
GHashTable *pack_index_mappings; GHashTable *pack_index_mappings;
GHashTable *pack_data_mappings; GHashTable *pack_data_mappings;
GHashTable *pending_transaction;
}; };
static void static void
@ -93,7 +91,6 @@ ostree_repo_finalize (GObject *object)
g_clear_object (&priv->config_file); g_clear_object (&priv->config_file);
g_hash_table_destroy (priv->pack_index_mappings); g_hash_table_destroy (priv->pack_index_mappings);
g_hash_table_destroy (priv->pack_data_mappings); g_hash_table_destroy (priv->pack_data_mappings);
g_hash_table_destroy (priv->pending_transaction);
if (priv->config) if (priv->config)
g_key_file_free (priv->config); g_key_file_free (priv->config);
@ -202,9 +199,6 @@ ostree_repo_init (OstreeRepo *self)
priv->pack_data_mappings = g_hash_table_new_full (g_str_hash, g_str_equal, priv->pack_data_mappings = g_hash_table_new_full (g_str_hash, g_str_equal,
g_free, g_free,
(GDestroyNotify)g_mapped_file_unref); (GDestroyNotify)g_mapped_file_unref);
priv->pending_transaction = g_hash_table_new_full (g_str_hash, g_str_equal,
g_free,
NULL);
} }
OstreeRepo* OstreeRepo*
@ -768,22 +762,6 @@ ostree_repo_get_file_object_path (OstreeRepo *self,
return ostree_repo_get_object_path (self, checksum, get_objtype_for_repo_file (self)); return ostree_repo_get_object_path (self, checksum, get_objtype_for_repo_file (self));
} }
static GFile *
get_pending_object_path (OstreeRepo *self,
const char *checksum,
OstreeObjectType objtype)
{
OstreeRepoPrivate *priv = GET_PRIVATE (self);
char *relpath;
GFile *ret;
relpath = ostree_get_relative_object_path (checksum, objtype);
ret = g_file_resolve_relative_path (priv->pending_dir, relpath);
g_free (relpath);
return ret;
}
static GFileInfo * static GFileInfo *
dup_file_info_owned_by_me (GFileInfo *file_info) dup_file_info_owned_by_me (GFileInfo *file_info)
{ {
@ -807,52 +785,40 @@ stage_object_impl (OstreeRepo *self,
GCancellable *cancellable, GCancellable *cancellable,
GError **error); GError **error);
static void
insert_into_transaction (OstreeRepo *self,
const char *checksum,
OstreeObjectType objtype)
{
OstreeRepoPrivate *priv = GET_PRIVATE (self);
char *key;
key = ostree_object_to_string (checksum, objtype);
/* Takes ownership */
g_hash_table_replace (priv->pending_transaction, key, NULL);
}
static gboolean static gboolean
stage_tmpfile_trusted (OstreeRepo *self, commit_tmpfile_trusted (OstreeRepo *self,
const char *checksum, const char *checksum,
OstreeObjectType objtype, OstreeObjectType objtype,
GFile *tempfile_path, GFile *tempfile_path,
GCancellable *cancellable,
GError **error) GError **error)
{ {
gboolean ret = FALSE; gboolean ret = FALSE;
GFile *pending_path = NULL; GFile *dest_file = NULL;
GFile *checksum_dir = NULL; GFile *checksum_dir = NULL;
pending_path = get_pending_object_path (self, checksum, objtype); dest_file = ostree_repo_get_object_path (self, checksum, objtype);
checksum_dir = g_file_get_parent (pending_path); checksum_dir = g_file_get_parent (dest_file);
if (!ot_gfile_ensure_directory (checksum_dir, TRUE, error)) if (!ot_gfile_ensure_directory (checksum_dir, FALSE, error))
goto out; goto out;
if (link (ot_gfile_get_path_cached (tempfile_path), ot_gfile_get_path_cached (pending_path)) < 0) if (link (ot_gfile_get_path_cached (tempfile_path), ot_gfile_get_path_cached (dest_file)) < 0)
{ {
if (errno != EEXIST) if (errno != EEXIST)
{ {
ot_util_set_error_from_errno (error, errno); ot_util_set_error_from_errno (error, errno);
g_prefix_error (error, "Storing file '%s': ",
ot_gfile_get_path_cached (dest_file));
goto out; goto out;
} }
} }
insert_into_transaction (self, checksum, objtype);
(void) unlink (ot_gfile_get_path_cached (tempfile_path)); (void) unlink (ot_gfile_get_path_cached (tempfile_path));
ret = TRUE; ret = TRUE;
out: out:
g_clear_object (&pending_path); g_clear_object (&dest_file);
g_clear_object (&checksum_dir); g_clear_object (&checksum_dir);
return ret; return ret;
} }
@ -933,12 +899,12 @@ impl_stage_archive_file_object_from_raw (OstreeRepo *self,
else else
actual_checksum = g_checksum_get_string (ret_checksum); actual_checksum = g_checksum_get_string (ret_checksum);
if (!stage_tmpfile_trusted (self, actual_checksum, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT, if (!commit_tmpfile_trusted (self, actual_checksum, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT,
content_temp_file, error)) content_temp_file, cancellable, error))
goto out; goto out;
if (!stage_tmpfile_trusted (self, actual_checksum, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META, if (!commit_tmpfile_trusted (self, actual_checksum, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META,
meta_temp_file, error)) meta_temp_file, cancellable, error))
goto out; goto out;
ret = TRUE; ret = TRUE;
@ -972,7 +938,6 @@ stage_object_impl (OstreeRepo *self,
GFileInfo *temp_info = NULL; GFileInfo *temp_info = NULL;
GFile *temp_file = NULL; GFile *temp_file = NULL;
GFile *stored_path = NULL; GFile *stored_path = NULL;
GFile *pending_path = NULL;
char *pack_checksum = NULL; char *pack_checksum = NULL;
guint64 pack_offset; guint64 pack_offset;
const char *actual_checksum; const char *actual_checksum;
@ -989,16 +954,14 @@ stage_object_impl (OstreeRepo *self,
if (!store_if_packed) if (!store_if_packed)
{ {
if (!ostree_repo_find_object (self, objtype, expected_checksum, if (!ostree_repo_find_object (self, objtype, expected_checksum,
&stored_path, &pending_path, &stored_path, &pack_checksum, &pack_offset,
&pack_checksum, &pack_offset,
cancellable, error)) cancellable, error))
goto out; goto out;
} }
else else
{ {
if (!ostree_repo_find_object (self, objtype, expected_checksum, if (!ostree_repo_find_object (self, objtype, expected_checksum,
&stored_path, &pending_path, &stored_path, NULL, NULL,
NULL, NULL,
cancellable, error)) cancellable, error))
goto out; goto out;
} }
@ -1007,7 +970,7 @@ stage_object_impl (OstreeRepo *self,
g_assert (objtype != OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT); g_assert (objtype != OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT);
g_assert (objtype != OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META); g_assert (objtype != OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META);
if (stored_path == NULL && pending_path == NULL && pack_checksum == NULL) if (stored_path == NULL && pack_checksum == NULL)
{ {
if (objtype == OSTREE_OBJECT_TYPE_RAW_FILE) if (objtype == OSTREE_OBJECT_TYPE_RAW_FILE)
{ {
@ -1055,17 +1018,12 @@ stage_object_impl (OstreeRepo *self,
} }
} }
if (!stage_tmpfile_trusted (self, actual_checksum, objtype, if (!commit_tmpfile_trusted (self, actual_checksum, objtype,
temp_file, error)) temp_file, cancellable, error))
goto out; goto out;
g_clear_object (&temp_file); g_clear_object (&temp_file);
} }
} }
else if (pending_path)
{
g_assert (expected_checksum);
insert_into_transaction (self, expected_checksum, objtype);
}
else else
{ {
g_assert (stored_path != NULL || pack_checksum != NULL); g_assert (stored_path != NULL || pack_checksum != NULL);
@ -1080,50 +1038,11 @@ stage_object_impl (OstreeRepo *self,
g_clear_object (&temp_file); g_clear_object (&temp_file);
g_clear_object (&temp_info); g_clear_object (&temp_info);
g_clear_object (&stored_path); g_clear_object (&stored_path);
g_clear_object (&pending_path);
g_free (pack_checksum); g_free (pack_checksum);
ot_clear_checksum (&ret_checksum); ot_clear_checksum (&ret_checksum);
return ret; return ret;
} }
static gboolean
commit_staged_file (OstreeRepo *self,
GFile *file,
const char *checksum,
OstreeObjectType objtype,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
GFile *dest_file = NULL;
GFile *checksum_dir = NULL;
dest_file = ostree_repo_get_object_path (self, checksum, objtype);
checksum_dir = g_file_get_parent (dest_file);
if (!ot_gfile_ensure_directory (checksum_dir, FALSE, error))
goto out;
if (link (ot_gfile_get_path_cached (file), ot_gfile_get_path_cached (dest_file)) < 0)
{
if (errno != EEXIST)
{
ot_util_set_error_from_errno (error, errno);
g_prefix_error (error, "Storing file '%s': ",
ot_gfile_get_path_cached (file));
goto out;
}
}
(void) unlink (ot_gfile_get_path_cached (file));
ret = TRUE;
out:
g_clear_object (&dest_file);
g_clear_object (&checksum_dir);
return ret;
}
gboolean gboolean
ostree_repo_prepare_transaction (OstreeRepo *self, ostree_repo_prepare_transaction (OstreeRepo *self,
GCancellable *cancellable, GCancellable *cancellable,
@ -1148,34 +1067,13 @@ ostree_repo_commit_transaction (OstreeRepo *self,
{ {
gboolean ret = FALSE; gboolean ret = FALSE;
OstreeRepoPrivate *priv = GET_PRIVATE (self); OstreeRepoPrivate *priv = GET_PRIVATE (self);
GFile *f = NULL;
GHashTableIter iter;
gpointer key, value;
char *checksum = NULL;
OstreeObjectType objtype;
g_return_val_if_fail (priv->in_transaction == TRUE, FALSE); g_return_val_if_fail (priv->in_transaction == TRUE, FALSE);
g_hash_table_iter_init (&iter, priv->pending_transaction);
while (g_hash_table_iter_next (&iter, &key, &value))
{
g_free (checksum);
ostree_object_from_string ((char*)key, &checksum, &objtype);
g_clear_object (&f);
f = get_pending_object_path (self, checksum, objtype);
if (!commit_staged_file (self, f, checksum, objtype, cancellable, error))
goto out;
}
ret = TRUE; ret = TRUE;
out: /* out: */
priv->in_transaction = FALSE; priv->in_transaction = FALSE;
g_free (checksum);
g_hash_table_remove_all (priv->pending_transaction);
g_clear_object (&f);
return ret; return ret;
} }
@ -1187,8 +1085,6 @@ ostree_repo_abort_transaction (OstreeRepo *self,
gboolean ret = FALSE; gboolean ret = FALSE;
OstreeRepoPrivate *priv = GET_PRIVATE (self); OstreeRepoPrivate *priv = GET_PRIVATE (self);
/* For now, let's not delete pending files */
g_hash_table_remove_all (priv->pending_transaction);
priv->in_transaction = FALSE; priv->in_transaction = FALSE;
ret = TRUE; ret = TRUE;
@ -3219,7 +3115,7 @@ ostree_repo_load_file (OstreeRepo *self,
/* Blah, right now we need to look up the content too to get the file size */ /* Blah, right now we need to look up the content too to get the file size */
if (!ostree_repo_find_object (self, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT, if (!ostree_repo_find_object (self, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT,
checksum, &content_loose_path, NULL, checksum, &content_loose_path,
&content_pack_checksum, &content_pack_offset, &content_pack_checksum, &content_pack_offset,
cancellable, error)) cancellable, error))
goto out; goto out;
@ -3475,7 +3371,6 @@ ostree_repo_find_object (OstreeRepo *self,
OstreeObjectType objtype, OstreeObjectType objtype,
const char *checksum, const char *checksum,
GFile **out_stored_path, GFile **out_stored_path,
GFile **out_pending_path,
char **out_pack_checksum, char **out_pack_checksum,
guint64 *out_pack_offset, guint64 *out_pack_offset,
GCancellable *cancellable, GCancellable *cancellable,
@ -3484,7 +3379,6 @@ ostree_repo_find_object (OstreeRepo *self,
gboolean ret = FALSE; gboolean ret = FALSE;
GFile *object_path = NULL; GFile *object_path = NULL;
GFile *ret_stored_path = NULL; GFile *ret_stored_path = NULL;
GFile *ret_pending_path = NULL;
char *ret_pack_checksum = NULL; char *ret_pack_checksum = NULL;
guint64 ret_pack_offset = 0; guint64 ret_pack_offset = 0;
struct stat stbuf; struct stat stbuf;
@ -3499,15 +3393,6 @@ ostree_repo_find_object (OstreeRepo *self,
else else
{ {
g_clear_object (&object_path); g_clear_object (&object_path);
if (out_pending_path)
{
object_path = get_pending_object_path (self, checksum, objtype);
if (lstat (ot_gfile_get_path_cached (object_path), &stbuf) == 0)
{
ret_pending_path = object_path;
object_path = NULL;
}
}
} }
if (out_pack_checksum) if (out_pack_checksum)
@ -3520,14 +3405,12 @@ ostree_repo_find_object (OstreeRepo *self,
ret = TRUE; ret = TRUE;
ot_transfer_out_value (out_stored_path, &ret_stored_path); ot_transfer_out_value (out_stored_path, &ret_stored_path);
ot_transfer_out_value (out_pending_path, &ret_pending_path);
ot_transfer_out_value (out_pack_checksum, &ret_pack_checksum); ot_transfer_out_value (out_pack_checksum, &ret_pack_checksum);
if (out_pack_offset) if (out_pack_offset)
*out_pack_offset = ret_pack_offset; *out_pack_offset = ret_pack_offset;
out: out:
g_clear_object (&object_path); g_clear_object (&object_path);
g_clear_object (&ret_stored_path); g_clear_object (&ret_stored_path);
g_clear_object (&ret_pending_path);
g_free (ret_pack_checksum); g_free (ret_pack_checksum);
return ret; return ret;
} }
@ -3541,7 +3424,6 @@ ostree_repo_load_variant (OstreeRepo *self,
{ {
gboolean ret = FALSE; gboolean ret = FALSE;
GFile *object_path = NULL; GFile *object_path = NULL;
GFile *pending_path = NULL;
GVariant *packed_object = NULL; GVariant *packed_object = NULL;
GVariant *ret_variant = NULL; GVariant *ret_variant = NULL;
char *pack_checksum = NULL; char *pack_checksum = NULL;
@ -3552,16 +3434,15 @@ ostree_repo_load_variant (OstreeRepo *self,
g_return_val_if_fail (OSTREE_OBJECT_TYPE_IS_META (objtype), FALSE); g_return_val_if_fail (OSTREE_OBJECT_TYPE_IS_META (objtype), FALSE);
if (!ostree_repo_find_object (self, objtype, sha256, &object_path, &pending_path, if (!ostree_repo_find_object (self, objtype, sha256, &object_path,
&pack_checksum, &object_offset, &pack_checksum, &object_offset,
cancellable, error)) cancellable, error))
goto out; goto out;
/* Prefer loose metadata for now */ /* Prefer loose metadata for now */
if (object_path != NULL || pending_path != NULL) if (object_path != NULL)
{ {
if (!ostree_map_metadata_file (object_path ? object_path : pending_path, if (!ostree_map_metadata_file (object_path, objtype, &ret_variant, error))
objtype, &ret_variant, error))
goto out; goto out;
} }
else if (pack_checksum != NULL) else if (pack_checksum != NULL)

1
src/libostree/ostree-repo.h
View File

@ -96,7 +96,6 @@ gboolean ostree_repo_find_object (OstreeRepo *self,
OstreeObjectType objtype, OstreeObjectType objtype,
const char *checksum, const char *checksum,
GFile **out_stored_path, GFile **out_stored_path,
GFile **out_pending_path,
char **out_pack_checksum, char **out_pack_checksum,
guint64 *out_pack_offset, guint64 *out_pack_offset,
GCancellable *cancellable, GCancellable *cancellable,

203
src/ostree/ostree-pull.c
View File

@ -62,6 +62,8 @@ typedef struct {
gboolean fetched_packs; gboolean fetched_packs;
GPtrArray *cached_pack_indexes; GPtrArray *cached_pack_indexes;
GHashTable *file_checksums_to_fetch;
} OtPullData; } OtPullData;
static SoupURI * static SoupURI *
@ -102,7 +104,6 @@ suburi_new (SoupURI *base,
return ret; return ret;
} }
typedef struct { typedef struct {
SoupSession *session; SoupSession *session;
GOutputStream *stream; GOutputStream *stream;
@ -436,58 +437,6 @@ fetch_loose_object (OtPullData *pull_data,
return ret; return ret;
} }
static gboolean
find_object (OtPullData *pull_data,
const char *checksum,
OstreeObjectType objtype,
gboolean *out_is_stored,
gboolean *out_is_pending,
char **out_remote_pack_checksum,
guint64 *out_offset,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
gboolean ret_is_stored;
gboolean ret_is_pending;
GFile *stored_path = NULL;
GFile *pending_path = NULL;
char *local_pack_checksum = NULL;
char *ret_remote_pack_checksum = NULL;
guint64 offset;
if (!ostree_repo_find_object (pull_data->repo, objtype, checksum,
&stored_path, &pending_path,
&local_pack_checksum, NULL,
cancellable, error))
goto out;
ret_is_stored = (stored_path != NULL || local_pack_checksum != NULL);
ret_is_pending = pending_path != NULL;
if (!(ret_is_stored || ret_is_pending))
{
if (!find_object_in_remote_packs (pull_data, checksum, objtype,
&ret_remote_pack_checksum, &offset,
cancellable, error))
goto out;
}
ret = TRUE;
if (out_is_stored)
*out_is_stored = ret_is_stored;
if (out_is_pending)
*out_is_pending = ret_is_pending;
ot_transfer_out_value (out_remote_pack_checksum, &ret_remote_pack_checksum);
if (out_offset)
*out_offset = offset;
out:
g_free (local_pack_checksum);
g_free (ret_remote_pack_checksum);
g_clear_object (&stored_path);
return ret;
}
static void static void
unlink_file_on_unref (GFile *f) unlink_file_on_unref (GFile *f)
{ {
@ -499,31 +448,39 @@ static gboolean
fetch_object_if_not_stored (OtPullData *pull_data, fetch_object_if_not_stored (OtPullData *pull_data,
const char *checksum, const char *checksum,
OstreeObjectType objtype, OstreeObjectType objtype,
gboolean *out_is_stored,
gboolean *out_is_pending,
GInputStream **out_input, GInputStream **out_input,
GCancellable *cancellable, GCancellable *cancellable,
GError **error) GError **error)
{ {
gboolean ret = FALSE; gboolean ret = FALSE;
gboolean ret_is_stored = FALSE;
gboolean ret_is_pending = FALSE;
GInputStream *ret_input = NULL; GInputStream *ret_input = NULL;
GFile *temp_path = NULL; GFile *temp_path = NULL;
GFile *stored_path = NULL;
GFile *pack_path = NULL; GFile *pack_path = NULL;
GMappedFile *pack_map = NULL; GMappedFile *pack_map = NULL;
char *local_pack_checksum = NULL;
char *remote_pack_checksum = NULL; char *remote_pack_checksum = NULL;
guint64 pack_offset = 0; guint64 pack_offset = 0;
GVariant *pack_entry = NULL; GVariant *pack_entry = NULL;
gboolean is_stored;
if (!find_object (pull_data, checksum, objtype, &ret_is_stored, if (!ostree_repo_find_object (pull_data->repo, objtype, checksum,
&ret_is_pending, &remote_pack_checksum, &stored_path, &local_pack_checksum, NULL,
&pack_offset, cancellable, error)) cancellable, error))
goto out; goto out;
is_stored = (stored_path != NULL || local_pack_checksum != NULL);
if (!is_stored)
{
if (!find_object_in_remote_packs (pull_data, checksum, objtype,
&remote_pack_checksum, &pack_offset,
cancellable, error))
goto out;
}
if (remote_pack_checksum != NULL) if (remote_pack_checksum != NULL)
{ {
g_assert (!(ret_is_stored || ret_is_pending)); g_assert (!is_stored);
if (!fetch_one_pack_file (pull_data, remote_pack_checksum, &pack_path, if (!fetch_one_pack_file (pull_data, remote_pack_checksum, &pack_path,
cancellable, error)) cancellable, error))
@ -539,12 +496,13 @@ fetch_object_if_not_stored (OtPullData *pull_data,
cancellable, error)) cancellable, error))
goto out; goto out;
/* Kind of a hack... */
ret_input = ostree_read_pack_entry_as_stream (pack_entry); ret_input = ostree_read_pack_entry_as_stream (pack_entry);
g_object_set_data_full ((GObject*)ret_input, "ostree-pull-pack-map", g_object_set_data_full ((GObject*)ret_input, "ostree-pull-pack-map",
pack_map, (GDestroyNotify) g_mapped_file_unref); pack_map, (GDestroyNotify) g_mapped_file_unref);
pack_map = NULL; /* Transfer ownership */ pack_map = NULL; /* Transfer ownership */
} }
else if (!(ret_is_stored || ret_is_pending)) else if (!is_stored)
{ {
if (!fetch_loose_object (pull_data, checksum, objtype, &temp_path, cancellable, error)) if (!fetch_loose_object (pull_data, checksum, objtype, &temp_path, cancellable, error))
goto out; goto out;
@ -559,11 +517,9 @@ fetch_object_if_not_stored (OtPullData *pull_data,
ret = TRUE; ret = TRUE;
ot_transfer_out_value (out_input, &ret_input); ot_transfer_out_value (out_input, &ret_input);
if (out_is_stored)
*out_is_stored = ret_is_stored;
if (out_is_pending)
*out_is_pending = ret_is_pending;
out: out:
g_free (local_pack_checksum);
g_clear_object (&stored_path);
g_clear_object (&temp_path); g_clear_object (&temp_path);
g_clear_object (&pack_path); g_clear_object (&pack_path);
if (pack_map) if (pack_map)
@ -578,27 +534,20 @@ static gboolean
fetch_and_store_object (OtPullData *pull_data, fetch_and_store_object (OtPullData *pull_data,
const char *checksum, const char *checksum,
OstreeObjectType objtype, OstreeObjectType objtype,
gboolean *out_was_stored,
GCancellable *cancellable, GCancellable *cancellable,
GError **error) GError **error)
{ {
gboolean ret = FALSE; gboolean ret = FALSE;
GFileInfo *file_info = NULL; GFileInfo *file_info = NULL;
GInputStream *input = NULL; GInputStream *input = NULL;
GFile *stored_path = NULL;
GFile *pending_path = NULL;
char *pack_checksum = NULL;
gboolean is_stored;
gboolean is_pending;
g_assert (objtype != OSTREE_OBJECT_TYPE_RAW_FILE); g_assert (objtype != OSTREE_OBJECT_TYPE_RAW_FILE);
if (!fetch_object_if_not_stored (pull_data, checksum, objtype, if (!fetch_object_if_not_stored (pull_data, checksum, objtype, &input,
&is_stored, &is_pending, &input,
cancellable, error)) cancellable, error))
goto out; goto out;
if (is_pending || input) if (input)
{ {
if (!ostree_repo_stage_object (pull_data->repo, objtype, checksum, NULL, NULL, if (!ostree_repo_stage_object (pull_data->repo, objtype, checksum, NULL, NULL,
input, cancellable, error)) input, cancellable, error))
@ -608,14 +557,9 @@ fetch_and_store_object (OtPullData *pull_data,
} }
ret = TRUE; ret = TRUE;
if (out_was_stored)
*out_was_stored = is_stored;
out: out:
g_clear_object (&file_info); g_clear_object (&file_info);
g_clear_object (&input); g_clear_object (&input);
g_clear_object (&stored_path);
g_clear_object (&pending_path);
g_free (pack_checksum);
return ret; return ret;
} }
@ -623,17 +567,15 @@ static gboolean
fetch_and_store_metadata (OtPullData *pull_data, fetch_and_store_metadata (OtPullData *pull_data,
const char *checksum, const char *checksum,
OstreeObjectType objtype, OstreeObjectType objtype,
gboolean *out_was_stored,
GVariant **out_variant, GVariant **out_variant,
GCancellable *cancellable, GCancellable *cancellable,
GError **error) GError **error)
{ {
gboolean ret = FALSE; gboolean ret = FALSE;
gboolean ret_was_stored;
GVariant *ret_variant = NULL; GVariant *ret_variant = NULL;
if (!fetch_and_store_object (pull_data, checksum, objtype, if (!fetch_and_store_object (pull_data, checksum, objtype,
&ret_was_stored, cancellable, error)) cancellable, error))
goto out; goto out;
if (!ostree_repo_load_variant (pull_data->repo, objtype, checksum, if (!ostree_repo_load_variant (pull_data->repo, objtype, checksum,
@ -642,8 +584,6 @@ fetch_and_store_metadata (OtPullData *pull_data,
ret = TRUE; ret = TRUE;
ot_transfer_out_value (out_variant, &ret_variant); ot_transfer_out_value (out_variant, &ret_variant);
if (out_was_stored)
*out_was_stored = ret_was_stored;
out: out:
ot_clear_gvariant (&ret_variant); ot_clear_gvariant (&ret_variant);
return ret; return ret;
@ -658,7 +598,6 @@ fetch_and_store_file (OtPullData *pull_data,
gboolean ret = FALSE; gboolean ret = FALSE;
GInputStream *input = NULL; GInputStream *input = NULL;
GFile *stored_path = NULL; GFile *stored_path = NULL;
GFile *pending_path = NULL;
char *pack_checksum = NULL; char *pack_checksum = NULL;
GVariant *archive_metadata_container = NULL; GVariant *archive_metadata_container = NULL;
GVariant *archive_metadata = NULL; GVariant *archive_metadata = NULL;
@ -672,23 +611,11 @@ fetch_and_store_file (OtPullData *pull_data,
if (ostree_repo_get_mode (pull_data->repo) == OSTREE_REPO_MODE_BARE) if (ostree_repo_get_mode (pull_data->repo) == OSTREE_REPO_MODE_BARE)
{ {
if (!ostree_repo_find_object (pull_data->repo, OSTREE_OBJECT_TYPE_RAW_FILE, if (!ostree_repo_find_object (pull_data->repo, OSTREE_OBJECT_TYPE_RAW_FILE,
checksum, &stored_path, &pending_path, &pack_checksum, checksum, &stored_path, &pack_checksum,
NULL, cancellable, error)) NULL, cancellable, error))
goto out; goto out;
if (stored_path || pack_checksum) skip_archive_fetch = (stored_path || pack_checksum);
skip_archive_fetch = TRUE;
else if (pending_path != NULL)
{
skip_archive_fetch = TRUE;
if (!ostree_repo_stage_object (pull_data->repo, OSTREE_OBJECT_TYPE_RAW_FILE,
checksum, NULL, NULL, NULL, cancellable, error))
goto out;
}
else
skip_archive_fetch = FALSE;
g_clear_object (&stored_path);
} }
else else
{ {
@ -699,7 +626,7 @@ fetch_and_store_file (OtPullData *pull_data,
{ {
if (!fetch_object_if_not_stored (pull_data, checksum, if (!fetch_object_if_not_stored (pull_data, checksum,
OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META,
NULL, NULL, &input, cancellable, error)) &input, cancellable, error))
goto out; goto out;
if (input != NULL) if (input != NULL)
@ -721,8 +648,7 @@ fetch_and_store_file (OtPullData *pull_data,
{ {
if (!fetch_object_if_not_stored (pull_data, checksum, if (!fetch_object_if_not_stored (pull_data, checksum,
OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT,
NULL, NULL, &input, &input, cancellable, error))
cancellable, error))
goto out; goto out;
} }
@ -737,7 +663,6 @@ fetch_and_store_file (OtPullData *pull_data,
out: out:
g_free (pack_checksum); g_free (pack_checksum);
g_clear_object (&stored_path); g_clear_object (&stored_path);
g_clear_object (&pending_path);
g_clear_object (&input); g_clear_object (&input);
ot_clear_gvariant (&archive_metadata_container); ot_clear_gvariant (&archive_metadata_container);
ot_clear_gvariant (&archive_metadata); ot_clear_gvariant (&archive_metadata);
@ -747,7 +672,7 @@ fetch_and_store_file (OtPullData *pull_data,
} }
static gboolean static gboolean
fetch_and_store_tree_recurse (OtPullData *pull_data, fetch_and_store_tree_metadata_recurse (OtPullData *pull_data,
const char *rev, const char *rev,
GCancellable *cancellable, GCancellable *cancellable,
GError **error) GError **error)
@ -756,20 +681,14 @@ fetch_and_store_tree_recurse (OtPullData *pull_data,
GVariant *tree = NULL; GVariant *tree = NULL;
GVariant *files_variant = NULL; GVariant *files_variant = NULL;
GVariant *dirs_variant = NULL; GVariant *dirs_variant = NULL;
gboolean was_stored;
int i, n; int i, n;
GFile *stored_path = NULL; GFile *stored_path = NULL;
GFile *pending_path = NULL;
char *pack_checksum = NULL; char *pack_checksum = NULL;
if (!fetch_and_store_metadata (pull_data, rev, OSTREE_OBJECT_TYPE_DIR_TREE, if (!fetch_and_store_metadata (pull_data, rev, OSTREE_OBJECT_TYPE_DIR_TREE,
&was_stored, &tree, cancellable, error)) &tree, cancellable, error))
goto out; goto out;
if (was_stored)
log_verbose ("Already have tree %s", rev);
else
{
/* PARSE OSTREE_SERIALIZED_TREE_VARIANT */ /* PARSE OSTREE_SERIALIZED_TREE_VARIANT */
files_variant = g_variant_get_child_value (tree, 2); files_variant = g_variant_get_child_value (tree, 2);
dirs_variant = g_variant_get_child_value (tree, 3); dirs_variant = g_variant_get_child_value (tree, 3);
@ -787,8 +706,11 @@ fetch_and_store_tree_recurse (OtPullData *pull_data,
if (!ostree_validate_checksum_string (checksum, error)) if (!ostree_validate_checksum_string (checksum, error))
goto out; goto out;
if (!fetch_and_store_file (pull_data, checksum, cancellable, error)) {
goto out; char *duped_key = g_strdup (checksum);
g_hash_table_replace (pull_data->file_checksums_to_fetch,
duped_key, duped_key);
}
} }
n = g_variant_n_children (dirs_variant); n = g_variant_n_children (dirs_variant);
@ -809,13 +731,12 @@ fetch_and_store_tree_recurse (OtPullData *pull_data,
goto out; goto out;
if (!fetch_and_store_object (pull_data, meta_checksum, OSTREE_OBJECT_TYPE_DIR_META, if (!fetch_and_store_object (pull_data, meta_checksum, OSTREE_OBJECT_TYPE_DIR_META,
NULL, cancellable, error)) cancellable, error))
goto out; goto out;
if (!fetch_and_store_tree_recurse (pull_data, tree_checksum, cancellable, error)) if (!fetch_and_store_tree_metadata_recurse (pull_data, tree_checksum, cancellable, error))
goto out; goto out;
} }
}
ret = TRUE; ret = TRUE;
out: out:
@ -823,7 +744,6 @@ fetch_and_store_tree_recurse (OtPullData *pull_data,
ot_clear_gvariant (&files_variant); ot_clear_gvariant (&files_variant);
ot_clear_gvariant (&dirs_variant); ot_clear_gvariant (&dirs_variant);
g_clear_object (&stored_path); g_clear_object (&stored_path);
g_clear_object (&pending_path);
g_free (pack_checksum); g_free (pack_checksum);
return ret; return ret;
} }
@ -838,28 +758,22 @@ fetch_and_store_commit_recurse (OtPullData *pull_data,
GVariant *commit = NULL; GVariant *commit = NULL;
const char *tree_contents_checksum; const char *tree_contents_checksum;
const char *tree_meta_checksum; const char *tree_meta_checksum;
gboolean was_stored;
if (!fetch_and_store_metadata (pull_data, rev, OSTREE_OBJECT_TYPE_COMMIT, if (!fetch_and_store_metadata (pull_data, rev, OSTREE_OBJECT_TYPE_COMMIT,
&was_stored, &commit, cancellable, error)) &commit, cancellable, error))
goto out; goto out;
if (was_stored)
log_verbose ("Already have commit %s", rev);
else
{
/* PARSE OSTREE_SERIALIZED_COMMIT_VARIANT */ /* PARSE OSTREE_SERIALIZED_COMMIT_VARIANT */
g_variant_get_child (commit, 6, "&s", &tree_contents_checksum); g_variant_get_child (commit, 6, "&s", &tree_contents_checksum);
g_variant_get_child (commit, 7, "&s", &tree_meta_checksum); g_variant_get_child (commit, 7, "&s", &tree_meta_checksum);
if (!fetch_and_store_object (pull_data, tree_meta_checksum, OSTREE_OBJECT_TYPE_DIR_META, if (!fetch_and_store_object (pull_data, tree_meta_checksum, OSTREE_OBJECT_TYPE_DIR_META,
NULL, cancellable, error))
goto out;
if (!fetch_and_store_tree_recurse (pull_data, tree_contents_checksum,
cancellable, error)) cancellable, error))
goto out; goto out;
}
if (!fetch_and_store_tree_metadata_recurse (pull_data, tree_contents_checksum,
cancellable, error))
goto out;
ret = TRUE; ret = TRUE;
out: out:
@ -897,6 +811,29 @@ fetch_ref_contents (OtPullData *pull_data,
return ret; return ret;
} }
static gboolean
fetch_files (OtPullData *pull_data,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
GHashTableIter hash_iter;
gpointer key, value;
g_hash_table_iter_init (&hash_iter, pull_data->file_checksums_to_fetch);
while (g_hash_table_iter_next (&hash_iter, &key, &value))
{
const char *checksum = key;
if (!fetch_and_store_file (pull_data, checksum, cancellable, error))
goto out;
}
ret = TRUE;
out:
return ret;
}
static gboolean static gboolean
pull_one_commit (OtPullData *pull_data, pull_one_commit (OtPullData *pull_data,
const char *branch, const char *branch,
@ -941,6 +878,9 @@ pull_one_commit (OtPullData *pull_data,
if (!fetch_and_store_commit_recurse (pull_data, rev, cancellable, error)) if (!fetch_and_store_commit_recurse (pull_data, rev, cancellable, error))
goto out; goto out;
if (!fetch_files (pull_data, cancellable, error))
goto out;
if (!ostree_repo_commit_transaction (pull_data->repo, cancellable, error)) if (!ostree_repo_commit_transaction (pull_data->repo, cancellable, error))
goto out; goto out;
@ -1026,7 +966,6 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error)
char *path = NULL; char *path = NULL;
char *baseurl = NULL; char *baseurl = NULL;
char *summary_data = NULL; char *summary_data = NULL;
SoupURI *base_uri = NULL;
SoupURI *summary_uri = NULL; SoupURI *summary_uri = NULL;
GKeyFile *config = NULL; GKeyFile *config = NULL;
GCancellable *cancellable = NULL; GCancellable *cancellable = NULL;
@ -1048,6 +987,7 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error)
memset (pull_data, 0, sizeof (*pull_data)); memset (pull_data, 0, sizeof (*pull_data));
pull_data->repo = repo; pull_data->repo = repo;
pull_data->file_checksums_to_fetch = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
if (argc < 2) if (argc < 2)
{ {
@ -1092,7 +1032,7 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error)
} }
else else
{ {
summary_uri = soup_uri_copy (base_uri); summary_uri = soup_uri_copy (pull_data->base_uri);
path = g_build_filename (soup_uri_get_path (summary_uri), "refs", "summary", NULL); path = g_build_filename (soup_uri_get_path (summary_uri), "refs", "summary", NULL);
soup_uri_set_path (summary_uri, path); soup_uri_set_path (summary_uri, path);
@ -1122,6 +1062,7 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error)
g_free (baseurl); g_free (baseurl);
g_free (summary_data); g_free (summary_data);
g_free (branch_rev); g_free (branch_rev);
ot_clear_hashtable (&(pull_data->file_checksums_to_fetch));
if (context) if (context)
g_option_context_free (context); g_option_context_free (context);
g_clear_object (&pull_data->session); g_clear_object (&pull_data->session);