packaging
/
ostree
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

repo: Further optimize `ostree_repo_list_objects_set()` 

In a prior change we discovered that for bad historical reasons
libostree was returning a mapping "object type+checksum" => "metadata"
but the "metadata" was redundant and pointless.

Optimize the prune API to use a (currently internal) object listing
API which returns a set, not a map.  This allows `GHashTable` to
avoid allocating a separate array for the values, neatly cutting
memory usage in half (from ~13MB to ~6MB) on my test case of a
dry-run prune of a FCOS build.
This commit is contained in:
Colin Walters 2022-06-07 21:41:08 -04:00
parent ef114db341
commit a71915e436
3 changed files with 84 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/libostree/ostree-repo-private.h
View File

@ -513,6 +513,12 @@ _ostree_repo_verify_bindings (const char *collection_id,
GVariant *commit, GVariant *commit,
GError **error); GError **error);
GHashTable *
ostree_repo_list_objects_set (OstreeRepo *self,
OstreeRepoListObjectsFlags flags,
GCancellable *cancellable,
GError **error);
/** /**
* OstreeRepoAutoTransaction: * OstreeRepoAutoTransaction:
* *

14
src/libostree/ostree-repo-prune.c
View File

@ -280,15 +280,8 @@ repo_prune_internal (OstreeRepo *self,
g_autoptr(GHashTable) reachable_owned = g_hash_table_ref (options->reachable); g_autoptr(GHashTable) reachable_owned = g_hash_table_ref (options->reachable);
data.reachable = reachable_owned; data.reachable = reachable_owned;
GLNX_HASH_TABLE_FOREACH_KV (objects, GVariant*, serialized_key, GVariant*, objdata) GLNX_HASH_TABLE_FOREACH (objects, GVariant*, serialized_key)
{ {
gboolean is_loose;
g_variant_get_child (objdata, 0, "b", &is_loose);
if (!is_loose)
continue;
if (!maybe_prune_loose_object (&data, options->flags, serialized_key, if (!maybe_prune_loose_object (&data, options->flags, serialized_key,
cancellable, error)) cancellable, error))
return FALSE; return FALSE;
@ -444,8 +437,9 @@ ostree_repo_prune (OstreeRepo *self,
return FALSE; return FALSE;
} }
if (!ostree_repo_list_objects (self, OSTREE_REPO_LIST_OBJECTS_ALL | OSTREE_REPO_LIST_OBJECTS_NO_PARENTS, objects = ostree_repo_list_objects_set (self, OSTREE_REPO_LIST_OBJECTS_ALL | OSTREE_REPO_LIST_OBJECTS_NO_PARENTS,
&objects, cancellable, error)) cancellable, error);
if (!objects)
return FALSE; return FALSE;
if (!refs_only) if (!refs_only)

110
src/libostree/ostree-repo.c
View File

@ -3974,8 +3974,11 @@ list_loose_objects_at (OstreeRepo *self,
key = ostree_object_name_serialize (buf, objtype); key = ostree_object_name_serialize (buf, objtype);
/* transfer ownership */ /* transfer ownership */
g_hash_table_replace (inout_objects, g_variant_ref_sink (key), if (dummy_value)
g_variant_ref (dummy_value)); g_hash_table_replace (inout_objects, g_variant_ref_sink (key),
g_variant_ref (dummy_value));
else
g_hash_table_add (inout_objects, g_variant_ref_sink (key));
} }
return TRUE; return TRUE;
@ -3983,15 +3986,13 @@ list_loose_objects_at (OstreeRepo *self,
static gboolean static gboolean
list_loose_objects (OstreeRepo *self, list_loose_objects (OstreeRepo *self,
GVariant *dummy_value,
GHashTable *inout_objects, GHashTable *inout_objects,
const char *commit_starting_with, const char *commit_starting_with,
GCancellable *cancellable, GCancellable *cancellable,
GError **error) GError **error)
{ {
static const gchar hexchars[] = "0123456789abcdef"; static const gchar hexchars[] = "0123456789abcdef";
// For unfortunate historical reasons we emit this dummy value.
g_autoptr(GVariant) dummy_loose_object_variant =
g_variant_ref_sink (g_variant_new ("(b@as)", TRUE, g_variant_new_strv (NULL, 0)));
for (guint c = 0; c < 256; c++) for (guint c = 0; c < 256; c++)
{ {
@ -3999,7 +4000,7 @@ list_loose_objects (OstreeRepo *self,
buf[0] = hexchars[c >> 4]; buf[0] = hexchars[c >> 4];
buf[1] = hexchars[c & 0xF]; buf[1] = hexchars[c & 0xF];
buf[2] = '\0'; buf[2] = '\0';
if (!list_loose_objects_at (self, dummy_loose_object_variant, if (!list_loose_objects_at (self, dummy_value,
inout_objects, self->objects_dir_fd, buf, inout_objects, self->objects_dir_fd, buf,
commit_starting_with, commit_starting_with,
cancellable, error)) cancellable, error))
@ -4905,6 +4906,65 @@ ostree_repo_load_commit (OstreeRepo *self,
out_variant, NULL, NULL, out_state, NULL, error); out_variant, NULL, NULL, out_state, NULL, error);
} }
static GHashTable *
repo_list_objects_impl (OstreeRepo *self,
OstreeRepoListObjectsFlags flags,
GVariant *dummy_value,
GCancellable *cancellable,
GError **error)
{
g_assert (error == NULL || *error == NULL);
g_assert (self->inited);
g_autoptr(GHashTable) ret_objects =
g_hash_table_new_full (ostree_hash_object_name, g_variant_equal,
(GDestroyNotify) g_variant_unref,
dummy_value ? (GDestroyNotify) g_variant_unref : NULL);
if (flags & OSTREE_REPO_LIST_OBJECTS_ALL)
flags |= (OSTREE_REPO_LIST_OBJECTS_LOOSE | OSTREE_REPO_LIST_OBJECTS_PACKED);
if (flags & OSTREE_REPO_LIST_OBJECTS_LOOSE)
{
if (!list_loose_objects (self, dummy_value, ret_objects, NULL, cancellable, error))
return FALSE;
if ((flags & OSTREE_REPO_LIST_OBJECTS_NO_PARENTS) == 0 && self->parent_repo)
{
if (!list_loose_objects (self->parent_repo, dummy_value, ret_objects, NULL, cancellable, error))
return FALSE;
}
}
if (flags & OSTREE_REPO_LIST_OBJECTS_PACKED)
{
/* Nothing for now... */
}
return g_steal_pointer (&ret_objects);
}
/* A currently-internal version of ostree_repo_list_objects which returns
* a set, and not a map (with a useless value).
*/
GHashTable *
ostree_repo_list_objects_set (OstreeRepo *self,
OstreeRepoListObjectsFlags flags,
GCancellable *cancellable,
GError **error)
{
return repo_list_objects_impl (self, flags, NULL, cancellable, error);
}
/* For unfortunate historical reasons we emit this dummy value.
* It was intended to provide additional information about the object (e.g. "is in a pack file")
* but we ended up not shipping pack files.
*/
static GVariant *
get_dummy_list_objects_variant (void)
{
return g_variant_ref_sink (g_variant_new ("(b@as)", TRUE, g_variant_new_strv (NULL, 0)));
}
/** /**
* ostree_repo_list_objects: * ostree_repo_list_objects:
* @self: Repo * @self: Repo
@ -4928,34 +4988,11 @@ ostree_repo_list_objects (OstreeRepo *self,
GCancellable *cancellable, GCancellable *cancellable,
GError **error) GError **error)
{ {
g_return_val_if_fail (error == NULL || *error == NULL, FALSE); g_autoptr(GVariant) dummy_value = get_dummy_list_objects_variant ();
g_return_val_if_fail (self->inited, FALSE); g_autoptr(GHashTable) ret = repo_list_objects_impl (self, flags, dummy_value, cancellable, error);
if (!ret)
g_autoptr(GHashTable) ret_objects = return FALSE;
g_hash_table_new_full (ostree_hash_object_name, g_variant_equal, ot_transfer_out_value (out_objects, &ret);
(GDestroyNotify) g_variant_unref,
(GDestroyNotify) g_variant_unref);
if (flags & OSTREE_REPO_LIST_OBJECTS_ALL)
flags |= (OSTREE_REPO_LIST_OBJECTS_LOOSE | OSTREE_REPO_LIST_OBJECTS_PACKED);
if (flags & OSTREE_REPO_LIST_OBJECTS_LOOSE)
{
if (!list_loose_objects (self, ret_objects, NULL, cancellable, error))
return FALSE;
if ((flags & OSTREE_REPO_LIST_OBJECTS_NO_PARENTS) == 0 && self->parent_repo)
{
if (!list_loose_objects (self->parent_repo, ret_objects, NULL, cancellable, error))
return FALSE;
}
}
if (flags & OSTREE_REPO_LIST_OBJECTS_PACKED)
{
/* Nothing for now... */
}
ot_transfer_out_value (out_objects, &ret_objects);
return TRUE; return TRUE;
} }
@ -4987,13 +5024,14 @@ ostree_repo_list_commit_objects_starting_with (OstreeRepo *self
g_hash_table_new_full (ostree_hash_object_name, g_variant_equal, g_hash_table_new_full (ostree_hash_object_name, g_variant_equal,
(GDestroyNotify) g_variant_unref, (GDestroyNotify) g_variant_unref,
(GDestroyNotify) g_variant_unref); (GDestroyNotify) g_variant_unref);
g_autoptr(GVariant) dummy_loose_object_variant = get_dummy_list_objects_variant ();
if (!list_loose_objects (self, ret_commits, start, cancellable, error)) if (!list_loose_objects (self, dummy_loose_object_variant, ret_commits, start, cancellable, error))
return FALSE; return FALSE;
if (self->parent_repo) if (self->parent_repo)
{ {
if (!list_loose_objects (self->parent_repo, ret_commits, start, if (!list_loose_objects (self->parent_repo, dummy_loose_object_variant, ret_commits, start,
cancellable, error)) cancellable, error))
return FALSE; return FALSE;
} }